Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
⏱️ feat: Run Scheduled Chats Through Durable Agent Triggers (#14939)
Danny Avilac5276fc63d26
diff --git a/.env.example b/.env.exampleindex 7091d3f9ad..22b4265882 100644--- a/.env.example+++ b/.env.example@@ -1059,6 +1059,11 @@ HELP_AND_FAQ_URL=https://librechat.ai# Enable Redis for resumable LLM streams (defaults to USE_REDIS value if not set)# Set to false to use in-memory storage for streams while keeping Redis for other caches# USE_REDIS_STREAMS=true+# Scheduled chats require shared Redis streams in multi-replica deployments.+# Set this only when the deployment truly runs one LibreChat process without Redis.+# SCHEDULES_SINGLE_PROCESS=true+# Emergency global stop for both automatic and manual scheduled runs.+# SCHEDULES_DISABLED=true# Generation stream wire/state protocol. Redis-backed deployments default to the# rolling-upgrade-safe v1 protocol when this is unset; in-memory deployments use v2.diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.jsindex 9821f47db7..040b2ae21b 100644--- a/api/server/controllers/UserController.js+++ b/api/server/controllers/UserController.js@@ -33,6 +33,11 @@ const {purgeAgentTriggerDeliveriesForUser,} = require('~/server/services/Agents/triggers');const { getAppConfig } = require('~/server/services/Config');+const { randomUUID } = require('node:crypto');+const {+ quiesceUserSchedules,+ restoreUserSchedulesFromDeletion,+} = require('~/server/services/Schedules');const { getLogStores } = require('~/cache');const db = require('~/models');@@ -360,6 +365,7 @@ const updateUserPluginsController = async (req, res) => {const deleteUserController = async (req, res) => {const { user } = req;let triggerDeletionFence;+ let scheduleSuspensionToken;let userDeleted = false;try {@@ -395,6 +401,14 @@ const deleteUserController = async (req, res) => {}await drainAgentTriggerDeliveriesForUser(user.id);await subagentThreadTaskStore.cancelAndDrainForOwner(user.id, user.tenantId);+ // Reversibly suspend the user's schedules under a per-attempt token BEFORE draining.+ // A later cascade step (or this drain) can still fail and cancel the deletion, and the+ // catch below restores exactly this attempt's rows — so a failed deletion never leaves+ // a live user with silently disabled, erasure-eligible schedules.+ scheduleSuspensionToken = randomUUID();+ if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) {+ throw new Error('Scheduled executions could not be confirmed stopped');+ }const activeAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(user.id,user.tenantId,@@ -446,6 +460,7 @@ const deleteUserController = async (req, res) => {await db.deleteTokens({ userId: user.id });await db.removeUserFromAllGroups(user.id);await db.deleteAclEntries({ principalId: user._id });+ await db.deleteSchedulesByUser(user.id);const deleteResult = await db.deleteUserById(user.id);if (deleteResult.deletedCount !== 1) {throw new Error('User disappeared before account deletion could commit');@@ -455,6 +470,33 @@ const deleteUserController = async (req, res) => {logger.info(`User deleted account. Email: ${user.email} ID: ${user.id}`);res.status(200).send({ message: 'User deleted' });} catch (err) {+ // The account survives this failed attempt, so its schedules must too: restore the+ // exact rows this attempt suspended (re-enabled/re-armed from their snapshot). Fenced+ // to the token, so a schedule the owner deleted meanwhile is not resurrected. A+ // successful deletion never reaches here (userDeleted short-circuits it).+ //+ // RESTORE BEFORE RELEASING THE DELETION FENCE. That fence is what refuses new schedule+ // writes/claims for this user; releasing it first opens a window where an owner PATCH+ // could edit a still-suspended row and then have its enabled/next-run state overwritten+ // by this older snapshot, and where a second deletion attempt could re-suspend these+ // rows under a new token — making this restore a no-op and stranding the disabled+ // snapshot permanently.+ if (scheduleSuspensionToken != null && !userDeleted) {+ try {+ await restoreUserSchedulesFromDeletion(user.id, scheduleSuspensionToken);+ } catch (restoreError) {+ // Every retry is exhausted at this point. The fence is still released below on+ // purpose: retaining it would refuse this live account's schedule writes AND make+ // `beginAgentTriggerUserDeletion` report `in_progress` forever, blocking the retry+ // that is the convergence path — a later attempt re-suspends by ADOPTING this+ // snapshot, so its cancel restores these exact rows. Log the token so the state is+ // recoverable directly if that never happens.+ logger.error(+ `[deleteUserController] Failed to restore suspended schedules after a cancelled deletion; they remain disabled for user ${user.id} under suspension token ${scheduleSuspensionToken}`,+ restoreError,+ );+ }+ }if (triggerDeletionFence != null && !userDeleted) {try {await cancelAgentTriggerUserPurge(user.id, triggerDeletionFence);diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.jsindex c9a3f6eb44..89af705a56 100644--- a/api/server/controllers/UserController.spec.js+++ b/api/server/controllers/UserController.spec.js@@ -8,6 +8,8 @@ const mockPrepareAgentTriggerUserPurge = jest.fn().mockResolvedValue(undefined);const mockCancelAgentTriggerUserPurge = jest.fn().mockResolvedValue(true);const mockPurgeAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);const mockCancelAndDrainSubagentThreads = jest.fn().mockResolvedValue(undefined);+const mockQuiesceUserSchedules = jest.fn().mockResolvedValue(true);+const mockRestoreUserSchedules = jest.fn().mockResolvedValue(undefined);jest.mock('@librechat/data-schemas', () => {const actual = jest.requireActual('@librechat/data-schemas');@@ -30,6 +32,7 @@ jest.mock('~/models', () => {deleteAllAgentApiKeys: jest.fn().mockResolvedValue(undefined),deleteConversationTags: jest.fn().mockResolvedValue(undefined),deleteAllUserMemories: jest.fn().mockResolvedValue(undefined),+ deleteSchedulesByUser: jest.fn().mockResolvedValue(undefined),deleteTransactions: jest.fn().mockResolvedValue(undefined),deleteAclEntries: jest.fn().mockResolvedValue(undefined),updateUserPlugins: jest.fn(),@@ -100,6 +103,11 @@ jest.mock('~/server/services/Endpoints/agents/subagentThreadStore', () => ({cancelAndDrainForOwner: (...args) => mockCancelAndDrainSubagentThreads(...args),}));+jest.mock('~/server/services/Schedules', () => ({+ quiesceUserSchedules: (...args) => mockQuiesceUserSchedules(...args),+ restoreUserSchedulesFromDeletion: (...args) => mockRestoreUserSchedules(...args),+}));+jest.mock('~/server/services/Files/process', () => ({processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),}));@@ -160,6 +168,7 @@ describe('verifyEmailController', () => {beforeEach(() => {jest.clearAllMocks();+ mockQuiesceUserSchedules.mockResolvedValue(true);});it('returns the generic verification error message from service failures', async () => {@@ -343,6 +352,7 @@ describe('deleteUserController', () => {beforeEach(() => {jest.clearAllMocks();+ mockQuiesceUserSchedules.mockResolvedValue(true);});it('should return 200 on successful deletion', async () => {@@ -361,6 +371,7 @@ describe('deleteUserController', () => {);expect(mockDrainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());expect(mockCancelAndDrainSubagentThreads).toHaveBeenCalledWith(userId.toString(), undefined);+ expect(mockQuiesceUserSchedules).toHaveBeenCalledWith(userId.toString(), expect.any(String));expect(beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],);@@ -371,6 +382,9 @@ describe('deleteUserController', () => {mockCancelAndDrainSubagentThreads.mock.invocationCallOrder[0],);expect(mockCancelAndDrainSubagentThreads.mock.invocationCallOrder[0]).toBeLessThan(+ mockQuiesceUserSchedules.mock.invocationCallOrder[0],+ );+ expect(mockQuiesceUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(deleteMessages.mock.invocationCallOrder[0],);expect(deleteMessages.mock.invocationCallOrder[0]).toBeLessThan(@@ -382,6 +396,8 @@ describe('deleteUserController', () => {expect(mockPurgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();+ // A successful deletion hard-deletes the schedules; it must never restore them.+ expect(mockRestoreUserSchedules).not.toHaveBeenCalled();});it('aborts generations admitted before the deletion fence before erasing messages', async () => {@@ -428,6 +444,17 @@ describe('deleteUserController', () => {expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);expect(deleteUserById).not.toHaveBeenCalled();+ // Account survives -> its suspended schedules are restored under the quiesce token.+ expect(mockRestoreUserSchedules).toHaveBeenCalledWith(+ userIdString,+ mockQuiesceUserSchedules.mock.calls[0][1],+ );+ // BEFORE the deletion fence is released: that fence is what refuses new schedule writes,+ // so restoring after it would let an owner PATCH — or a second deletion attempt+ // re-suspending under a new token — race the restore and strand the disabled snapshot.+ expect(mockRestoreUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(+ cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],+ );});it('fails closed before data cleanup when detached subagents do not drain', async () => {@@ -453,6 +480,41 @@ describe('deleteUserController', () => {expect(deleteUserById).not.toHaveBeenCalled();});+ it('fails closed and releases deletion fences when schedules cannot be quiesced', async () => {+ const userId = new mongoose.Types.ObjectId();+ const userIdString = userId.toString();+ mockQuiesceUserSchedules.mockResolvedValueOnce(false);+ const req = {+ user: {+ id: userIdString,+ _id: userId,+ email: '[email protected]',+ tenantId: 'tenant-1',+ },+ };++ await deleteUserController(req, mockRes);++ expect(mockRes.status).toHaveBeenCalledWith(500);+ expect(deleteMessages).not.toHaveBeenCalled();+ expect(mockGetActiveJobIdsForUser).not.toHaveBeenCalled();+ const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);+ expect(deleteUserById).not.toHaveBeenCalled();+ // Account survives -> its suspended schedules are restored under the quiesce token.+ expect(mockRestoreUserSchedules).toHaveBeenCalledWith(+ userIdString,+ mockQuiesceUserSchedules.mock.calls[0][1],+ );+ // BEFORE the deletion fence is released: that fence is what refuses new schedule writes,+ // so restoring after it would let an owner PATCH — or a second deletion attempt+ // re-suspending under a new token — race the restore and strand the disabled snapshot.+ expect(mockRestoreUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(+ cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],+ );+ });+it('should remove the user from all groups via $pullAll', async () => {const userId = new mongoose.Types.ObjectId();const userIdStr = userId.toString();diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.jsindex f6777172a6..c64a40de8d 100644--- a/api/server/controllers/__tests__/deleteUser.spec.js+++ b/api/server/controllers/__tests__/deleteUser.spec.js@@ -28,6 +28,8 @@ const mockPurgeAgentTriggerDeliveriesForUser = jest.fn();const mockBeginAgentTriggerUserDeletion = jest.fn();const mockCancelAgentTriggerUserDeletion = jest.fn();const mockCancelAndDrainSubagentThreads = jest.fn();+const mockQuiesceUserSchedules = jest.fn();+const mockDeleteSchedulesByUser = jest.fn();jest.mock('@librechat/data-schemas', () => ({logger: { error: jest.fn(), info: jest.fn() },@@ -85,6 +87,7 @@ jest.mock('~/models', () => ({deleteTokens: jest.fn(),removeUserFromAllGroups: jest.fn(),deleteAclEntries: jest.fn(),+ deleteSchedulesByUser: (...args) => mockDeleteSchedulesByUser(...args),getSoleOwnedResourceIds: jest.fn().mockResolvedValue([]),}));@@ -127,6 +130,10 @@ jest.mock('~/server/services/Endpoints/agents/subagentThreadStore', () => ({cancelAndDrainForOwner: (...args) => mockCancelAndDrainSubagentThreads(...args),}));+jest.mock('~/server/services/Schedules', () => ({+ quiesceUserSchedules: (...args) => mockQuiesceUserSchedules(...args),+}));+jest.mock('~/server/services/Config', () => ({getAppConfig: jest.fn(),}));@@ -171,6 +178,8 @@ function stubDeletionMocks() {mockBeginAgentTriggerUserDeletion.mockResolvedValue('acquired');mockCancelAgentTriggerUserDeletion.mockResolvedValue(true);mockCancelAndDrainSubagentThreads.mockResolvedValue();+ mockQuiesceUserSchedules.mockResolvedValue(true);+ mockDeleteSchedulesByUser.mockResolvedValue();}beforeEach(() => {diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.jsindex c2f1173257..7428c2287f 100644--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js@@ -47,6 +47,8 @@ jest.mock('@librechat/data-schemas', () => ({jest.mock('@librechat/api', () => ({sendEvent: jest.fn(),+ isScheduleFireRequest: jest.fn(() => false),+ exemptFromConcurrencyLimiter: jest.fn(() => false),toPendingSteer: jest.fn((item) => item),isSteerPreemptSupported: jest.fn(() => true),buildRecoveredSteerPayload: jest.fn(() => null),diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex e659242977..5b6fb7dab6 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -64,6 +64,10 @@ const mockGetConvo = jest.fn();const mockGetMessages = jest.fn();const mockSaveMessage = jest.fn();const mockIsAgentTriggerPrincipalActive = jest.fn();+const mockIsScheduleFireRequest = jest.fn();+const mockExemptFromConcurrencyLimiter = jest.fn();+const mockRecordScheduleOutcome = jest.fn();+const mockIsScheduleLive = jest.fn();const mockDeleteAgentCheckpoint = jest.fn();const mockStartupTelemetry = {mark: jest.fn(),@@ -132,6 +136,8 @@ jest.mock('@librechat/data-schemas', () => ({jest.mock('@librechat/api', () => ({sendEvent: jest.fn(),+ isScheduleFireRequest: (...args) => mockIsScheduleFireRequest(...args),+ exemptFromConcurrencyLimiter: (...args) => mockExemptFromConcurrencyLimiter(...args),toPendingSteer: jest.fn((item) => item),/** Recorded onto the job so the steer route can honour the OWNING replica's* seal capability rather than its own probe. */@@ -214,6 +220,11 @@ jest.mock('~/models', () => ({isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),}));+jest.mock('~/server/services/Schedules', () => ({+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),+ isScheduleLive: (...args) => mockIsScheduleLive(...args),+}));+const AgentController = require('../request');const { ErrorTypes } = require('librechat-data-provider');const { disposeClient: mockDisposeClient } = require('~/server/cleanup');@@ -250,6 +261,12 @@ describe('ResumableAgentController resume metadata', () => {mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });mockGetMessages.mockResolvedValue([]);mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);+ mockIsScheduleFireRequest.mockImplementation((req) => req?._isScheduledFire === true);+ mockExemptFromConcurrencyLimiter.mockImplementation(+ (req) => req?._isScheduledFire === true && req?._isManualScheduledFire !== true,+ );+ mockRecordScheduleOutcome.mockResolvedValue(true);+ mockIsScheduleLive.mockResolvedValue(true);mockGenerationJobManager.createJob.mockResolvedValue({createdAt: 1000,metadata: {@@ -286,7 +303,7 @@ describe('ResumableAgentController resume metadata', () => {}),);mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);- mockGenerationJobManager.completeJob.mockResolvedValue(undefined);+ mockGenerationJobManager.completeJob.mockResolvedValue(true);mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);@@ -797,6 +814,73 @@ describe('ResumableAgentController resume metadata', () => {expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');});+ it('rejects a superseded automatic occurrence after durable job creation and preserves its outcome', async () => {+ const conversationId = 'scheduled-conversation-123';+ mockGenerationJobManager.claimGeneration.mockResolvedValue(+ wonGenerationClaim({ streamId: conversationId, conversationId }),+ );+ mockIsScheduleLive.mockResolvedValue(false);+ const req = {+ _isScheduledFire: true,+ _isManualScheduledFire: false,+ user: { id: 'user-123' },+ body: {+ text: 'Run the scheduled digest.',+ messageId: 'scheduled-user-message',+ clientRequestId: 'sched:schedule-1:2026-08-17T12:00:00-000Z',+ conversationId: 'new',+ newConversationId: conversationId,+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleConfigRevision: 7,+ endpointOption: { endpoint: 'agents', agent_id: 'agent-1' },+ },+ config: {},+ };+ const res = createResumableResponse();+ const initializeClient = jest.fn();++ await AgentController(req, res, jest.fn(), initializeClient, null);++ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(+ conversationId,+ 'user-123',+ conversationId,+ expect.objectContaining({+ initialMetadata: expect.objectContaining({+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleConfigRevision: 7,+ preserveForScheduleReconcile: true,+ }),+ }),+ );+ expect(mockIsScheduleLive).toHaveBeenCalledWith('schedule-1', 7, {+ automatic: true,+ policy: true,+ });+ expect(initializeClient).not.toHaveBeenCalled();+ expect(res.status).toHaveBeenCalledWith(409);+ expect(res.json).toHaveBeenCalledWith({+ status: 409,+ code: 'SCHEDULE_NO_LONGER_ACTIVE',+ error: 'This scheduled occurrence is no longer active',+ generationProtocolVersion: 1,+ });+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ streamId: conversationId,+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId,+ clearConversationId: false,+ error: 'This scheduled occurrence is no longer active',+ });+ expect(mockDecrementPendingRequest).not.toHaveBeenCalled();+ });+it('does not start a provider when account deletion or replacement wins the startup CAS', async () => {mockGenerationJobManager.beginProviderExecution.mockResolvedValue(false);const req = {diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.jsindex eb30808054..034c53189e 100644--- a/api/server/controllers/agents/__tests__/resume.spec.js+++ b/api/server/controllers/agents/__tests__/resume.spec.js@@ -60,6 +60,7 @@ const mockGenerationJobManager = {publishTerminalClaim: jest.fn(),finishTerminalJob: jest.fn(),completeJob: jest.fn(),+ abortJob: jest.fn(),beginProviderExecution: jest.fn(),markProviderExecutionDrained: jest.fn(),failPausePersistence: jest.fn(),@@ -82,6 +83,12 @@ const mockGetMessages = jest.fn();const mockDisposeClient = jest.fn();const mockGetMCPRequestContext = jest.fn();const mockCleanupMCPRequestContextForReq = jest.fn();+const mockRecordScheduleOutcome = jest.fn();+const mockIsScheduleLive = jest.fn();+const mockClaimScheduleResume = jest.fn();+const mockReleaseScheduleResumeClaim = jest.fn();+const mockFinalizeScheduleResumeClaim = jest.fn();+const mockReleaseScheduleResumeFence = jest.fn();jest.mock('@librechat/data-schemas', () => ({...jest.requireActual('@librechat/data-schemas'),@@ -104,6 +111,15 @@ jest.mock('~/models', () => ({getMessages: (...args) => mockGetMessages(...args),}));+jest.mock('~/server/services/Schedules', () => ({+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),+ claimScheduleResume: (...args) => mockClaimScheduleResume(...args),+ releaseScheduleResumeClaim: (...args) => mockReleaseScheduleResumeClaim(...args),+ finalizeScheduleResumeClaim: (...args) => mockFinalizeScheduleResumeClaim(...args),+ releaseScheduleResumeFence: (...args) => mockReleaseScheduleResumeFence(...args),+ isScheduleLive: (...args) => mockIsScheduleLive(...args),+}));+jest.mock('~/server/cleanup', () => ({disposeClient: (...args) => mockDisposeClient(...args),}));@@ -248,12 +264,23 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {);mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);mockGenerationJobManager.completeJob.mockResolvedValue(true);+ mockGenerationJobManager.abortJob.mockResolvedValue({ success: true });mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);+ mockRecordScheduleOutcome.mockResolvedValue(true);+ mockIsScheduleLive.mockResolvedValue(true);+ mockClaimScheduleResume.mockResolvedValue({+ capacitySlot: 0,+ claimToken: 'resume-token',+ leaseBy: 'resume:resume-token',+ });+ mockReleaseScheduleResumeClaim.mockResolvedValue(true);+ mockFinalizeScheduleResumeClaim.mockResolvedValue(true);+ mockReleaseScheduleResumeFence.mockResolvedValue(undefined);// `decrementPendingRequest` runs in the controller's `finally` on every// post-ACK path, so resolving on it signals the async continuation is done.@@ -304,6 +331,222 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {...extra,});+ describe('scheduled occurrence lifecycle', () => {+ const scheduledFor = '2026-08-17T12:00:00.000Z';+ const makeScheduledJob = () =>+ makeToolApprovalJob({+ metadata: {+ scheduleId: 'schedule-1',+ scheduledFor,+ scheduleConfigRevision: 4,+ checkpointNamespace: '1000',+ },+ });++ it('stops and settles an occurrence that became inactive while awaiting approval', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockIsScheduleLive.mockResolvedValue(false);++ const res = await post(approveBody());++ expect(res.status).toBe(409);+ expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });+ expect(mockIsScheduleLive).toHaveBeenCalledWith('schedule-1', 4, {+ automatic: true,+ policy: true,+ });+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {+ expectedCreatedAt: 1000,+ awaitProviderDrain: true,+ });+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId: CONVO_ID,+ error: 'Schedule was disabled, changed, or deleted before approval',+ });+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(+ CONVO_ID,+ { type: 'mongo' },+ undefined,+ { checkpointNamespace: '1000' },+ );+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();+ });++ it('fails closed without settling or pruning when provider drain cannot be confirmed', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockIsScheduleLive.mockResolvedValue(false);+ mockGenerationJobManager.abortJob.mockRejectedValue(new Error('drain timed out'));++ const res = await post(approveBody());++ expect(res.status).toBe(503);+ expect(res.headers['retry-after']).toBe('1');+ expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });+ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();+ });++ it('records success after resumed persistence and before terminal publication', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());++ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;++ expect(mockClaimScheduleResume).toHaveBeenCalledWith('schedule-1', scheduledFor, {+ expectedConfigRevision: 4,+ automatic: true,+ });+ expect(mockFinalizeScheduleResumeClaim).toHaveBeenCalledWith(+ 'schedule-1',+ 'resume-token',+ 'resume:resume-token',+ { expectedConfigRevision: 4, automatic: true },+ );++ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'success',+ conversationId: CONVO_ID,+ });+ expect(mockRecordScheduleOutcome.mock.invocationCallOrder[0]).toBeLessThan(+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],+ );+ });++ it('keeps the approval paused when global scheduled-run capacity is full', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockClaimScheduleResume.mockResolvedValue({ conflict: 'capacity' });++ const res = await post(approveBody());++ expect(res.status).toBe(429);+ expect(res.headers['retry-after']).toBe('1');+ expect(res.body).toMatchObject({ code: 'SCHEDULE_CAPACITY' });+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);+ });++ it('rolls back scheduled capacity when the approval CAS does not consume the action', async () => {+ const job = makeScheduledJob();+ mockGenerationJobManager.getJob.mockResolvedValue(job);+ mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);++ const res = await post(approveBody());++ expect(res.status).toBe(409);+ expect(mockReleaseScheduleResumeClaim).toHaveBeenCalledWith('schedule-1', scheduledFor, 0);+ expect(mockReleaseScheduleResumeFence).toHaveBeenCalledWith(+ 'schedule-1',+ 'resume:resume-token',+ );+ });++ it('stops before provider execution when an edit wins the final resume handoff', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockFinalizeScheduleResumeClaim.mockResolvedValue(false);++ const res = await post(approveBody());++ expect(res.status).toBe(409);+ expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalled();+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {+ expectedCreatedAt: 1000,+ awaitProviderDrain: true,+ });+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId: CONVO_ID,+ error: 'Schedule was disabled, changed, or deleted before approval',+ });+ expect(mockInitializeClient).not.toHaveBeenCalled();+ });++ it('settles a scheduled continuation stopped during its resumed segment', async () => {+ const job = makeScheduledJob();+ mockGenerationJobManager.getJob.mockResolvedValue(job);+ mockInitializeClient.mockImplementation(async () => {+ job.abortController.abort();+ return { client: makeClient(), userMCPAuthMap: {} };+ });++ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;+ await flush();++ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId: CONVO_ID,+ error: 'Scheduled run was stopped',+ });+ });++ it('records an empty-preempt resumed segment as interrupted, not successful', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockInitializeClient.mockResolvedValue({+ client: makeClient({+ run: {+ getPreemptStats: () => ({ emptyBoundaries: 1 }),+ getHaltReason: () => 'preempt_incomplete',+ },+ }),+ userMCPAuthMap: {},+ });++ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;+ await flush();++ expect(mockSaveMessage).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({ unfinished: true }),+ expect.anything(),+ );+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId: CONVO_ID,+ error: 'Scheduled run was interrupted before completion',+ });+ });++ it('does not overwrite the terminal winner when failed-resume finalization loses its CAS', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());+ mockGenerationJobManager.completeJob.mockResolvedValue(false);+ mockInitializeClient.mockRejectedValue(new Error('resume reconstruction failed'));++ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;++ expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();+ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();+ });+ });+describe('temporal context restore', () => {it('restores req.conversationCreatedAt from the convo before initializeClient', async () => {// Temporal prompt vars must resolve against the paused anchor, not resume wall-clock.diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex 3da58904e8..bb9201ab89 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -2884,7 +2884,9 @@ class AgentClient extends BaseClient {// the flag; if it fails here, the teardown still releases (it checks the flag).if (!this.pendingRequestReleased) {try {- await decrementPendingRequest(this.options.req?.user?.id);+ if (this.options.req?._scheduleConcurrencyExempt !== true) {+ await decrementPendingRequest(this.options.req?.user?.id);+ }this.pendingRequestReleased = true;} catch (err) {logger.error(`[AgentClient] Failed to release request slot on pause ${streamId}`, err);diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex 580b545f4c..5c99383180 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -18,6 +18,8 @@ const {decrementPendingRequest,sanitizeMessageForTransmit,checkAndIncrementPendingRequest,+ exemptFromConcurrencyLimiter,+ isScheduleFireRequest,isUnpersistedPreliminaryParent,resolveConversationAnchor,getAgentStartupTelemetry,@@ -33,6 +35,7 @@ const {cleanupMCPRequestContextForReq,} = require('~/server/services/MCPRequestContext');const { logViolation } = require('~/cache');+const { recordScheduleOutcome, isScheduleLive } = require('~/server/services/Schedules');const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models');const {GENERATION_PROTOCOL_HEADER,@@ -182,8 +185,20 @@ async function finishResumableRequest(req, userId) {try {await cleanupMCPRequestContextForReq(req);} finally {- await decrementPendingRequest(userId);+ if (req._scheduleConcurrencyExempt !== true) {+ await decrementPendingRequest(userId);+ }+ }+}++function classifyScheduledFailure(error, aborted = false) {+ if (aborted || error?.code === 'SCHEDULE_NO_LONGER_ACTIVE') {+ return { status: 'interrupted', error: error?.message };}+ if (error?.message?.includes(ViolationTypes.TOKEN_BALANCE)) {+ return { status: 'skipped_balance' };+ }+ return { status: 'error', error: error?.message || 'Generation failed' };}const JOB_RECORD_WAIT_ATTEMPTS = 5;@@ -338,8 +353,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitparentMessageId = null,overrideParentMessageId = null,responseMessageId: editedResponseMessageId = null,+ scheduleId: bodyScheduleId = null,+ scheduledFor: bodyScheduledFor = null,+ scheduleConfigRevision: bodyScheduleConfigRevision = null,} = req.body;+ const isScheduledFire = isScheduleFireRequest(req);+ const scheduleId = isScheduledFire ? bodyScheduleId : null;+ const scheduledFor = isScheduledFire ? bodyScheduledFor : null;+ const scheduleConfigRevision = isScheduledFire ? bodyScheduleConfigRevision : undefined;+const userId = req.user.id;const tenantId = req.user.tenantId;const rawClientRequestId = req.body?.clientRequestId;@@ -446,12 +469,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitreq.body.overrideUserMessageId = `${recoveredSteerId}${Constants.COMMON_DIVIDER}0`;}const isNewConvo = !reqConversationId || reqConversationId === 'new';+ const scheduledNewConversationId =+ isScheduledFire && typeof req.body?.newConversationId === 'string'+ ? req.body.newConversationId+ : null;let conversationId = reqConversationId;if (isNewConvo) {conversationId =- typeof clientRequestId === 'string' && clientRequestId.length > 0+ scheduledNewConversationId ??+ (typeof clientRequestId === 'string' && clientRequestId.length > 0? uuidv5(`${userId}:${clientRequestId}`, NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE)- : crypto.randomUUID();+ : crypto.randomUUID());}const conversationAnchorPromise = resolveConversationCreatedAt({userId,@@ -936,26 +964,53 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit}}- const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);- if (!allowed) {- if (ownedIdempotencyClaim) {- await GenerationJobManager.releaseGeneration(- userId,- clientRequestId,- streamId,- ownedIdempotencyClaim,- ).catch(() => {});+ const scheduleConcurrencyExempt = exemptFromConcurrencyLimiter(req);+ req._scheduleConcurrencyExempt = scheduleConcurrencyExempt;+ if (!scheduleConcurrencyExempt) {+ const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);+ if (!allowed) {+ if (ownedIdempotencyClaim) {+ await GenerationJobManager.releaseGeneration(+ userId,+ clientRequestId,+ streamId,+ ownedIdempotencyClaim,+ ).catch(() => {});+ }+ const violationInfo = getViolationInfo(pendingRequests, limit);+ await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);+ startupTelemetry?.end('rejected');+ return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);}- const violationInfo = getViolationInfo(pendingRequests, limit);- await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);- startupTelemetry?.end('rejected');- return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);}startupTelemetry?.mark('request_admitted');let client = null;let jobCreatedAt;let providerExecutionId;+ let scheduleTerminalOutcomeRecorded = false;+ const settleScheduledRun = async ({ status, error, clearConversationId = false }) => {+ if (!scheduleId) {+ return true;+ }+ if (status !== 'requires_action' && scheduleTerminalOutcomeRecorded) {+ return true;+ }+ const recorded = await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt,+ status,+ conversationId,+ clearConversationId,+ error,+ });+ if (recorded && status !== 'requires_action') {+ scheduleTerminalOutcomeRecorded = true;+ }+ return recorded;+ };try {logger.debug(`[ResumableAgentController] Creating job`, {@@ -995,6 +1050,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// Persist temporary-chat state so a HITL resume keeps the resumed response// non-persisted instead of trusting the resume request to re-send the flag.isTemporary: req.body?.isTemporary,+ ...(scheduleId+ ? {+ scheduleId,+ scheduledFor,+ preserveForScheduleReconcile: true,+ ...(Number.isSafeInteger(scheduleConfigRevision) && {+ scheduleConfigRevision,+ }),+ ...(req._isManualScheduledFire === true && { scheduleManual: true }),+ }+ : {}),responseMessageId: preliminaryResponseMessageId,userMessage: preliminaryUserMessage,},@@ -1016,6 +1082,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitstatus: 409,});}+ if (+ scheduleId &&+ !(await isScheduleLive(scheduleId, scheduleConfigRevision, {+ automatic: req._isManualScheduledFire !== true,+ policy: true,+ }))+ ) {+ throw Object.assign(new Error('This scheduled occurrence is no longer active'), {+ code: 'SCHEDULE_NO_LONGER_ACTIVE',+ status: 409,+ });+ }if (providerExecutionId &&!(await GenerationJobManager.beginProviderExecution(@@ -1187,6 +1265,11 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitcompleteErr,);});+ await settleScheduledRun({+ status: 'interrupted',+ error: 'Request aborted during initialization',+ clearConversationId: job.createdEventEmitted !== true,+ });startupTelemetry?.end('aborted');try {await finishResumableRequest(req, userId);@@ -1260,6 +1343,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit* may call completeJob: the pause may already have been replaced by a newer* action or generation by the time the persistence failure is observed. */let pausePersistenceFailed = false;+ let pausePersistenceFailureFinalized = false;const finishOwnedTerminalClaim = async () => {if (!terminalClaim || terminalClaimFinished) {return;@@ -1586,21 +1670,21 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitawait commitRecoveredSteer();} catch (pausePersistenceError) {pausePersistenceFailed = true;- let failed;try {- failed = await GenerationJobManager.failPausePersistence(- streamId,- pauseActionId,- pausePersistenceError?.message ?? 'Pause persistence failed',- pauseCreatedAt,- );+ pausePersistenceFailureFinalized =+ (await GenerationJobManager.failPausePersistence(+ streamId,+ pauseActionId,+ pausePersistenceError?.message ?? 'Pause persistence failed',+ pauseCreatedAt,+ )) === true;} catch (failError) {logger.error(`[ResumableAgentController] Failed to terminalize pause persistence error for ${streamId}`,failError,);}- if (failed === true) {+ if (pausePersistenceFailureFinalized) {/** Namespaced checkpoints belong exclusively to this epoch,* so the exact pause-failure CAS winner can safely remove the* now-unresumable graph state. Legacy shared namespaces are@@ -1621,7 +1705,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit);}}- } else if (failed === false) {+ } else if (pausePersistenceFailureFinalized === false) {logger.warn(`[ResumableAgentController] Skipping stale pause persistence failure — ${streamId} no longer owns its barrier`,);@@ -1638,6 +1722,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit`[ResumableAgentController] Pause persistence barrier changed before release: ${streamId}`,);}+ // The pause projection is what moves the run row off `started` and frees its+ // GLOBAL capacity slot. recordScheduleOutcome already retried it; a `false`+ // here means every attempt failed, leaving the row `started` while the job+ // sits `requires_action`. Surface it — the armed engine's reconciler replays+ // this state, and the clustered sweep now converges it too, but a silent drop+ // gave neither a reason to look.+ if (!(await settleScheduledRun({ status: 'requires_action' }))) {+ logger.error(+ `[ResumableAgentController] Failed to project the scheduled pause for ${streamId}; run stays active until reconciliation replays it`,+ );+ }} else {logger.debug(`[ResumableAgentController] Skipping stale pause persistence — ${streamId} no longer owns its barrier`,@@ -1650,7 +1745,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// (so a fast /resume isn't 429'd); only release here if that didn't happen.// Always run the MCP request-context cleanup.await cleanupMCPRequestContextForReq(req);- if (!client?.pendingRequestReleased) {+ if (!client?.pendingRequestReleased && req._scheduleConcurrencyExempt !== true) {await decrementPendingRequest(userId);}if (client) {@@ -1781,6 +1876,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitawait titleEventPromise;}+ let scheduleCompletionError;+ if (terminalWasAborted) {+ scheduleCompletionError = 'Scheduled run was stopped';+ } else if (preemptIncomplete) {+ scheduleCompletionError = 'Scheduled run was interrupted before completion';+ }+ await settleScheduledRun({+ status: terminalWasAborted || preemptIncomplete ? 'interrupted' : 'success',+ ...(scheduleCompletionError != null && { error: scheduleCompletionError }),+ });+let terminalPublicationStarted = false;try {const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);@@ -1893,7 +1999,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// transition can win. Settle its pending marker with conservative// reconciliation on any required-write/final-construction failure,// then release exactly that claim.+ let ownsScheduledFailure = false;if (terminalClaim && !terminalClaimFinished) {+ ownsScheduledFailure = true;try {await GenerationJobManager.publishTerminalClaim(terminalClaim, null);} catch (publishError) {@@ -1915,6 +2023,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit);startupTelemetry?.end('error', error);} else if (pausePersistenceFailed) {+ ownsScheduledFailure = pausePersistenceFailureFinalized;// failPausePersistence owns the only legal requires_action -> error// transition for this exact action/epoch. Never fall through to// completeJob, which could race a newer action or replacement job.@@ -1924,6 +2033,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit);startupTelemetry?.end('error', error);} else if (job.abortController.signal.aborted || error.message?.includes('abort')) {+ ownsScheduledFailure = true;logger.debug(`[ResumableAgentController] Generation aborted for ${streamId}`);startupTelemetry?.end('aborted');// abortJob already handled emitDone and completeJob@@ -1933,7 +2043,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTittry {// completeJob first wins running -> error and atomically parks// steers, then publishes. A competing abort/pause emits nothing.- await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt);+ ownsScheduledFailure =+ (await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt)) ===+ true;} catch (completeErr) {logger.warn('[ResumableAgentController] completeJob failed during generation-error cleanup',@@ -1944,6 +2056,14 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit}}+ if (ownsScheduledFailure && !scheduleTerminalOutcomeRecorded) {+ const scheduledFailure = classifyScheduledFailure(+ error,+ job.abortController.signal.aborted,+ );+ await settleScheduledRun(scheduledFailure);+ }+try {await finishResumableRequest(req, userId);} finally {@@ -1962,15 +2082,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit`[ResumableAgentController] Unhandled error in background generation: ${err.message}`,);startupTelemetry?.end('error', err);+ let errorFinalized = false;if (!pausePersistenceFailed) {- await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(- (completeErr) => {- logger.warn(- '[ResumableAgentController] completeJob failed during background-error cleanup',- completeErr,- );- },- );+ errorFinalized =+ (await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(+ (completeErr) => {+ logger.warn(+ '[ResumableAgentController] completeJob failed during background-error cleanup',+ completeErr,+ );+ return false;+ },+ )) === true;+ }+ if (+ (errorFinalized || (pausePersistenceFailed && pausePersistenceFailureFinalized)) &&+ !scheduleTerminalOutcomeRecorded+ ) {+ await settleScheduledRun(classifyScheduledFailure(err));}try {await finishResumableRequest(req, userId);@@ -2084,18 +2213,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// release + pending-request decrement below, or the retry stays wedged behind the claim// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not// finalize anything, so releasing afterward can't let it abort a later replacement.)+ let initializationFinalized = jobCreatedAt == null;if (jobCreatedAt != null) {const initializationError = initializationFailure? JSON.stringify(initializationFailure): error.message || 'Failed to start generation';- await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(- (completeErr) => {- logger.warn(- '[ResumableAgentController] completeJob failed during init-error cleanup',- completeErr,- );- },- );+ initializationFinalized =+ (await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(+ (completeErr) => {+ logger.warn(+ '[ResumableAgentController] completeJob failed during init-error cleanup',+ completeErr,+ );+ return false;+ },+ )) === true;+ }+ if (initializationFinalized && !scheduleTerminalOutcomeRecorded) {+ await settleScheduledRun(classifyScheduledFailure(error));}if (ownedIdempotencyClaim) {await GenerationJobManager.releaseGeneration(diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.jsindex cb3ea82908..e4eb76e295 100644--- a/api/server/controllers/agents/resume.js+++ b/api/server/controllers/agents/resume.js@@ -1,6 +1,6 @@const { randomUUID } = require('crypto');const { logger } = require('@librechat/data-schemas');-const { Constants, EModelEndpoint } = require('librechat-data-provider');+const { Constants, EModelEndpoint, ViolationTypes } = require('librechat-data-provider');const {GenerationJobManager,isPendingActionStale,@@ -30,6 +30,14 @@ const {cleanupMCPRequestContextForReq,} = require('~/server/services/MCPRequestContext');const { saveMessage, getConvo, getMessages } = require('~/models');+const {+ recordScheduleOutcome,+ claimScheduleResume,+ releaseScheduleResumeClaim,+ finalizeScheduleResumeClaim,+ releaseScheduleResumeFence,+ isScheduleLive,+} = require('~/server/services/Schedules');const {GENERATION_PROTOCOL_HEADER,negotiateNewGenerationProtocol,@@ -422,6 +430,20 @@ async function finalizeResumedTurn({}conversation.title = conversation.title || 'New Chat';+ if (meta.scheduleId) {+ await recordScheduleOutcome({+ scheduleId: meta.scheduleId,+ scheduledFor: meta.scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: preemptIncomplete ? 'interrupted' : 'success',+ conversationId,+ ...(preemptIncomplete && {+ error: 'Scheduled run was interrupted before completion',+ }),+ });+ }+const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);const finalEvent = {final: true,@@ -571,6 +593,65 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle));}+ const scheduleId = job.metadata?.scheduleId;+ const scheduledFor = job.metadata?.scheduledFor;+ if (+ scheduleId &&+ !(await isScheduleLive(scheduleId, job.metadata?.scheduleConfigRevision, {+ automatic: job.metadata?.scheduleManual !== true,+ policy: true,+ }))+ ) {+ let stopped = false;+ try {+ const abortResult = await GenerationJobManager.abortJob(streamId, {+ expectedCreatedAt: job.createdAt,+ awaitProviderDrain: true,+ });+ stopped = abortResult != null && abortResult.failureReason == null;+ } catch (error) {+ logger.warn('[ResumeAgentController] Failed to stop inactive scheduled run', error);+ }+ if (!stopped) {+ res.set('Retry-After', '1');+ return sendGenerationJson(+ res,+ 503,+ {+ code: 'SCHEDULE_STOP_UNCONFIRMED',+ error: 'The inactive scheduled run could not be confirmed stopped. Please retry.',+ },+ generationProtocolVersion,+ );+ }+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'interrupted',+ conversationId,+ error: 'Schedule was disabled, changed, or deleted before approval',+ });+ const checkpointNamespace = job.metadata?.checkpointNamespace;+ if (typeof checkpointNamespace === 'string' && checkpointNamespace !== '') {+ await deleteAgentCheckpoint(+ conversationId,+ req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,+ undefined,+ { checkpointNamespace },+ ).catch((error) => {+ logger.warn('[ResumeAgentController] Failed to prune inactive schedule checkpoint', error);+ });+ }+ return sendGenerationJson(+ res,+ 409,+ { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' },+ generationProtocolVersion,+ );+ }+const pendingAction = job.metadata?.pendingAction;if (job.status !== 'requires_action') {return sendGenerationJson(@@ -720,6 +801,109 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle));}+ // Finish the legacy checkpoint snapshot before claiming scheduled capacity.+ // It is independent of the approval claim, and holding a deployment-wide slot+ // while an indexed saver read stalls would unnecessarily block other schedules+ // and lengthen the Mongo-claim -> approval-CAS hand-off window below.+ const checkpointGeneration = await checkpointGenerationPromise;++ // A pause frees its scheduled-run capacity slot. Before consuming the approval,+ // atomically promote the run row back to `started` and claim a fresh global slot.+ // The database's partial unique indexes arbitrate both deployment capacity and a+ // concurrent active occurrence of the same schedule.+ let scheduleCapacitySlot;+ let scheduleResumeClaimToken;+ let scheduleResumeLeaseBy;+ const scheduleResumeOptions = {+ expectedConfigRevision: job.metadata?.scheduleConfigRevision,+ automatic: job.metadata?.scheduleManual !== true,+ };+ if (scheduleId) {+ let scheduleClaim;+ try {+ scheduleClaim = await claimScheduleResume(scheduleId, scheduledFor, scheduleResumeOptions);+ } catch (err) {+ await decrementPendingRequest(userId);+ logger.error('[ResumeAgentController] Failed to claim scheduled resume capacity', err);+ return sendGenerationJson(+ res,+ 500,+ { error: 'Failed to reserve scheduled-run capacity' },+ generationProtocolVersion,+ );+ }+ if ('conflict' in scheduleClaim) {+ await decrementPendingRequest(userId);+ if (scheduleClaim.conflict === 'capacity' || scheduleClaim.conflict === 'overlap') {+ res.set('Retry-After', '1');+ return sendGenerationJson(+ res,+ 429,+ {+ code:+ scheduleClaim.conflict === 'capacity'+ ? 'SCHEDULE_CAPACITY'+ : 'SCHEDULE_OCCURRENCE_ACTIVE',+ error:+ scheduleClaim.conflict === 'capacity'+ ? 'Scheduled-run capacity is currently full. Please retry.'+ : 'Another occurrence of this schedule is still running. Please retry.',+ },+ generationProtocolVersion,+ );+ }+ return sendGenerationJson(+ res,+ 409,+ {+ code:+ scheduleClaim.conflict === 'inactive'+ ? 'SCHEDULE_NO_LONGER_ACTIVE'+ : 'SCHEDULE_RUN_NOT_PAUSED',+ error: 'This scheduled run can no longer be resumed',+ },+ generationProtocolVersion,+ );+ }+ scheduleCapacitySlot = scheduleClaim.capacitySlot;+ scheduleResumeClaimToken = scheduleClaim.claimToken;+ scheduleResumeLeaseBy = scheduleClaim.leaseBy;+ }++ const releaseScheduleFence = async () => {+ if (scheduleId == null || scheduleResumeLeaseBy == null) {+ return;+ }+ try {+ await releaseScheduleResumeFence(scheduleId, scheduleResumeLeaseBy);+ } catch (releaseError) {+ logger.warn('[ResumeAgentController] Failed to release scheduled resume fence', releaseError);+ }+ };++ /** Release only when the exact generation demonstrably remains paused. If the+ * approval CAS reply is ambiguous and the job cannot be read, retaining the slot+ * until reconciliation is the safe direction: releasing it could exceed the cap+ * while a committed continuation is already running. */+ const rollbackUnconsumedScheduleClaim = async (currentJob) => {+ if (+ scheduleId == null ||+ scheduleCapacitySlot == null ||+ currentJob?.createdAt !== job.createdAt ||+ currentJob?.status !== 'requires_action'+ ) {+ return;+ }+ try {+ await releaseScheduleResumeClaim(scheduleId, scheduledFor, scheduleCapacitySlot);+ } catch (rollbackError) {+ logger.warn(+ '[ResumeAgentController] Failed to release unconsumed scheduled resume capacity',+ rollbackError,+ );+ }+ };+// Atomically claim the resume. The single winner drives the run; a racing second// submit (double-click, two tabs) gets false and must not re-drive — that would// re-execute tools and double-bill.@@ -729,10 +913,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)// would leak the concurrency slot until the counter TTL expires — spuriously 429'ing// the user when they retry the still-paused approval. Release the slot on that path too.let claimed;- let checkpointGeneration;const providerExecutionId = randomUUID();try {- checkpointGeneration = await checkpointGenerationPromise;/** The CAS that reopens steering must also publish THIS owner's seal* capability. A separate write after status=`running` leaves a window in* which steer/arm requests read the previous replica's capability. */@@ -748,6 +930,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)job.createdAt,);} catch (err) {+ const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);+ await rollbackUnconsumedScheduleClaim(currentJob);+ await releaseScheduleFence();await decrementPendingRequest(userId);logger.error('[ResumeAgentController] Failed to claim resume', err);return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion);@@ -755,6 +940,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)if (!claimed) {await decrementPendingRequest(userId);const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);+ await rollbackUnconsumedScheduleClaim(currentJob);+ await releaseScheduleFence();if (currentJob != null && currentJob.createdAt !== job.createdAt) {return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion);}@@ -766,6 +953,73 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle));}+ // Linearize the consumed approval against the schedule's live config. The schedule+ // document fence was acquired only after all async policy reads, and this atomic+ // consume checks its token/revision/enabled state immediately after the approval CAS.+ // An edit/disable that won first makes this fail; one that lands afterward is ordered+ // after the continuation has started. Never begin provider execution on a stale claim.+ if (scheduleId) {+ let scheduleClaimCurrent = false;+ try {+ scheduleClaimCurrent = await finalizeScheduleResumeClaim(+ scheduleId,+ scheduleResumeClaimToken,+ scheduleResumeLeaseBy,+ scheduleResumeOptions,+ );+ } catch (error) {+ logger.error('[ResumeAgentController] Failed to finalize scheduled resume fence', error);+ await releaseScheduleFence();+ }+ if (!scheduleClaimCurrent) {+ await decrementPendingRequest(userId);+ let stopped = false;+ try {+ const abortResult = await GenerationJobManager.abortJob(streamId, {+ expectedCreatedAt: job.createdAt,+ awaitProviderDrain: true,+ });+ stopped = abortResult != null && abortResult.failureReason == null;+ } catch (error) {+ logger.warn('[ResumeAgentController] Failed to stop stale scheduled resume', error);+ }+ if (!stopped) {+ res.set('Retry-After', '1');+ return sendGenerationJson(+ res,+ 503,+ {+ code: 'SCHEDULE_STOP_UNCONFIRMED',+ error: 'The stale scheduled resume could not be confirmed stopped.',+ },+ generationProtocolVersion,+ );+ }+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'interrupted',+ conversationId,+ error: 'Schedule was disabled, changed, or deleted before approval',+ });+ if (checkpointNamespace !== '') {+ await deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, {+ checkpointNamespace,+ }).catch((error) => {+ logger.warn('[ResumeAgentController] Failed to prune stale schedule checkpoint', error);+ });+ }+ return sendGenerationJson(+ res,+ 409,+ { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' },+ generationProtocolVersion,+ );+ }+ }+/*** An interrupt steer enqueued just before the pause survives durably with* its `preempt` flag, but the ARM lived only in the previous owner's@@ -1018,6 +1272,16 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)`[ResumeAgentController] Re-pause persistence barrier changed before release: ${streamId}`,);}+ if (scheduleId) {+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'requires_action',+ conversationId,+ });+ }} else {logger.debug(`[ResumeAgentController] Skipping stale re-pause persistence — ${streamId} no longer owns its barrier`,@@ -1027,11 +1291,25 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)}// If the user aborted mid-resume, the abort route already emitted the terminal- // event and finalized the job — don't double-save / double-finalize here.+ // event and finalized the job — don't double-save / double-finalize here. This+ // continuation is nevertheless the scheduled-run owner, so it must settle the+ // run row after observing its own abort; the generic Stop route deliberately+ // delegates a running generation's settlement to that generation owner.if (job.abortController.signal.aborted) {logger.debug(`[ResumeAgentController] Aborted during resume; abort route finalizes: ${streamId}`,);+ if (scheduleId) {+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'interrupted',+ conversationId,+ error: 'Scheduled run was stopped',+ });+ }return;}@@ -1061,6 +1339,17 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)'re-pause persistence failure',);}+ if (scheduleId && pausePersistenceFailureFinalized) {+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'error',+ conversationId,+ error: err?.message ?? 'Re-pause persistence failed',+ });+ }return;}// Job-replacement guard (mirrors finalizeResumedTurn's success-path guard): if a@@ -1104,6 +1393,18 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)'failed resume finalization',);}+ if (scheduleId && errorFinalized) {+ const balanceRefusal = err?.message?.includes(ViolationTypes.TOKEN_BALANCE);+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: balanceRefusal ? 'skipped_balance' : 'error',+ conversationId,+ ...(!balanceRefusal && { error: err?.message ?? 'Resume failed' }),+ });+ }}} finally {try {diff --git a/api/server/experimental.js b/api/server/experimental.jsindex 698cb52ff2..8d03f8ae59 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -28,6 +28,7 @@ const {setupGracefulShutdown,configureMessageFilterRegexValidator,configureFileConfigRegexEngine,+ GenerationJobManager,waitForKeyvRedisClient,} = require('@librechat/api');const { connectDb, indexSync } = require('~/db');@@ -37,6 +38,10 @@ const createValidateImageRequest = require('./middleware/validateImageRequest');const { startExpiredFileSweep } = require('./services/Files/process');const { initializeGitHubSkillSync } = require('./services/Skills/sync');const { initializeAgentTriggerService } = require('./services/Agents/triggers');+const {+ recordExpiredScheduleApproval,+ initializeScheduleErasureSweep,+} = require('./services/Schedules');const { configureSubagentTaskRouting } = require('./services/Endpoints/agents/subagentThreadStore');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api');@@ -276,6 +281,11 @@ if (cluster.isMaster) {* Each worker runs a full Express server instance*/const app = express();+ // The clustered entrypoint deliberately does not arm the v1 schedule engine,+ // but an already-fired scheduled generation can still reach HITL here. Settle+ // its durable run when the generic approval runtime expires it.+ GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);+ GenerationJobManager.initialize();/*** The master may assign the sweep worker before or after this worker has* loaded app config. These flags join the IPC assignment with config@@ -284,6 +294,17 @@ if (cluster.isMaster) {let shouldStartExpiredFileSweep = false;let expiredFileSweepOptions = null;let expiredFileSweepStarted = false;+ const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);++ const rejectScheduleWritesUntilReady = (req, res, next) => {+ if (SCHEDULE_ENGINE_OPTIONAL_METHODS.has(req.method)) {+ return next();+ }+ return res.status(501).json({+ code: 'SCHEDULES_NOT_SUPPORTED',+ error: 'Scheduled chats are not available in clustered mode.',+ });+ };const startExpiredFileSweepOnce = () => {if (!shouldStartExpiredFileSweep || expiredFileSweepStarted || !expiredFileSweepOptions) {@@ -322,6 +343,13 @@ if (cluster.isMaster) {logger.error(`[Worker ${process.pid}][indexSync] Background sync failed:`, err);});+ // This entrypoint deliberately does not arm the schedule engine, but DELETE stays+ // open — so soft-deleted rows still accrue with no reconciler to erase them. Start+ // the erasure-ONLY sweep (Mongo is up, GenerationJobManager was initialized above):+ // it never claims, fires, advances, or infers owner death from a process-local+ // missing job, and its idempotent guard makes this safe once per worker.+ initializeScheduleErasureSweep();+app.disable('x-powered-by');app.set('trust proxy', trusted_proxy);@@ -488,6 +516,7 @@ if (cluster.isMaster) {app.use('/api/agents', routes.agents);app.use('/api/banner', routes.banner);app.use('/api/memories', routes.memories);+ app.use('/api/schedules', rejectScheduleWritesUntilReady, routes.schedules);app.use('/api/permissions', routes.accessPermissions);app.use('/api/tags', routes.tags);app.use('/api/mcp', routes.mcp);diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.jsindex 4ff9d6b924..58349d3091 100644--- a/api/server/experimental.spec.js+++ b/api/server/experimental.spec.js@@ -23,6 +23,34 @@ describe('Experimental server configuration', () => {expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);});+ it('starts approval expiry after installing the scheduled-run callback', () => {+ const handlerIndex = source.indexOf(+ 'GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);',+ );+ const initializeIndex = source.indexOf('GenerationJobManager.initialize();');++ expect(handlerIndex).toBeGreaterThan(-1);+ expect(initializeIndex).toBeGreaterThan(handlerIndex);+ });++ it('starts erasure-only schedule maintenance after connecting to Mongo, once per worker', () => {+ const connectIndex = source.indexOf('await connectDb();');+ const sweepIndex = source.indexOf('initializeScheduleErasureSweep();');++ expect(connectIndex).toBeGreaterThan(-1);+ expect(sweepIndex).toBeGreaterThan(-1);+ // Mongo must be up before the sweep reads soft-deleted rows.+ expect(sweepIndex).toBeGreaterThan(connectIndex);+ // Idempotent guard lives in the service; started exactly once from this entrypoint.+ expect(source.match(/initializeScheduleErasureSweep\(\);/g)).toHaveLength(1);+ });++ it('never arms the full schedule engine in a clustered worker', () => {+ // The clustered entrypoint runs erasure-only maintenance: arming the engine here+ // would claim/fire/absence-reconcile runs whose peer generations it cannot see.+ expect(source).not.toContain('initializeScheduleEngine(');+ });+it('runs cross-tenant startup work in the system context', () => {expect(source).toContain('await runAsSystem(seedDatabase);');expect(source).toMatch(diff --git a/api/server/index.js b/api/server/index.jsindex 0768a27c50..9203250d70 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -57,6 +57,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilitiesconst createValidateImageRequest = require('./middleware/validateImageRequest');const { initializeGitHubSkillSync } = require('./services/Skills/sync');const { initializeAgentTriggerService } = require('./services/Agents/triggers');+const { initializeScheduleEngine, recordExpiredScheduleApproval } = require('./services/Schedules');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { startExpiredFileSweep } = require('./services/Files/process');const { checkMigrations } = require('./services/start/migration');@@ -85,9 +86,12 @@ const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by defaultconst app = express();let serverReady = false;+let schedulesReady = false;const SERVER_NOT_READY_CODE = 'SERVER_NOT_READY';const CHAT_START_RETRY_AFTER_SECONDS = '1';+const SCHEDULES_NOT_READY_CODE = 'SCHEDULES_NOT_READY';+const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);const rejectChatStartsUntilReady = (req, res, next) => {if (serverReady || req.method !== 'POST' || req.path === '/abort') {@@ -101,12 +105,24 @@ const rejectChatStartsUntilReady = (req, res, next) => {});};+const rejectScheduleWritesUntilReady = (req, res, next) => {+ if (schedulesReady || SCHEDULE_ENGINE_OPTIONAL_METHODS.has(req.method)) {+ return next();+ }+ res.set('Retry-After', CHAT_START_RETRY_AFTER_SECONDS);+ return res.status(503).json({+ code: SCHEDULES_NOT_READY_CODE,+ error: 'Scheduler is still starting. Please retry shortly.',+ });+};+const configureGenerationStreams = () => {const streamServices = createStreamServices();GenerationJobManager.configure({...streamServices,cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),});+ GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);GenerationJobManager.initialize();// Stop active generations and close their SSE streams while the HTTP server drains.registerShutdownTask(@@ -345,6 +361,7 @@ const startServer = async () => {app.use('/api/agents', routes.agents);app.use('/api/banner', routes.banner);app.use('/api/memories', routes.memories);+ app.use('/api/schedules', rejectScheduleWritesUntilReady, routes.schedules);app.use('/api/permissions', routes.accessPermissions);app.use('/api/tags', routes.tags);@@ -402,6 +419,10 @@ const startServer = async () => {memoryDiagnostics.start();}await initializeAgentTriggerService({ address: server.address() });+ schedulesReady = (await initializeScheduleEngine()) != null;+ if (!schedulesReady) {+ logger.warn('[schedules] write routes remain unavailable because the engine did not arm.');+ }serverReady = true;logger.info('Server readiness checks passing.');} catch (initErr) {diff --git a/api/server/index.metrics.spec.js b/api/server/index.metrics.spec.jsindex 6b95104c3d..80a386c607 100644--- a/api/server/index.metrics.spec.js+++ b/api/server/index.metrics.spec.js@@ -38,6 +38,10 @@ jest.mock('~/server/services/Agents/triggers', () => ({initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),}));+jest.mock('~/server/services/Schedules', () => ({+ initializeScheduleEngine: jest.fn().mockResolvedValue(undefined),+}));+describe('Server metrics route', () => {jest.setTimeout(30_000);diff --git a/api/server/index.spec.js b/api/server/index.spec.jsindex 013ece204e..73ad042865 100644--- a/api/server/index.spec.js+++ b/api/server/index.spec.js@@ -39,6 +39,10 @@ jest.mock('~/server/services/Agents/triggers', () => ({initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),}));+jest.mock('~/server/services/Schedules', () => ({+ initializeScheduleEngine: jest.fn().mockResolvedValue(undefined),+}));+jest.mock('@librechat/api/telemetry',() => ({diff --git a/api/server/routes/__tests__/mcp.spec.js b/api/server/routes/__tests__/mcp.spec.jsindex 44dcfe2628..99ebff511a 100644--- a/api/server/routes/__tests__/mcp.spec.js+++ b/api/server/routes/__tests__/mcp.spec.js@@ -228,6 +228,12 @@ describe('MCP Routes', () => {currentUser = undefined;mockResolveAllMcpConfigs.mockResolvedValue({});mockResolveMcpConfigNames.mockResolvedValue([]);+ // `clearAllMocks` preserves queued `mockReturnValueOnce` entries. A callback+ // test can legitimately leave one unconsumed, which then masks a later test's+ // default implementation when this file shares a CI shard with other suites.+ // Reset the cache accessor itself so every test starts from a deterministic+ // default and can opt into a one-shot value deliberately.+ require('~/cache').getLogStores.mockReset().mockReturnValue({});const { MCPOAuthHandler } = require('@librechat/api');const { getTenantId } = require('@librechat/data-schemas');getTenantId.mockReturnValue(undefined);diff --git a/api/server/routes/agents/__tests__/abort.spec.js b/api/server/routes/agents/__tests__/abort.spec.jsindex f6a3ca559f..43516fbb16 100644--- a/api/server/routes/agents/__tests__/abort.spec.js+++ b/api/server/routes/agents/__tests__/abort.spec.js@@ -25,6 +25,10 @@ const mockGenerationJobManager = {const mockSaveMessage = jest.fn();+const mockRecordScheduleOutcome = jest.fn();+const mockBeginScheduledStop = jest.fn();+const mockAcknowledgeScheduledStopPersistence = jest.fn();+jest.mock('@librechat/data-schemas', () => ({...jest.requireActual('@librechat/data-schemas'),logger: mockLogger,@@ -33,6 +37,11 @@ jest.mock('@librechat/data-schemas', () => ({jest.mock('@librechat/api', () => ({...jest.requireActual('@librechat/api'),isEnabled: jest.fn().mockReturnValue(false),+ isAgentTriggerRequest: jest.fn(() => false),+ captureScheduleFireContext: jest.fn((req) => {+ req._isScheduledFire = false;+ req._isManualScheduledFire = false;+ }),GenerationJobManager: mockGenerationJobManager,}));@@ -40,6 +49,13 @@ jest.mock('~/models', () => ({saveMessage: (...args) => mockSaveMessage(...args),}));+jest.mock('~/server/services/Schedules', () => ({+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),+ beginScheduledStop: (...args) => mockBeginScheduledStop(...args),+ acknowledgeScheduledStopPersistence: (...args) =>+ mockAcknowledgeScheduledStopPersistence(...args),+}));+jest.mock('~/server/middleware', () => ({uaParser: (req, res, next) => next(),checkBan: (req, res, next) => next(),@@ -81,6 +97,12 @@ describe('Agent Abort Endpoint', () => {mockGenerationJobManager.getActiveJobIdsForUser.mockReset();mockSaveMessage.mockReset();mockSaveMessage.mockImplementation(async (_context, message) => message);+ mockRecordScheduleOutcome.mockReset();+ mockRecordScheduleOutcome.mockResolvedValue(true);+ mockBeginScheduledStop.mockReset();+ mockBeginScheduledStop.mockResolvedValue(true);+ mockAcknowledgeScheduledStopPersistence.mockReset();+ mockAcknowledgeScheduledStopPersistence.mockResolvedValue(undefined);});describe('POST /chat/abort', () => {@@ -841,5 +863,140 @@ describe('Agent Abort Endpoint', () => {});});});++ describe('Scheduled Stop persistence protocol', () => {+ const scheduledJob = {+ status: 'running',+ createdAt: 111,+ metadata: {+ userId: 'test-user-123',+ generationProtocolVersion: 2,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ conversationId: 'conv-1',+ },+ };++ it('stamps the Stop before signalling abort, then acknowledges after persistence', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);+ mockGenerationJobManager.abortJob.mockResolvedValue({+ success: true,+ jobData: { conversationId: 'conv-1' },+ content: [],+ text: '',+ });++ const response = await request(app)+ .post('/api/agents/chat/abort')+ .set('X-LibreChat-Generation-Protocol', '2')+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });++ expect(response.status).toBe(200);+ expect(mockBeginScheduledStop).toHaveBeenCalledWith({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ });+ // Stamp BEFORE the abort signal; acknowledgement AFTER it (persistence done).+ expect(mockBeginScheduledStop.mock.invocationCallOrder[0]).toBeLessThan(+ mockGenerationJobManager.abortJob.mock.invocationCallOrder[0],+ );+ // The ack also carries a terminal outcome to re-drive: the owner calls+ // recordScheduleOutcome once, and if that call's Stop barrier deferred (slow+ // beforePublish), nothing would settle the run where no reconciler is armed.+ expect(mockAcknowledgeScheduledStopPersistence).toHaveBeenCalledWith(+ expect.objectContaining({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ settle: expect.objectContaining({ status: 'interrupted' }),+ }),+ );+ expect(mockGenerationJobManager.abortJob.mock.invocationCallOrder[0]).toBeLessThan(+ mockAcknowledgeScheduledStopPersistence.mock.invocationCallOrder[0],+ );+ });++ it('returns STOP_IN_PROGRESS and never signals a second abort when a Stop is already live', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);+ mockBeginScheduledStop.mockResolvedValue('in_progress');++ const response = await request(app)+ .post('/api/agents/chat/abort')+ .set('X-LibreChat-Generation-Protocol', '2')+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });++ expect(response.status).toBe(409);+ expect(response.body.code).toBe('STOP_IN_PROGRESS');+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();+ });++ it('does NOT acknowledge when the abort persistence failed (run stays preserved)', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);+ mockGenerationJobManager.abortJob.mockResolvedValue({+ success: true,+ persistenceFailed: true,+ jobData: { conversationId: 'conv-1' },+ content: [],+ text: '',+ });++ const response = await request(app)+ .post('/api/agents/chat/abort')+ .set('X-LibreChat-Generation-Protocol', '2')+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });++ expect(response.status).toBe(200);+ expect(response.body.persistenceFailed).toBe(true);+ expect(mockBeginScheduledStop).toHaveBeenCalled();+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();+ });++ it('does not re-drive settlement for a PAUSED job, which settles explicitly', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue({+ ...scheduledJob,+ status: 'requires_action',+ });+ mockGenerationJobManager.abortJob.mockResolvedValue({+ success: true,+ jobData: { conversationId: 'conv-1' },+ content: [],+ text: '',+ });++ const response = await request(app)+ .post('/api/agents/chat/abort')+ .set('X-LibreChat-Generation-Protocol', '2')+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });++ expect(response.status).toBe(200);+ const ack = mockAcknowledgeScheduledStopPersistence.mock.calls[0][0];+ expect(ack.settle).toBeUndefined();+ // The paused path records its own outcome explicitly.+ expect(mockRecordScheduleOutcome).toHaveBeenCalled();+ });++ it('leaves non-scheduled aborts untouched by the Stop protocol', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue({+ status: 'running',+ createdAt: 1,+ metadata: { userId: 'test-user-123', generationProtocolVersion: 2 },+ });+ mockGenerationJobManager.abortJob.mockResolvedValue({+ success: true,+ jobData: { conversationId: 'conv-x' },+ content: [],+ text: '',+ });++ const response = await request(app)+ .post('/api/agents/chat/abort')+ .set('X-LibreChat-Generation-Protocol', '2')+ .send({ conversationId: 'conv-x', generationProtocolVersion: 2 });++ expect(response.status).toBe(200);+ expect(mockBeginScheduledStop).not.toHaveBeenCalled();+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();+ });+ });});});diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.jsindex ee3ed4e7bb..1d73ea5d98 100644--- a/api/server/routes/agents/index.js+++ b/api/server/routes/agents/index.js@@ -15,6 +15,8 @@ const {createMessageFilterPii,isAgentTriggerRequest,exemptAgentTriggerFromIpLimiter,+ captureScheduleFireContext,+ exemptFromUserLimiter: exemptScheduleFromUserLimiter,} = require('@librechat/api');const { createSseStreamTelemetry } = require('@librechat/api/telemetry');const { logger } = require('@librechat/data-schemas');@@ -36,6 +38,11 @@ const {negotiateExistingGenerationProtocol,} = require('~/server/controllers/agents/protocol');const { saveMessage } = require('~/models');+const {+ recordScheduleOutcome,+ beginScheduledStop,+ acknowledgeScheduledStopPersistence,+} = require('~/server/services/Schedules');const responses = require('./responses');const openai = require('./openai');const { v1 } = require('./v1');@@ -119,6 +126,7 @@ router.use(requireJwtAuth);// middleware reads this stable decision instead of re-verifying an expired token.router.use((req, _res, next) => {req._isAgentTrigger = isAgentTriggerRequest(req);+ captureScheduleFireContext(req);next();});router.use(checkBan);@@ -669,6 +677,27 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {throwOnError: true,}): undefined;+ // Stamp a scheduled run's Stop BEFORE signalling the abort. `abortJob` flips the job+ // to `aborted` immediately, then runs the partial-message/checkpoint persistence in+ // `beforePublish`. Without this stamp the owner settlement, reconciliation, and+ // schedule/account deletion could observe `aborted` and terminalize/erase the run+ // mid-write. The stamp is serialized: a fresh Stop already owning it means another+ // request is persisting, so we must not signal a second abort.+ const stopScheduleId = job.metadata?.scheduleId;+ const stopScheduledFor = job.metadata?.scheduledFor;+ const isScheduledStop = stopScheduleId != null && stopScheduledFor != null;+ let scheduledStopStamped = false;+ if (isScheduledStop) {+ const stopStamp = await beginScheduledStop({+ scheduleId: stopScheduleId,+ scheduledFor: stopScheduledFor,+ });+ if (stopStamp === 'in_progress') {+ res.set('Retry-After', '1');+ return res.status(409).json({ code: 'STOP_IN_PROGRESS', generationProtocolVersion });+ }+ scheduledStopStamped = stopStamp === true;+ }const abortResult = await GenerationJobManager.abortJob(jobStreamId, {expectedCreatedAt: job.createdAt,transformAbortContent: (content, abortJobData) => {@@ -807,6 +836,18 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {}},});+ // The abort did not land (replaced/still-active/already-settled), so no persistence+ // is in flight: release the Stop barrier we armed rather than deferring this+ // occurrence's settlement for the full stale-owner window. A retry or a replacement+ // generation re-stamps its own; the acknowledgement is fenced to the occurrence, not+ // a generation, so it cannot settle a successor through its predecessor.+ if (scheduledStopStamped && !abortResult.success) {+ await acknowledgeScheduledStopPersistence({+ scheduleId: stopScheduleId,+ scheduledFor: stopScheduledFor,+ });+ scheduledStopStamped = false;+ }if (abortResult.failureReason === 'generation_replaced') {return res.status(409).json({ code: 'RUN_REPLACED', generationProtocolVersion });}@@ -858,6 +899,53 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {abortResultResponseMessageId: abortResult.jobData?.responseMessageId,});+ // `beforePublish` has run: its partial-message and checkpoint writes have either+ // landed or failed. Acknowledge the Stop ONLY on success — that releases the owner's+ // settlement barrier so the run can terminalize. On a persistence failure we leave+ // the barrier unresolved: the run stays preserved (client retries; the stale-owner+ // timeout is the bounded recovery) rather than settling over an incomplete write.+ if (scheduledStopStamped && !abortResult.persistenceFailed) {+ await acknowledgeScheduledStopPersistence({+ scheduleId: stopScheduleId,+ scheduledFor: stopScheduledFor,+ // Re-drive the terminal outcome from here for a RUNNING generation: its owner+ // calls recordScheduleOutcome once, and if that call's Stop barrier deferred+ // (slow beforePublish), nothing would settle the run where no schedule+ // reconciler is armed. recordRunOutcome is match-guarded and idempotent, so an+ // owner that already settled makes this a no-op. A paused job is settled+ // explicitly below and needs no re-drive here.+ ...(job.status !== 'requires_action' && {+ settle: {+ status: 'interrupted',+ conversationId: job.metadata?.conversationId ?? jobStreamId,+ error: 'Scheduled run was stopped',+ },+ }),+ });+ }++ // A paused generation has no live provider owner left to report the stop.+ // Persist its scheduled occurrence here, after abortJob's required partial+ // response/checkpoint work AND its acknowledgement above, while running generations+ // continue to settle from their owning request/resume controller. A failed+ // persistence skips settlement so the incomplete run is not terminalized.+ if (+ job.status === 'requires_action' &&+ job.metadata?.scheduleId &&+ !abortResult.persistenceFailed+ ) {+ await recordScheduleOutcome({+ scheduleId: job.metadata.scheduleId,+ scheduledFor: job.metadata.scheduledFor,+ streamId: jobStreamId,+ jobCreatedAt: job.createdAt,+ status: 'interrupted',+ conversationId: job.metadata.conversationId ?? jobStreamId,+ clearConversationId: abortResult.jobData?.createdEventEmitted !== true,+ error: 'Scheduled run was stopped while awaiting approval',+ });+ }+if (abortResult.persistenceFailed && generationProtocolVersion < GENERATION_PROTOCOL_V2) {res.set('Retry-After', '1');return res.status(409).json({@@ -977,7 +1065,7 @@ if (isEnabled(LIMIT_MESSAGE_IP)) {}if (isEnabled(LIMIT_MESSAGE_USER)) {- chatRouter.use(messageUserLimiter);+ chatRouter.use(unless(exemptScheduleFromUserLimiter, messageUserLimiter));}chatRouter.use('/', chat);diff --git a/api/server/routes/index.js b/api/server/routes/index.jsindex 96baaf92b6..6b305686b6 100644--- a/api/server/routes/index.js+++ b/api/server/routes/index.js@@ -17,6 +17,7 @@ const memories = require('./memories');const presets = require('./presets');const projects = require('./projects');const prompts = require('./prompts');+const schedules = require('./schedules');const skills = require('./skills');const balance = require('./balance');const actions = require('./actions');@@ -69,6 +70,7 @@ module.exports = {models,prompts,projects,+ schedules,skills,actions,presets,diff --git a/api/server/routes/schedules.js b/api/server/routes/schedules.jsnew file mode 100644index 0000000000..b426710990--- /dev/null+++ b/api/server/routes/schedules.js@@ -0,0 +1,102 @@+const express = require('express');+const { Permissions, PermissionTypes } = require('librechat-data-provider');+const {+ isEnabled,+ SCHEDULE_FILE_HOLD,+ generateCheckAccess,+ createSchedulesHandlers,+} = require('@librechat/api');+const { requireJwtAuth, configMiddleware, messageIpLimiter } = require('~/server/middleware');+const {+ getLimits,+ fireScheduleNow,+ deleteScheduleForOwner,+ isUserDeleting,+} = require('~/server/services/Schedules');+const { resolveAgentFireAccess } = require('~/server/services/Schedules/access');+const methods = require('~/models');++const { getRoleByName } = methods;++const router = express.Router();+router.use(requireJwtAuth);+router.use(configMiddleware);++const checkSchedulesAccess = generateCheckAccess({+ permissionType: PermissionTypes.SCHEDULES,+ permissions: [Permissions.USE],+ getRoleByName,+});+const checkSchedulesCreate = generateCheckAccess({+ permissionType: PermissionTypes.SCHEDULES,+ permissions: [Permissions.USE, Permissions.CREATE],+ getRoleByName,+});++const handlers = createSchedulesHandlers({+ methods,+ getLimits,+ // Full fire-equivalent access check (not mere existence): the body-based agent+ // middleware is a no-op for schedule payloads (no `endpoint: 'agents'`), so+ // enforce the same role AGENTS:USE + resource VIEW (with manage:agents bypass)+ // the actual fire requires — otherwise a role without AGENTS:USE could schedule+ // runs the chat route rejects and walks toward auto-disable.+ canViewAgent: async (agentId, req) => (await resolveAgentFireAccess(agentId, req.user)) === 'ok',+ filterOwnedFileIds: async (fileIds, userId) => {+ const files = await methods.getFiles({ file_id: { $in: fileIds }, user: userId }, null, {+ file_id: 1,+ });+ return (files ?? []).map((file) => file.file_id);+ },+ markFilesUsed: async (fileIds, userId) => {+ // BOUNDED renewable hold (extendFilesTTL), not a permanent `$unset` of the upload+ // TTL: permanence made a schedule deleted before its first run, an edit that+ // replaced file_ids, or a failed creation leak the upload forever, since nothing+ // ever restored an expiry. The first fire that actually SENDS the file clears its+ // TTL through the ordinary consumption path; until then the hold is renewed at+ // create/edit and each fire preflight, and lapses when the schedule stops touching+ // it. Files already made permanent by a real send are skipped by construction.+ // Idempotent, and never touches the usage counter (a retention is not a consumption).+ // Then VERIFY every requested file still exists: one can be deleted between the+ // ownership check and here, and a silent success would persist a schedule whose+ // attachments the first fire drops. Existence is the check — not the hold's+ // modified-count, which reads 0 for an already-permanent or already-held file.+ const unique = [...new Set(fileIds)];+ await methods.extendFilesTTL(unique, SCHEDULE_FILE_HOLD, { user: userId });+ const files = await methods.getFiles({ file_id: { $in: unique }, user: userId }, null, {+ file_id: 1,+ });+ const present = (files ?? []).length;+ if (present !== unique.length) {+ throw new Error(`attachment retention incomplete: ${present}/${unique.length} files exist`);+ }+ },+ fireNow: fireScheduleNow,+ // Quiesce-then-erase delete: stops new claims, settles provably job-less runs+ // synchronously, aborts live ones, and reports honestly (see ScheduleDeleteResult);+ // a delivered abort erases on the generation's own outcome write, in any topology.+ deleteSchedule: deleteScheduleForOwner,+ // Durable account-deletion barrier. A one-shot disable scan cannot close the+ // create race, so every scheduling WRITE consults the user-level flag instead.+ isUserDeleting,+});++router.get('/', checkSchedulesAccess, handlers.listSchedules);+router.get('/:id', checkSchedulesAccess, handlers.getSchedule);+router.post('/', checkSchedulesCreate, handlers.createSchedule);+router.patch('/:id', checkSchedulesCreate, handlers.updateSchedule);+router.delete('/:id', checkSchedulesCreate, handlers.deleteSchedule);+// Run-now mutates runtime state; gate it on CREATE like the UI does (not USE).+// This is also where LIMIT_MESSAGE_IP has to apply to a manual run: the fire itself is a+// loopback POST carrying the server's address, so limiting by IP there would pool every+// user into one bucket. Here the initiating client's address is still on the request.+// The USER limiter is NOT duplicated here; the fire token carries the authenticated id,+// so the chat router applies it to the loopback exactly once.+router.post(+ '/:id/run',+ ...(isEnabled(process.env.LIMIT_MESSAGE_IP) ? [messageIpLimiter] : []),+ checkSchedulesCreate,+ handlers.runScheduleNow,+);++module.exports = router;diff --git a/api/server/services/Schedules/access.js b/api/server/services/Schedules/access.jsnew file mode 100644index 0000000000..a278b4e339--- /dev/null+++ b/api/server/services/Schedules/access.js@@ -0,0 +1,18 @@+const mongoose = require('mongoose');+const { createResolveAgentFireAccess } = require('@librechat/api');+const { checkPermission } = require('~/server/services/PermissionService');+const { hasCapability } = require('~/server/middleware/roles/capabilities');+const { getRoleByName } = require('~/models');++// Thin wiring over the TypeScript implementation in packages/api: inject the+// api-layer lookups (agent id, role, capability, resource ACL); the authorization+// logic itself lives in @librechat/api so it stays type-checked and in-boundary.+const resolveAgentFireAccess = createResolveAgentFireAccess({+ findAgentObjectId: (agentId) =>+ mongoose.models.Agent.findOne({ id: agentId }).select('_id').lean(),+ getRoleByName,+ hasCapability,+ checkPermission,+});++module.exports = { resolveAgentFireAccess };diff --git a/api/server/services/Schedules/index.js b/api/server/services/Schedules/index.jsnew file mode 100644index 0000000000..72f76fa7dc--- /dev/null+++ b/api/server/services/Schedules/index.js@@ -0,0 +1,112 @@+let service;++/**+ * Build the schedule service on first use. Controllers import this facade in many+ * focused test/runtime paths that never use scheduling; eagerly loading Config and+ * the trigger worker made those paths initialize two unrelated service graphs just+ * by requiring a controller.+ */+function getService() {+ if (service != null) {+ return service;+ }+ const mongoose = require('mongoose');+ const { createSchedulesService } = require('@librechat/api');+ const { getAppConfig } = require('~/server/services/Config/app');+ const {+ enqueueAgentTrigger,+ getAgentTriggerDelivery,+ } = require('~/server/services/Agents/triggers');+ const { resolveAgentFireAccess } = require('./access');+ const methods = require('~/models');+ const isUserDeleting = async (userId) => !(await methods.isAgentTriggerPrincipalActive(userId));++ service = createSchedulesService({+ methods,+ getAppConfig,+ findUserById: (userId) =>+ mongoose.models.User.findById(userId).select('_id tenantId role').lean(),+ findBalance: (userId) => mongoose.models.Balance.findOne({ user: userId }).lean(),+ upsertBalance: (userId, { set, setOnInsert }) =>+ mongoose.models.Balance.findOneAndUpdate(+ { user: userId },+ {+ ...(set && Object.keys(set).length > 0 ? { $set: set } : {}),+ ...(setOnInsert && Object.keys(setOnInsert).length > 0+ ? { $setOnInsert: setOnInsert }+ : {}),+ },+ { upsert: true, new: true },+ ).lean(),+ // Compare-and-set: only initialize an existing record while its credit is still null.+ // No upsert — a CAS miss must re-read the winner, not insert a fresh balance. `null`+ // in the filter also matches a legacy record whose `tokenCredits` field is absent.+ initializeNullBalance: (userId, { tokenCredits, sync }) =>+ mongoose.models.Balance.findOneAndUpdate(+ { user: userId, tokenCredits: null },+ { $set: { tokenCredits, ...(sync && Object.keys(sync).length > 0 ? sync : {}) } },+ { new: true },+ ).lean(),+ enqueueAgentTrigger,+ // Reconciliation reads the durable delivery to tell a still-live admission (a+ // deferred Retry-After) or a dead-letter apart from a genuinely orphaned run.+ getTriggerDelivery: getAgentTriggerDelivery,+ resolveAgentFireAccess,+ // Reuse the merged trigger/deletion admission fence. A schedule fire and every+ // generic trigger now fail closed on the same durable principal state.+ isUserDeleting,+ });+ return service;+}++const invoke =+ (method) =>+ (...args) =>+ getService()[method](...args);++/** Host adapter for the generic generation runtime's approval-expiry seam. The+ * callback is intentionally idempotent: a replica relay or schedule reconciliation+ * may re-drive the same retained terminal evidence after a transient failure. */+async function recordExpiredScheduleApproval(streamId, job) {+ if (!job?.scheduleId || !job?.scheduledFor) {+ return;+ }+ const recorded = await getService().recordScheduleOutcome({+ scheduleId: job.scheduleId,+ scheduledFor: job.scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'interrupted',+ conversationId: job.conversationId ?? streamId,+ error: 'Approval expired before a decision was made',+ });+ if (!recorded) {+ throw new Error(`Failed to settle expired scheduled approval ${job.scheduleId}`);+ }+}++module.exports = {+ getLimits: invoke('getLimits'),+ fireScheduleNow: invoke('fireScheduleNow'),+ recordScheduleOutcome: invoke('recordScheduleOutcome'),+ beginScheduledStop: invoke('beginScheduledStop'),+ acknowledgeScheduledStopPersistence: invoke('acknowledgeScheduledStopPersistence'),+ claimScheduleResume: invoke('claimScheduleResume'),+ releaseScheduleResumeClaim: invoke('releaseScheduleResumeClaim'),+ finalizeScheduleResumeClaim: invoke('finalizeScheduleResumeClaim'),+ releaseScheduleResumeFence: invoke('releaseScheduleResumeFence'),+ isScheduleLive: invoke('isScheduleLive'),+ deleteScheduleForOwner: invoke('deleteScheduleForOwner'),+ quiesceUserSchedules: invoke('quiesceUserSchedules'),+ restoreUserSchedulesFromDeletion: invoke('restoreUserSchedulesFromDeletion'),+ initializeScheduleEngine: invoke('initializeScheduleEngine'),+ initializeScheduleErasureSweep: invoke('initializeScheduleErasureSweep'),+ recordExpiredScheduleApproval,+ isUserDeleting: async (userId) => {+ const methods = require('~/models');+ return !(await methods.isAgentTriggerPrincipalActive(userId));+ },+ // Identity-fenced delete for a retained job whose run has since been settled+ // inline; reconciliation only reaches jobs whose run is still active.+ clearScheduledJob: (...args) => getService().engineDeps.clearReconciledJob(...args),+};diff --git a/api/server/services/Schedules/index.spec.js b/api/server/services/Schedules/index.spec.jsnew file mode 100644index 0000000000..194af1ea68--- /dev/null+++ b/api/server/services/Schedules/index.spec.js@@ -0,0 +1,60 @@+const mockRecordScheduleOutcome = jest.fn();+const mockCreateSchedulesService = jest.fn(() => ({+ recordScheduleOutcome: mockRecordScheduleOutcome,+}));++jest.mock('@librechat/api', () => ({+ createSchedulesService: (...args) => mockCreateSchedulesService(...args),+}));+jest.mock('mongoose', () => ({ models: {} }));+jest.mock('~/server/services/Config/app', () => ({ getAppConfig: jest.fn() }));+jest.mock('~/server/services/Agents/triggers', () => ({ enqueueAgentTrigger: jest.fn() }));+jest.mock('./access', () => ({ resolveAgentFireAccess: jest.fn() }));+jest.mock('~/models', () => ({ isAgentTriggerPrincipalActive: jest.fn() }));++const { recordExpiredScheduleApproval } = require('./index');++describe('recordExpiredScheduleApproval', () => {+ beforeEach(() => {+ mockRecordScheduleOutcome.mockReset().mockResolvedValue(true);+ });++ it('ignores ordinary generation approvals', async () => {+ await expect(+ recordExpiredScheduleApproval('conversation-1', { createdAt: 1000 }),+ ).resolves.toBeUndefined();++ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();+ });++ it('settles the exact scheduled generation as interrupted', async () => {+ await recordExpiredScheduleApproval('conversation-1', {+ createdAt: 1000,+ conversationId: 'conversation-1',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ });++ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ streamId: 'conversation-1',+ jobCreatedAt: 1000,+ status: 'interrupted',+ conversationId: 'conversation-1',+ error: 'Approval expired before a decision was made',+ });+ });++ it('surfaces a failed durable settlement instead of reporting false success', async () => {+ mockRecordScheduleOutcome.mockResolvedValue(false);++ await expect(+ recordExpiredScheduleApproval('conversation-1', {+ createdAt: 1000,+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ }),+ ).rejects.toThrow('Failed to settle expired scheduled approval schedule-1');+ });+});diff --git a/client/src/components/SidePanel/Agents/Instructions.tsx b/client/src/components/SidePanel/Agents/Instructions.tsxindex e4b0ae58f6..347ad0a1d8 100644--- a/client/src/components/SidePanel/Agents/Instructions.tsx+++ b/client/src/components/SidePanel/Agents/Instructions.tsx@@ -1,185 +1,42 @@-import { useState, useId } from 'react';-import * as Menu from '@ariakit/react/menu';-import { PlusCircle, Maximize2 } from 'lucide-react';-import { specialVariables } from 'librechat-data-provider';import { Controller, useFormContext } from 'react-hook-form';-import {- Label,- Button,- OGDialog,- Textarea,- DropdownPopup,- OGDialogClose,- TooltipAnchor,- OGDialogTitle,- OGDialogHeader,- OGDialogContent,-} from '@librechat/client';-import type { TSpecialVarLabel } from 'librechat-data-provider';import type { AgentForm } from '~/common';+import { VariableEditor } from '~/components/Variables';import { useLocalize } from '~/hooks';-interface VariableOption {- label: TSpecialVarLabel;- value: string;-}--const variableOptions: VariableOption[] = Object.keys(specialVariables).map((key) => ({- label: `com_ui_special_var_${key}` as TSpecialVarLabel,- value: `{{${key}}}`,-}));-export default function Instructions() {- const menuId = useId();- const dialogMenuId = useId();const localize = useLocalize();- const methods = useFormContext<AgentForm>();- const { control, setValue, getValues } = methods;-- const [isMenuOpen, setIsMenuOpen] = useState(false);- const [isDialogMenuOpen, setIsDialogMenuOpen] = useState(false);- const [isDialogOpen, setIsDialogOpen] = useState(false);-- const handleAddVariable = (label: TSpecialVarLabel, value: string) => {- const currentInstructions = getValues('instructions') || '';- const spacer = currentInstructions.length > 0 ? '\n' : '';- const prefix = localize(label);- setValue('instructions', currentInstructions + spacer + prefix + ': ' + value);- setIsMenuOpen(false);- setIsDialogMenuOpen(false);- };-- const variableItems = variableOptions.map((option) => ({- label: localize(option.label) || option.label,- onClick: () => handleAddVariable(option.label, option.value),- }));+ const { control } = useFormContext<AgentForm>();return (- <div className="mb-3 flex flex-col">- <div className="mb-1 flex items-center justify-between">- <Label- className="block text-[11px] font-medium uppercase tracking-wide text-text-secondary"- htmlFor="instructions"- >- {localize('com_ui_instructions')}- </Label>- <div className="flex items-center gap-0.5">- <DropdownPopup- portal={true}- mountByState={true}- unmountOnHide={true}- preserveTabOrder={true}- isOpen={isMenuOpen}- setIsOpen={setIsMenuOpen}- trigger={- <Menu.MenuButton- id="variables-menu-button"- aria-label={localize('com_ui_variables')}- title={localize('com_ui_variables')}- className="inline-flex h-7 w-7 items-center justify-center rounded-lg text-text-secondary transition-colors hover:bg-surface-secondary hover:text-text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring-primary"- >- <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />- </Menu.MenuButton>- }- items={variableItems}- menuId={menuId}- className="pointer-events-auto z-30"- />- <TooltipAnchor- description={localize('com_ui_expand_editor')}- render={- <Button- variant="ghost"- onClick={() => setIsDialogOpen(true)}- aria-label={localize('com_ui_expand_editor')}- className="h-7 w-7 p-0 text-text-secondary hover:bg-surface-secondary hover:text-text-primary"- >- <Maximize2 className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />- </Button>- }+ <Controller+ name="instructions"+ control={control}+ render={({ field, fieldState: { error } }) => (+ <div className="mb-3 flex flex-col">+ <VariableEditor+ id="instructions"+ label={localize('com_ui_instructions')}+ value={field.value ?? ''}+ onChange={field.onChange}+ onBlur={field.onBlur}+ inputRef={field.ref}+ placeholder={localize('com_agents_instructions_placeholder')}+ className="min-h-[88px] resize-y"+ labelClassName="block text-[11px] font-medium uppercase tracking-wide text-text-secondary"+ rows={3}+ required={true}+ invalid={error != null}/>+ {error && (+ <span+ className="mt-1 text-xs text-text-destructive transition duration-300 ease-in-out"+ role="alert"+ >+ {localize('com_ui_field_required')}+ </span>+ )}</div>- </div>- <Controller- name="instructions"- control={control}- render={({ field, fieldState: { error } }) => (- <>- <Textarea- {...field}- value={field.value ?? ''}- className="min-h-[88px] resize-y"- id="instructions"- placeholder={localize('com_agents_instructions_placeholder')}- rows={3}- aria-label={localize('com_ui_instructions')}- aria-required="true"- aria-invalid={error ? 'true' : 'false'}- />- {error && (- <span- className="mt-1 text-xs text-text-destructive transition duration-300 ease-in-out"- role="alert"- >- {localize('com_ui_field_required')}- </span>- )}- </>- )}- />-- <OGDialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>- <OGDialogContent- className="flex h-[85vh] max-h-[85vh] w-11/12 max-w-6xl flex-col gap-4 p-6"- showCloseButton={false}- >- <OGDialogHeader className="mb-2 pr-14">- <OGDialogTitle className="text-left text-2xl font-semibold">- {localize('com_ui_instructions')}- </OGDialogTitle>- </OGDialogHeader>- <Controller- name="instructions"- control={control}- render={({ field }) => (- <Textarea- {...field}- value={field.value ?? ''}- className="min-h-0 flex-1 resize-none text-base leading-relaxed"- placeholder={localize('com_agents_instructions_placeholder')}- aria-label={localize('com_ui_instructions')}- />- )}- />- <div className="flex items-center justify-between">- <DropdownPopup- portal={true}- mountByState={true}- unmountOnHide={true}- preserveTabOrder={true}- isOpen={isDialogMenuOpen}- setIsOpen={setIsDialogMenuOpen}- trigger={- <Menu.MenuButton- id="variables-menu-button-dialog"- render={- <Button variant="outline" className="gap-1.5">- <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />- {localize('com_ui_variables')}- </Button>- }- />- }- items={variableItems}- menuId={dialogMenuId}- className="pointer-events-auto z-[200]"- />- <OGDialogClose asChild>- <Button>{localize('com_ui_done')}</Button>- </OGDialogClose>- </div>- </OGDialogContent>- </OGDialog>- </div>+ )}+ />);}diff --git a/client/src/components/SidePanel/Schedules/ScheduleCard.tsx b/client/src/components/SidePanel/Schedules/ScheduleCard.tsxnew file mode 100644index 0000000000..03cad7310a--- /dev/null+++ b/client/src/components/SidePanel/Schedules/ScheduleCard.tsx@@ -0,0 +1,271 @@+import { useId, useRef, useMemo, useState, useCallback } from 'react';+import * as Ariakit from '@ariakit/react';+import { useNavigate } from 'react-router-dom';+import { Trans, useTranslation } from 'react-i18next';+import { Play, Trash, Pencil, Ellipsis } from 'lucide-react';+import { PermissionTypes, Permissions } from 'librechat-data-provider';+import {+ Label,+ Chip,+ Switch,+ Spinner,+ OGDialog,+ DropdownPopup,+ OGDialogTemplate,+ useToastContext,+} from '@librechat/client';+import type { TSchedule, ScheduleRunStatus, ScheduleDisabledReason } from 'librechat-data-provider';+import type { TranslationKeys } from '~/hooks';+import {+ useGetAgentByIdQuery,+ useDeleteScheduleMutation,+ useUpdateScheduleMutation,+ useRunScheduleNowMutation,+} from '~/data-provider';+import { useLocalize, useHasAccess } from '~/hooks';+import { useAgentsMapContext } from '~/Providers';+import { getMessageTimestamp } from '~/utils';+import ScheduleDialog from './ScheduleDialog';+import { describeCadence } from './cadence';++interface ScheduleCardProps {+ schedule: TSchedule;+}++type StatusTone = 'neutral' | 'success' | 'warning' | 'error';++const STATUS_CHIPS: Record<ScheduleRunStatus, { label: TranslationKeys; tone: StatusTone }> = {+ success: { label: 'com_ui_schedule_last_run', tone: 'success' },+ error: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },+ interrupted: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },+ requires_action: { label: 'com_ui_schedule_needs_approval', tone: 'warning' },+ started: { label: 'com_ui_schedule_run_started', tone: 'neutral' },+ skipped_overlap: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },+ skipped_balance: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },+};++const DISABLED_REASON_LABELS: Record<ScheduleDisabledReason, TranslationKeys> = {+ too_many_failures: 'com_ui_schedule_disabled_too_many_failures',+ agent_deleted: 'com_ui_schedule_disabled_agent_deleted',+ invalid_schedule: 'com_ui_schedule_disabled_invalid',+ permission_revoked: 'com_ui_schedule_disabled_permission_revoked',+ insufficient_balance: 'com_ui_schedule_disabled_insufficient_balance',+};++export default function ScheduleCard({ schedule }: ScheduleCardProps) {+ const localize = useLocalize();+ const navigate = useNavigate();+ const { i18n } = useTranslation();+ const { showToast } = useToastContext();+ const agentsMap = useAgentsMapContext();+ // Enable/disable, run-now, edit and delete all hit CREATE-gated routes, so a+ // USE-only viewer sees a read-only card instead of controls that 403.+ const canWrite = useHasAccess({+ permissionType: PermissionTypes.SCHEDULES,+ permission: Permissions.CREATE,+ });++ const menuId = useId();+ const [menuOpen, setMenuOpen] = useState(false);+ const [editOpen, setEditOpen] = useState(false);+ const [deleteOpen, setDeleteOpen] = useState(false);+ const editButtonRef = useRef<HTMLButtonElement>(null);+ const deleteButtonRef = useRef<HTMLButtonElement>(null);++ const mappedAgent = agentsMap?.[schedule.agent_id];+ const { data: fetchedAgent } = useGetAgentByIdQuery(schedule.agent_id, {+ enabled: agentsMap !== undefined && mappedAgent == null,+ });+ const agentName = mappedAgent?.name || fetchedAgent?.name || schedule.agent_id;++ const updateSchedule = useUpdateScheduleMutation({+ onError: () => {+ showToast({ message: localize('com_ui_error'), status: 'error' });+ },+ });+ const deleteSchedule = useDeleteScheduleMutation();+ const runSchedule = useRunScheduleNowMutation();++ const handleToggle = useCallback(+ (checked: boolean) => {+ updateSchedule.mutate({ id: schedule.id, payload: { enabled: checked } });+ },+ [schedule.id, updateSchedule],+ );++ const handleRunNow = useCallback(() => {+ runSchedule.mutate(schedule.id, {+ onSuccess: () => {+ showToast({ message: localize('com_ui_schedule_run_now_started'), status: 'success' });+ setMenuOpen(false);+ },+ onError: () => {+ showToast({ message: localize('com_ui_error'), status: 'error' });+ },+ });+ }, [schedule.id, runSchedule, showToast, localize]);++ const confirmDelete = useCallback(() => {+ deleteSchedule.mutate(schedule.id, {+ onSuccess: () => {+ showToast({ message: localize('com_ui_deleted'), status: 'success' });+ setDeleteOpen(false);+ },+ onError: () => {+ showToast({ message: localize('com_ui_error'), status: 'error' });+ },+ });+ }, [schedule.id, deleteSchedule, showToast, localize]);++ const cadenceText = describeCadence(schedule.cadence, localize, i18n.language);++ const nextRunText = useMemo(() => {+ if (!schedule.enabled || schedule.nextRunAt == null) {+ return null;+ }+ const timestamp = getMessageTimestamp(schedule.nextRunAt, i18n.language);+ if (!timestamp) {+ return null;+ }+ return localize('com_ui_schedule_next_run', { time: timestamp.relative });+ }, [schedule.enabled, schedule.nextRunAt, i18n.language, localize]);++ const dropdownItems = useMemo(+ () => [+ {+ label: localize('com_ui_schedule_run_now'),+ onClick: handleRunNow,+ hideOnClick: false,+ disabled: runSchedule.isLoading,+ icon: runSchedule.isLoading ? (+ <Spinner className="size-4" />+ ) : (+ <Play className="icon-sm mr-2 text-text-primary" aria-hidden="true" />+ ),+ },+ {+ label: localize('com_ui_edit'),+ onClick: () => setEditOpen(true),+ icon: <Pencil className="icon-sm mr-2 text-text-primary" aria-hidden="true" />,+ ariaHasPopup: 'dialog' as const,+ hideOnClick: false,+ ref: editButtonRef,+ render: (props) => <button {...props} />,+ },+ {+ label: localize('com_ui_delete'),+ onClick: () => setDeleteOpen(true),+ icon: <Trash className="icon-sm mr-2 text-text-primary" aria-hidden="true" />,+ ariaHasPopup: 'dialog' as const,+ hideOnClick: false,+ ref: deleteButtonRef,+ render: (props) => <button {...props} />,+ },+ ],+ [localize, handleRunNow, runSchedule.isLoading],+ );++ const statusChip = schedule.lastRun ? STATUS_CHIPS[schedule.lastRun.status] : null;+ const lastRunConvoId = schedule.lastRun?.conversationId;++ return (+ <div+ data-testid="schedule-card"+ className="rounded-lg border border-border-light bg-transparent px-3 py-2.5 transition-colors duration-theme-fast hover:bg-surface-secondary"+ >+ <div className="flex items-center gap-2">+ <span className="min-w-0 flex-1 truncate text-sm font-semibold text-text-primary">+ {schedule.name}+ </span>+ {canWrite && (+ <>+ <Switch+ checked={schedule.enabled}+ onCheckedChange={handleToggle}+ disabled={updateSchedule.isLoading}+ aria-label={`${localize('com_ui_schedule_enabled')}: ${schedule.name}`}+ className="shrink-0"+ />+ <DropdownPopup+ portal={true}+ menuId={menuId}+ focusLoop={true}+ className="z-[125]"+ unmountOnHide={true}+ isOpen={menuOpen}+ setIsOpen={setMenuOpen}+ trigger={+ <Ariakit.MenuButton+ id={`schedule-menu-${schedule.id}`}+ aria-label={`${localize('com_ui_schedule_options')}: ${schedule.name}`}+ className="inline-flex size-7 shrink-0 items-center justify-center rounded-md text-text-secondary hover:bg-surface-tertiary hover:text-text-primary focus:outline-none focus-visible:ring-2 focus-visible:ring-border-heavy"+ >+ <Ellipsis className="size-4" aria-hidden={true} />+ </Ariakit.MenuButton>+ }+ items={dropdownItems}+ />+ </>+ )}+ </div>+ <p className="mt-0.5 truncate text-xs text-text-secondary" title={agentName}>+ {agentName}+ </p>+ <p className="mt-1 text-sm text-text-primary">{cadenceText}</p>+ {nextRunText != null && <p className="mt-0.5 text-xs text-text-secondary">{nextRunText}</p>}+ {(statusChip != null || schedule.disabledReason != null) && (+ <div className="mt-1.5 flex flex-wrap items-center gap-1.5">+ {statusChip != null &&+ (lastRunConvoId != null && lastRunConvoId !== '' ? (+ <button+ type="button"+ className="rounded-full hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary"+ onClick={() => navigate(`/c/${lastRunConvoId}`)}+ >+ <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>+ </button>+ ) : (+ <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>+ ))}+ {schedule.disabledReason != null && (+ <Chip tone="error">{localize(DISABLED_REASON_LABELS[schedule.disabledReason])}</Chip>+ )}+ </div>+ )}+ {editOpen && (+ <ScheduleDialog+ open={editOpen}+ onOpenChange={setEditOpen}+ schedule={schedule}+ triggerRef={editButtonRef as React.MutableRefObject<HTMLButtonElement | null>}+ />+ )}+ <OGDialog+ open={deleteOpen}+ onOpenChange={setDeleteOpen}+ triggerRef={deleteButtonRef as React.MutableRefObject<HTMLButtonElement | null>}+ >+ <OGDialogTemplate+ showCloseButton={false}+ title={localize('com_ui_schedule_delete')}+ className="w-11/12 max-w-lg"+ main={+ <Label className="text-left text-sm font-medium">+ <Trans+ i18nKey="com_ui_delete_confirm_strong"+ values={{ title: schedule.name }}+ components={{ strong: <strong /> }}+ />+ </Label>+ }+ selection={{+ selectHandler: confirmDelete,+ selectClasses:+ 'bg-surface-destructive text-text-on-status hover:bg-surface-destructive-hover',+ selectText: localize('com_ui_delete'),+ }}+ />+ </OGDialog>+ </div>+ );+}diff --git a/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx b/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsxnew file mode 100644index 0000000000..5f71c241a6--- /dev/null+++ b/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx@@ -0,0 +1,24 @@+import { Skeleton } from '@librechat/client';++/** Mirrors the schedule card's title controls, agent, cadence, and next-run rows. */+export default function ScheduleCardSkeleton({ count = 5 }: { count?: number }) {+ return (+ <div className="flex flex-col gap-2" aria-hidden="true">+ {Array.from({ length: count }, (_, index) => (+ <div+ key={index}+ className="rounded-lg border border-border-light bg-transparent px-3 py-2.5"+ >+ <div className="flex items-center gap-2">+ <Skeleton className="h-4 min-w-0 flex-1 rounded" />+ <Skeleton className="h-5 w-9 shrink-0 rounded-full" />+ <Skeleton className="size-7 shrink-0 rounded-md" />+ </div>+ <Skeleton className="mt-1 h-3 w-2/5 rounded" />+ <Skeleton className="mt-2 h-4 w-3/4 rounded" />+ <Skeleton className="mt-1 h-3 w-1/2 rounded" />+ </div>+ ))}+ </div>+ );+}diff --git a/client/src/components/SidePanel/Schedules/ScheduleDialog.tsx b/client/src/components/SidePanel/Schedules/ScheduleDialog.tsxnew file mode 100644index 0000000000..16d1356e18--- /dev/null+++ b/client/src/components/SidePanel/Schedules/ScheduleDialog.tsx@@ -0,0 +1,575 @@+import { useMemo, useRef } from 'react';+import { v4 } from 'uuid';+import { useTranslation } from 'react-i18next';+import { useForm, Controller } from 'react-hook-form';+import { PermissionBits, scheduleFrequencies } from 'librechat-data-provider';+import {+ Input,+ Label,+ Radio,+ Button,+ FieldMessage,+ Spinner,+ Dropdown,+ OGDialog,+ ControlCombobox,+ OGDialogTemplate,+ useToastContext,+} from '@librechat/client';+import type {+ TSchedule,+ TCreateSchedule,+ TScheduleCadence,+ ScheduleFrequency,+} from 'librechat-data-provider';+import type { TranslationKeys } from '~/hooks';+import type { Meridiem } from './cadence';+import {+ useListAgentsQuery,+ useCreateScheduleMutation,+ useUpdateScheduleMutation,+} from '~/data-provider';+import { to12Hour, to24Hour, describeCadence, formatScheduleDay } from './cadence';+import { VariableEditor } from '~/components/Variables';+import { useLocalize } from '~/hooks';+import { cn } from '~/utils';++interface ScheduleDialogProps {+ open: boolean;+ onOpenChange: (open: boolean) => void;+ schedule?: TSchedule;+ triggerRef?: React.MutableRefObject<HTMLButtonElement | null>;+}++type ScheduleFormValues = {+ name: string;+ prompt: string;+ agent_id: string;+ frequency: ScheduleFrequency;+ hour12: number;+ minute: number;+ meridiem: Meridiem;+ dayOfWeek: number;+};++const FREQUENCY_LABELS: Record<ScheduleFrequency, TranslationKeys> = {+ hourly: 'com_ui_schedule_hourly',+ daily: 'com_ui_schedule_daily',+ weekdays: 'com_ui_schedule_weekdays',+ weekly: 'com_ui_schedule_weekly',+};++const BASE_MINUTES = [0, 15, 30, 45];++const FORM_ID = 'schedule-form';++const getDefaultValues = (schedule?: TSchedule): ScheduleFormValues => {+ if (!schedule) {+ return {+ name: '',+ prompt: '',+ agent_id: '',+ frequency: 'daily',+ hour12: 9,+ minute: 0,+ meridiem: 'AM',+ dayOfWeek: 1,+ };+ }+ const { hour12, meridiem } = to12Hour(schedule.cadence.hour);+ return {+ name: schedule.name,+ prompt: schedule.prompt,+ agent_id: schedule.agent_id,+ frequency: schedule.cadence.frequency,+ hour12,+ minute: schedule.cadence.minute,+ meridiem,+ dayOfWeek: schedule.cadence.daysOfWeek?.[0] ?? 1,+ };+};++type CadenceFormValues = Pick<+ ScheduleFormValues,+ 'frequency' | 'hour12' | 'minute' | 'meridiem' | 'dayOfWeek'+>;++const buildCadence = (values: CadenceFormValues, overrideDays?: number[]): TScheduleCadence => {+ if (values.frequency === 'hourly') {+ return { frequency: 'hourly', hour: 0, minute: values.minute };+ }+ const hour = to24Hour(values.hour12, values.meridiem);+ if (values.frequency === 'weekly') {+ // `overrideDays` preserves a stored multi-day set (which this single-day+ // picker can't represent) when only the time — not the day — was changed.+ return {+ frequency: 'weekly',+ hour,+ minute: values.minute,+ daysOfWeek: overrideDays ?? [values.dayOfWeek],+ };+ }+ return { frequency: values.frequency, hour, minute: values.minute };+};++export default function ScheduleDialog({+ open,+ onOpenChange,+ schedule,+ triggerRef,+}: ScheduleDialogProps) {+ const localize = useLocalize();+ const { i18n } = useTranslation();+ const { showToast } = useToastContext();+ const locale = i18n.language;++ const {+ control,+ register,+ watch,+ handleSubmit,+ formState: { dirtyFields, errors },+ } = useForm<ScheduleFormValues>({+ defaultValues: getDefaultValues(schedule),+ mode: 'onChange',+ });+ /** The revision the form defaults were built from, captured at the same instant.+ * The `schedule` prop keeps refreshing while the dialog is open (the schedules+ * query polls), so fencing with the LIVE prop's revision would stamp a freshly+ * refreshed revision onto a cadence rebuilt from this stale snapshot — exactly+ * the overwrite the fence exists to refuse. */+ const openedConfigRevision = useRef(schedule?.configRevision);+ const frequency = watch('frequency');+ const hour12 = watch('hour12');+ const minute = watch('minute');+ const meridiem = watch('meridiem');+ const dayOfWeek = watch('dayOfWeek');++ const { data: agents } = useListAgentsQuery(+ { requiredPermission: PermissionBits.VIEW },+ { select: (res) => res.data },+ );++ const agentItems = useMemo(+ () => (agents ?? []).map((agent) => ({ label: agent.name || agent.id, value: agent.id })),+ [agents],+ );++ const frequencyOptions = useMemo(+ () =>+ scheduleFrequencies.map((value) => ({+ value,+ label: localize(FREQUENCY_LABELS[value]),+ })),+ [localize],+ );++ const hourOptions = useMemo(+ () => Array.from({ length: 12 }, (_, index) => String(index + 1)),+ [],+ );++ const minuteOptions = useMemo(() => {+ const minutes = new Set(BASE_MINUTES);+ if (schedule) {+ minutes.add(schedule.cadence.minute);+ }+ return [...minutes]+ .sort((a, b) => a - b)+ .map((minute) => ({ value: String(minute), label: String(minute).padStart(2, '0') }));+ }, [schedule]);++ const meridiemOptions = useMemo(+ () => [+ { value: 'AM', label: localize('com_ui_schedule_am') },+ { value: 'PM', label: localize('com_ui_schedule_pm') },+ ],+ [localize],+ );++ const dayOptions = useMemo(+ () =>+ Array.from({ length: 7 }, (_, day) => ({+ value: String(day),+ label: formatScheduleDay(day, locale),+ })),+ [locale],+ );++ const timezone = useMemo(+ () => schedule?.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone,+ [schedule],+ );++ /** Idempotency key for the create being attempted. Held in a ref so every retry of+ * the same intent reuses it, and rotated when the attempt SUCCEEDS or when the+ * form content changes after a failed one (the server pins each key to the exact+ * payload it first saw, so retrying edited content under the old key answers 409).+ * `uuid` rather than `crypto.randomUUID`, which secure contexts gate — an HTTP/IP+ * deployment would throw here and never open the dialog. */+ const createRequestId = useRef(v4());+ /** Fingerprint of the payload the current key was last attempted with. */+ const lastAttemptedPayload = useRef<string | null>(null);++ const createSchedule = useCreateScheduleMutation({+ onSuccess: () => {+ createRequestId.current = v4();+ lastAttemptedPayload.current = null;+ showToast({ message: localize('com_ui_schedule_created'), status: 'success' });+ onOpenChange(false);+ },+ onError: () => {+ showToast({ message: localize('com_ui_error'), status: 'error' });+ },+ });++ const updateSchedule = useUpdateScheduleMutation({+ onSuccess: () => {+ showToast({ message: localize('com_ui_schedule_updated'), status: 'success' });+ onOpenChange(false);+ },+ onError: (error) => {+ const status = (error as { response?: { status?: number } } | undefined)?.response?.status;+ showToast({+ message: localize(status === 409 ? 'com_ui_schedule_conflict' : 'com_ui_error'),+ status: 'error',+ });+ },+ });++ const isLoading = createSchedule.isLoading || updateSchedule.isLoading;++ /** The stored weekly days an edit deliberately keeps: the form's single-day picker+ * only holds `daysOfWeek[0]`, so an untouched picker must not collapse a+ * multi-day (API-created) weekly schedule. ONE rule, used by both the submitted+ * cadence and the summary — a summary built without it told the user the schedule+ * runs on one day while the submit preserved (and kept firing) all of them. */+ const resolvePreservedWeeklyDays = (nextFrequency: ScheduleFormValues['frequency']) =>+ schedule &&+ !dirtyFields.dayOfWeek &&+ !dirtyFields.frequency &&+ schedule.cadence.frequency === 'weekly' &&+ nextFrequency === 'weekly'+ ? schedule.cadence.daysOfWeek+ : undefined;++ const onSubmit = (values: ScheduleFormValues) => {+ if (schedule) {+ // Preserve the stored cadence entirely on a pure rename (no cadence control+ // touched).+ const cadenceTouched =+ dirtyFields.frequency ||+ dirtyFields.hour12 ||+ dirtyFields.minute ||+ dirtyFields.meridiem ||+ dirtyFields.dayOfWeek;+ const cadence = buildCadence(values, resolvePreservedWeeklyDays(values.frequency));+ // PATCH only the fields the user actually touched, like the cadence handling+ // above: submitting the whole form snapshot silently overwrites fields another+ // tab or session edited while this dialog sat open (the server's revision fence+ // reads the CURRENT revision, so it cannot catch a stale full-form write).+ const payload = {+ ...(dirtyFields.name ? { name: values.name.trim() } : {}),+ ...(dirtyFields.prompt ? { prompt: values.prompt.trim() } : {}),+ ...(dirtyFields.agent_id ? { agent_id: values.agent_id } : {}),+ ...(cadenceTouched ? { cadence } : {}),+ };+ // Nothing touched: a field-less PATCH is refused server-side (it would rotate+ // the schedule's fencing for a request that changes nothing), so just close.+ if (Object.keys(payload).length === 0) {+ onOpenChange(false);+ return;+ }+ updateSchedule.mutate({+ id: schedule.id,+ // Fence on the revision this dialog OPENED with (not the live prop, which+ // polling refreshes under the stale form): cadence is rebuilt whole from+ // the opening snapshot, so an edit from another tab would otherwise be+ // silently overwritten — the server answers 409 instead.+ payload: {+ ...payload,+ ...(openedConfigRevision.current != null+ ? { expectedConfigRevision: openedConfigRevision.current }+ : {}),+ },+ });+ return;+ }+ const content = {+ name: values.name.trim(),+ prompt: values.prompt.trim(),+ agent_id: values.agent_id,+ cadence: buildCadence(values),+ timezone,+ target: 'new' as const,+ enabled: true,+ };+ // STABLE across retries of the SAME intent, which is the whole point: the server+ // commits the row and arms it in two writes, so a failure between them leaves the+ // client unable to tell what persisted, and a fresh key per attempt would make+ // each retry a new schedule. But the server pins each key to the exact payload it+ // first saw — so once the user EDITS the form after a failed attempt, this is a+ // new intent and needs a new key, or the retry answers 409 until the dialog is+ // reopened.+ const fingerprint = JSON.stringify(content);+ if (lastAttemptedPayload.current != null && lastAttemptedPayload.current !== fingerprint) {+ createRequestId.current = v4();+ }+ lastAttemptedPayload.current = fingerprint;+ const payload: TCreateSchedule = {+ ...content,+ clientRequestId: createRequestId.current,+ };+ createSchedule.mutate(payload);+ };++ const summaryCadence = buildCadence(+ { frequency, hour12, minute, meridiem, dayOfWeek },+ resolvePreservedWeeklyDays(frequency),+ );+ const summary = `${describeCadence(summaryCadence, localize, locale)} · ${timezone}`;++ return (+ <OGDialog open={open} onOpenChange={onOpenChange} triggerRef={triggerRef}>+ <OGDialogTemplate+ title={localize(schedule ? 'com_ui_schedule_edit' : 'com_ui_schedule_new')}+ showCloseButton={false}+ // The agent and time popovers cannot portal out of a focus-trapping dialog+ // (see below), so `overflow-visible` keeps them from being clipped. Only from+ // `md` up: the two-column layout fits well inside 90vh there, while narrow+ // viewports keep the template's scrolling so the footer stays reachable.+ className="w-11/12 md:max-w-3xl md:overflow-visible"+ main={+ <form id={FORM_ID} onSubmit={handleSubmit(onSubmit)} className="space-y-4">+ <div className="grid gap-4 md:grid-cols-2">+ <div className="space-y-2">+ <Label htmlFor="schedule-name" className="text-sm font-medium text-text-primary">+ {localize('com_ui_name')}+ </Label>+ <Input+ id="schedule-name"+ className="w-full"+ placeholder={localize('com_ui_schedule_name_placeholder')}+ aria-invalid={errors.name != null}+ aria-describedby="schedule-name-message"+ {...register('name', { required: localize('com_ui_field_required') })}+ />+ <FieldMessage id="schedule-name-message" message={errors.name?.message} />+ </div>+ <div className="space-y-2">+ <Label htmlFor="schedule-agent" className="text-sm font-medium text-text-primary">+ {localize('com_ui_agent')}+ </Label>+ <Controller+ name="agent_id"+ control={control}+ rules={{ required: localize('com_ui_field_required') }}+ render={({ field }) => (+ <ControlCombobox+ selectedValue={field.value}+ displayValue={+ agentItems.find((item) => item.value === field.value)?.label ?? ''+ }+ selectPlaceholder={localize('com_ui_select_agent')}+ searchPlaceholder={localize('com_agents_search_name')}+ setValue={field.onChange}+ onBlur={field.onBlur}+ items={agentItems}+ ariaLabel={localize('com_ui_agent')}+ ariaInvalid={errors.agent_id != null}+ ariaDescribedBy="schedule-agent-message"+ selectId="schedule-agent"+ isCollapsed={false}+ showCarat={true}+ placement="bottom-start"+ // Radix traps focus inside the dialog, so a popover portaled to+ // the body cannot be clicked, tabbed into, or typed in — and the+ // trap fighting Ariakit for focus locks the page up.+ portal={false}+ matchTriggerWidth={true}+ variant="field"+ />+ )}+ />+ <FieldMessage id="schedule-agent-message" message={errors.agent_id?.message} />+ <p className="text-xs text-text-secondary">+ {localize('com_ui_schedule_target_new_chat')}+ </p>+ </div>+ </div>++ <Controller+ name="prompt"+ control={control}+ rules={{ required: localize('com_ui_field_required') }}+ render={({ field }) => (+ <div className="space-y-2">+ <VariableEditor+ id="schedule-prompt"+ label={localize('com_ui_prompt')}+ value={field.value}+ onChange={field.onChange}+ onBlur={field.onBlur}+ inputRef={field.ref}+ placeholder={localize('com_ui_schedule_prompt_placeholder')}+ className="min-h-[120px] resize-none bg-transparent"+ labelClassName="text-sm font-medium text-text-primary"+ rows={4}+ required={true}+ invalid={errors.prompt != null}+ describedBy="schedule-prompt-message"+ portal={false}+ />+ <FieldMessage id="schedule-prompt-message" message={errors.prompt?.message} />+ </div>+ )}+ />++ <fieldset className="space-y-2">+ <legend>+ <Label+ id="schedule-frequency-label"+ className="text-sm font-medium text-text-primary"+ >+ {localize('com_ui_schedule_frequency')}+ </Label>+ </legend>+ <Controller+ name="frequency"+ control={control}+ render={({ field }) => (+ <Radio+ options={frequencyOptions}+ value={field.value}+ onChange={(value) => field.onChange(value as ScheduleFrequency)}+ fullWidth+ aria-labelledby="schedule-frequency-label"+ />+ )}+ />+ </fieldset>++ <div className="grid gap-4 md:grid-cols-2">+ {frequency === 'weekly' && (+ <fieldset className="space-y-2">+ <legend>+ <Label+ id="schedule-day-label"+ className="text-sm font-medium text-text-primary"+ >+ {localize('com_ui_schedule_day')}+ </Label>+ </legend>+ <Controller+ name="dayOfWeek"+ control={control}+ render={({ field }) => (+ <Dropdown+ value={String(field.value)}+ onChange={(value) => field.onChange(Number(value))}+ options={dayOptions}+ variant="field"+ portal={false}+ aria-labelledby="schedule-day-label"+ testId="schedule-day-select"+ />+ )}+ />+ </fieldset>+ )}+ <fieldset className="space-y-2">+ <legend>+ <Label id="schedule-time-label" className="text-sm font-medium text-text-primary">+ {localize(+ frequency === 'hourly'+ ? 'com_ui_schedule_minutes_past_hour'+ : 'com_ui_schedule_time',+ )}+ </Label>+ </legend>+ <div+ className={cn(+ 'grid gap-2',+ frequency === 'hourly' ? 'max-w-[8rem] grid-cols-1' : 'grid-cols-3',+ )}+ >+ {frequency !== 'hourly' && (+ <Controller+ name="hour12"+ control={control}+ render={({ field }) => (+ <Dropdown+ value={String(field.value)}+ onChange={(value) => field.onChange(Number(value))}+ options={hourOptions}+ variant="field"+ portal={false}+ ariaLabel={localize('com_ui_schedule_hour')}+ testId="schedule-hour-select"+ />+ )}+ />+ )}+ <Controller+ name="minute"+ control={control}+ render={({ field }) => (+ <Dropdown+ value={String(field.value)}+ onChange={(value) => field.onChange(Number(value))}+ options={minuteOptions}+ variant="field"+ portal={false}+ ariaLabel={localize('com_ui_schedule_minute')}+ testId="schedule-minute-select"+ />+ )}+ />+ {frequency !== 'hourly' && (+ <Controller+ name="meridiem"+ control={control}+ render={({ field }) => (+ <Dropdown+ value={field.value}+ onChange={field.onChange}+ options={meridiemOptions}+ variant="field"+ portal={false}+ ariaLabel={localize('com_ui_schedule_meridiem')}+ testId="schedule-meridiem-select"+ />+ )}+ />+ )}+ </div>+ </fieldset>+ </div>++ <p+ className="break-words rounded-lg bg-surface-secondary px-3 py-2 text-sm text-text-secondary"+ data-testid="schedule-summary"+ >+ {summary}+ </p>+ </form>+ }+ buttons={+ <Button+ type="submit"+ form={FORM_ID}+ variant="submit"+ disabled={isLoading}+ aria-label={localize(schedule ? 'com_ui_save' : 'com_ui_create')}+ >+ {isLoading ? (+ <Spinner className="size-4" />+ ) : (+ localize(schedule ? 'com_ui_save' : 'com_ui_create')+ )}+ </Button>+ }+ />+ </OGDialog>+ );+}diff --git a/client/src/components/SidePanel/Schedules/SchedulePanel.tsx b/client/src/components/SidePanel/Schedules/SchedulePanel.tsxnew file mode 100644index 0000000000..006eae48c3--- /dev/null+++ b/client/src/components/SidePanel/Schedules/SchedulePanel.tsx@@ -0,0 +1,105 @@+import { useState, type ReactNode } from 'react';+import { Plus } from 'lucide-react';+import { Button, TooltipAnchor } from '@librechat/client';+import { PermissionTypes, Permissions } from 'librechat-data-provider';+import { PanelContent, PanelFooter } from '~/components/ui';+import ScheduleCardSkeleton from './ScheduleCardSkeleton';+import { useSchedulesQuery } from '~/data-provider';+import { useLocalize, useHasAccess } from '~/hooks';+import ScheduleDialog from './ScheduleDialog';+import ScheduleCard from './ScheduleCard';++export default function SchedulePanel() {+ const localize = useLocalize();+ const { data, isLoading, isError, refetch } = useSchedulesQuery();+ const [createOpen, setCreateOpen] = useState(false);++ const hasCreateAccess = useHasAccess({+ permissionType: PermissionTypes.SCHEDULES,+ permission: Permissions.CREATE,+ });++ const schedules = data?.schedules ?? [];+ const maxPerUser = data?.limits.maxPerUser;+ const atLimit = maxPerUser !== undefined && schedules.length >= maxPerUser;+ let panelContent: ReactNode;++ if (isError) {+ panelContent = (+ <div className="flex min-h-40 flex-col items-center justify-center gap-2 p-4">+ <p className="text-center text-sm text-text-secondary">+ {localize('com_ui_schedules_error')}+ </p>+ <Button type="button" variant="outline" size="sm" onClick={() => refetch()}>+ {localize('com_ui_retry')}+ </Button>+ </div>+ );+ } else if (schedules.length === 0) {+ panelContent = (+ <div className="rounded-lg border border-dashed border-border-light p-4 text-center">+ <p className="text-sm text-text-secondary">{localize('com_ui_schedules_empty')}</p>+ </div>+ );+ } else {+ panelContent = (+ <div className="space-y-2" role="list" aria-label={localize('com_ui_schedules')}>+ {schedules.map((schedule) => (+ <div key={schedule.id} role="listitem">+ <ScheduleCard schedule={schedule} />+ </div>+ ))}+ </div>+ );+ }++ return (+ <div+ role="region"+ aria-label={localize('com_ui_schedules')}+ className="flex h-full w-full flex-col overflow-hidden pt-2"+ >+ <div className="shrink-0 px-3 pb-2">+ <div className="flex items-center justify-between gap-2">+ <span className="text-sm font-medium text-text-primary">+ {localize('com_ui_schedules')}+ </span>+ {hasCreateAccess && (+ <TooltipAnchor+ description={localize('com_ui_schedule_new')}+ side="bottom"+ render={+ <Button+ variant="outline"+ size="icon"+ className="size-9 shrink-0 bg-transparent"+ aria-label={localize('com_ui_schedule_new')}+ disabled={atLimit || isError || isLoading}+ onClick={() => setCreateOpen(true)}+ >+ <Plus className="size-4" aria-hidden="true" />+ </Button>+ }+ />+ )}+ </div>+ </div>++ <PanelContent isLoading={isLoading} skeleton={<ScheduleCardSkeleton />} className="px-3 pb-3">+ {/* A failed query must not masquerade as an empty list or enable creation+ against limits that could not be loaded. */}+ {panelContent}+ </PanelContent>++ {!isLoading && !isError && maxPerUser !== undefined && (+ <PanelFooter className="justify-start">+ <p className="text-xs text-text-secondary">+ {localize('com_ui_schedules_used', { used: schedules.length, max: maxPerUser })}+ </p>+ </PanelFooter>+ )}++ {createOpen && <ScheduleDialog open={createOpen} onOpenChange={setCreateOpen} />}+ </div>+ );+}diff --git a/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsx b/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsxnew file mode 100644index 0000000000..1facc9e110--- /dev/null+++ b/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsx@@ -0,0 +1,109 @@+import { createElement } from 'react';+import { ToastProvider } from '@librechat/client';+import userEvent from '@testing-library/user-event';+import { render, screen, within } from '@testing-library/react';+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import type { ReactNode } from 'react';+import ScheduleDialog from '../ScheduleDialog';++jest.mock('~/hooks', () => ({+ useLocalize: () => (key: string) => key,+}));++/** `@librechat/client` primitives localize through their own `useLocalize`, so the+ * shared `t` has to resolve too — not just the `i18n` this dialog reads for locale. */+jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key, i18n: { language: 'en-US' } }),+}));++const mockMutate = jest.fn();++jest.mock('~/data-provider', () => ({+ useListAgentsQuery: () => ({+ data: [+ { id: 'agent-1', name: 'Research Agent' },+ { id: 'agent-2', name: 'Digest Agent' },+ ],+ }),+ useCreateScheduleMutation: () => ({ mutate: mockMutate, isLoading: false }),+ useUpdateScheduleMutation: () => ({ mutate: mockMutate, isLoading: false }),+}));++const renderDialog = () => {+ const queryClient = new QueryClient({+ defaultOptions: { queries: { retry: false }, mutations: { retry: false } },+ });+ function Wrapper({ children }: { children: ReactNode }) {+ return createElement(+ QueryClientProvider,+ { client: queryClient },+ createElement(ToastProvider, null, children),+ );+ }+ return render(<ScheduleDialog open={true} onOpenChange={jest.fn()} />, { wrapper: Wrapper });+};++describe('ScheduleDialog', () => {+ afterEach(() => jest.clearAllMocks());++ /**+ * The agent list must render INSIDE the dialog. Portaled to the body it lands+ * outside Radix's focus trap, where it cannot be clicked, tabbed into, or typed+ * in — and the trap fighting Ariakit for focus locks the page up.+ */+ it('opens the agent list inside the dialog rather than portaling it away', async () => {+ const user = userEvent.setup();+ renderDialog();++ const dialog = screen.getByRole('dialog');+ await user.click(within(dialog).getByRole('combobox', { name: 'com_ui_agent' }));++ expect(within(dialog).getByPlaceholderText('com_agents_search_name')).toBeInTheDocument();+ });++ it('exposes frequency as a radiogroup with the default selected', () => {+ renderDialog();++ const group = screen.getByRole('radiogroup', { name: 'com_ui_schedule_frequency' });+ expect(within(group).getByRole('radio', { name: 'com_ui_schedule_daily' })).toBeChecked();+ expect(within(group).getByRole('radio', { name: 'com_ui_schedule_weekly' })).not.toBeChecked();+ });++ it('switches the day picker on only for weekly schedules', async () => {+ const user = userEvent.setup();+ renderDialog();++ expect(screen.queryByTestId('schedule-day-select')).not.toBeInTheDocument();+ await user.click(screen.getByRole('radio', { name: 'com_ui_schedule_weekly' }));+ expect(screen.getByTestId('schedule-day-select')).toBeInTheDocument();+ });++ /** The submit button sits in the dialog footer, outside the <form>, so it has to+ * reach it by id — that association is what lets Enter submit from a text field. */+ it('wires the footer submit button to the form', () => {+ renderDialog();++ const form = document.getElementById('schedule-form');+ expect(form).toBeInstanceOf(HTMLFormElement);+ const submit = screen.getByRole('button', { name: 'com_ui_create' });+ expect(submit).toHaveAttribute('type', 'submit');+ expect(submit).toHaveAttribute('form', 'schedule-form');+ });++ it('describes both text fields with placeholders', () => {+ renderDialog();++ expect(screen.getByPlaceholderText('com_ui_schedule_name_placeholder')).toBeInTheDocument();+ expect(screen.getByPlaceholderText('com_ui_schedule_prompt_placeholder')).toBeInTheDocument();+ });++ it('offers the special-variable menu on the prompt field', async () => {+ const user = userEvent.setup();+ renderDialog();++ await user.click(screen.getByRole('button', { name: 'com_ui_variables' }));+ expect(+ await screen.findByRole('menuitem', { name: 'com_ui_special_var_current_date' }),+ ).toBeInTheDocument();+ });+});diff --git a/client/src/components/SidePanel/Schedules/cadence.ts b/client/src/components/SidePanel/Schedules/cadence.tsnew file mode 100644index 0000000000..cbcb3dde22--- /dev/null+++ b/client/src/components/SidePanel/Schedules/cadence.ts@@ -0,0 +1,58 @@+import type { TScheduleCadence } from 'librechat-data-provider';+import type { LocalizeFunction } from '~/common';++export type Meridiem = 'AM' | 'PM';++const DAY_MS = 24 * 60 * 60 * 1000;++/** Mirrors the server's default weekly day (Monday) when a weekly cadence omits+ * daysOfWeek, so an API-created/migrated `frequency: 'weekly'` renders as weekly. */+const WEEKLY_DEFAULT_DAY = 1;++/** August 1st, 2021 was a Sunday; anchors day-of-week indices 0-6 to real dates */+const SUNDAY_UTC = Date.UTC(2021, 7, 1);++export const to12Hour = (hour: number): { hour12: number; meridiem: Meridiem } => ({+ hour12: hour % 12 === 0 ? 12 : hour % 12,+ meridiem: hour >= 12 ? 'PM' : 'AM',+});++export const to24Hour = (hour12: number, meridiem: Meridiem): number =>+ meridiem === 'PM' ? (hour12 % 12) + 12 : hour12 % 12;++export const formatScheduleTime = (hour: number, minute: number, locale?: string): string =>+ new Intl.DateTimeFormat(locale, { hour: 'numeric', minute: '2-digit' }).format(+ new Date(2000, 0, 1, hour, minute),+ );++export const formatScheduleDay = (day: number, locale?: string): string =>+ new Intl.DateTimeFormat(locale, { weekday: 'long', timeZone: 'UTC' }).format(+ new Date(SUNDAY_UTC + day * DAY_MS),+ );++export const describeCadence = (+ cadence: TScheduleCadence,+ localize: LocalizeFunction,+ locale?: string,+): string => {+ const { frequency, hour, minute, daysOfWeek } = cadence;+ if (frequency === 'hourly') {+ return localize('com_ui_schedule_runs_hourly', {+ minute: String(minute).padStart(2, '0'),+ });+ }++ const time = formatScheduleTime(hour, minute, locale);+ if (frequency === 'weekdays') {+ return localize('com_ui_schedule_runs_weekdays', { time });+ }+ if (frequency === 'weekly') {+ // A weekly cadence with no daysOfWeek is valid — the server fires it on the+ // default weekly day — so render it as weekly (not daily) using that same day.+ const effectiveDays =+ daysOfWeek != null && daysOfWeek.length > 0 ? daysOfWeek : [WEEKLY_DEFAULT_DAY];+ const days = effectiveDays.map((day) => formatScheduleDay(day, locale)).join(', ');+ return localize('com_ui_schedule_runs_weekly', { days, time });+ }+ return localize('com_ui_schedule_runs_daily', { time });+};diff --git a/client/src/components/SidePanel/Schedules/index.ts b/client/src/components/SidePanel/Schedules/index.tsnew file mode 100644index 0000000000..4b0457b900--- /dev/null+++ b/client/src/components/SidePanel/Schedules/index.ts@@ -0,0 +1,3 @@+export { default as SchedulePanel } from './SchedulePanel';+export { default as ScheduleCard } from './ScheduleCard';+export { default as ScheduleDialog } from './ScheduleDialog';diff --git a/client/src/components/Variables/Editor.tsx b/client/src/components/Variables/Editor.tsxnew file mode 100644index 0000000000..27b5820bcb--- /dev/null+++ b/client/src/components/Variables/Editor.tsx@@ -0,0 +1,203 @@+import { useId, useState } from 'react';+import * as Menu from '@ariakit/react/menu';+import { PlusCircle, Maximize2 } from 'lucide-react';+import { specialVariables } from 'librechat-data-provider';+import {+ Label,+ Button,+ OGDialog,+ Textarea,+ DropdownPopup,+ OGDialogClose,+ TooltipAnchor,+ OGDialogTitle,+ OGDialogHeader,+ OGDialogContent,+} from '@librechat/client';+import type { TSpecialVarLabel } from 'librechat-data-provider';+import { useLocalize } from '~/hooks';+import { cn } from '~/utils';++interface VariableOption {+ label: TSpecialVarLabel;+ value: string;+}++const variableOptions: VariableOption[] = Object.keys(specialVariables).map((key) => ({+ label: `com_ui_special_var_${key}` as TSpecialVarLabel,+ value: `{{${key}}}`,+}));++interface VariableEditorProps {+ id: string;+ /** Already-localized heading rendered above the field. */+ label: string;+ value: string;+ onChange: (value: string) => void;+ onBlur?: () => void;+ inputRef?: React.Ref<HTMLTextAreaElement>;+ placeholder?: string;+ rows?: number;+ required?: boolean;+ invalid?: boolean;+ describedBy?: string;+ className?: string;+ labelClassName?: string;+ containerClassName?: string;+ /**+ * Ariakit popovers portal to the body by default, which puts them outside a Radix+ * dialog's focus trap. Pass `false` from inside a dialog, and give that dialog+ * `overflow-visible` so the inline menu is not clipped.+ */+ portal?: boolean;+}++/**+ * A textarea for model-facing text with a menu that appends special variables+ * (`{{current_date}}` and friends) and an expand-to-fullscreen editor. Controlled so+ * each caller binds it to its own form field — insertions go through `onChange`, which+ * keeps them visible to react-hook-form's dirty tracking.+ */+export default function VariableEditor({+ id,+ label,+ value,+ onChange,+ onBlur,+ inputRef,+ placeholder,+ rows = 3,+ required,+ invalid,+ describedBy,+ className,+ labelClassName,+ containerClassName,+ portal = true,+}: VariableEditorProps) {+ const menuId = useId();+ const dialogMenuId = useId();+ const localize = useLocalize();++ const [isMenuOpen, setIsMenuOpen] = useState(false);+ const [isDialogMenuOpen, setIsDialogMenuOpen] = useState(false);+ const [isDialogOpen, setIsDialogOpen] = useState(false);++ const handleAddVariable = (variableLabel: TSpecialVarLabel, variableValue: string) => {+ const spacer = value.length > 0 ? '\n' : '';+ onChange(value + spacer + localize(variableLabel) + ': ' + variableValue);+ setIsMenuOpen(false);+ setIsDialogMenuOpen(false);+ };++ const variableItems = variableOptions.map((option) => ({+ label: localize(option.label) || option.label,+ onClick: () => handleAddVariable(option.label, option.value),+ }));++ return (+ <div className={cn('flex flex-col', containerClassName)}>+ <div className="mb-1 flex items-center justify-between">+ <Label className={labelClassName} htmlFor={id}>+ {label}+ </Label>+ <div className="flex items-center gap-0.5">+ <DropdownPopup+ portal={portal}+ mountByState={true}+ unmountOnHide={true}+ preserveTabOrder={true}+ isOpen={isMenuOpen}+ setIsOpen={setIsMenuOpen}+ trigger={+ <Menu.MenuButton+ id={`${id}-variables-menu-button`}+ aria-label={localize('com_ui_variables')}+ title={localize('com_ui_variables')}+ className="inline-flex h-7 w-7 items-center justify-center rounded-lg text-text-secondary transition-colors hover:bg-surface-secondary hover:text-text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring-primary"+ >+ <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />+ </Menu.MenuButton>+ }+ items={variableItems}+ menuId={menuId}+ className="pointer-events-auto z-30"+ />+ <TooltipAnchor+ description={localize('com_ui_expand_editor')}+ render={+ <Button+ type="button"+ variant="ghost"+ onClick={() => setIsDialogOpen(true)}+ aria-label={localize('com_ui_expand_editor')}+ className="h-7 w-7 p-0 text-text-secondary hover:bg-surface-secondary hover:text-text-primary"+ >+ <Maximize2 className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />+ </Button>+ }+ />+ </div>+ </div>+ <Textarea+ id={id}+ ref={inputRef}+ value={value}+ onChange={(event) => onChange(event.target.value)}+ onBlur={onBlur}+ className={className}+ placeholder={placeholder}+ rows={rows}+ aria-label={label}+ aria-required={required}+ aria-invalid={invalid}+ aria-describedby={describedBy}+ />++ <OGDialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>+ <OGDialogContent+ className="flex h-[85vh] max-h-[85vh] w-11/12 max-w-6xl flex-col gap-4 p-6"+ showCloseButton={false}+ >+ <OGDialogHeader className="mb-2 pr-14">+ <OGDialogTitle className="text-left text-2xl font-semibold">{label}</OGDialogTitle>+ </OGDialogHeader>+ <Textarea+ value={value}+ onChange={(event) => onChange(event.target.value)}+ className="min-h-0 flex-1 resize-none text-base leading-relaxed"+ placeholder={placeholder}+ aria-label={label}+ />+ <div className="flex items-center justify-between">+ <DropdownPopup+ portal={portal}+ mountByState={true}+ unmountOnHide={true}+ preserveTabOrder={true}+ isOpen={isDialogMenuOpen}+ setIsOpen={setIsDialogMenuOpen}+ trigger={+ <Menu.MenuButton+ id={`${id}-variables-menu-button-dialog`}+ render={+ <Button type="button" variant="outline" className="gap-1.5">+ <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />+ {localize('com_ui_variables')}+ </Button>+ }+ />+ }+ items={variableItems}+ menuId={dialogMenuId}+ className="pointer-events-auto z-[200]"+ />+ <OGDialogClose asChild>+ <Button type="button">{localize('com_ui_done')}</Button>+ </OGDialogClose>+ </div>+ </OGDialogContent>+ </OGDialog>+ </div>+ );+}diff --git a/client/src/components/Variables/index.ts b/client/src/components/Variables/index.tsnew file mode 100644index 0000000000..59c4636d36--- /dev/null+++ b/client/src/components/Variables/index.ts@@ -0,0 +1 @@+export { default as VariableEditor } from './Editor';diff --git a/client/src/data-provider/Schedules/__tests__/mutations.spec.tsx b/client/src/data-provider/Schedules/__tests__/mutations.spec.tsxnew file mode 100644index 0000000000..d193acfa10--- /dev/null+++ b/client/src/data-provider/Schedules/__tests__/mutations.spec.tsx@@ -0,0 +1,48 @@+import { createElement } from 'react';+import { QueryKeys } from 'librechat-data-provider';+import { act, renderHook } from '@testing-library/react';+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import type { ReactNode } from 'react';+import { useDeleteScheduleMutation } from '../mutations';++const mockDeleteSchedule = jest.fn();++jest.mock('librechat-data-provider', () => {+ const actual = jest.requireActual('librechat-data-provider');+ return {+ ...actual,+ dataService: {+ ...actual.dataService,+ deleteSchedule: (...args: unknown[]) => mockDeleteSchedule(...args),+ },+ };+});++const createWrapper = (queryClient: QueryClient) =>+ function Wrapper({ children }: { children: ReactNode }) {+ return createElement(QueryClientProvider, { client: queryClient }, children);+ };++describe('useDeleteScheduleMutation', () => {+ it('invalidates the schedules list when deletion returns an error after mutating state', async () => {+ const queryClient = new QueryClient({+ logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },+ defaultOptions: {+ queries: { retry: false },+ mutations: { retry: false },+ },+ });+ const invalidateQueries = jest.spyOn(queryClient, 'invalidateQueries');+ mockDeleteSchedule.mockRejectedValueOnce(new Error('Could not confirm active run stopped'));+ const { result } = renderHook(() => useDeleteScheduleMutation(), {+ wrapper: createWrapper(queryClient),+ });++ await act(async () => {+ await result.current.mutateAsync('schedule-1').catch(() => undefined);+ });++ expect(invalidateQueries).toHaveBeenCalledWith([QueryKeys.schedules]);+ queryClient.clear();+ });+});diff --git a/client/src/data-provider/Schedules/index.ts b/client/src/data-provider/Schedules/index.tsnew file mode 100644index 0000000000..53e6ebc9a8--- /dev/null+++ b/client/src/data-provider/Schedules/index.ts@@ -0,0 +1,3 @@+/* Scheduled chats */+export * from './queries';+export * from './mutations';diff --git a/client/src/data-provider/Schedules/mutations.ts b/client/src/data-provider/Schedules/mutations.tsnew file mode 100644index 0000000000..a16e48e2ef--- /dev/null+++ b/client/src/data-provider/Schedules/mutations.ts@@ -0,0 +1,87 @@+/* Scheduled chats */+import { useMutation, useQueryClient } from '@tanstack/react-query';+import { QueryKeys, MutationKeys, dataService } from 'librechat-data-provider';+import type {+ TSchedule,+ TCreateSchedule,+ TUpdateSchedule,+ TScheduleRunNowResponse,+} from 'librechat-data-provider';+import type { UseMutationOptions } from '@tanstack/react-query';++export const useCreateScheduleMutation = (+ options?: UseMutationOptions<TSchedule, Error, TCreateSchedule>,+) => {+ const queryClient = useQueryClient();+ return useMutation<TSchedule, Error, TCreateSchedule>(+ [MutationKeys.createSchedule],+ (payload: TCreateSchedule) => dataService.createSchedule(payload),+ {+ ...options,+ onSuccess: (...args) => {+ queryClient.invalidateQueries([QueryKeys.schedules]);+ options?.onSuccess?.(...args);+ },+ },+ );+};++export type UpdateScheduleParams = { id: string; payload: TUpdateSchedule };+export const useUpdateScheduleMutation = (+ options?: UseMutationOptions<TSchedule, Error, UpdateScheduleParams>,+) => {+ const queryClient = useQueryClient();+ return useMutation<TSchedule, Error, UpdateScheduleParams>(+ [MutationKeys.updateSchedule],+ ({ id, payload }: UpdateScheduleParams) => dataService.updateSchedule(id, payload),+ {+ ...options,+ onSuccess: (...args) => {+ queryClient.invalidateQueries([QueryKeys.schedules]);+ options?.onSuccess?.(...args);+ },+ },+ );+};++export const useDeleteScheduleMutation = (+ options?: UseMutationOptions<{ id: string }, Error, string>,+) => {+ const queryClient = useQueryClient();+ return useMutation<{ id: string }, Error, string>(+ [MutationKeys.deleteSchedule],+ (id: string) => dataService.deleteSchedule(id),+ {+ ...options,+ // An `unconfirmed` 503 is still a server-side mutation: the schedule has+ // already been disabled, marked deleting, and hidden from list reads.+ // Refresh on either outcome so the card cannot remain actionable in cache.+ onSettled: (...args) => {+ queryClient.invalidateQueries([QueryKeys.schedules]);+ options?.onSettled?.(...args);+ },+ },+ );+};++export const useRunScheduleNowMutation = (+ options?: UseMutationOptions<TScheduleRunNowResponse, Error, string>,+) => {+ const queryClient = useQueryClient();+ return useMutation<TScheduleRunNowResponse, Error, string>(+ [MutationKeys.runSchedule],+ (id: string) => dataService.runScheduleNow(id),+ {+ ...options,+ // Invalidate on SETTLED, not just success: several run-now 409 paths are still+ // server-side mutations (a balance skip updates lastRun/counters and can+ // auto-disable; agent/permission/invalid-schedule skips disable the schedule+ // before returning), so the card must refresh on those errors too rather than+ // wait for the polling interval.+ onSettled: (...args) => {+ queryClient.invalidateQueries([QueryKeys.schedules]);+ options?.onSettled?.(...args);+ },+ },+ );+};diff --git a/client/src/data-provider/Schedules/queries.ts b/client/src/data-provider/Schedules/queries.tsnew file mode 100644index 0000000000..a5fd8fee9e--- /dev/null+++ b/client/src/data-provider/Schedules/queries.ts@@ -0,0 +1,18 @@+/* Scheduled chats */+import { useQuery } from '@tanstack/react-query';+import { QueryKeys, dataService } from 'librechat-data-provider';+import type { UseQueryOptions, QueryObserverResult } from '@tanstack/react-query';+import type { TSchedulesResponse } from 'librechat-data-provider';++export const useSchedulesQuery = (+ config?: UseQueryOptions<TSchedulesResponse>,+): QueryObserverResult<TSchedulesResponse> => {+ return useQuery<TSchedulesResponse>([QueryKeys.schedules], () => dataService.getSchedules(), {+ // Automatic runs mutate nextRunAt/lastRun/auto-disable server-side while the+ // panel is open; refresh on focus and on a modest interval so it stays current.+ refetchOnWindowFocus: true,+ refetchOnReconnect: true,+ refetchInterval: 60_000,+ ...config,+ });+};diff --git a/client/src/data-provider/index.ts b/client/src/data-provider/index.tsindex a9f6799033..8285caa905 100644--- a/client/src/data-provider/index.ts+++ b/client/src/data-provider/index.ts@@ -10,6 +10,8 @@ export * from './Memories';export * from './Messages';export * from './Misc';export * from './Projects';+/* Scheduled chats */+export * from './Schedules';export * from './Tools';export * from './connection';export * from './Favorites';diff --git a/client/src/hooks/Nav/useSideNavLinks.ts b/client/src/hooks/Nav/useSideNavLinks.tsindex 04cf146f7a..20ccf3c321 100644--- a/client/src/hooks/Nav/useSideNavLinks.ts+++ b/client/src/hooks/Nav/useSideNavLinks.ts@@ -6,6 +6,7 @@ import {Bookmark,NotebookPen,ScrollText,+ CalendarClock,ArrowRightToLine,SlidersHorizontal,} from 'lucide-react';@@ -29,6 +30,7 @@ import MCPBuilderPanel from '~/components/SidePanel/MCPBuilder/MCPBuilderPanel';import AgentPanelSwitch from '~/components/SidePanel/Agents/AgentPanelSwitch';import BookmarkPanel from '~/components/SidePanel/Bookmarks/BookmarkPanel';import PanelSwitch from '~/components/SidePanel/Builder/PanelSwitch';+import { SchedulePanel } from '~/components/SidePanel/Schedules';import Parameters from '~/components/SidePanel/Parameters/Panel';import { MemoryPanel } from '~/components/SidePanel/Memories';import FilesPanel from '~/components/SidePanel/Files/Panel';@@ -88,6 +90,10 @@ export default function useSideNavLinks({permissionType: PermissionTypes.MCP_SERVERS,permission: Permissions.CREATE,});+ const hasAccessToSchedules = useHasAccess({+ permissionType: PermissionTypes.SCHEDULES,+ permission: Permissions.USE,+ });const { availableMCPServers } = useMCPServerManager();const { agentsConfig } = useGetAgentsConfig({ endpointsConfig });@@ -140,6 +146,26 @@ export default function useSideNavLinks({});}+ // Scheduled chats are EXPERIMENTAL and default-OFF: the server enables them only+ // when an admin opts in explicitly, so ABSENT config means disabled here too.+ // Mirrors getLimits exactly — absent/null/`false` are all off, `true` is on, and the+ // object form is on unless it sets `use: false`. Any mismatch would show an entry+ // whose create/run operations the backend rejects.+ const schedulesConfig = interfaceConfig.schedules;+ const schedulesEnabled =+ schedulesConfig != null &&+ schedulesConfig !== false &&+ !(typeof schedulesConfig === 'object' && schedulesConfig.use === false);+ if (hasAccessToSchedules && schedulesEnabled) {+ links.push({+ title: 'com_ui_schedules',+ label: '',+ icon: CalendarClock,+ id: 'schedules',+ Component: SchedulePanel,+ });+ }+if (hasAccessToPrompts) {links.push({title: 'com_ui_prompts',@@ -228,6 +254,8 @@ export default function useSideNavLinks({skillsEnabled,hasAccessToMemories,hasAccessToReadMemories,+ hasAccessToSchedules,+ interfaceConfig.schedules,interfaceConfig.parameters,endpointType,hasAccessToBookmarks,diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.jsonindex 3f2405e8a5..7a47979387 100644--- a/client/src/locales/en/translation.json+++ b/client/src/locales/en/translation.json@@ -1852,6 +1852,51 @@"com_ui_save_message_error": "The message could not be saved. Your changes are still in the editor.","com_ui_saved": "Saved!","com_ui_saving": "Saving...",+ "com_ui_schedule_am": "AM",+ "com_ui_schedule_conflict": "This schedule was changed elsewhere. Reopen it to edit the latest version.",+ "com_ui_schedule_created": "Schedule created",+ "com_ui_schedule_daily": "Daily",+ "com_ui_schedule_day": "Day of week",+ "com_ui_schedule_delete": "Delete schedule",+ "com_ui_schedule_disabled_agent_deleted": "Disabled: agent deleted",+ "com_ui_schedule_disabled_insufficient_balance": "Disabled: insufficient balance",+ "com_ui_schedule_disabled_invalid": "Disabled: invalid schedule",+ "com_ui_schedule_disabled_permission_revoked": "Disabled: permission revoked",+ "com_ui_schedule_disabled_too_many_failures": "Disabled: too many failures",+ "com_ui_schedule_edit": "Edit schedule",+ "com_ui_schedule_enabled": "Enabled",+ "com_ui_schedule_frequency": "Frequency",+ "com_ui_schedule_hour": "Hour",+ "com_ui_schedule_hourly": "Hourly",+ "com_ui_schedule_last_run": "Last run",+ "com_ui_schedule_last_run_failed": "Last run failed",+ "com_ui_schedule_meridiem": "AM or PM",+ "com_ui_schedule_minute": "Minute",+ "com_ui_schedule_minutes_past_hour": "Minutes past the hour",+ "com_ui_schedule_name_placeholder": "Morning news briefing",+ "com_ui_schedule_needs_approval": "Needs approval",+ "com_ui_schedule_new": "New schedule",+ "com_ui_schedule_next_run": "Next run {{time}}",+ "com_ui_schedule_options": "Schedule options",+ "com_ui_schedule_pm": "PM",+ "com_ui_schedule_prompt_placeholder": "Summarize the top AI news from the last 24 hours and highlight anything worth acting on.",+ "com_ui_schedule_run_now": "Run now",+ "com_ui_schedule_run_now_started": "Run started",+ "com_ui_schedule_run_skipped": "Skipped",+ "com_ui_schedule_run_started": "Running",+ "com_ui_schedule_runs_daily": "Runs daily at {{time}}",+ "com_ui_schedule_runs_hourly": "Runs every hour at :{{minute}}",+ "com_ui_schedule_runs_weekdays": "Runs weekdays at {{time}}",+ "com_ui_schedule_runs_weekly": "Runs weekly on {{days}} at {{time}}",+ "com_ui_schedule_target_new_chat": "Each run starts a new chat with this agent",+ "com_ui_schedule_time": "Time",+ "com_ui_schedule_updated": "Schedule updated",+ "com_ui_schedule_weekdays": "Weekdays",+ "com_ui_schedule_weekly": "Weekly",+ "com_ui_schedules": "Scheduled chats",+ "com_ui_schedules_error": "Couldn't load your scheduled chats",+ "com_ui_schedules_empty": "No scheduled chats yet",+ "com_ui_schedules_used": "{{used}} of {{max}} schedules used","com_ui_schema": "Schema","com_ui_scope": "Scope","com_ui_scroll_left": "Scroll left",diff --git a/config/__tests__/delete-user.spec.js b/config/__tests__/delete-user.spec.jsnew file mode 100644index 0000000000..282fe37767--- /dev/null+++ b/config/__tests__/delete-user.spec.js@@ -0,0 +1,216 @@+const mockModelRegistry = {};+const mockModelFor = (name) => {+ if (!mockModelRegistry[name]) {+ mockModelRegistry[name] = {+ findOne: jest.fn(async () => null),+ deleteMany: jest.fn(async () => ({ deletedCount: 0 })),+ updateMany: jest.fn(async () => ({ modifiedCount: 0 })),+ };+ }+ return mockModelRegistry[name];+};++const mockMethods = {+ beginAgentTriggerUserDeletion: jest.fn(),+ recoverStaleAgentTriggerUserDeletion: jest.fn(),+ prepareAgentTriggerUserPurge: jest.fn(),+ suspendUserSchedulesForDeletion: jest.fn(),+ restoreUserSchedulesFromDeletion: jest.fn(),+ countActiveAgentTriggerDeliveriesByUser: jest.fn(),+ deleteSchedulesByUser: jest.fn(),+ deleteUserById: jest.fn(),+ deleteAgentTriggerDeliveriesByUser: jest.fn(),+ cancelAgentTriggerUserPurge: jest.fn(),+ cancelAgentTriggerUserDeletion: jest.fn(),+};+const mockGetCleanupBlockingJobIdsForUser = jest.fn();+const mockAbortJob = jest.fn();+const mockDestroy = jest.fn();+const mockSilentExit = jest.fn();+const mockAskQuestion = jest.fn();++jest.mock('../connect', () => jest.fn(async () => undefined));+jest.mock('mongoose', () => ({ disconnect: jest.fn(async () => undefined) }));+jest.mock('@librechat/data-schemas', () => ({+ createModels: () => new Proxy({}, { get: (_target, prop) => mockModelFor(prop) }),+ createMethods: () => mockMethods,+ runAsSystem: (operation) => operation(),+}));+jest.mock('@librechat/api', () => ({+ waitForKeyvRedisClient: jest.fn(async () => undefined),+ createStreamServices: jest.fn(() => ({ isRedis: true })),+ GenerationJobManager: {+ configure: jest.fn(),+ initialize: jest.fn(),+ getCleanupBlockingJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),+ abortJob: (...args) => mockAbortJob(...args),+ destroy: (...args) => mockDestroy(...args),+ },+}));+jest.mock('~/cache/getLogStores', () => jest.fn());+jest.mock('../helpers', () => ({+ ...jest.requireActual('../helpers'),+ askQuestion: mockAskQuestion,+ silentExit: mockSilentExit,+}));++const USER_ID = 'user-being-deleted';++const runCli = () =>+ new Promise((resolve, reject) => {+ mockSilentExit.mockImplementation((code = 0) => resolve(code));+ jest.spyOn(process, 'exit').mockImplementation((code) => {+ resolve(code);+ });+ jest.isolateModules(() => {+ try {+ require('../delete-user');+ } catch (error) {+ reject(error);+ }+ });+ });++describe('Delete user CLI', () => {+ const originalArgv = process.argv;+ let logSpy;++ beforeEach(() => {+ process.argv = ['node', 'delete-user.js', '[email protected]'];+ logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined);+ jest.spyOn(console, 'error').mockImplementation(() => undefined);+ for (const model of Object.values(mockModelRegistry)) {+ for (const method of Object.values(model)) {+ method.mockClear();+ }+ }+ mockModelFor('User').findOne.mockResolvedValue({+ _id: { toString: () => USER_ID },+ email: '[email protected]',+ tenantId: 'tenant-1',+ });+ mockMethods.beginAgentTriggerUserDeletion.mockReset().mockResolvedValue('acquired');+ mockMethods.recoverStaleAgentTriggerUserDeletion.mockReset().mockResolvedValue('acquired');+ mockMethods.prepareAgentTriggerUserPurge.mockReset().mockResolvedValue(undefined);+ mockMethods.suspendUserSchedulesForDeletion.mockReset().mockResolvedValue(undefined);+ mockMethods.restoreUserSchedulesFromDeletion.mockReset().mockResolvedValue(undefined);+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(0);+ mockMethods.deleteSchedulesByUser.mockReset().mockResolvedValue(undefined);+ mockMethods.deleteUserById.mockReset().mockResolvedValue({ deletedCount: 1 });+ mockMethods.deleteAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(undefined);+ mockMethods.cancelAgentTriggerUserPurge.mockReset().mockResolvedValue(true);+ mockMethods.cancelAgentTriggerUserDeletion.mockReset().mockResolvedValue(true);+ mockGetCleanupBlockingJobIdsForUser.mockReset().mockResolvedValue([]);+ mockAbortJob.mockReset().mockResolvedValue({ success: true });+ mockDestroy.mockReset().mockResolvedValue(undefined);+ mockAskQuestion.mockReset().mockResolvedValueOnce('y').mockResolvedValueOnce('n');+ });++ afterEach(() => {+ process.argv = originalArgv;+ logSpy.mockRestore();+ jest.restoreAllMocks();+ });++ it('fences triggers and schedules before inspecting active work', async () => {+ expect(await runCli()).toBe(0);++ expect(mockMethods.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(+ USER_ID,+ expect.any(Date),+ );+ expect(mockMethods.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(+ USER_ID,+ expect.any(Date),+ 'tenant-1',+ );+ expect(mockMethods.suspendUserSchedulesForDeletion).toHaveBeenCalledWith(+ USER_ID,+ expect.any(String),+ );+ expect(mockMethods.beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(+ mockMethods.prepareAgentTriggerUserPurge.mock.invocationCallOrder[0],+ );+ expect(mockMethods.prepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(+ mockMethods.suspendUserSchedulesForDeletion.mock.invocationCallOrder[0],+ );+ expect(mockMethods.suspendUserSchedulesForDeletion.mock.invocationCallOrder[0]).toBeLessThan(+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mock.invocationCallOrder[0],+ );+ expect(+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mock.invocationCallOrder[0],+ ).toBeLessThan(mockGetCleanupBlockingJobIdsForUser.mock.invocationCallOrder[0]);+ });++ it('aborts provider work before deleting account-owned records', async () => {+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);++ expect(await runCli()).toBe(0);++ expect(mockGetCleanupBlockingJobIdsForUser).toHaveBeenCalledWith(USER_ID, 'tenant-1');+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });+ expect(mockAbortJob.mock.invocationCallOrder[0]).toBeLessThan(+ mockModelFor('Message').deleteMany.mock.invocationCallOrder[0],+ );+ expect(mockMethods.deleteSchedulesByUser).toHaveBeenCalledWith(USER_ID);+ expect(mockMethods.deleteSchedulesByUser.mock.invocationCallOrder[0]).toBeLessThan(+ mockMethods.deleteUserById.mock.invocationCallOrder[0],+ );+ });++ it('deletes nothing and releases both fences when schedule quiescing fails', async () => {+ mockMethods.suspendUserSchedulesForDeletion.mockRejectedValueOnce(+ new Error('schedule write failed'),+ );++ expect(await runCli()).toBe(1);++ expect(mockModelFor('Message').deleteMany).not.toHaveBeenCalled();+ expect(mockMethods.deleteSchedulesByUser).not.toHaveBeenCalled();+ expect(mockMethods.deleteUserById).not.toHaveBeenCalled();+ const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];+ expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);+ expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);+ });++ /**+ * The suspension is reversible, so a CLI attempt that does not commit must restore the+ * rows it suspended — and must do so while the user-deletion fence still refuses new+ * schedule writes, or an owner edit (or a second attempt re-suspending under a new token)+ * can race the restore and strand the disabled snapshot.+ */+ it('restores suspended schedules BEFORE releasing the deletion fence on failure', async () => {+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain failed'));++ expect(await runCli()).toBe(1);++ const token = mockMethods.suspendUserSchedulesForDeletion.mock.calls[0][1];+ expect(token).toEqual(expect.any(String));+ expect(mockMethods.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(USER_ID, token);+ expect(mockMethods.restoreUserSchedulesFromDeletion.mock.invocationCallOrder[0]).toBeLessThan(+ mockMethods.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],+ );+ });++ it('never restores schedules after a committed deletion', async () => {+ expect(await runCli()).toBe(0);++ expect(mockMethods.deleteSchedulesByUser).toHaveBeenCalledWith(USER_ID);+ expect(mockMethods.restoreUserSchedulesFromDeletion).not.toHaveBeenCalled();+ });++ it('deletes nothing and releases both fences when provider drain fails', async () => {+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain failed'));++ expect(await runCli()).toBe(1);++ expect(mockModelFor('Message').deleteMany).not.toHaveBeenCalled();+ expect(mockMethods.deleteSchedulesByUser).not.toHaveBeenCalled();+ expect(mockMethods.deleteUserById).not.toHaveBeenCalled();+ const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];+ expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);+ expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);+ });+});diff --git a/config/delete-user.js b/config/delete-user.jsindex fec8df7590..533a239f2f 100644--- a/config/delete-user.js+++ b/config/delete-user.js@@ -3,6 +3,7 @@// @ts-nocheckconst path = require('path');const mongoose = require('mongoose');+const { randomUUID } = require('node:crypto');const { createModels, createMethods, runAsSystem } = require('@librechat/data-schemas');const {Key,@@ -118,6 +119,7 @@ async function gracefulExit(code = 0) {}let deletionFence;+ let scheduleSuspensionToken;let userDeleted = false;try {@@ -158,6 +160,13 @@ async function gracefulExit(code = 0) {await runAsSystem(() =>methods.prepareAgentTriggerUserPurge(uid, deletionFence, user.tenantId),);+ // Reversible, token-fenced suspension (the same protocol the HTTP controller uses):+ // an attempt that does not commit restores exactly these rows in the finally block,+ // rather than leaving a surviving account with disabled, erasure-eligible schedules.+ scheduleSuspensionToken = randomUUID();+ await runAsSystem(() =>+ methods.suspendUserSchedulesForDeletion(uid, scheduleSuspensionToken),+ );if (hasSharedGenerationStore) {const deadline = Date.now() + TRIGGER_DRAIN_TIMEOUT_MS;while (@@ -214,6 +223,7 @@ async function gracefulExit(code = 0) {}await Promise.all(tasks);+ await runAsSystem(() => methods.deleteSchedulesByUser(uid));// 6) Remove user from all groupsawait Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });@@ -226,6 +236,22 @@ async function gracefulExit(code = 0) {userDeleted = true;await runAsSystem(() => methods.deleteAgentTriggerDeliveriesByUser(uid));} finally {+ // RESTORE BEFORE RELEASING THE FENCE. While the user-deletion fence is still armed, new+ // schedule writes/claims are refused, so this restore cannot race an owner PATCH nor be+ // superseded by a second deletion attempt re-suspending these rows under a new token.+ if (scheduleSuspensionToken != null && !userDeleted) {+ // Retried inside the method; the fence is still released below on purpose, since+ // retaining it would block the retry that is the convergence path. Print the token+ // so a restore that never converges stays recoverable by hand.+ await runAsSystem(() =>+ methods.restoreUserSchedulesFromDeletion(uid, scheduleSuspensionToken),+ ).catch((error) =>+ console.error(+ `Failed to restore suspended schedules; they remain disabled for user ${uid} under suspension token ${scheduleSuspensionToken}:`,+ error,+ ),+ );+ }if (deletionFence != null && !userDeleted) {await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch((error) => console.error('Failed to disarm trigger purge recovery:', error),diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yamlindex 7b8dc5a218..17421f502b 100644--- a/e2e/config/librechat.e2e.yaml+++ b/e2e/config/librechat.e2e.yaml@@ -13,6 +13,13 @@ interface:# value the permission falls through to the seeded role default and# `handlePlusCommand` returns before opening the popover.multiConvo: true+ # Scheduled chats are EXPERIMENTAL and default-off, so the e2e harness must opt in+ # explicitly before schedule writes and fires are accepted.+ # minIntervalMinutes is lowered so the spec can create a schedule without waiting+ # out the production 60-minute floor.+ schedules:+ use: true+ minIntervalMinutes: 1# Enables the memory feature so the MEMORIES.USE permission is granted and the# ephemeral memory badge (inline set_memory/delete_memory tools) is available.diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.tsindex d215f09256..0230984726 100644--- a/e2e/playwright.config.mock.ts+++ b/e2e/playwright.config.mock.ts@@ -241,6 +241,12 @@ export default defineConfig({// network fixtures so inspection and persistent connections agree.command: `node ${serverPath}`,cwd: rootPath,+ // Only the one-replica harness may assert the scheduler's single-process topology.+ // The two-replica MCP suite must leave scheduled writes disabled.+ env: {+ ...process.env,+ ...(replicaCount === 1 ? { SCHEDULES_SINGLE_PROCESS: 'true' } : {}),+ },url: baseURL,stdout: 'pipe',ignoreHTTPSErrors: true,diff --git a/e2e/specs/mock/schedules-execution.spec.ts b/e2e/specs/mock/schedules-execution.spec.tsnew file mode 100644index 0000000000..bfad84e4fb--- /dev/null+++ b/e2e/specs/mock/schedules-execution.spec.ts@@ -0,0 +1,301 @@+import { expect, test } from '@playwright/test';+import type { Page } from '@playwright/test';+import { getAccessToken, requestJson, replyPrompt, replyText } from './helpers';++const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;++type AgentSummary = { id: string; name?: string };+type AgentList = { data?: AgentSummary[] };+type Schedule = {+ id: string;+ name: string;+ nextRunAt?: string;+ enabled?: boolean;+ cadence?: { frequency: string; hour?: number; minute?: number };+ lastRun?: { status: string; conversationId?: string };+ runCount?: number;+};+type ScheduleList = { schedules: Schedule[] };+type RunNowResult = { scheduleId: string; conversationId?: string; status?: string };++async function ensureAgent(page: Page, token: string): Promise<AgentSummary> {+ const list = await requestJson<AgentList>(page, { path: '/api/agents?limit=1', token }).catch(+ () => ({}) as AgentList,+ );+ const existing = list.data?.[0];+ if (existing?.id) {+ return existing;+ }+ const agent = await requestJson<AgentSummary>(page, {+ path: '/api/agents',+ token,+ method: 'POST',+ body: { name: uniqueName('E2E Agent'), provider: 'Mock Provider A', model: 'mock-model-a' },+ });+ expect(agent.id).toBeTruthy();+ return agent;+}++async function createSchedule(+ page: Page,+ token: string,+ body: Record<string, unknown>,+): Promise<Schedule> {+ const schedule = await requestJson<Schedule>(page, {+ path: '/api/schedules',+ token,+ method: 'POST',+ body,+ });+ expect(schedule.id).toBeTruthy();+ return schedule;+}++async function readSchedule(page: Page, token: string, id: string): Promise<Schedule | undefined> {+ const list = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });+ return list.schedules.find((s) => s.id === id);+}++async function openSchedulesPanel(page: Page) {+ const navButton = page.getByRole('button', { name: 'Scheduled chats' });+ await expect(navButton).toBeVisible();+ if ((await navButton.getAttribute('aria-pressed')) !== 'true') {+ await navButton.click();+ }+ const panel = page.getByRole('region', { name: 'Scheduled chats' });+ await expect(panel).toBeVisible({ timeout: 15000 });+ return panel;+}++const scheduleBody = (agentId: string, over: Record<string, unknown> = {}) => ({+ name: uniqueName('E2E Schedule'),+ prompt: 'Summarize what happened today',+ agent_id: agentId,+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ clientRequestId: uniqueName('e2e-intent'),+ ...over,+});++test.describe('scheduled chat execution', () => {+ /**+ * Run Now is the one path that dispatches a real generation on demand, so it is+ * where a broken loopback URL, a rejected fire token, or a lost schedule identity+ * surfaces. The smoke spec only proves the card renders.+ */+ test('Run Now generates a conversation and records it on the schedule', async ({ page }) => {+ test.setTimeout(120000);+ await page.goto('/c/new', { timeout: 15000 });+ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);+ const label = `sched-${Date.now()}`;+ const schedule = await createSchedule(+ page,+ token,+ scheduleBody(agent.id, { prompt: replyPrompt(label) }),+ );++ // A skip/throttle answers 409/429, which requestJson surfaces as a throw.+ const result = await requestJson<RunNowResult>(page, {+ path: `/api/schedules/${schedule.id}/run`,+ token,+ method: 'POST',+ });+ expect(result.status).toBe('started');+ expect(result.conversationId).toBeTruthy();++ // Poll the schedule until the run's own completion hook records its outcome.+ await expect+ .poll(async () => (await readSchedule(page, token, schedule.id))?.lastRun?.status, {+ timeout: 60000,+ intervals: [1000],+ })+ .toBe('success');++ const settled = await readSchedule(page, token, schedule.id);+ expect(settled?.runCount).toBe(1);+ const conversationId = settled?.lastRun?.conversationId;+ expect(conversationId).toBeTruthy();++ // The generated chat is real and reachable: the agent's reply is persisted.+ await page.goto(`/c/${conversationId}`, { timeout: 15000 });+ await expect(page.getByTestId('messages-view')).toContainText(replyText(label), {+ timeout: 20000,+ });+ });++ /**+ * A due schedule must fire on the engine's own tick — no user action — exactly once,+ * and then advance past the occurrence. TICK_MS is 30s, so budget for one tick.+ */+ test('a due schedule fires automatically, once, and advances', async ({ page }) => {+ test.setTimeout(600000);+ await page.goto('/c/new', { timeout: 15000 });+ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);+ const label = `auto-${Date.now()}`;+ // Hourly ignores `hour` (cron `m * * * *`) but the payload schema still requires it.+ const schedule = await createSchedule(+ page,+ token,+ scheduleBody(agent.id, {+ prompt: replyPrompt(label),+ cadence: { frequency: 'hourly', hour: 0, minute: (new Date().getUTCMinutes() + 1) % 60 },+ timezone: 'UTC',+ }),+ );+ const before = await readSchedule(page, token, schedule.id);+ expect(before?.nextRunAt).toBeTruthy();++ // Budget from the server's OWN nextRunAt rather than a guessed constant: it already+ // includes this schedule's deterministic jitter (up to SCHEDULE_JITTER_WINDOW_MS,+ // 120s), which no fixed timeout can safely assume away. Add the engine tick+ // (30s + 2s jitter) plus room for the generation.+ const dueIn = Math.max(new Date(before!.nextRunAt!).getTime() - Date.now(), 0);+ const budget = dueIn + 120000;++ await expect+ .poll(async () => (await readSchedule(page, token, schedule.id))?.lastRun?.status, {+ timeout: budget,+ intervals: [2000],+ })+ .toBe('success');++ const after = await readSchedule(page, token, schedule.id);+ // Exactly one run, and the occurrence was advanced rather than re-fired.+ expect(after?.runCount).toBe(1);+ expect(new Date(after!.nextRunAt!).getTime()).toBeGreaterThan(+ new Date(before!.nextRunAt!).getTime(),+ );+ });++ test('rejects an invalid timezone before persisting anything', async ({ page }) => {+ await page.goto('/c/new', { timeout: 15000 });+ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);++ const before = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });+ const rejected = await requestJson<unknown>(page, {+ path: '/api/schedules',+ token,+ method: 'POST',+ body: scheduleBody(agent.id, { timezone: 'Not/AZone' }),+ }).then(+ () => null,+ (err: Error) => err.message,+ );++ expect(rejected).toMatch(/400/);+ const after = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });+ expect(after.schedules).toHaveLength(before.schedules.length);+ });++ /**+ * Edits through the real dialog and proves the change round-trips the backend.+ *+ * Seeded over the API rather than created through the UI on purpose: creation+ * requires the agent picker, whose list comes from a React Query cache that an+ * API-created agent does not invalidate, and which renders virtualized. That made+ * the create half brittle for reasons that have nothing to do with schedules. UI+ * CREATION is therefore still uncovered — worth a follow-up that seeds the agent+ * before first paint.+ */+ test('edits a schedule through the UI with the cadence persisted', async ({ page }) => {+ test.setTimeout(120000);+ await page.goto('/c/new', { timeout: 15000 });+ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);+ const name = uniqueName('UI Schedule');+ await createSchedule(+ page,+ token,+ scheduleBody(agent.id, { name, cadence: { frequency: 'weekly', hour: 8, minute: 0 } }),+ );++ await openSchedulesPanel(page);+ const persisted = page.getByTestId('schedule-card').filter({ hasText: name });+ await expect(persisted).toContainText(/Runs weekly/i, { timeout: 15000 });++ // EDIT through the dialog: rename and move the cadence to daily. The dialog+ // pre-populates the agent from the schedule, so no picker interaction is needed.+ await persisted.getByRole('button', { name: 'Schedule options' }).click();+ await page.getByRole('menuitem', { name: 'Edit' }).click();+ const editDialog = page.getByRole('dialog');+ const renamed = `${name} edited`;+ await editDialog.locator('#schedule-name').fill(renamed);+ await editDialog.getByRole('radio', { name: 'Daily' }).click();+ await editDialog.getByRole('button', { name: 'Save' }).click();++ await page.reload();+ await openSchedulesPanel(page);+ const edited = page.getByTestId('schedule-card').filter({ hasText: renamed });+ await expect(edited).toBeVisible({ timeout: 15000 });+ await expect(edited).toContainText(/Runs daily/i);+ });++ /**+ * Deleting a schedule mid-run must quiesce it: the in-flight generation is aborted and+ * the run settles, rather than the row lingering `started` and holding a global+ * capacity slot until the orphan sweep.+ */+ test('deleting a schedule while its run is active aborts the generation', async ({ page }) => {+ test.setTimeout(180000);+ await page.goto('/c/new', { timeout: 15000 });+ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);+ const schedule = await createSchedule(+ page,+ token,+ // A slow reply keeps the generation in flight long enough to delete underneath it.+ scheduleBody(agent.id, { prompt: `E2E_SLOW_REPLY:del-${Date.now()}` }),+ );++ const started = await requestJson<RunNowResult>(page, {+ path: `/api/schedules/${schedule.id}/run`,+ token,+ method: 'POST',+ });+ const conversationId = started.conversationId!;+ expect(conversationId).toBeTruthy();++ // PROVE the run is actually generating before deleting. The schedule row is hidden+ // from the owner the instant it is soft-deleted, so its disappearance is no evidence+ // that the abort was delivered, the run settled, or the row was erased.+ await expect+ .poll(+ async () =>+ (+ await requestJson<{ active?: boolean }>(page, {+ path: `/api/agents/chat/status/${conversationId}`,+ token,+ })+ ).active,+ { timeout: 60000, intervals: [500] },+ )+ .toBe(true);++ await requestJson<unknown>(page, {+ path: `/api/schedules/${schedule.id}`,+ token,+ method: 'DELETE',+ });++ // The delete has to reach the loopback generation, not just hide the row.+ await expect+ .poll(+ async () =>+ (+ await requestJson<{ active?: boolean }>(page, {+ path: `/api/agents/chat/status/${conversationId}`,+ token,+ })+ ).active,+ { timeout: 60000, intervals: [1000] },+ )+ .toBe(false);++ expect(await readSchedule(page, token, schedule.id)).toBeUndefined();+ });+});diff --git a/e2e/specs/mock/schedules.spec.ts b/e2e/specs/mock/schedules.spec.tsnew file mode 100644index 0000000000..309e828176--- /dev/null+++ b/e2e/specs/mock/schedules.spec.ts@@ -0,0 +1,112 @@+import { expect, test } from '@playwright/test';+import type { Page } from '@playwright/test';+import { getAccessToken, requestJson } from './helpers';++const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;++type AgentSummary = { id: string; name?: string };+type AgentList = { data?: AgentSummary[] };+type Schedule = { id: string; name: string };++/**+ * Ensures an agent exists (reusing one if present, else creating). Uses a real+ * Bearer token because the app authenticates with an access token, not a plain+ * cookie. Requires the page to be on the app origin. Returns id + name.+ */+async function ensureAgent(page: Page, token: string): Promise<AgentSummary> {+ const list = await requestJson<AgentList>(page, { path: '/api/agents?limit=1', token }).catch(+ () => ({}) as AgentList,+ );+ const existing = list.data?.[0];+ if (existing?.id) {+ return existing;+ }+ const name = uniqueName('E2E Agent');+ const agent = await requestJson<AgentSummary>(page, {+ path: '/api/agents',+ token,+ method: 'POST',+ body: { name, provider: 'Mock Provider A', model: 'mock-model-a' },+ });+ expect(agent.id).toBeTruthy();+ return agent;+}++/** Seeds a schedule via the API so the UI test can exercise its lifecycle. */+async function seedSchedule(page: Page, token: string, agentId: string): Promise<Schedule> {+ const name = uniqueName('E2E Schedule');+ const schedule = await requestJson<Schedule>(page, {+ path: '/api/schedules',+ token,+ method: 'POST',+ body: {+ name,+ prompt: 'Summarize what happened today',+ agent_id: agentId,+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ clientRequestId: uniqueName('e2e-intent'),+ },+ });+ expect(schedule.id).toBeTruthy();+ return schedule;+}++async function openSchedulesPanel(page: Page) {+ const navButton = page.getByRole('button', { name: 'Scheduled chats' });+ await expect(navButton).toBeVisible();+ // The nav button TOGGLES: clicking it while schedules is the active, expanded+ // panel collapses the sidebar. The active panel persists to localStorage, so+ // after a reload schedules may already be active but still loading its query+ // (a spinner, no region yet). Clicking then would collapse it. Branch on+ // aria-pressed: only click to activate when it isn't already the active panel,+ // then wait for the query to resolve and render the region.+ const isActive = (await navButton.getAttribute('aria-pressed')) === 'true';+ if (!isActive) {+ await navButton.click();+ }+ const panel = page.getByRole('region', { name: 'Scheduled chats' });+ await expect(panel).toBeVisible({ timeout: 15000 });+ return panel;+}++test.describe('scheduled chats', () => {+ test('lists a schedule and toggles + deletes it through the panel', async ({ page }) => {+ test.setTimeout(120000);+ await page.goto('/c/new', { timeout: 15000 });++ const token = await getAccessToken(page);+ const agent = await ensureAgent(page, token);+ const schedule = await seedSchedule(page, token, agent.id);++ // The seeded schedule renders as a card in the panel (full backend round-trip).+ await openSchedulesPanel(page);+ const card = page.getByTestId('schedule-card').filter({ hasText: schedule.name });+ await expect(card).toBeVisible({ timeout: 15000 });++ // It survives a full reload (persisted through the real backend + DB).+ await page.reload();+ await openSchedulesPanel(page);+ const reloadedCard = page.getByTestId('schedule-card').filter({ hasText: schedule.name });+ await expect(reloadedCard).toBeVisible({ timeout: 15000 });++ // Toggling the enabled switch round-trips and re-reads from the server.+ const toggle = reloadedCard.getByRole('switch', { name: 'Enabled' });+ await expect(toggle).toBeChecked();+ await toggle.click();+ await expect(toggle).not.toBeChecked();++ // Delete via the kebab menu + confirmation dialog.+ await reloadedCard.getByRole('button', { name: 'Schedule options' }).click();+ await page.getByRole('menuitem', { name: 'Delete' }).click();+ const confirm = page.getByRole('dialog', { name: /delete schedule/i });+ await confirm.getByRole('button', { name: 'Delete', exact: true }).click();++ await expect(page.getByTestId('schedule-card').filter({ hasText: schedule.name })).toHaveCount(+ 0,+ { timeout: 15000 },+ );+ });+});diff --git a/librechat.example.yaml b/librechat.example.yamlindex 53f0955429..b6b4f3183c 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -196,6 +196,17 @@ interface:create: trueshare: falsepublic: false+ # Scheduled chats are experimental and disabled unless explicitly enabled.+ # A deployment with Redis-backed resumable streams can run this on every replica.+ # A single-process deployment without Redis must also set+ # `SCHEDULES_SINGLE_PROCESS=true`; unsafe multi-replica writes fail closed.+ # schedules:+ # use: true+ # create: true+ # maxPerUser: 10+ # minIntervalMinutes: 60+ # autoDisableAfterFailures: 5+ # fireConcurrency: 5peoplePicker:users: truegroups: truediff --git a/package-lock.json b/package-lock.jsonindex 7f4c6826a7..887ae7adfc 100644--- a/package-lock.json+++ b/package-lock.json@@ -23269,6 +23269,25 @@"resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz","integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="},+ "node_modules/croner": {+ "version": "10.0.1",+ "resolved": "https://registry.npmjs.org/croner/-/croner-10.0.1.tgz",+ "integrity": "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==",+ "funding": [+ {+ "type": "other",+ "url": "https://paypal.me/hexagonpp"+ },+ {+ "type": "github",+ "url": "https://github.com/sponsors/hexagon"+ }+ ],+ "license": "MIT",+ "engines": {+ "node": ">=18.0"+ }+ },"node_modules/cross-env": {"version": "7.0.3","resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",@@ -42822,6 +42841,7 @@"@langchain/langgraph-checkpoint": "^1.1.2","@langchain/langgraph-checkpoint-mongodb": "^1.4.0","cluster-key-slot": "^1.1.2",+ "croner": "^10.0.1","proxy-from-env": "^2.1.0"},"devDependencies": {diff --git a/packages/api/package.json b/packages/api/package.jsonindex 210f8572ad..fbef1b1a81 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -169,6 +169,7 @@"@langchain/langgraph-checkpoint": "^1.1.2","@langchain/langgraph-checkpoint-mongodb": "^1.4.0","cluster-key-slot": "^1.1.2",+ "croner": "^10.0.1","proxy-from-env": "^2.1.0"}}diff --git a/packages/api/src/admin/config.handler.spec.ts b/packages/api/src/admin/config.handler.spec.tsindex 7d44f65a12..2b037bda76 100644--- a/packages/api/src/admin/config.handler.spec.ts+++ b/packages/api/src/admin/config.handler.spec.ts@@ -521,6 +521,25 @@ describe('createAdminConfigHandlers', () => {expect(savedOverrides.interface).toEqual({ modelSelect: false });});+ it('collapses an explicit schedules disable to the boolean form in overrides', async () => {+ const { handlers, deps } = createHandlers({+ upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }),+ });+ const req = mockReq({+ params: { principalType: 'role', principalId: 'admin' },+ body: {+ overrides: { interface: { schedules: { use: false, maxPerUser: 2 } } },+ },+ });+ const res = mockRes();++ await handlers.upsertConfigOverrides(req, res);++ expect(res.statusCode).toBe(201);+ const savedOverrides = deps.upsertConfig.mock.calls[0][3];+ expect(savedOverrides.interface).toEqual({ schedules: false });+ });+it('preserves skillSync sections in admin overrides', async () => {const { handlers, deps } = createHandlers({upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }),@@ -1046,6 +1065,43 @@ describe('createAdminConfigHandlers', () => {expect(patchedFields['interface.prompts']).toBeUndefined();});+ /** `use` is both a permission bit and the runtime disable for dual-purpose fields.+ * Stripping it alone leaves an object, which `getLimits` reads as ENABLED — so an+ * override meant to stop scheduled billing would start it. */+ it('collapses an explicit schedules disable to the boolean form in patches', async () => {+ const { handlers, deps } = createHandlers();+ const req = mockReq({+ params: { principalType: 'role', principalId: 'admin' },+ body: {+ entries: [{ fieldPath: 'interface.schedules', value: { use: false, maxPerUser: 2 } }],+ },+ });+ const res = mockRes();++ await handlers.patchConfigField(req, res);++ expect(res.statusCode).toBe(200);+ const patchedFields = deps.patchConfigFields.mock.calls[0][3];+ expect(patchedFields['interface.schedules']).toBe(false);+ });++ it('keeps a schedules object that only narrows limits', async () => {+ const { handlers, deps } = createHandlers();+ const req = mockReq({+ params: { principalType: 'role', principalId: 'admin' },+ body: {+ entries: [{ fieldPath: 'interface.schedules', value: { maxPerUser: 2 } }],+ },+ });+ const res = mockRes();++ await handlers.patchConfigField(req, res);++ expect(res.statusCode).toBe(200);+ const patchedFields = deps.patchConfigFields.mock.calls[0][3];+ expect(patchedFields['interface.schedules']).toEqual({ maxPerUser: 2 });+ });+it('preserves skillSync field entries in patches', async () => {const { handlers, deps } = createHandlers();const req = mockReq({diff --git a/packages/api/src/admin/config.ts b/packages/api/src/admin/config.tsindex 20852cc1a3..979aaafa19 100644--- a/packages/api/src/admin/config.ts+++ b/packages/api/src/admin/config.ts@@ -5,6 +5,7 @@ import {PrincipalType,PrincipalModel,INTERFACE_PERMISSION_FIELDS,+ RUNTIME_CONFIG_INTERFACE_FIELDS,PERMISSION_SUB_KEYS,hasProcessMCPServerConfig,isProcessMCPServerConfig,@@ -124,14 +125,47 @@ function isInterfacePermissionPath(fieldPath: string): boolean {if (!INTERFACE_PERMISSION_FIELDS.has(parts[1])) {return false;}- // "interface.<permField>" with no sub-key → permission (blocks the whole field)+ // "interface.<permField>" with no sub-key → permission (blocks the whole field),+ // EXCEPT dual-purpose runtime fields (e.g. schedules) whose bare top-level value+ // is a runtime enable toggle, not a permission — those must pass through so admin+ // field patches/tombstones can set or clear them (their .use/.create permission+ // sub-keys are still blocked below).if (parts.length === 2) {- return true;+ return !RUNTIME_CONFIG_INTERFACE_FIELDS.has(parts[1]);}// "interface.<permField>.<subKey>" → only block if sub-key is a permission bitreturn PERMISSION_SUB_KEYS.has(parts[2]);}+/**+ * Collapses an explicit disable on a dual-purpose runtime interface field (e.g. `schedules`)+ * to its boolean form.+ *+ * For these fields `use` is BOTH a permission bit — stripped from DB overrides — and the+ * runtime disable signal. Stripping it alone would leave `{ maxPerUser: 2 }`, which+ * `getLimits` reads as ENABLED because an object opts in unless it sets `use: false`. An+ * override written to stop scheduled billing for a principal would therefore start it.+ * Other object forms are left alone so a principal can still narrow limits.+ */+function normalizeRuntimeInterfaceValue(field: string, value: unknown): unknown {+ if (!RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {+ return value;+ }+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {+ return value;+ }+ return (value as Record<string, unknown>).use === false ? false : value;+}++/** Applies {@link normalizeRuntimeInterfaceValue} to a bare `interface.<field>` patch. */+function normalizeInterfaceFieldPatch(fieldPath: string, value: unknown): unknown {+ const parts = fieldPath.split('.');+ if (parts[0] !== 'interface' || parts.length !== 2) {+ return value;+ }+ return normalizeRuntimeInterfaceValue(parts[1], value);+}+export interface AdminConfigDeps {listAllConfigs: (filter?: { isActive?: boolean }, session?: ClientSession) => Promise<IConfig[]>;findConfigByPrincipal: (@@ -612,7 +646,8 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {const iface = (overrides as Record<string, unknown>).interface;if (iface != null && typeof iface === 'object' && !Array.isArray(iface)) {const filteredIface: Record<string, unknown> = {};- for (const [field, val] of Object.entries(iface as Record<string, unknown>)) {+ for (const [field, rawVal] of Object.entries(iface as Record<string, unknown>)) {+ const val = normalizeRuntimeInterfaceValue(field, rawVal);if (!INTERFACE_PERMISSION_FIELDS.has(field)) {filteredIface[field] = val;} else if (val != null && typeof val === 'object' && !Array.isArray(val)) {@@ -631,6 +666,10 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {if (Object.keys(uiOnly).length > 0) {filteredIface[field] = uiOnly;}+ } else if (RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {+ // Dual-purpose field: the boolean form is a runtime disable, not a+ // permission toggle, so preserve it (e.g. schedules: false).+ filteredIface[field] = val;} else {logger.warn(`[adminConfig] Stripping interface permission field "${field}" — use role permissions instead`,@@ -794,27 +833,32 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {return res.status(401).json({ error: 'Authentication required' });}- const validEntries = entries.filter((entry) => {- if (isBaseOnlyFieldPath(entry.fieldPath)) {- logger.warn(- `[adminConfig] Stripping base-only config field "${entry.fieldPath}" - configure it in librechat.yaml instead`,- );- return false;- }- if (BASE_PRINCIPAL_OVERRIDE_SECTIONS.has(getTopLevelSection(entry.fieldPath))) {- logger.warn(- `[adminConfig] Stripping dedicated tenant-wide config field "${entry.fieldPath}" from the generic config API`,- );- return false;- }- if (isInterfacePermissionPath(entry.fieldPath)) {- logger.warn(- `[adminConfig] Stripping interface permission field "${entry.fieldPath}" — use role permissions instead`,- );- return false;- }- return true;- });+ const validEntries = entries+ .map((entry) => ({+ ...entry,+ value: normalizeInterfaceFieldPatch(entry.fieldPath, entry.value),+ }))+ .filter((entry) => {+ if (isBaseOnlyFieldPath(entry.fieldPath)) {+ logger.warn(+ `[adminConfig] Stripping base-only config field "${entry.fieldPath}" - configure it in librechat.yaml instead`,+ );+ return false;+ }+ if (BASE_PRINCIPAL_OVERRIDE_SECTIONS.has(getTopLevelSection(entry.fieldPath))) {+ logger.warn(+ `[adminConfig] Stripping dedicated tenant-wide config field "${entry.fieldPath}" from the generic config API`,+ );+ return false;+ }+ if (isInterfacePermissionPath(entry.fieldPath)) {+ logger.warn(+ `[adminConfig] Stripping interface permission field "${entry.fieldPath}" — use role permissions instead`,+ );+ return false;+ }+ return true;+ });const hasBroadManage = await hasConfigCapability(user, null, 'manage');diff --git a/packages/api/src/agents/triggers/envelope.spec.ts b/packages/api/src/agents/triggers/envelope.spec.tsindex da869e1c63..80c1980d73 100644--- a/packages/api/src/agents/triggers/envelope.spec.ts+++ b/packages/api/src/agents/triggers/envelope.spec.ts@@ -19,6 +19,12 @@ describe('createAgentTriggerEnvelope', () => {tenantId: 'tenant-1',},target: { agentId: 'agent-1' },+ run: {+ conversationId: 'conversation-1',+ timezone: 'America/New_York',+ files: [{ file_id: 'file-1' }],+ metadata: { adapter: 'test' },+ },event: {id: 'event-1',type: 'work.ready',@@ -49,6 +55,12 @@ describe('createAgentTriggerEnvelope', () => {receivedAt: 1_725_000_000_010,principal: { userId: 'user-1', role: 'USER', tenantId: 'tenant-1' },target: { agentId: 'agent-1' },+ run: {+ conversationId: 'conversation-1',+ timezone: 'America/New_York',+ files: [{ file_id: 'file-1' }],+ metadata: { adapter: 'test' },+ },event: {id: 'event-1',type: 'work.ready',@@ -208,6 +220,33 @@ describe('createAgentTriggerEnvelope', () => {).toThrow('target.agentId must be a non-empty string');});+ it('validates and detaches trusted fire run context', () => {+ const input = createFireInput();+ const envelope = createAgentTriggerEnvelope(input);+ const files = input.mode === 'fire' ? input.run?.files : undefined;+ if (files != null) {+ (files[0] as { file_id: string }).file_id = 'changed';+ }++ expect(envelope.mode).toBe('fire');+ if (envelope.mode === 'fire') {+ expect(envelope.run?.files).toEqual([{ file_id: 'file-1' }]);+ }++ expect(() =>+ createAgentTriggerEnvelope({+ ...createFireInput(),+ run: { files: 'not-an-array' },+ } as unknown as CreateAgentTriggerEnvelopeInput),+ ).toThrow('run.files must be an array');+ expect(() =>+ createAgentTriggerEnvelope({+ ...createFireInput(),+ run: { metadata: { callback: () => undefined } },+ } as unknown as CreateAgentTriggerEnvelopeInput),+ ).toThrow('run.metadata.callback contains a non-JSON function value');+ });+it('rejects non-JSON and circular event payloads', () => {expect(() =>createAgentTriggerEnvelope({diff --git a/packages/api/src/agents/triggers/envelope.ts b/packages/api/src/agents/triggers/envelope.tsindex 2391fb0293..c599df6b80 100644--- a/packages/api/src/agents/triggers/envelope.ts+++ b/packages/api/src/agents/triggers/envelope.ts@@ -30,6 +30,20 @@ interface AgentTriggerTarget {agentId: string;}+/**+ * Trusted host controls for a new fire. These values shape LibreChat's own+ * generation request; they are never sourced directly from model output.+ * `metadata` is opaque to the trigger core and lets an authenticated adapter+ * identify its delivery to host-side lifecycle hooks without widening the+ * public chat request surface.+ */+export interface AgentFireRunContext {+ conversationId?: string;+ timezone?: string;+ files?: JsonValue[];+ metadata?: JsonValue;+}+/*** One fire delivery represents one new conversation for this agent; retries* reuse the delivery id instead of starting another conversation.@@ -68,6 +82,7 @@ interface AgentTriggerEnvelopeBase {export interface AgentFireTriggerEnvelope extends AgentTriggerEnvelopeBase {mode: 'fire';target: AgentFireTarget;+ run?: AgentFireRunContext;}export interface AgentContinueTriggerEnvelope extends AgentTriggerEnvelopeBase {@@ -98,6 +113,7 @@ export type CreateAgentTriggerEnvelopeInput =| (CreateAgentTriggerEnvelopeBase & {mode: 'fire';target: AgentFireTarget;+ run?: AgentFireRunContext;})| (CreateAgentTriggerEnvelopeBase & {mode: 'continue';@@ -168,6 +184,32 @@ function createEvent(input: AgentTriggerEvent | null | undefined): AgentTriggerE};}+function createFireRunContext(+ input: AgentFireRunContext | null | undefined,+): AgentFireRunContext | undefined {+ if (input == null) {+ return undefined;+ }+ const run = requireRecord(input, 'run');+ const context: AgentFireRunContext = {};+ if (run.conversationId != null) {+ context.conversationId = requireString(run.conversationId, 'run.conversationId');+ }+ if (run.timezone != null) {+ context.timezone = requireString(run.timezone, 'run.timezone');+ }+ if (run.files != null) {+ if (!Array.isArray(run.files)) {+ throw error('run.files must be an array');+ }+ context.files = cloneJsonValue(run.files, 'run.files', error) as JsonValue[];+ }+ if (run.metadata !== undefined) {+ context.metadata = cloneJsonValue(run.metadata, 'run.metadata', error) as JsonValue;+ }+ return context;+}+export function createAgentTriggerEnvelope(input: CreateAgentTriggerEnvelopeInput,): AgentTriggerEnvelope {@@ -183,10 +225,12 @@ export function createAgentTriggerEnvelope(};if (input.mode === 'fire') {+ const run = createFireRunContext(input.run);return {...base,mode: input.mode,target: { agentId: requireString(input.target?.agentId, 'target.agentId') },+ ...(run != null && { run }),};}@@ -276,10 +320,14 @@ export function parseAgentTriggerEnvelope(input: unknown): AgentTriggerEnvelopeconst target = requireRecord(envelope.target, 'target');if (mode === 'fire') {+ const run = createFireRunContext(+ envelope.run == null ? undefined : (envelope.run as AgentFireRunContext),+ );return {...base,mode,target: { agentId: requireString(target.agentId, 'target.agentId') },+ ...(run != null && { run }),};}diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.tsindex c9c9624064..cf97319b5d 100644--- a/packages/api/src/agents/triggers/host.spec.ts+++ b/packages/api/src/agents/triggers/host.spec.ts@@ -12,6 +12,12 @@ const createFireEnvelope = () =>receivedAt: 20,principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },target: { agentId: 'agent-1' },+ run: {+ conversationId: 'scheduled-conversation-1',+ timezone: 'Europe/Paris',+ files: [{ file_id: 'file-1' }],+ metadata: { manual: false },+ },event: {id: 'event-1',type: 'resource.ready',@@ -154,7 +160,20 @@ describe('createAgentTriggerExecutionHost fire adapter', () => {isRegenerate: false,clientRequestId: idempotencyKey,generationProtocolVersion: 2,- timezone: 'America/New_York',+ agentTrigger: {+ version: envelope.version,+ deliveryId: envelope.deliveryId,+ event: {+ id: envelope.event.id,+ type: envelope.event.type,+ occurredAt: envelope.event.occurredAt,+ source: envelope.event.source,+ },+ metadata: { manual: false },+ },+ newConversationId: 'scheduled-conversation-1',+ files: [{ file_id: 'file-1' }],+ timezone: 'Europe/Paris',});expect(mintToken).toHaveBeenCalledWith(envelope.principal, envelope);});diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.tsindex c3a33464d7..869d326b86 100644--- a/packages/api/src/agents/triggers/host.ts+++ b/packages/api/src/agents/triggers/host.ts@@ -556,7 +556,7 @@ async function startRun(}const input = preparation?.status === 'ready' ? preparation.input : envelope.input;const parentMessageId = resolveParentMessageId(preparation, envelope);- const [token, timezone, baseUrl] = await Promise.all([+ const [token, resolvedTimezone, baseUrl] = await Promise.all([setupValue(() => deps.mintToken(envelope.principal, envelope),mode,@@ -574,6 +574,8 @@ async function startRun(scope.abort();throw error;});+ const run = envelope.mode === 'fire' ? envelope.run : undefined;+ const timezone = run?.timezone ?? resolvedTimezone;const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;let response: Response;@@ -602,6 +604,25 @@ async function startRun(isRegenerate: false,clientRequestId: context.idempotencyKey,generationProtocolVersion: 2,+ ...(envelope.mode === 'fire' && {+ agentTrigger: {+ version: envelope.version,+ deliveryId: envelope.deliveryId,+ event: {+ id: envelope.event.id,+ type: envelope.event.type,+ occurredAt: envelope.event.occurredAt,+ source: envelope.event.source,+ },+ ...(run?.metadata !== undefined && {+ metadata: run.metadata,+ }),+ },+ }),+ ...(run?.conversationId != null && {+ newConversationId: run.conversationId,+ }),+ ...(run?.files != null && { files: run.files }),...(typeof timezone === 'string' && timezone.trim().length > 0? { timezone: timezone.trim() }: {}),diff --git a/packages/api/src/app/permissions.spec.ts b/packages/api/src/app/permissions.spec.tsindex b93efe64d3..d12fe0e3fe 100644--- a/packages/api/src/app/permissions.spec.ts+++ b/packages/api/src/app/permissions.spec.ts@@ -118,6 +118,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -177,6 +181,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -300,6 +308,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -359,6 +371,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -468,6 +484,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -527,6 +547,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -649,6 +673,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -708,6 +736,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -817,6 +849,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -876,6 +912,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -990,6 +1030,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -1037,6 +1081,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -1168,6 +1216,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};const expectedPermissionsForAdmin = {@@ -1219,6 +1271,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },};expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);@@ -2777,6 +2833,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },},});@@ -2814,6 +2874,10 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE]: false,[Permissions.SHARE_PUBLIC]: false,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },},});@@ -2842,4 +2906,66 @@ describe('updateInterfacePermissions - permissions', () => {[Permissions.SHARE_PUBLIC]: true,});});++ it('does not re-enable a DB-disabled schedules permission when only runtime limits are configured', async () => {+ // The role already has schedules USE disabled in the DB.+ mockGetRoleByName.mockResolvedValue({+ permissions: {+ [PermissionTypes.SCHEDULES]: { [Permissions.USE]: false, [Permissions.CREATE]: false },+ },+ });+ // Runtime-only limits — NOT a permission config; must not re-derive USE.+ const config = {+ interface: {+ schedules: { maxPerUser: 20, fireConcurrency: 2 },+ },+ };+ const configDefaults = { interface: {} } as TConfigDefaults;+ const interfaceConfig = await loadDefaultInterface({ config, configDefaults });+ const appConfig = { config, interfaceConfig } as unknown as AppConfig;++ await updateInterfacePermissions({+ appConfig,+ getRoleByName: mockGetRoleByName,+ updateAccessPermissions: mockUpdateAccessPermissions,+ });++ // Schedules must be preserved (omitted from every role update), not silently+ // re-enabled to USE:true from defaults just because limits were tuned.+ for (const call of mockUpdateAccessPermissions.mock.calls) {+ expect(call[1][PermissionTypes.SCHEDULES]).toBeUndefined();+ }+ });++ it('treats a boolean schedules kill switch as runtime-only (does not touch the permission)', async () => {+ // The role currently has schedules enabled.+ mockGetRoleByName.mockResolvedValue({+ permissions: {+ [PermissionTypes.SCHEDULES]: { [Permissions.USE]: true, [Permissions.CREATE]: true },+ },+ });+ // `schedules: false` is the RUNTIME kill switch read by getLimits, NOT a permission+ // config: it must not write SCHEDULES into the role docs, so removing it later can+ // never leave USE stuck false (forbidden) until manual repair.+ const config = {+ interface: {+ schedules: false,+ },+ };+ const configDefaults = { interface: {} } as TConfigDefaults;+ const interfaceConfig = await loadDefaultInterface({ config, configDefaults });+ const appConfig = { config, interfaceConfig } as unknown as AppConfig;++ await updateInterfacePermissions({+ appConfig,+ getRoleByName: mockGetRoleByName,+ updateAccessPermissions: mockUpdateAccessPermissions,+ });++ // The kill switch is runtime-only: SCHEDULES is omitted from every role update,+ // leaving the existing USE:true untouched.+ for (const call of mockUpdateAccessPermissions.mock.calls) {+ expect(call[1][PermissionTypes.SCHEDULES]).toBeUndefined();+ }+ });});diff --git a/packages/api/src/app/permissions.ts b/packages/api/src/app/permissions.tsindex dcc626143c..c4ea6ed610 100644--- a/packages/api/src/app/permissions.ts+++ b/packages/api/src/app/permissions.ts@@ -49,6 +49,19 @@ function hasExplicitConfig(return interfaceConfig?.skills !== undefined;case PermissionTypes.SHARED_LINKS:return interfaceConfig?.sharedLinks !== undefined;+ case PermissionTypes.SCHEDULES: {+ // `schedules` is dual-purpose. The BOOLEAN form is the RUNTIME kill switch read+ // by getLimits, NOT a permission config: treating it as explicit would write+ // SCHEDULES.USE into the role docs, and removing the kill switch later would+ // leave that disabled permission stuck (forbidden) until manual repair. Only an+ // OBJECT carrying explicit use/create is permission intent; runtime-only limits+ // (maxPerUser, fireConcurrency, …) are not.+ const schedules = interfaceConfig?.schedules;+ if (typeof schedules !== 'object' || schedules == null) {+ return false;+ }+ return schedules.use !== undefined || schedules.create !== undefined;+ }default:return false;}@@ -199,6 +212,12 @@ export async function updateInterfacePermissions({typeof defaults.agents === 'object' ? defaults.agents?.public : undefined;const skillsDefaultPublic =typeof defaults.skills === 'object' ? defaults.skills?.public : undefined;+ // `schedules` is intentionally absent from the interface DEFAULTS (it is+ // experimental/default-off at runtime), so the PERMISSION defaults are stated here.+ // Runtime availability and the role permission are separate concerns: a user may hold+ // the permission while the feature stays off until an admin enables it.+ const schedulesDefaultUse = true;+ const schedulesDefaultCreate = true;const sharedLinksDefaultCreate =typeof defaults.sharedLinks === 'boolean' ? undefined : defaults.sharedLinks?.create;const sharedLinksDefaultShare =@@ -528,6 +547,29 @@ export async function updateInterfacePermissions({}: {}),},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: getPermissionValue(+ // Only an OBJECT `use` drives the permission; the boolean form is the runtime+ // kill switch and must not seed SCHEDULES.USE (see hasExplicitConfig), so a+ // removed kill switch can never leave USE stuck false.+ typeof loadedInterface.schedules === 'object'+ ? loadedInterface.schedules?.use+ : undefined,+ defaultPerms[PermissionTypes.SCHEDULES]?.[Permissions.USE],+ schedulesDefaultUse,+ ),+ ...((typeof interfaceConfig?.schedules === 'object' &&+ 'create' in interfaceConfig.schedules) ||+ !existingPermissions?.[PermissionTypes.SCHEDULES]+ ? {+ [Permissions.CREATE]: getPermissionValue(+ getConfigCreate(loadedInterface.schedules),+ defaultPerms[PermissionTypes.SCHEDULES]?.[Permissions.CREATE],+ schedulesDefaultCreate ?? true,+ ),+ }+ : {}),+ },};// Check and add each permission type if neededdiff --git a/packages/api/src/index.ts b/packages/api/src/index.tsindex eb54509115..f1e99a01bf 100644--- a/packages/api/src/index.ts+++ b/packages/api/src/index.ts@@ -58,6 +58,8 @@ export * from './projects';/* Conversations */export * from './conversations';/* Skills */+export * from './schedules';+export * from './schedules/service';export * from './skills';export * from './favorites';/* User */diff --git a/packages/api/src/middleware/balance.ts b/packages/api/src/middleware/balance.tsindex 453d7c66b0..c6e8978b9e 100644--- a/packages/api/src/middleware/balance.ts+++ b/packages/api/src/middleware/balance.ts@@ -55,7 +55,7 @@ async function runBalanceUpdate(userId: string, task: () => Promise<void>): Prom* @param userId - The user's ID* @returns Fields that need updating*/-function buildUpdateFields(+export function buildBalanceUpdateFields(config: BalanceConfig,userRecord: IBalance | null,userId: string,@@ -144,7 +144,7 @@ export function createSetBalanceConfig({const userId = typeof user._id === 'string' ? user._id : user._id.toString();await runBalanceUpdate(userId, async () => {const userBalanceRecord = await findBalanceByUser(userId);- const updateFields = buildUpdateFields(balanceConfig, userBalanceRecord, userId);+ const updateFields = buildBalanceUpdateFields(balanceConfig, userBalanceRecord, userId);if (Object.keys(updateFields).length === 0) {balanceLocals.balanceData = userBalanceRecord;diff --git a/packages/api/src/schedules/FOLLOWUPS.md b/packages/api/src/schedules/FOLLOWUPS.mdnew file mode 100644index 0000000000..4ebeee3368--- /dev/null+++ b/packages/api/src/schedules/FOLLOWUPS.md@@ -0,0 +1,32 @@+# Scheduled chats: deferred scope++Scheduled chats remain experimental and default-off.++## Must-have product follow-ups++- Add one-time and monthly schedules; current cadences are hourly, daily, weekdays, and weekly.+- Add non-agent chat targets and existing-conversation continuation; current schedules require+ `agent_id` and always target `new`.++## Client/API parity++- Return `minIntervalMinutes` in schedule limits and surface it in the dialog.+- Add active-run and run-history discovery. The list response currently projects only `lastRun`,+ which is written on pause, skip, or terminal outcomes, so the client's `started` chip is+ unreachable.+- Add multi-day weekly, timezone, and attachment controls. The API supports these, but the dialog+ does not.++## Runtime scope++- The standard server supports multi-replica scheduling only with a confirmed Redis-backed+ `GenerationJobManager`; in-memory mode requires `SCHEDULES_SINGLE_PROCESS`. The legacy+ `experimental.js` clustered entrypoint does not arm the engine and rejects schedule writes.+- Deferred account deletion blocks new remote API admission but cannot yet drain already-admitted+ OpenAI-compatible or Responses requests; see+ [#14594](https://github.com/danny-avila/LibreChat/issues/14594).++## Test debt++- Add Playwright coverage for creating a schedule through the UI. The execution E2E currently+ seeds schedules through the API.diff --git a/packages/api/src/schedules/access.ts b/packages/api/src/schedules/access.tsnew file mode 100644index 0000000000..83cb793124--- /dev/null+++ b/packages/api/src/schedules/access.ts@@ -0,0 +1,77 @@+import { logger, ResourceCapabilityMap } from '@librechat/data-schemas';+import {+ ResourceType,+ Permissions,+ PermissionBits,+ PermissionTypes,+} from 'librechat-data-provider';+import type { SystemCapability } from '@librechat/data-schemas';+import type { Types } from 'mongoose';+import type { ScheduleUserContext } from './types';++type AgentAccess = 'ok' | 'missing' | 'forbidden';++export interface AgentFireAccessDeps {+ /** Resolves an agent's internal `_id` by its custom id, or null when it doesn't exist. */+ findAgentObjectId: (agentId: string) => Promise<{ _id: Types.ObjectId } | null>;+ /** Loads a role's permission map by name. */+ getRoleByName: (+ role?: string,+ ) => Promise<{ permissions?: Record<string, Record<string, boolean | undefined>> } | null>;+ /** Whether the user's role grants a system capability (the manage:agents bypass). */+ hasCapability: (user: ScheduleUserContext, capability: SystemCapability) => Promise<boolean>;+ /** Resource ACL check for a specific permission bit. */+ checkPermission: (params: {+ userId: string;+ role?: string;+ resourceType: ResourceType;+ resourceId: Types.ObjectId;+ requiredPermission: PermissionBits;+ }) => Promise<boolean>;+}++/**+ * Resolves a user's live access to a schedule's target agent, mirroring the loopback+ * chat route's authorization EXACTLY so the create/update precheck and the fire-time+ * precheck accept a schedule iff the actual fire would be accepted:+ * 1) role-level AGENTS:USE (checkAccess on the route; admins do NOT bypass)+ * 2) resource VIEW with the manage:agents capability bypass+ * The two prechecks must never diverge — a create-time VIEW-only check would let a+ * role without AGENTS:USE schedule runs that every fire then rejects, burning failures.+ */+export function createResolveAgentFireAccess(deps: AgentFireAccessDeps) {+ return async function resolveAgentFireAccess(+ agentId: string,+ user: ScheduleUserContext,+ ): Promise<AgentAccess> {+ const agent = await deps.findAgentObjectId(agentId);+ if (agent == null) {+ return 'missing';+ }+ // Mirror the chat route's checkAccess, which reads the role's AGENTS:USE directly+ // and does NOT special-case admins: an admin whose role has AGENTS:USE disabled is+ // rejected there, so the precheck must reject too — otherwise every fire 403s.+ const role = await deps.getRoleByName(user.role);+ if (role?.permissions?.[PermissionTypes.AGENTS]?.[Permissions.USE] !== true) {+ return 'forbidden';+ }+ const capability = ResourceCapabilityMap[ResourceType.AGENT];+ try {+ if (capability != null && (await deps.hasCapability(user, capability))) {+ return 'ok';+ }+ } catch (err) {+ logger.warn(+ `[schedules] agent capability check failed, denying bypass: ${(err as Error).message}`,+ );+ }+ const allowed = await deps.checkPermission({+ userId: user.id,+ role: user.role,+ resourceType: ResourceType.AGENT,+ resourceId: agent._id,+ requiredPermission: PermissionBits.VIEW,+ });+ return allowed ? 'ok' : 'forbidden';+ };+}diff --git a/packages/api/src/schedules/cadence.spec.ts b/packages/api/src/schedules/cadence.spec.tsnew file mode 100644index 0000000000..9b7011f35b--- /dev/null+++ b/packages/api/src/schedules/cadence.spec.ts@@ -0,0 +1,322 @@+import type { TScheduleCadence } from 'librechat-data-provider';+import {+ cadenceToCron,+ computeNextRunAt,+ isValidTimezone,+ scheduleJitterMs,+ cadenceIntervalMinutes,+ SCHEDULE_JITTER_WINDOW_MS,+} from './cadence';++const NEW_YORK = 'America/New_York';++function cadence(overrides: Partial<TScheduleCadence>): TScheduleCadence {+ return { frequency: 'daily', hour: 0, minute: 0, ...overrides };+}++function wallClock(date: Date, timeZone: string): string {+ const parts = new Intl.DateTimeFormat('en-CA', {+ timeZone,+ weekday: 'short',+ year: 'numeric',+ month: '2-digit',+ day: '2-digit',+ hour: '2-digit',+ minute: '2-digit',+ hourCycle: 'h23',+ }).formatToParts(date);+ const get = (type: Intl.DateTimeFormatPartTypes): string =>+ parts.find((part) => part.type === type)?.value ?? '';+ return `${get('weekday')} ${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}`;+}++function nextRun(params: {+ cadence: TScheduleCadence;+ timezone: string;+ after: Date;+ scheduleId?: string;+}): Date {+ const result = computeNextRunAt({+ cadence: params.cadence,+ timezone: params.timezone,+ scheduleId: params.scheduleId ?? 'sched-test',+ after: params.after,+ disableJitter: true,+ });+ if (result == null) {+ throw new Error('expected computeNextRunAt to return a Date');+ }+ return result;+}++describe('cadenceToCron', () => {+ it('maps hourly to a minute-only pattern', () => {+ expect(cadenceToCron(cadence({ frequency: 'hourly', minute: 15 }))).toBe('15 * * * *');+ });++ it('maps daily to minute + hour', () => {+ expect(cadenceToCron(cadence({ frequency: 'daily', hour: 8, minute: 0 }))).toBe('0 8 * * *');+ });++ it('maps weekdays to Monday through Friday', () => {+ expect(cadenceToCron(cadence({ frequency: 'weekdays', hour: 9, minute: 30 }))).toBe(+ '30 9 * * 1-5',+ );+ });++ it('maps weekly multi-day with days sorted ascending', () => {+ expect(+ cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45, daysOfWeek: [5, 1, 3] })),+ ).toBe('45 7 * * 1,3,5');+ });++ it('defaults weekly to Monday when daysOfWeek is missing or empty', () => {+ expect(cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45 }))).toBe('45 7 * * 1');+ expect(+ cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45, daysOfWeek: [] })),+ ).toBe('45 7 * * 1');+ });+});++describe('computeNextRunAt', () => {+ const daily8amNewYork = {+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),+ timezone: NEW_YORK,+ };++ it('fires later the same day when the occurrence is still ahead', () => {+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T09:00:00Z') });+ expect(wallClock(next, NEW_YORK)).toBe('Wed 2026-07-15 08:00');+ expect(next.toISOString()).toBe('2026-07-15T12:00:00.000Z');+ });++ it('rolls to the next day once the occurrence has passed', () => {+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T13:00:00Z') });+ expect(wallClock(next, NEW_YORK)).toBe('Thu 2026-07-16 08:00');+ expect(next.toISOString()).toBe('2026-07-16T12:00:00.000Z');+ });++ it('returns null for a malformed stored timezone instead of throwing', () => {+ // A bad schedule reaching the engine must not throw out of a tick — null is+ // the "uncomputable" signal the engine disables (invalid_schedule) on.+ expect(() =>+ computeNextRunAt({+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),+ timezone: 'Not/AZone',+ scheduleId: 'sched_bad_tz',+ }),+ ).not.toThrow();+ expect(+ computeNextRunAt({+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),+ timezone: 'Not/AZone',+ scheduleId: 'sched_bad_tz',+ }),+ ).toBeNull();+ });++ it('skips an occurrence exactly at `after` (strictly-after semantics)', () => {+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T12:00:00Z') });+ expect(wallClock(next, NEW_YORK)).toBe('Thu 2026-07-16 08:00');+ });++ describe('DST spring-forward (US 2026-03-08)', () => {+ const daily230amNewYork = {+ cadence: cadence({ frequency: 'daily', hour: 2, minute: 30 }),+ timezone: NEW_YORK,+ };++ it('fires the gap occurrence shifted to 03:30 EDT rather than dropping March 8', () => {+ const next = nextRun({ ...daily230amNewYork, after: new Date('2026-03-07T17:00:00Z') });+ expect(next.toISOString()).toBe('2026-03-08T07:30:00.000Z');+ expect(wallClock(next, NEW_YORK)).toBe('Sun 2026-03-08 03:30');+ });++ it('resumes at 02:30 EDT on March 9 after the gap occurrence', () => {+ const march8 = nextRun({ ...daily230amNewYork, after: new Date('2026-03-07T17:00:00Z') });+ const next = nextRun({ ...daily230amNewYork, after: march8 });+ expect(next.toISOString()).toBe('2026-03-09T06:30:00.000Z');+ expect(wallClock(next, NEW_YORK)).toBe('Mon 2026-03-09 02:30');+ });+ });++ describe('DST fall-back (US 2026-11-01)', () => {+ it('fires the repeated 01:30 exactly once, at the first (EDT) occurrence', () => {+ const daily130amNewYork = {+ cadence: cadence({ frequency: 'daily', hour: 1, minute: 30 }),+ timezone: NEW_YORK,+ };+ const runs: Date[] = [];+ let after = new Date('2026-10-31T16:00:00Z');+ for (let i = 0; i < 3; i++) {+ const next = nextRun({ ...daily130amNewYork, after });+ runs.push(next);+ after = next;+ }+ expect(runs.map((run) => run.toISOString())).toEqual([+ '2026-11-01T05:30:00.000Z',+ '2026-11-02T06:30:00.000Z',+ '2026-11-03T06:30:00.000Z',+ ]);+ expect(runs.map((run) => wallClock(run, NEW_YORK))).toEqual([+ 'Sun 2026-11-01 01:30',+ 'Mon 2026-11-02 01:30',+ 'Tue 2026-11-03 01:30',+ ]);+ });+ });++ it('weekdays cadence skips weekends', () => {+ const weekdays9amNewYork = {+ cadence: cadence({ frequency: 'weekdays', hour: 9, minute: 0 }),+ timezone: NEW_YORK,+ };+ const fromFriday = nextRun({ ...weekdays9amNewYork, after: new Date('2026-07-17T15:00:00Z') });+ expect(wallClock(fromFriday, NEW_YORK)).toBe('Mon 2026-07-20 09:00');+ const fromSaturday = nextRun({+ ...weekdays9amNewYork,+ after: new Date('2026-07-18T12:00:00Z'),+ });+ expect(wallClock(fromSaturday, NEW_YORK)).toBe('Mon 2026-07-20 09:00');+ });++ it('hourly cadence fires at the next :MM', () => {+ const hourly45 = { cadence: cadence({ frequency: 'hourly', minute: 45 }), timezone: 'UTC' };+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:30:00Z') }).toISOString()).toBe(+ '2026-07-15T10:45:00.000Z',+ );+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:50:00Z') }).toISOString()).toBe(+ '2026-07-15T11:45:00.000Z',+ );+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:45:00Z') }).toISOString()).toBe(+ '2026-07-15T11:45:00.000Z',+ );+ });++ it('weekly multi-day cadence fires only on the configured days', () => {+ const weekly = {+ cadence: cadence({ frequency: 'weekly', hour: 10, minute: 0, daysOfWeek: [3, 6] }),+ timezone: 'UTC',+ };+ const first = nextRun({ ...weekly, after: new Date('2026-07-16T00:00:00Z') });+ const second = nextRun({ ...weekly, after: first });+ const third = nextRun({ ...weekly, after: second });+ expect([first, second, third].map((run) => run.toISOString())).toEqual([+ '2026-07-18T10:00:00.000Z',+ '2026-07-22T10:00:00.000Z',+ '2026-07-25T10:00:00.000Z',+ ]);+ expect([first, second, third].map((run) => wallClock(run, 'UTC').slice(0, 3))).toEqual([+ 'Sat',+ 'Wed',+ 'Sat',+ ]);+ });+});++describe('jitter', () => {+ it('is deterministic for the same schedule id', () => {+ expect(scheduleJitterMs('sched-abc')).toBe(scheduleJitterMs('sched-abc'));+ });++ it('stays within the jitter window', () => {+ for (let i = 0; i < 100; i++) {+ const value = scheduleJitterMs(`schedule-${i}`);+ expect(value).toBeGreaterThanOrEqual(0);+ expect(value).toBeLessThan(SCHEDULE_JITTER_WINDOW_MS);+ }+ });++ it('respects a custom window', () => {+ for (let i = 0; i < 20; i++) {+ const value = scheduleJitterMs(`schedule-${i}`, 1000);+ expect(value).toBeGreaterThanOrEqual(0);+ expect(value).toBeLessThan(1000);+ }+ });++ it('spreads different ids across the window', () => {+ const values = new Set(Array.from({ length: 50 }, (_, i) => scheduleJitterMs(`schedule-${i}`)));+ expect(values.size).toBeGreaterThan(40);+ });++ it('computeNextRunAt offsets the cron instant by the schedule jitter', () => {+ const params = {+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),+ timezone: NEW_YORK,+ scheduleId: 'sched-jitter',+ after: new Date('2026-07-15T09:00:00Z'),+ };+ const jittered = computeNextRunAt(params);+ const bare = computeNextRunAt({ ...params, disableJitter: true });+ expect(jittered).not.toBeNull();+ expect(bare).not.toBeNull();+ expect((jittered?.getTime() ?? 0) - (bare?.getTime() ?? 0)).toBe(+ scheduleJitterMs('sched-jitter'),+ );+ });++ it('does not skip a still-future occurrence inside the jitter window', () => {+ const scheduleId = 'sched-window';+ const jitter = scheduleJitterMs(scheduleId);+ expect(jitter).toBeGreaterThan(1000); // ensure the window is meaningful+ const cadenceHourly = cadence({ frequency: 'hourly', hour: 0, minute: 0 });+ const cronInstant = new Date('2026-07-15T12:00:00Z'); // an hourly :00 boundary (UTC)+ // `after` is just past the unjittered instant but before the jittered one:+ // the jittered occurrence (cronInstant + jitter) is still in the future.+ const after = new Date(cronInstant.getTime() + 1000);+ const next = computeNextRunAt({ cadence: cadenceHourly, timezone: 'UTC', scheduleId, after });+ // Must return THIS hour's jittered instant, not next hour's.+ expect(next?.getTime()).toBe(cronInstant.getTime() + jitter);+ });+});++describe('isValidTimezone', () => {+ it('accepts IANA zone names', () => {+ expect(isValidTimezone('America/New_York')).toBe(true);+ expect(isValidTimezone('Europe/Stockholm')).toBe(true);+ expect(isValidTimezone('UTC')).toBe(true);+ });++ it('rejects invalid zones', () => {+ expect(isValidTimezone('Not/AZone')).toBe(false);+ expect(isValidTimezone('EST5EDT-Bogus')).toBe(false);+ expect(isValidTimezone('')).toBe(false);+ });+});++describe('cadenceIntervalMinutes', () => {+ it('returns the minimum minutes between occurrences per frequency', () => {+ expect(cadenceIntervalMinutes(cadence({ frequency: 'hourly' }))).toBe(60);+ // Day-and-longer gaps report the DST-compressed minimum (largest real-world+ // transition is 2h — Antarctica/Troll): a floor set at the nominal value would+ // admit a schedule that genuinely violates it once a year.+ expect(cadenceIntervalMinutes(cadence({ frequency: 'daily' }))).toBe(24 * 60 - 120);+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekdays' }))).toBe(24 * 60 - 120);+ });++ it('uses the shortest gap between weekly days, incl. the week wrap-around', () => {+ // single day → full week+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly' }))).toBe(7 * 24 * 60 - 120);+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [3] }))).toBe(+ 7 * 24 * 60 - 120,+ );+ // adjacent days fire 24h apart — the floor must reflect that, not the average+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 2] }))).toBe(+ 24 * 60 - 120,+ );+ // evenly spaced Mon/Wed/Fri → 2-day min gap+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 3, 5] }))).toBe(+ 2 * 24 * 60 - 120,+ );+ // duplicates are one selection, not a zero-day gap: [1, 1] fires once a week,+ // and reading it as 0 minutes rejected the schedule against every valid floor+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 1] }))).toBe(+ 7 * 24 * 60 - 120,+ );+ // wrap-around: Sun + Sat are 1 day apart across the week boundary+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [0, 6] }))).toBe(+ 24 * 60 - 120,+ );+ });+});diff --git a/packages/api/src/schedules/cadence.ts b/packages/api/src/schedules/cadence.tsnew file mode 100644index 0000000000..f03563c67f--- /dev/null+++ b/packages/api/src/schedules/cadence.ts@@ -0,0 +1,133 @@+import { Cron } from 'croner';+import type { TScheduleCadence } from 'librechat-data-provider';++export const SCHEDULE_JITTER_WINDOW_MS = 120_000;++/**+ * Spring-forward compresses consecutive wall-clock occurrences, so the ENFORCEABLE+ * minimum for day-and-longer gaps is the nominal gap minus the largest real-world+ * transition: two hours (Antarctica/Troll; every other zone shifts at most one).+ * A floor set exactly at the nominal value would otherwise admit a schedule that+ * genuinely violates it once a year. Hourly gaps are unaffected (the skipped hours+ * lengthen, never shorten, the gap between occurrences).+ */+const DST_COMPRESSION_MINUTES = 120;++const WEEKLY_DEFAULT_DAY = 1;++/**+ * Compiles a structured cadence to a 5-field cron expression. The cadence+ * object stays canonical (UI-native, no cron round-tripping); cron exists only+ * as croner's input.+ */+export function cadenceToCron(cadence: TScheduleCadence): string {+ const { frequency, hour, minute } = cadence;+ if (frequency === 'hourly') {+ return `${minute} * * * *`;+ }+ if (frequency === 'daily') {+ return `${minute} ${hour} * * *`;+ }+ if (frequency === 'weekdays') {+ return `${minute} ${hour} * * 1-5`;+ }+ const days = cadence.daysOfWeek?.length ? cadence.daysOfWeek : [WEEKLY_DEFAULT_DAY];+ return `${minute} ${hour} * * ${[...days].sort((a, b) => a - b).join(',')}`;+}++export function isValidTimezone(timezone: string): boolean {+ try {+ new Intl.DateTimeFormat('en-US', { timeZone: timezone });+ return true;+ } catch {+ return false;+ }+}++/** Minimum minutes between occurrences, for the admin interval floor. */+export function cadenceIntervalMinutes(cadence: TScheduleCadence): number {+ if (cadence.frequency === 'hourly') {+ return 60;+ }+ if (cadence.frequency === 'daily' || cadence.frequency === 'weekdays') {+ return 24 * 60 - DST_COMPRESSION_MINUTES;+ }+ // Deduped defensively: the payload schema normalizes new writes, but a legacy+ // stored [1, 1] would otherwise read as a zero-day gap and fail every floor.+ const days = cadence.daysOfWeek?.length+ ? Array.from(new Set(cadence.daysOfWeek))+ : [WEEKLY_DEFAULT_DAY];+ if (days.length <= 1) {+ return 7 * 24 * 60 - DST_COMPRESSION_MINUTES;+ }+ // The interval floor must reflect the SHORTEST gap between selected days+ // (incl. the week wrap-around), not the average — e.g. [Mon, Tue] fires 24h+ // apart, so it must be rejected against a >1440-minute floor.+ const sorted = [...days].sort((a, b) => a - b);+ let minGapDays = 7;+ for (let i = 0; i < sorted.length; i++) {+ const gap = i + 1 < sorted.length ? sorted[i + 1] - sorted[i] : 7 - sorted[i] + sorted[0];+ minGapDays = Math.min(minGapDays, gap);+ }+ return minGapDays * 24 * 60 - DST_COMPRESSION_MINUTES;+}++/**+ * Deterministic per-schedule jitter so fleet-wide fire spikes (everyone at+ * 9:00) spread across a window while each schedule's displayed next-run time+ * stays stable across recomputations.+ */+export function scheduleJitterMs(+ scheduleId: string,+ windowMs: number = SCHEDULE_JITTER_WINDOW_MS,+): number {+ let hash = 5381;+ for (let i = 0; i < scheduleId.length; i++) {+ hash = ((hash << 5) + hash + scheduleId.charCodeAt(i)) | 0;+ }+ return Math.abs(hash) % windowMs;+}++export interface ComputeNextRunParams {+ cadence: TScheduleCadence;+ timezone: string;+ scheduleId: string;+ /** Occurrences at or before this instant are skipped (misfire skip-forward). */+ after?: Date;+ disableJitter?: boolean;+}++/**+ * Next fire instant strictly after `after` (default now), jitter applied.+ * DST semantics are croner's behavior (locked by cadence.spec.ts): a+ * spring-forward gap occurrence fires shifted to the first valid instant+ * (02:30 → 03:30); fall-back ambiguity fires the first occurrence only.+ */+export function computeNextRunAt(params: ComputeNextRunParams): Date | null {+ // A malformed stored schedule (e.g. an invalid timezone inserted before+ // validation or by an admin script) makes croner throw at construction. Treat+ // an uncomputable next run as null — the same signal as "no future occurrence"+ // — so the engine disables it (`invalid_schedule`) rather than throwing out of+ // a tick every time the lease expires and starving other due schedules.+ try {+ const cron = new Cron(cadenceToCron(params.cadence), {+ timezone: params.timezone,+ paused: true,+ });+ const base = params.after ?? new Date();+ const jitter = params.disableJitter === true ? 0 : scheduleJitterMs(params.scheduleId);+ // The jittered instant is `cronOccurrence + jitter`. To return the first one+ // strictly after `base`, find the first cron occurrence after `base - jitter`+ // (occurrence O > base - jitter ⇒ O + jitter > base). Querying from `base`+ // directly would skip an occurrence whose jittered time is still in the future+ // but whose unjittered time already passed (e.g. an hourly :00 created at+ // 12:00:30 with 90s jitter must fire at 12:01:30, not 13:01:30).+ const next = cron.nextRun(new Date(base.getTime() - jitter));+ if (next == null) {+ return null;+ }+ return new Date(next.getTime() + jitter);+ } catch {+ return null;+ }+}diff --git a/packages/api/src/schedules/capacity.ts b/packages/api/src/schedules/capacity.tsnew file mode 100644index 0000000000..79770b5e5d--- /dev/null+++ b/packages/api/src/schedules/capacity.ts@@ -0,0 +1,53 @@+/** Occupancy of the global scheduled-run capacity slots. `unslotted` counts legacy+ * `started` rows written before slots existed; they shrink the effective cap so the+ * bound stays conservative during rollout instead of transiently overshooting. */+export interface CapacityOccupancy {+ takenSlots: number[];+ unslotted: number;+}++/** A claim either succeeded (carrying the caller's own result) or lost the slot race. */+export type SlotClaimResult<T> = { claimed: T } | 'slot-taken';++/**+ * Allocates the lowest free global capacity slot and hands it to `claim`, retrying the+ * next free slot when the DB rejects a collision on the unique partial index.+ *+ * This replaces "count active runs, compare to the cap, then insert": the count is a+ * read-then-write race, so two admissions of DIFFERENT schedules could both observe+ * cap-1 and both proceed. Here the slot itself is the contended resource and the+ * unique index is the arbiter, so the cap is enforced by the database.+ *+ * Bounded by cap+1 attempts: every collision advances to a strictly higher free slot.+ */+export async function withCapacitySlot<T>(+ cap: number,+ readOccupancy: () => Promise<CapacityOccupancy>,+ claim: (slot: number) => Promise<SlotClaimResult<T>>,+): Promise<{ claimed: T } | 'capacity'> {+ if (cap <= 0) {+ return 'capacity';+ }+ for (let attempt = 0; attempt <= cap; attempt++) {+ const { takenSlots, unslotted } = await readOccupancy();+ if (takenSlots.length + unslotted >= cap) {+ return 'capacity';+ }+ const taken = new Set(takenSlots);+ let slot = -1;+ for (let candidate = 0; candidate < cap; candidate++) {+ if (!taken.has(candidate)) {+ slot = candidate;+ break;+ }+ }+ if (slot < 0) {+ return 'capacity';+ }+ const result = await claim(slot);+ if (result !== 'slot-taken') {+ return result;+ }+ }+ return 'capacity';+}diff --git a/packages/api/src/schedules/engine.spec.ts b/packages/api/src/schedules/engine.spec.tsnew file mode 100644index 0000000000..4e1a582712--- /dev/null+++ b/packages/api/src/schedules/engine.spec.ts@@ -0,0 +1,737 @@+import type { ScheduleEngineDeps, ScheduleLimits, ScheduleUserContext } from './types';+import type { FireableSchedule } from './types';+import { startScheduleEngine } from './engine';++const LEASE_MS = 5 * 60_000;+const OWNER: ScheduleUserContext = { id: 'user-1', tenantId: 't1', role: 'USER' };+const LIMITS: ScheduleLimits = {+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+};++/**+ * A schedule due 60s ago whose lease was just taken, so the tick reads it as due+ * rather than misfired (`dbNow` is derived from `leaseUntil - LEASE_MS`).+ */+function makeClaimedSchedule(overrides: Partial<FireableSchedule> = {}): FireableSchedule {+ const now = Date.now();+ return {+ id: 'sched-1',+ user: 'user-1' as never,+ tenantId: 't1',+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ claimToken: 'ct-1',+ leaseBy: 'inst-1',+ nextRunAt: new Date(now - 60_000),+ leaseUntil: new Date(now + LEASE_MS),+ runCount: 0,+ failureCount: 0,+ balanceSkipCount: 0,+ ...overrides,+ } as FireableSchedule;+}++function makeMethods(schedule: FireableSchedule) {+ let claims = 0;+ return {+ // One claimable occurrence per tick, then the claim scan comes up empty.+ claimDueSchedule: jest.fn(async () => (claims++ === 0 ? schedule : null)),+ countActiveRuns: jest.fn(async () => 0),+ advanceSchedule: jest.fn(+ async (_id: string, _next: Date | null, _from?: Date, _token?: string) => true,+ ),+ releaseLeaseByHolder: jest.fn(async () => undefined),+ disableSchedule: jest.fn(async (_id: string, _reason: string, _token?: string) => undefined),+ releaseLease: jest.fn(async () => true),+ holdsLease: jest.fn(async () => true),+ scheduleExists: jest.fn(async () => true),+ deleteScheduleRun: jest.fn(async () => undefined),+ getRunsForReconciliation: jest.fn(async () => []),+ markRunsReconciled: jest.fn(async () => undefined),+ getUnbookkeptRuns: jest.fn(async () => []),+ getDeletingSchedules: jest.fn(async () => []),+ markEraseAttempted: jest.fn(async () => undefined),+ recordRunOutcome: jest.fn(async () => undefined),+ getUnarmedSchedules: jest.fn(async () => []),+ armSchedule: jest.fn(async () => undefined),+ eraseScheduleIfDrained: jest.fn(async () => true),+ finalizeBookkeeping: jest.fn(async () => undefined),+ };+}++function makeDeps(+ methods: ReturnType<typeof makeMethods>,+ over: Partial<ScheduleEngineDeps> = {},+): ScheduleEngineDeps {+ return {+ methods: methods as unknown as ScheduleEngineDeps['methods'],+ getLimits: async () => LIMITS,+ getUserContext: async () => OWNER,+ isOutOfBalance: async () => false,+ agentAccess: async () => 'ok',+ hasScheduleAccess: async () => true,+ resolveFiles: async () => [],+ enqueueTrigger: jest.fn(async () => undefined),+ getTriggerDelivery: async () => null,+ runInTenantContext: (_user, fn) => fn(),+ getJobStatus: async () => null,+ abortScheduledJob: async () => undefined,+ clearReconciledJob: async () => undefined,+ isOwnerDeleting: async () => false,+ isGloballyDisabled: async () => false,+ countActiveRunsGlobal: async () => 0,+ withGlobalCapacitySlot: (_cap: number, claim: (slot: number) => Promise<unknown>) => claim(0),+ ...over,+ } as ScheduleEngineDeps;+}++/** Builds the engine and immediately parks its timer; tests drive `runTick` directly. */+async function tickOnce(deps: ScheduleEngineDeps): Promise<void> {+ const engine = startScheduleEngine(deps);+ engine.stop();+ await engine.runTick();+}++/** Drives one AWAITED reconciliation pass (startup's own is fire-and-forget). */+async function reconcileOnce(deps: ScheduleEngineDeps): Promise<void> {+ const engine = startScheduleEngine(deps);+ engine.stop();+ await engine.reconcile();+}++afterEach(() => jest.restoreAllMocks());++/** Overdue past MISFIRE_GRACE_MS (15m), so the tick skips it forward instead of firing. */+const staleAt = () => new Date(Date.now() - 20 * 60_000);++describe('runTick misfire skip-forward', () => {+ it('advances a stale occurrence to the next future one', async () => {+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt() });+ const methods = makeMethods(schedule);+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as never;+ await tickOnce(makeDeps(methods));+ // Skipping forward is the whole point of the branch. Leaving nextRunAt on the+ // stale occurrence would make every later tick reclaim and skip the same one,+ // so a schedule overdue past an outage would never fire again.+ expect(methods.advanceSchedule).toHaveBeenCalledTimes(1);+ const [id, next, expectedFrom, token] = methods.advanceSchedule.mock.calls[0];+ expect(id).toBe('sched-1');+ expect(next).toBeInstanceOf(Date);+ expect(next?.getTime()).toBeGreaterThan(Date.now());+ // Fenced on the claimed occurrence and the claim token, so a re-claim or an+ // owner edit is not clobbered.+ expect(expectedFrom).toBe(schedule.nextRunAt);+ expect(token).toBe('ct-1');+ // Being overdue is not a fault: the schedule stays enabled and is not fired.+ expect(methods.disableSchedule).not.toHaveBeenCalled();+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it('releases its old holder when an owner edit fences the misfire advance', async () => {+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt() });+ const methods = makeMethods(schedule);+ // The edit rotated claimToken/nextRunAt after this worker claimed but preserved+ // its unique lease holder, so the token-fenced advance no longer matches.+ methods.advanceSchedule.mockResolvedValueOnce(false);++ await tickOnce(makeDeps(methods));++ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');+ });++ it('disables and clears a stale occurrence whose cadence is uncomputable', async () => {+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt(), timezone: 'Not/AZone' });+ const methods = makeMethods(schedule);+ await tickOnce(makeDeps(methods));+ expect(methods.disableSchedule).toHaveBeenCalledWith('sched-1', 'invalid_schedule', 'ct-1');+ expect(methods.advanceSchedule).toHaveBeenCalledWith(+ 'sched-1',+ null,+ schedule.nextRunAt,+ 'ct-1',+ );+ });++ it('leaves a stale occurrence alone when its disable failed', async () => {+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt(), timezone: 'Not/AZone' });+ const methods = makeMethods(schedule);+ methods.disableSchedule.mockRejectedValue(new Error('mongo unavailable') as never);+ await tickOnce(makeDeps(methods));+ // Clearing nextRunAt without the disable landing would leave `enabled: true`+ // with no nextRunAt and no disabledReason: unclaimable and invisible.+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ });+});++describe('runTick error handling', () => {+ it('retains the due occurrence when a preflight query throws', async () => {+ const schedule = makeClaimedSchedule();+ const methods = makeMethods(schedule);+ await tickOnce(+ makeDeps(methods, {+ // A transient infrastructure failure on one of the preflight reads.+ getUserContext: async () => {+ throw new Error('mongo unavailable');+ },+ }),+ );+ // Advancing schedules the NEXT recurrence, so the due one would be discarded+ // permanently with no ScheduleRun row and no evidence it was ever attempted.+ // The lease is left to expire instead, which re-claims THIS occurrence.+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ // A transient read failure says nothing about the schedule's validity.+ expect(methods.disableSchedule).not.toHaveBeenCalled();+ });++ it('disables and clears an uncomputable cadence whose in-fire disable threw', async () => {+ // An unparseable timezone makes the next run uncomputable, so fireSchedule+ // disables the schedule up front — and throws out to the tick when that write+ // fails. Retrying is pointless here: the occurrence can never be computed, and+ // leaving the lease would re-claim it forever.+ const schedule = makeClaimedSchedule({ timezone: 'Not/AZone' });+ const methods = makeMethods(schedule);+ methods.disableSchedule+ .mockRejectedValueOnce(new Error('mongo unavailable') as never)+ .mockResolvedValueOnce(undefined as never);+ await tickOnce(makeDeps(methods));+ expect(methods.disableSchedule).toHaveBeenCalledTimes(2);+ // null clears nextRunAt, so the disabled schedule is no longer due.+ expect(methods.advanceSchedule).toHaveBeenCalledWith(+ 'sched-1',+ null,+ schedule.nextRunAt,+ 'ct-1',+ );+ });++ it('does not clear nextRunAt when the disable itself keeps failing', async () => {+ const schedule = makeClaimedSchedule({ timezone: 'Not/AZone' });+ const methods = makeMethods(schedule);+ methods.disableSchedule.mockRejectedValue(new Error('mongo unavailable') as never);+ await tickOnce(makeDeps(methods));+ // Advancing on a failed disable would leave `enabled: true` with no nextRunAt+ // and no disabledReason: permanently unclaimable and invisible to its owner.+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ });+});++describe('reconciliation consults the durable trigger delivery', () => {+ const YOUNG = () => new Date(Date.now() - 5 * 60_000); // past reconcile min, before orphan cutoff+ const OLD = () => new Date(Date.now() - 60 * 60_000); // past the 30-minute orphan cutoff++ function joblessRun(firedAt: Date, deliveryKey: string | undefined = 'dk1') {+ return {+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ user: 'u1',+ status: 'started',+ conversationId: 'conv-1',+ firedAt,+ ...(deliveryKey != null ? { deliveryKey } : {}),+ };+ }++ async function runReconcile(+ run: ReturnType<typeof joblessRun>,+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'],+ ) {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([run]);+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null, getTriggerDelivery }));+ return methods;+ }++ it('settles a DEAD delivery as error promptly, before the 30-minute orphan age', async () => {+ const methods = await runReconcile(joblessRun(YOUNG()), async () => ({+ status: 'dead',+ // The durable record's lastError is an AgentTriggerDeliveryFailure OBJECT, not a+ // string — the recorded outcome must carry its `message`, or the String-typed+ // Mongoose field rejects the cast and the run keeps its capacity slot.+ lastError: {+ code: 'rate_limited',+ message: 'rate limited',+ certainty: 'definite' as const,+ retryable: false,+ attemptedAt: new Date(),+ },+ }));+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'error', error: 'rate limited' }),+ );+ });++ it('does NOT orphan a PENDING delivery past the cutoff (Retry-After may still fire it)', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'pending' }));+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('does NOT orphan a LEASED delivery past the cutoff', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'leased' }));+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('does NOT orphan a STAGING delivery past the cutoff', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'staging' }));+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('falls back to the legacy interrupted orphan for a SUCCEEDED delivery past the cutoff', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'succeeded' }));+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),+ );+ });++ it('does not prematurely orphan a SUCCEEDED delivery before the cutoff', async () => {+ const methods = await runReconcile(joblessRun(YOUNG()), async () => ({ status: 'succeeded' }));+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('uses the legacy orphan policy when the delivery record is gone (null)', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => null);+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'interrupted' }),+ );+ });++ it('DEFERS (stays reconcilable) when the delivery lookup fails', async () => {+ const methods = await runReconcile(joblessRun(OLD()), async () => {+ throw new Error('delivery store unavailable');+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });+});++describe('reconciliation is isolated per row', () => {+ /**+ * getRunsForReconciliation returns the OLDEST rows first, so a row that always throws+ * came back every tick and starved every run behind it — in the one component whose+ * entire job is settling states nothing else will.+ */+ it('keeps reconciling after a row throws', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ const rows = [+ { scheduleId: 'poison', scheduledFor: new Date(0), user: 'u1', status: 'started' },+ { scheduleId: 'healthy', scheduledFor: new Date(0), user: 'u1', status: 'started' },+ ];+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue(rows);+ (methods.recordRunOutcome as jest.Mock).mockImplementation(async (params) => {+ if (params.scheduleId === 'poison') {+ throw new Error('permanently broken row');+ }+ });++ await tickOnce(+ makeDeps(methods, {+ // Both rows are orphans well past the cutoff, so both reach recordRunOutcome.+ getJobStatus: async () => null,+ }),+ );++ const settled = (methods.recordRunOutcome as jest.Mock).mock.calls.map(+ ([params]) => params.scheduleId,+ );+ expect(settled).toContain('poison');+ expect(settled).toContain('healthy');+ });+});++describe('unarmed recovery sweep', () => {+ it('disables an unarmed schedule whose cadence is uncomputable', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ methods.claimDueSchedule.mockResolvedValue(null);+ const broken = makeClaimedSchedule({+ id: 'sched-broken',+ timezone: 'Not/AZone',+ claimToken: 'ct-9',+ configRevision: 4,+ nextRunAt: undefined,+ });+ const healthy = makeClaimedSchedule({+ id: 'sched-ok',+ claimToken: 'ct-10',+ nextRunAt: undefined,+ });+ (methods.getUnarmedSchedules as jest.Mock).mockResolvedValue([broken, healthy]);++ await reconcileOnce(makeDeps(methods));++ // Skipping the broken row left it enabled-but-unarmed forever: holding the+ // owner's slot, never firing, and re-filling the bounded window ahead of+ // valid crash-left rows. Fenced on token + revision so a concurrent edit+ // that repairs the cadence wins over this observation of the broken one.+ expect(methods.disableSchedule).toHaveBeenCalledWith(+ 'sched-broken',+ 'invalid_schedule',+ 'ct-9',+ 4,+ );+ expect(methods.armSchedule).toHaveBeenCalledWith('sched-ok', expect.any(Date));+ });+});++describe('bookkeeping replay rotation', () => {+ it('stamps every replayed row so persistent failures rotate out of the window', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ methods.claimDueSchedule.mockResolvedValue(null);+ const poison = {+ _id: 'row-1',+ scheduleId: 'poison',+ scheduledFor: new Date(0),+ user: 'u1',+ status: 'success',+ };+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([poison]);+ (methods.finalizeBookkeeping as jest.Mock).mockRejectedValue(new Error('permanently broken'));++ await reconcileOnce(makeDeps(methods));++ // getUnbookkeptRuns reads least-recently-attempted first; without the stamp a+ // batch of failing rows re-fills the bounded window and every later terminal+ // row's counters never land.+ expect(methods.markRunsReconciled).toHaveBeenCalledWith([poison]);+ });++ /**+ * A terminal run reaches this pass only because its OWNER crashed before bookkeeping —+ * which is also before it could release the job it retained for exactly this recovery.+ * A preserved job is kept WITHOUT `completedAt` so the store's finished-job sweep cannot+ * reap it early, so if this pass does not clear it, nothing ever does.+ */+ it('releases the retained job once replayed bookkeeping is durable', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ methods.claimDueSchedule.mockResolvedValue(null);+ const unbookkept = {+ _id: 'row-2',+ scheduleId: 'sched-crashed',+ scheduledFor: new Date(0),+ user: 'u1',+ status: 'success',+ conversationId: 'convo-crashed',+ };+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([unbookkept]);+ const clearReconciledJob = jest.fn(async () => undefined);++ await reconcileOnce(makeDeps(methods, { clearReconciledJob }));++ expect(methods.finalizeBookkeeping).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 'sched-crashed' }),+ );+ expect(clearReconciledJob).toHaveBeenCalledWith('convo-crashed', {+ scheduleId: 'sched-crashed',+ scheduledFor: new Date(0),+ });+ });++ /** The retained job is the only surviving evidence when bookkeeping fails, so it must+ * outlive a failed replay rather than be cleared alongside it. */+ it('keeps the retained job when the bookkeeping replay itself fails', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ methods.claimDueSchedule.mockResolvedValue(null);+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([+ {+ _id: 'row-3',+ scheduleId: 'sched-broken-bk',+ scheduledFor: new Date(0),+ user: 'u1',+ status: 'success',+ conversationId: 'convo-broken',+ },+ ]);+ (methods.finalizeBookkeeping as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const clearReconciledJob = jest.fn(async () => undefined);++ await reconcileOnce(makeDeps(methods, { clearReconciledJob }));++ expect(clearReconciledJob).not.toHaveBeenCalled();+ });+});++describe('reconciliation abort fence', () => {+ const scheduledFor = new Date(0);+ const abortedRun = (abortAgeMs: number, status = 'started', jobGone = false) => ({+ scheduleId: 'sched-1',+ scheduledFor,+ user: 'u1',+ status,+ conversationId: jobGone ? 'c1' : 'c1',+ firedAt: new Date(Date.now() - 60 * 60_000),+ abortRequestedAt: new Date(Date.now() - abortAgeMs),+ });+ const abortedJob = async () => ({+ status: 'aborted',+ scheduleId: 'sched-1',+ scheduledFor: scheduledFor.toISOString(),+ });++ /**+ * A job reads `aborted` the moment abortJob wins its status CAS — BEFORE the+ * generation owner has unwound and persisted (partial response, user message).+ * Settling here releases the run mid-persistence: an account-deletion drain then+ * observes zero active runs and destroys data a pending save recreates. The owner's+ * own outcome write is the only settlement while its abort is in flight.+ */+ it('defers an aborted job while its abort is in flight', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(60_000)]);+ const clearReconciledJob = jest.fn(async () => undefined);+ await tickOnce(makeDeps(methods, { getJobStatus: abortedJob, clearReconciledJob }));++ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ expect(clearReconciledJob).not.toHaveBeenCalled();+ });++ it('finalizes an aborted job once its owner is presumed dead', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);+ const clearReconciledJob = jest.fn(async () => undefined);+ await tickOnce(makeDeps(methods, { getJobStatus: abortedJob, clearReconciledJob }));++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 'sched-1', status: 'interrupted' }),+ );+ expect(clearReconciledJob).toHaveBeenCalled();+ });++ it('omits the conversation link when the aborted job never started', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);+ const preStartAbortedJob = async () => ({+ status: 'aborted',+ scheduleId: 'sched-1',+ scheduledFor: scheduledFor.toISOString(),+ createdEventEmitted: false,+ });+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: preStartAbortedJob,+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ // A pre-start abort reserved an id but no conversation ever came to exist;+ // projecting it hands the card a link to a missing chat, and the row's+ // reserved id is erased so the crash-retry replay cannot restore it either.+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ status: 'interrupted',+ conversationId: undefined,+ clearConversationId: true,+ }),+ );+ });++ /** Account-deletion quiesce DELETES the aborted job, so post-abort absence carries+ * the same fence: the orphan branch must not settle a run whose owner is still+ * unwinding its persistence. */+ it('defers a vanished job while its abort is in flight', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(60_000)]);+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null }));++ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('still reaps a vanished job once the abort fence lapses', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null }));++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'interrupted' }),+ );+ });+});++describe('reconciliation resume hand-off fence', () => {+ const scheduledFor = new Date(0);+ const resumedRun = (claimAgeMs: number) => ({+ scheduleId: 'sched-1',+ scheduledFor,+ user: 'u1',+ status: 'started',+ conversationId: 'c1',+ firedAt: new Date(Date.now() - 60 * 60_000),+ resumeClaimedAt: new Date(Date.now() - claimAgeMs),+ });+ const pausedJob = async () => ({+ status: 'requires_action',+ scheduleId: 'sched-1',+ scheduledFor: scheduledFor.toISOString(),+ });++ it('does not release freshly reacquired capacity while approval claiming is in flight', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(60_000)]);++ await reconcileOnce(makeDeps(methods, { getJobStatus: pausedJob }));++ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('does not treat a temporarily missing hand-off job as an orphan', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(60_000)]);++ await reconcileOnce(makeDeps(methods, { getJobStatus: async () => null }));++ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('re-pauses a crashed resume after the hand-off fence expires', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(11 * 60_000)]);++ await reconcileOnce(makeDeps(methods, { getJobStatus: pausedJob }));++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 'sched-1', status: 'requires_action' }),+ );+ });+});++describe('reconciliation preserves the intended outcome', () => {+ const scheduledFor = new Date(0);+ /** A run whose inline outcome write never landed: old enough to reconcile, still+ * `started`, with the owner's retained terminal job as the only evidence. */+ const unsettledRun = () => ({+ scheduleId: 'sched-1',+ scheduledFor,+ user: 'u1',+ status: 'started',+ conversationId: 'c1',+ firedAt: new Date(Date.now() - 60 * 60_000),+ });+ const retainedComplete =+ (extra: Record<string, string> = {}) =>+ async () => ({+ status: 'complete',+ scheduleId: 'sched-1',+ scheduledFor: scheduledFor.toISOString(),+ ...extra,+ });++ /**+ * A terminal `complete` is generic: it covers a clean finish, a mid-run balance+ * refusal, and a provider failure the client swallowed into an error part. Deriving+ * `success` from it turned a transient outcome-write failure into a reset of the very+ * streaks that drive insufficient_balance and too_many_failures auto-disable.+ */+ it('recovers a balance refusal as skipped_balance, not success', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ const clearReconciledJob = jest.fn(async () => undefined);+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: retainedComplete({ scheduleOutcome: 'skipped_balance' }),+ clearReconciledJob,+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 'sched-1', status: 'skipped_balance' }),+ );+ expect(clearReconciledJob).toHaveBeenCalled();+ });++ it('preserves an interrupted owner outcome instead of converting it to success', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: retainedComplete({+ scheduleOutcome: 'interrupted',+ scheduleOutcomeError: 'Stopped by owner',+ }),+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'interrupted', error: 'Stopped by owner' }),+ );+ });++ it('recovers a balance refusal that claimed an ERROR terminal as skipped_balance', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ await tickOnce(+ makeDeps(methods, {+ // A mid-continuation balance refusal finalizes through completeJob('error'),+ // so the terminal is `error` — but the stamp still routes it to the+ // insufficient_balance streak instead of too_many_failures.+ getJobStatus: async () => ({+ status: 'error',+ scheduleId: 'sched-1',+ scheduledFor: scheduledFor.toISOString(),+ scheduleOutcome: 'skipped_balance',+ }),+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'skipped_balance' }),+ );+ });++ it('recovers a swallowed generation failure as error, with the owner’s message', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: retainedComplete({+ scheduleOutcome: 'error',+ scheduleOutcomeError: 'upstream 503',+ }),+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', error: 'upstream 503' }),+ );+ });++ it('still records success when the owner left no stamp', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: retainedComplete(),+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'success' }),+ );+ });++ // The stamp crosses a serialization boundary, and recordRunOutcome would reject a+ // status outside its union: degrade to success rather than fail the recovery write.+ it('degrades an unrecognized stamp to success', async () => {+ const methods = makeMethods(makeClaimedSchedule());+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);+ await tickOnce(+ makeDeps(methods, {+ getJobStatus: retainedComplete({ scheduleOutcome: 'not_a_status' }),+ clearReconciledJob: jest.fn(async () => undefined),+ }),+ );++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'success' }),+ );+ });+});diff --git a/packages/api/src/schedules/engine.ts b/packages/api/src/schedules/engine.tsnew file mode 100644index 0000000000..92a884447a--- /dev/null+++ b/packages/api/src/schedules/engine.ts@@ -0,0 +1,630 @@+import { logger, runAsSystem } from '@librechat/data-schemas';+import type { IScheduleRun } from '@librechat/data-schemas';+import type { ScheduleEngineDeps, JobState } from './types';+import { hasAbortInFlight, hasResumeHandoffInFlight, retainedOutcome } from './types';+import { isShutdownInProgress, registerShutdownTask } from '~/app/shutdown';+import { fireSchedule, BALANCE_SKIP_DISABLE_THRESHOLD } from './fire';+import { computeNextRunAt } from './cadence';++const TICK_MS = 30_000;+const TICK_JITTER_MS = 2_000;+const LEASE_MS = 5 * 60_000;+const RECONCILE_MIN_RUN_AGE_MS = 2 * 60_000;+const ORPHAN_RUN_AGE_MS = 30 * 60_000;+const ABANDONED_PAUSE_AGE_MS = 25 * 60 * 60_000;+const RECONCILE_BATCH = 100;+// Occurrences due more than this long ago (server downtime / paused engine) are+// skipped forward instead of fired, so a restart doesn't burst stale chats.+const MISFIRE_GRACE_MS = 15 * 60_000;++/**+ * Whether the job currently at a run's conversationId is THIS occurrence's+ * generation. A replacement turn reuses the conversationId but strips the+ * scheduleId/scheduledFor metadata, so an identity mismatch means the original+ * job is gone and its status must not be attributed to (or its hash deleted for)+ * this scheduled run.+ */+function jobIdentityMatches(jobState: JobState | null, run: IScheduleRun): boolean {+ if (jobState == null || jobState.scheduleId !== run.scheduleId || jobState.scheduledFor == null) {+ return false;+ }+ const jobFor = new Date(jobState.scheduledFor).getTime();+ return jobFor === run.scheduledFor.getTime();+}++export type ScheduleEngine = {+ stop: () => void;+ /** Exposed for tests and the run-now handler: one full claim/fire pass. */+ runTick: () => Promise<number>;+ /** Exposed for tests: one awaitable reconciliation pass (the tick loop's is fire-and-forget). */+ reconcile: () => Promise<void>;+};++export function startScheduleEngine(deps: ScheduleEngineDeps): ScheduleEngine {+ let stopped = false;+ let timer: NodeJS.Timeout | undefined;+ let ticks = 0;+ const instanceId = `${process.pid}-${Math.random().toString(36).slice(2, 8)}`;++ /**+ * Job-store-aware run reconciliation: pauses (`requires_action`) surface on+ * the run doc so overlap-skip ignores them; crashed runs become+ * `interrupted`; runs resumed elsewhere become `success` when the job store+ * still shows completion. Long-running generations are left alone.+ */+ async function reconcile() {+ try {+ const limits = await deps.getLimits();+ const runs = await runAsSystem(() =>+ deps.methods.getRunsForReconciliation(+ new Date(Date.now() - RECONCILE_MIN_RUN_AGE_MS),+ RECONCILE_BATCH,+ ),+ );+ await runAsSystem(async () => {+ for (const run of runs) {+ // PER-ROW isolation. A single throwing row used to abort the whole pass, and+ // since this query returns the OLDEST rows first, that row came back every+ // tick and starved every run behind it indefinitely. Reconciliation is the+ // backstop for exactly the states nothing else settles, so it has to make+ // progress on the rest.+ try {+ // Identity-fence the job lookup: a replacement user turn reuses this+ // conversationId but sheds the scheduleId/scheduledFor metadata. Only+ // trust the job's status when it still carries THIS occurrence's identity;+ // otherwise treat the job as gone (null) so a replacement generation's+ // status can never finalize — or its hash be deleted for — this run.+ const jobState = run.conversationId+ ? await deps.getJobStatus(run.conversationId)+ : null;+ const jobStatus = jobIdentityMatches(jobState, run) ? jobState!.status : null;+ const ageMs = Date.now() - (run.firedAt?.getTime() ?? 0);+ // Resolve the run owner's limits so crash-reconciled auto-disable uses+ // the same per-principal threshold as an inline completion. Must run in+ // the OWNER's tenant context: getLimits resolves config via the ALS+ // tenant, and this loop is under runAsSystem (system tenant).+ const owner = await deps.getUserContext(run.user);+ const runLimits = owner+ ? await deps.runInTenantContext(owner, () => deps.getLimits(owner))+ : limits;+ // All transitions go through recordRunOutcome so the schedule's lastRun+ // (and the card's status chip) tracks the run, including the pause.+ const finalize = (+ status: 'success' | 'interrupted' | 'error' | 'requires_action' | 'skipped_balance',+ error?: string,+ opts?: { omitConversationId?: boolean },+ ) =>+ deps.methods.recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status,+ // Pre-start aborts have a reserved id but no conversation was ever+ // created; projecting it gives the card a link to a missing chat.+ conversationId: opts?.omitConversationId ? undefined : run.conversationId,+ clearConversationId: opts?.omitConversationId,+ error,+ autoDisableAfterFailures: runLimits.autoDisableAfterFailures,+ });+ // The clear runs AFTER finalize (the retained job is the only evidence if+ // the finalize write fails), which means a clear that keeps failing has no+ // natural retry: the now-terminal run never rescans, so nothing else would+ // reap the retained job. Retry the transient case inline; the store's own+ // aged-retained-job backstop bounds anything more persistent.+ const clearRetainedJob = async () => {+ for (let attempt = 1; attempt <= 2; attempt++) {+ try {+ await deps.clearReconciledJob(run.conversationId as string, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ });+ return;+ } catch (clearError) {+ logger.warn(+ `[schedules] failed to clear retained job for ${run.scheduleId} (attempt ${attempt}/2):`,+ clearError,+ );+ }+ }+ };+ if (jobStatus === 'running') {+ continue;+ }+ // Surface a pause on the card (lastRun → requires_action). Also re-invoked for+ // a row ALREADY `requires_action`: recordRunOutcome flips the row before+ // projecting the card, so a crash between the two leaves the pause invisible+ // until this replays it. Both writes are idempotent.+ if (+ (run.status === 'started' || run.status === 'requires_action') &&+ jobStatus === 'requires_action'+ ) {+ // A resumed run claims capacity in Mongo BEFORE the approval CAS flips+ // the job back to `running`. During that short hand-off, the old paused+ // job is expected and must not re-pause the run (which would release the+ // newly claimed capacity slot while the continuation is about to run).+ // A stale stamp still falls through so a crashed resume can recover.+ if (run.status === 'started' && hasResumeHandoffInFlight(run, Date.now())) {+ continue;+ }+ await finalize('requires_action');+ continue;+ }+ // A retained terminal job whose inline outcome hook failed transiently —+ // finalize the run from the retained status, then delete the job. The+ // delete is the cleanup path for `preserveForReconcile` jobs (kept+ // without `completedAt`, so the store's finished-job sweep never reaps+ // them); `conversationId` is guaranteed here since jobStatus was fetched.+ if (jobStatus === 'complete') {+ // Finalize either a paused OR a still-started run so it stops consuming+ // capacity / blocking overlap — but from the OWNER'S intended outcome when+ // it left one. A terminal `complete` covers a clean finish, a mid-run+ // balance refusal, and a provider failure the client swallowed into an+ // error part alike, so re-deriving `success` here turned a transient+ // outcome-write failure into a reset of the very streaks that drive+ // insufficient_balance and too_many_failures auto-disable.+ const intended = retainedOutcome(jobState, 'success');+ await finalize(intended.status, intended.error);+ await clearRetainedJob();+ continue;+ }+ if (jobStatus === 'error') {+ // Honor the stamp here too: a mid-continuation balance refusal claims an+ // `error` terminal but must walk the insufficient_balance streak.+ const intended = retainedOutcome(jobState, 'error');+ await finalize(intended.status, intended.error);+ await clearRetainedJob();+ continue;+ }+ if (jobStatus === 'aborted') {+ // ABORT FENCE, same rule as the deletion drains: a job flips `aborted`+ // the moment abortJob wins its status CAS, BEFORE the generation owner+ // has unwound and persisted (partial response, user message). Settling+ // here would release the run mid-persistence — the exact drain-mid-write+ // hazard the deletion paths defer on — so while the abort is in flight+ // the owner's own outcome write is the only settlement. Past the window+ // the owner is presumed dead and this backstop finalizes the run.+ if (hasAbortInFlight(run, Date.now())) {+ continue;+ }+ await finalize('interrupted', undefined, {+ omitConversationId: jobState?.createdEventEmitted !== true,+ });+ await clearRetainedJob();+ continue;+ }+ // A `started` run whose job is gone (jobStatus null). Every run records its+ // conversationId up front, so getJobStatus above already liveness-checked it: a+ // live long-running fire reads as `running` and is left alone. A fresh abort+ // also DELETES the job (account-deletion quiesce), so absence carries the same+ // abort fence as `aborted` above.+ //+ // But a jobless run is NOT automatically an orphan: fireSchedule reserves the+ // run and its capacity BEFORE the durable trigger delivery reaches the chat+ // route, and that delivery can be deferred (a `Retry-After` up to 24h, far past+ // the 30-minute cutoff) or dead-lettered by a pre-generation rejection (an+ // interactive limiter, PII, or moderation). Consult the durable delivery keyed+ // by the reservation before deciding.+ if (+ run.status === 'started' &&+ jobStatus == null &&+ !hasAbortInFlight(run, Date.now()) &&+ !hasResumeHandoffInFlight(run, Date.now())+ ) {+ let delivery: Awaited<ReturnType<typeof deps.getTriggerDelivery>> | null = null;+ try {+ delivery = run.deliveryKey ? await deps.getTriggerDelivery(run.deliveryKey) : null;+ } catch (deliveryError) {+ // Unknown is not gone: defer rather than orphan a possibly-live delivery.+ logger.warn(+ `[schedules] delivery lookup failed for ${run.scheduleId}@${run.scheduledFor?.toISOString?.() ?? run.scheduledFor}; deferring:`,+ deliveryError,+ );+ continue;+ }+ const deliveryStatus = delivery?.status;+ if (+ deliveryStatus === 'staging' ||+ deliveryStatus === 'pending' ||+ deliveryStatus === 'leased'+ ) {+ // Admission is still live; the delivery will fire or dead-letter later.+ continue;+ }+ if (deliveryStatus === 'dead') {+ // Definitively failed before a generation ever existed. Record the error+ // PROMPTLY from the durable lastError and release capacity through the+ // ordinary outcome path — no need to wait the 30-minute orphan age.+ await finalize(+ 'error',+ delivery?.lastError?.message ?? 'Scheduled delivery failed before running',+ );+ continue;+ }+ // `succeeded` (a generation ran; its own job/outcome evidence is the+ // authority) or NO delivery record (a legacy fire): only a genuinely aged+ // jobless run is an orphan.+ if (ageMs > ORPHAN_RUN_AGE_MS) {+ await finalize('interrupted');+ }+ continue;+ }+ if (+ run.status === 'requires_action' &&+ jobStatus == null &&+ ageMs > ABANDONED_PAUSE_AGE_MS &&+ !hasAbortInFlight(run, Date.now())+ ) {+ await finalize('interrupted');+ }+ } catch (rowError) {+ logger.error(+ `[schedules] reconciliation failed for run ${run.scheduleId}@${run.scheduledFor?.toISOString?.() ?? run.scheduledFor}:`,+ rowError,+ );+ }+ }+ // Stamp EVERY row this pass looked at, including the ones that threw — a row+ // that keeps failing must still rotate to the back, or it re-fills the window+ // and starves the rest, which is the same starvation the ordering exists to+ // prevent (and the per-row try/catch above already exists because of).+ await deps.methods+ .markRunsReconciled(runs)+ .catch((err) => logger.warn('[schedules] failed to stamp reconciled runs:', err));+ });++ // Catch terminal runs whose schedule bookkeeping never landed (a crash+ // between the run-row terminalization and the schedule counter update).+ const unbookkept = await runAsSystem(() =>+ deps.methods.getUnbookkeptRuns(+ new Date(Date.now() - RECONCILE_MIN_RUN_AGE_MS),+ RECONCILE_BATCH,+ ),+ );+ await runAsSystem(async () => {+ for (const run of unbookkept) {+ try {+ const owner = await deps.getUserContext(run.user);+ const runLimits = owner+ ? await deps.runInTenantContext(owner, () => deps.getLimits(owner))+ : limits;+ await deps.methods.finalizeBookkeeping({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: run.status as+ | 'success'+ | 'error'+ | 'interrupted'+ | 'skipped_balance'+ | 'skipped_overlap',+ conversationId: run.conversationId,+ error: run.error,+ autoDisableAfterFailures: runLimits.autoDisableAfterFailures,+ balanceSkipDisableThreshold: BALANCE_SKIP_DISABLE_THRESHOLD,+ });+ // The run is terminal but its OWNER crashed before bookkeeping, which is also+ // before it could release the job it retained for exactly this recovery. The+ // active-run pass clears its own; this one never did, and a preserved job is+ // kept WITHOUT `completedAt` precisely so the store's finished-job sweep+ // cannot reap it early — so nothing else would ever have. Identity-guarded,+ // and a no-op when no retained job is there.+ if (run.conversationId) {+ await deps+ .clearReconciledJob(run.conversationId, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ })+ .catch((clearError) =>+ logger.warn(+ `[schedules] failed to clear retained job after bookkeeping replay for ${run.scheduleId}:`,+ clearError,+ ),+ );+ }+ } catch (rowError) {+ logger.error(+ `[schedules] bookkeeping replay failed for run ${run.scheduleId}:`,+ rowError,+ );+ }+ }+ // Same rotation as the active-run pass above: stamp every row this pass+ // looked at so persistent failures rotate to the back of the bounded+ // window instead of starving the rows behind them. Successful replays+ // left the query via bookkept:true; re-stamping them is harmless.+ await deps.methods+ .markRunsReconciled(unbookkept)+ .catch((err) => logger.warn('[schedules] failed to stamp replayed runs:', err));+ });++ // Erase soft-deleted schedules once their active runs have drained. Delete+ // disabled + marked them `deleting` and aborted in-flight jobs; the run+ // reconciliation above finalizes those runs, so here we erase only the+ // schedules with no run still active — their evidence is preserved until+ // settled. eraseScheduleIfDrained is idempotent, so concurrent workers are safe.+ await runAsSystem(async () => {+ const deleting = await deps.methods.getDeletingSchedules(RECONCILE_BATCH);+ for (const schedule of deleting) {+ await deps.methods.eraseScheduleIfDrained(schedule.id).catch(() => undefined);+ }+ // Rotate the window (never-attempted first) so a batch of undrainable rows+ // cannot re-fill it every pass and starve the rows behind them.+ await deps.methods+ .markEraseAttempted(deleting.map((schedule) => schedule.id))+ .catch((err) => logger.warn('[schedules] failed to stamp erase attempts:', err));+ });++ // Re-arm schedules that are enabled but carry no nextRunAt. Creation arms in a+ // second write, so a crash or a failed arm leaves a row that LOOKS enabled to its+ // owner and occupies a slot while claimDueSchedule (which sorts on nextRunAt) can+ // never select it. Recovering here means the owner does not have to notice and+ // edit it; armSchedule is conditional on still being unarmed, so it cannot+ // disturb one that armed itself meanwhile.+ await runAsSystem(async () => {+ const unarmed = await deps.methods.getUnarmedSchedules(RECONCILE_BATCH);+ for (const schedule of unarmed) {+ const nextRunAt = computeNextRunAt({+ cadence: schedule.cadence,+ timezone: schedule.timezone,+ scheduleId: schedule.id,+ });+ if (nextRunAt == null) {+ // Uncomputable cadence is NOT transient (same policy as the claim path):+ // skipping left the row enabled-but-unarmed forever — holding the owner's+ // slot while never firing, and re-filling this bounded window until valid+ // crash-left rows behind it could never be recovered. Fenced on both the+ // claim token and revision so a concurrent edit that repairs the cadence+ // wins over this observation of the broken one.+ await deps.methods+ .disableSchedule(+ schedule.id,+ 'invalid_schedule',+ schedule.claimToken,+ schedule.configRevision,+ )+ .catch((err) => {+ logger.warn(`[schedules] failed to disable unarmed ${schedule.id}:`, err);+ });+ continue;+ }+ await deps.methods.armSchedule(schedule.id, nextRunAt).catch((err) => {+ logger.warn(`[schedules] failed to re-arm ${schedule.id}:`, err);+ });+ }+ });+ } catch (error) {+ logger.error('[schedules] run reconciliation failed:', error);+ }+ }++ /**+ * One claim/fire pass. Claims and cross-tenant reconciliation run in a system+ * tenant context (the claim scans all tenants; strict tenant isolation would+ * otherwise throw on the unscoped query). The fire itself re-enters the+ * owner's context via `runInTenantContext`.+ */+ async function runTick(): Promise<number> {+ // GLOBAL kill switch: stop claiming entirely. This is the operator's hard stop+ // (SCHEDULES_DISABLED, or `interface.schedules: false` in the BASE config), which+ // no principal override can widen — distinct from per-principal availability below.+ // Deliberately gates CLAIMS only; reconcile() is never gated, because in-flight+ // runs must still be settled while the feature is off or they strand `started`+ // rows and leak capacity forever.+ if (await deps.isGloballyDisabled()) {+ return 0;+ }+ // Do NOT gate claims on the per-principal `enabled`: schedules can be enabled per+ // user/role/tenant, so gating here would silently never fire those users'+ // occurrences. The fire path re-resolves the OWNER's limits and skips ('disabled')+ // any occurrence whose owner has the feature off, so an owner-scoped disable is+ // still honored. The base config only supplies the per-tick claim budget.+ const limits = await deps.getLimits();+ let fired = 0;+ // Cap on ACTIVE scheduled runs, not just per-tick starts: the loopback chat+ // endpoint returns as soon as the generation starts and scheduled fires+ // bypass the interactive limiter, so without this the in-flight count would+ // grow by fireConcurrency every tick. Only claim up to the free headroom.+ const active = await runAsSystem(() => deps.methods.countActiveRuns());+ const budget = Math.max(0, limits.fireConcurrency - active);+ for (let i = 0; i < budget; i++) {+ // Claim + the fire's pre-owner-context bookkeeping (disable/advance,+ // cross-tenant reads) run as system; fireSchedule re-enters owner context+ // internally for the run-specific work.+ const done = await runAsSystem(async () => {+ const schedule = await deps.methods.claimDueSchedule({ instanceId, leaseMs: LEASE_MS });+ if (schedule == null) {+ return true;+ }+ const scheduledFor = schedule.nextRunAt ?? new Date();+ /**+ * Advances this claimed occurrence without preserving a superseded holder.+ * Owner edits rotate the token/next occurrence but intentionally leave the old+ * lease in place; a fenced miss therefore releases only this worker's unique+ * holder. Infrastructure errors keep the lease as retry backoff, as before.+ */+ const advanceClaim = async (nextRunAt: Date | null): Promise<void> => {+ try {+ const advanced = await deps.methods.advanceSchedule(+ schedule.id,+ nextRunAt,+ scheduledFor,+ schedule.claimToken,+ );+ if (!advanced && schedule.leaseBy != null) {+ await deps.methods.releaseLeaseByHolder(schedule.id, schedule.leaseBy);+ }+ } catch {+ // Leave the claim lease as bounded backoff for transient storage failures.+ }+ };+ // Use the CLAIM's clock for the misfire cutoff: the claim wrote+ // leaseUntil = now + LEASE_MS from the claiming worker's clock, so+ // leaseUntil - LEASE_MS is that worker's "now" at claim — usually this very+ // process, making cutoff and claim self-consistent. (DocumentDB rules out the+ // server-clock `$$NOW` CAS; skew between REPLICAS shifts fire timing by at+ // most the skew and can never double-fire — the lease CAS and the unique+ // occurrence index arbitrate that regardless of clocks.)+ const dbNow = schedule.leaseUntil ? schedule.leaseUntil.getTime() - LEASE_MS : Date.now();+ // Misfire skip-forward: an occurrence overdue past the grace window (the+ // engine was down/paused) is advanced to the next FUTURE occurrence+ // without firing, so a restart doesn't launch stale or bursty chats.+ if (dbNow - scheduledFor.getTime() > MISFIRE_GRACE_MS) {+ const next = computeNextRunAt({+ cadence: schedule.cadence,+ timezone: schedule.timezone,+ scheduleId: schedule.id,+ // The claim's own clock (see dbNow above): a clock-behind worker would+ // otherwise compute another already-due occurrence and reclaim the row.+ after: new Date(dbNow),+ });+ if (next == null) {+ // Uncomputable cadence is NOT transient, so this occurrence can never run.+ // The advance below would clear nextRunAt AND the lease; doing that after a+ // transiently failed disable would leave `enabled: true` with no nextRunAt+ // and no disabledReason, permanently unclaimable and invisible. Bail+ // instead: the lease expires and the occurrence is retried.+ const disabled = await deps.methods+ .disableSchedule(schedule.id, 'invalid_schedule', schedule.claimToken)+ .then(() => true)+ .catch(() => false);+ if (!disabled) {+ return false;+ }+ }+ // The SKIP-FORWARD itself, and the whole point of this branch: move to the+ // next FUTURE occurrence. Without it nextRunAt keeps pointing at the stale+ // one, so every later tick reclaims and skips the same occurrence forever and+ // a schedule overdue past an outage never fires again.+ await advanceClaim(next);+ logger.info(`[schedules] skipped stale occurrence for ${schedule.id} (misfire grace)`);+ return false;+ }+ try {+ const result = await fireSchedule(+ // The dispatch boundary observes shutdown from BOTH signals: the+ // coordinator flag flips before the listener starts closing (ahead of+ // any pre-drain task ordering), and the engine's own stop covers direct+ // runTick callers outside a coordinated shutdown.+ { ...deps, isShuttingDown: () => stopped || isShutdownInProgress() },+ schedule,+ limits,+ scheduledFor,+ {+ dbNow: new Date(dbNow),+ },+ );+ if (result.fired) {+ fired += 1;+ }+ } catch (error) {+ // A transient preflight throw (user/config/permission/balance/capacity query)+ // must NOT advance past this occurrence: advancing schedules the NEXT+ // recurrence, so the due one is discarded permanently with no ScheduleRun row+ // and no evidence it was ever attempted. Leave nextRunAt alone and keep the+ // claim lease as backoff — the lease expires and the SAME occurrence is+ // re-claimed and retried, matching how the fire path already treats a+ // transient file-resolution failure.+ logger.error(`[schedules] unexpected fire error for ${schedule.id} (will retry):`, error);+ // An UNCOMPUTABLE cadence is the one non-transient case: retrying can never+ // make progress, and leaving the lease would re-claim the same occurrence+ // forever. Disable so it stops being due, and only then clear nextRunAt —+ // advancing on a failed disable would leave `enabled: true` with no nextRunAt+ // and no disabledReason (permanently unclaimable and invisible). Everything+ // else falls through untouched: nextRunAt and the lease both stand.+ const next = computeNextRunAt({+ cadence: schedule.cadence,+ timezone: schedule.timezone,+ scheduleId: schedule.id,+ // The claim's clock (see the misfire branch) so a skewed worker doesn't+ // reschedule to an already-due occurrence and reclaim the same row.+ after: new Date(dbNow),+ });+ if (next == null) {+ const disabled = await deps.methods+ .disableSchedule(schedule.id, 'invalid_schedule', schedule.claimToken)+ .then(() => true)+ .catch(() => false);+ if (disabled) {+ await advanceClaim(null);+ }+ }+ }+ return false;+ });+ if (done) {+ break;+ }+ }+ return fired;+ }++ const scheduleNext = () => {+ if (stopped) {+ return;+ }+ timer = setTimeout(tick, TICK_MS + Math.floor(Math.random() * TICK_JITTER_MS));+ timer.unref?.();+ };++ let activePass: Promise<void> | null = null;++ async function tick() {+ if (stopped) {+ return;+ }+ const pass = (async () => {+ try {+ if (ticks % 4 === 0) {+ await reconcile();+ }+ ticks += 1;+ await runTick();+ } catch (error) {+ logger.error('[schedules] tick failed:', error);+ }+ })().finally(() => {+ activePass = null;+ });+ activePass = pass;+ await pass;+ scheduleNext();+ }++ void reconcile();+ scheduleNext();++ const engine: ScheduleEngine = {+ stop: () => {+ stopped = true;+ if (timer) {+ clearTimeout(timer);+ timer = undefined;+ }+ },+ runTick,+ reconcile,+ };++ // PRE-DRAIN: the default post-drain phase leaves this timer armed while the HTTP+ // listener is closing and the generation manager has already begun refusing new jobs.+ // A tick in that window claims a due occurrence, fails its loopback POST against a+ // server that is shutting down, and books the failure against the schedule — walking a+ // healthy schedule toward auto-disable for nothing more than a restart. Stopping the+ // timer alone only narrowed the window: a pass ALREADY in flight could still claim a+ // due occurrence and lose its loopback POST against the closing listener, so the+ // shutdown also AWAITS the active pass — the coordinator's drain then holds the+ // listener open until that pass's fire completes or its claim is released.+ // Occurrences skipped by stopping early are simply still due at restart, within the+ // misfire grace.+ registerShutdownTask(+ 'schedule engine',+ async () => {+ engine.stop();+ if (activePass) {+ await activePass;+ }+ },+ { phase: 'pre-drain' },+ );++ logger.info('[schedules] engine started');+ return engine;+}diff --git a/packages/api/src/schedules/erasure.spec.ts b/packages/api/src/schedules/erasure.spec.tsnew file mode 100644index 0000000000..0617201ad6--- /dev/null+++ b/packages/api/src/schedules/erasure.spec.ts@@ -0,0 +1,402 @@+import type { IScheduleRun, ScheduleMethods } from '@librechat/data-schemas';+import type { JobState } from './types';+import { startScheduleErasureSweep } from './erasure';++const oldRun = (overrides: Partial<IScheduleRun> = {}): IScheduleRun =>+ ({+ scheduleId: 'schedule-1',+ user: 'user-1',+ scheduledFor: new Date('2026-08-17T12:00:00.000Z'),+ firedAt: new Date(Date.now() - 31 * 60_000),+ conversationId: 'conversation-1',+ status: 'started',+ ...overrides,+ }) as unknown as IScheduleRun;++async function sweepOnce(options: {+ canInferOwnerDeathFromMissingJob: boolean;+ job?: JobState | null;+}) {+ const methods = {+ getDeletingSchedules: jest.fn(async () => [{ id: 'schedule-1' }]),+ getActiveRunsForSchedule: jest.fn(async () => [oldRun()]),+ recordRunOutcome: jest.fn(async () => undefined),+ eraseScheduleIfDrained: jest.fn(async () => false),+ markEraseAttempted: jest.fn(async () => undefined),+ getRunsForReconciliation: jest.fn(async () => []),+ };+ const sweep = startScheduleErasureSweep({+ methods: methods as unknown as ScheduleMethods,+ getJobStatus: jest.fn(async () => options.job ?? null),+ getTriggerDelivery: jest.fn(async () => null),+ clearReconciledJob: jest.fn(async () => undefined),+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,+ });+ await jest.advanceTimersByTimeAsync(5 * 60_000);+ sweep.stop();+ return methods;+}++describe('schedule erasure fallback owner-death evidence', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ jest.spyOn(Math, 'random').mockReturnValue(0);+ });++ afterEach(() => {+ jest.useRealTimers();+ jest.restoreAllMocks();+ });++ it('does not settle a peer-owned run from process-local job absence', async () => {+ const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: false });++ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('schedule-1');+ });++ it('retains owner-death recovery when job absence is deployment-authoritative', async () => {+ const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: true });++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ scheduleId: 'schedule-1',+ status: 'interrupted',+ error: 'Schedule deleted',+ }),+ );+ });++ it('still trusts an identity-matched terminal job in an unsafe topology', async () => {+ const methods = await sweepOnce({+ canInferOwnerDeathFromMissingJob: false,+ job: {+ status: 'complete',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ },+ });++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 'schedule-1', status: 'success' }),+ );+ });+});++const DEFINITE = {+ code: 'rejected',+ message: 'rejected before running',+ certainty: 'definite' as const,+};++describe('topology-safe dead-delivery convergence', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ jest.spyOn(Math, 'random').mockReturnValue(0);+ });++ afterEach(() => {+ jest.useRealTimers();+ jest.restoreAllMocks();+ });++ async function convergeOnce(options: {+ run?: Partial<IScheduleRun>;+ delivery?: {+ status: string;+ lastError?: { code: string; message: string; certainty: 'definite' | 'ambiguous' };+ } | null;+ job?: JobState | null;+ canInferOwnerDeathFromMissingJob?: boolean;+ }) {+ const methods = {+ getDeletingSchedules: jest.fn(async () => []),+ getActiveRunsForSchedule: jest.fn(async () => []),+ recordRunOutcome: jest.fn(async () => undefined),+ eraseScheduleIfDrained: jest.fn(async () => false),+ markEraseAttempted: jest.fn(async () => undefined),+ getRunsForReconciliation: jest.fn(async () => [+ oldRun({ deliveryKey: 'dk-1', ...options.run } as Partial<IScheduleRun>),+ ]),+ };+ const getTriggerDelivery = jest.fn(async () => options.delivery ?? null);+ const clearReconciledJob = jest.fn(async () => undefined);+ const sweep = startScheduleErasureSweep({+ methods: methods as unknown as ScheduleMethods,+ getJobStatus: jest.fn(async () => options.job ?? null),+ getTriggerDelivery: getTriggerDelivery as never,+ clearReconciledJob,+ // Defaults to the UNSAFE topology to prove this path never depends on it.+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob ?? false,+ });+ await jest.advanceTimersByTimeAsync(5 * 60_000);+ sweep.stop();+ return { methods, getTriggerDelivery, clearReconciledJob };+ }++ it('settles a dead delivery as error even in an unsafe topology (positive evidence)', async () => {+ const { methods } = await convergeOnce({+ delivery: {+ status: 'dead',+ lastError: { code: 'blocked', message: 'blocked by moderation', certainty: 'definite' },+ },+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ scheduleId: 'schedule-1',+ status: 'error',+ error: 'blocked by moderation',+ }),+ );+ });++ it('leaves a live delivery alone', async () => {+ const { methods } = await convergeOnce({ delivery: { status: 'pending' } });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('never settles while an identity-matched generation still owns the reservation', async () => {+ const { methods, getTriggerDelivery } = await convergeOnce({+ delivery: { status: 'dead', lastError: DEFINITE },+ job: {+ status: 'running',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ } as unknown as JobState,+ });+ expect(getTriggerDelivery).not.toHaveBeenCalled();+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('defers a run whose abort is still in flight', async () => {+ const { methods } = await convergeOnce({+ run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,+ delivery: { status: 'dead', lastError: DEFINITE },+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ /**+ * The gap this closes: a generation that FINISHED but whose owner exhausted every Mongo+ * outcome retry leaves a preserved terminal job as the only evidence. The armed engine's+ * reconciler replays that; the clustered entrypoint arms no engine, so before this the run+ * kept its `started` row — and its global capacity slot — until the job store expired it.+ */+ it('settles a live schedule from its retained terminal job, then releases the evidence', async () => {+ const { methods, clearReconciledJob } = await convergeOnce({+ job: {+ status: 'complete',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ scheduleId: 'schedule-1',+ status: 'success',+ conversationId: 'conversation-1',+ }),+ );+ expect(clearReconciledJob).toHaveBeenCalledWith('conversation-1', {+ scheduleId: 'schedule-1',+ scheduledFor: new Date('2026-08-17T12:00:00.000Z'),+ });+ });++ /** Presence of an identity-matched job is positive evidence in EVERY topology, so this+ * must not depend on the owner-death inference the unsafe fallback refuses. */+ it('honors the owner-stamped outcome over the generic terminal status', async () => {+ const { methods } = await convergeOnce({+ job: {+ status: 'complete',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleOutcome: 'skipped_balance',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'skipped_balance' }),+ );+ });++ it('carries the stamped failure reason from a retained error job', async () => {+ const { methods } = await convergeOnce({+ job: {+ status: 'error',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleOutcomeError: 'provider exploded',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', error: 'provider exploded' }),+ );+ });++ /** A pre-start abort reserves a conversationId but never creates the conversation, so+ * projecting it would point the schedule card at a chat that does not exist. */+ it('clears the reserved conversationId when the generation never emitted its created event', async () => {+ const { methods } = await convergeOnce({+ job: {+ status: 'aborted',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ createdEventEmitted: false,+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'interrupted', clearConversationId: true }),+ );+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.not.objectContaining({ conversationId: 'conversation-1' }),+ );+ });++ /** A terminal job belonging to a REPLACEMENT turn (same conversationId, no scheduled+ * identity) must never finalize this run, nor have its evidence deleted for it. */+ it("ignores a terminal job that lost this occurrence's identity", async () => {+ const { methods, clearReconciledJob } = await convergeOnce({+ job: { status: 'complete' } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ expect(clearReconciledJob).not.toHaveBeenCalled();+ });++ /**+ * A pause whose `requires_action` projection failed every retry leaves the row `started`+ * while the job is durably paused. The armed reconciler replays that; with no engine the+ * row held its GLOBAL capacity slot forever, since a paused job is not terminal and the+ * dead-delivery path never inspects an identity-matched job.+ */+ it('projects a pause the owner never managed to record, freeing the slot', async () => {+ const { methods, clearReconciledJob } = await convergeOnce({+ job: {+ status: 'requires_action',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ scheduleId: 'schedule-1',+ status: 'requires_action',+ conversationId: 'conversation-1',+ // Every replica runs this sweep, so the write itself must be fenced against a+ // resume that claimed the row after this snapshot was taken — as a staleness+ // CUTOFF, so an abandoned claim still recovers.+ resumeClaimStaleBefore: expect.any(Date),+ }),+ );+ // The job is still LIVE awaiting approval — its evidence must not be released.+ expect(clearReconciledJob).not.toHaveBeenCalled();+ });++ /** A resume claims capacity before flipping the job back to `running`; re-projecting the+ * pause during that hand-off would release the slot the continuation just took. */+ it('defers a paused job while a resume hand-off is in flight', async () => {+ const { methods } = await convergeOnce({+ run: { resumeClaimedAt: new Date() } as Partial<IScheduleRun>,+ job: {+ status: 'requires_action',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('never settles a terminal job while its abort is still in flight', async () => {+ const { methods, clearReconciledJob } = await convergeOnce({+ run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,+ job: {+ status: 'aborted',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ } as unknown as JobState,+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ expect(clearReconciledJob).not.toHaveBeenCalled();+ });++ it('ignores a legacy reservation carrying no deliveryKey', async () => {+ const { methods, getTriggerDelivery } = await convergeOnce({+ run: { deliveryKey: undefined } as Partial<IScheduleRun>,+ delivery: { status: 'dead', lastError: DEFINITE },+ });+ expect(getTriggerDelivery).not.toHaveBeenCalled();+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });+});++describe('dead-delivery certainty fence', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ jest.spyOn(Math, 'random').mockReturnValue(0);+ });++ afterEach(() => {+ jest.useRealTimers();+ jest.restoreAllMocks();+ });++ async function sweepWithDelivery(options: {+ certainty: 'definite' | 'ambiguous';+ canInferOwnerDeathFromMissingJob: boolean;+ }) {+ const methods = {+ getDeletingSchedules: jest.fn(async () => []),+ getActiveRunsForSchedule: jest.fn(async () => []),+ recordRunOutcome: jest.fn(async () => undefined),+ eraseScheduleIfDrained: jest.fn(async () => false),+ markEraseAttempted: jest.fn(async () => undefined),+ getRunsForReconciliation: jest.fn(async () => [oldRun({ deliveryKey: 'dk-1' } as never)]),+ };+ const sweep = startScheduleErasureSweep({+ methods: methods as unknown as ScheduleMethods,+ getJobStatus: jest.fn(async () => null),+ getTriggerDelivery: (async () => ({+ status: 'dead',+ lastError: { code: 'x', message: 'timed out', certainty: options.certainty },+ })) as never,+ clearReconciledJob: jest.fn(async () => undefined),+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,+ });+ await jest.advanceTimersByTimeAsync(5 * 60_000);+ sweep.stop();+ return methods;+ }++ /**+ * `dead` is not proof of rejection: the trigger host marks response timeouts and invalid+ * success responses `ambiguous`, and the engine dead-letters those after exhausting+ * retries — so an ambiguous dead letter can sit over a generation a peer accepted and is+ * still running. Settling it here would release capacity mid-run.+ */+ it('refuses an AMBIGUOUS dead letter when absence is not deployment-authoritative', async () => {+ const methods = await sweepWithDelivery({+ certainty: 'ambiguous',+ canInferOwnerDeathFromMissingJob: false,+ });+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('settles an ambiguous dead letter once job absence IS deployment-authoritative', async () => {+ const methods = await sweepWithDelivery({+ certainty: 'ambiguous',+ canInferOwnerDeathFromMissingJob: true,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', error: 'timed out' }),+ );+ });++ it('settles a DEFINITE rejection even in an unsafe topology', async () => {+ const methods = await sweepWithDelivery({+ certainty: 'definite',+ canInferOwnerDeathFromMissingJob: false,+ });+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', error: 'timed out' }),+ );+ });+});diff --git a/packages/api/src/schedules/erasure.ts b/packages/api/src/schedules/erasure.tsnew file mode 100644index 0000000000..37dc92ea01--- /dev/null+++ b/packages/api/src/schedules/erasure.ts@@ -0,0 +1,376 @@+import { logger, runAsSystem } from '@librechat/data-schemas';+import type { ScheduleMethods, IScheduleRun } from '@librechat/data-schemas';+import type { JobState, ScheduleEngineDeps } from './types';+import {+ hasResumeHandoffInFlight,+ hasAbortInFlight,+ retainedOutcome,+ RESUME_HANDOFF_STALE_MS,+} from './types';+import { registerShutdownTask } from '~/app/shutdown';++const SWEEP_MS = 5 * 60_000;+const SWEEP_JITTER_MS = 30_000;+const SWEEP_BATCH = 100;+/** Runs with no readable job older than this are presumed owner-dead (matches the+ * engine reconciler's orphan cutoff). */+const ABANDONED_RUN_AGE_MS = 30 * 60_000;+/** Grace before a live schedule's reservation is converged, so an accepted delivery still+ * creating its generation — or an owner still writing its terminal outcome — is never+ * settled mid-handoff. */+const STRANDED_RUN_MIN_AGE_MS = 2 * 60_000;++/** Terminal job status → the run outcome it proves (mirror of the quiesce map). */+const TERMINAL_JOB_OUTCOMES: Record<string, 'success' | 'error' | 'interrupted' | undefined> = {+ complete: 'success',+ error: 'error',+ aborted: 'interrupted',+};++export interface ScheduleErasureSweep {+ stop: () => void;+}++export interface ScheduleErasureDeps {+ methods: Pick<+ ScheduleMethods,+ | 'getDeletingSchedules'+ | 'eraseScheduleIfDrained'+ | 'markEraseAttempted'+ | 'getActiveRunsForSchedule'+ | 'getRunsForReconciliation'+ | 'recordRunOutcome'+ >;+ /** Job state at a run's conversationId; null = confirmed absent, throw = unknown. */+ getJobStatus: (conversationId: string) => Promise<JobState | null>;+ /** Durable trigger delivery for a reservation's `deliveryKey`. A `dead` delivery is+ * POSITIVE shared evidence that no generation owns the reservation, so settling on it+ * is topology-safe (unlike absence). */+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'];+ /** Deletes a run's retained terminal job once its outcome is durable. Identity-guarded,+ * so a replacement generation reusing the conversationId is never destroyed. */+ clearReconciledJob: ScheduleEngineDeps['clearReconciledJob'];+ /** Whether absence in this process's job store proves absence deployment-wide.+ * False for the process-local fallback whose scheduler refused unsafe topology. */+ canInferOwnerDeathFromMissingJob: boolean;+}++/** Whether the observed job still carries THIS occurrence's scheduled identity. */+function jobMatchesRun(job: JobState | null, run: IScheduleRun): boolean {+ if (job == null || job.scheduleId !== run.scheduleId || job.scheduledFor == null) {+ return false;+ }+ return new Date(job.scheduledFor).getTime() === run.scheduledFor.getTime();+}++/**+ * Settles the abandoned active runs of a DELETING schedule so the erase below can+ * proceed. The clustered entrypoint runs no engine reconciler, and the run TTL now+ * (correctly) never expires active rows — so a deleting schedule whose generation+ * owner died would otherwise retain the run and the owner's prompt indefinitely.+ * Same evidence discipline as the quiesce paths: settle only on positive evidence+ * (a terminal identity-matched job, or a confirmed-absent job past the owner-death+ * cutoff), and defer anything fenced by an in-flight abort or resume hand-off.+ */+async function settleAbandonedRuns(deps: ScheduleErasureDeps, scheduleId: string): Promise<void> {+ const runs = await deps.methods.getActiveRunsForSchedule(scheduleId);+ const now = Date.now();+ for (const run of runs) {+ try {+ if (hasAbortInFlight(run, now) || hasResumeHandoffInFlight(run, now)) {+ continue;+ }+ const job = run.conversationId+ ? await deps.getJobStatus(run.conversationId).then(+ (state) => ({ known: true, state }),+ () => ({ known: false, state: null }),+ )+ : { known: true, state: null };+ if (!job.known) {+ continue;+ }+ const identity = jobMatchesRun(job.state, run);+ if (identity && job.state!.status === 'running') {+ continue;+ }+ if (identity && job.state!.status === 'requires_action') {+ // A paused run of a DELETING schedule: its approval can never be consumed,+ // but a fresh pause hand-off may still be writing — the started-row gate+ // and the resume fence above already deferred those; a settled-state+ // paused row is safe to interrupt.+ if (run.status === 'started') {+ continue;+ }+ }+ const settledPause =+ identity && job.state!.status === 'requires_action' && run.status === 'requires_action';+ const retained = identity ? TERMINAL_JOB_OUTCOMES[job.state!.status] : undefined;+ if (retained == null) {+ // In the unsafe-topology fallback, a peer-owned live job is indistinguishable+ // from an absent one. Never turn that local absence (or identity mismatch) into+ // owner-death evidence and free its globally visible run/capacity slot.+ // An identity-matched, durably paused row is positive local evidence, so it+ // remains eligible for the existing age-based deleting-schedule cleanup.+ if (!settledPause && !deps.canInferOwnerDeathFromMissingJob) {+ continue;+ }+ // No terminal evidence: only presume the owner dead past the cutoff.+ const age = now - (run.firedAt?.getTime() ?? 0);+ if (age < ABANDONED_RUN_AGE_MS) {+ continue;+ }+ }+ await deps.methods.recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: retained ?? 'interrupted',+ conversationId: run.conversationId,+ ...(retained == null ? { error: 'Schedule deleted' } : {}),+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ });+ } catch (err) {+ logger.warn(`[schedules] abandoned-run settle failed for ${scheduleId}:`, err);+ }+ }+}++/**+ * Converges a run from its OWN identity-matched job — the clustered mirror of the engine+ * reconciler's retained-job and pause branches.+ */+async function settleFromObservedJob(+ deps: ScheduleErasureDeps,+ run: IScheduleRun,+ job: JobState,+ now: number,+): Promise<void> {+ // A PAUSE the owner never managed to project. `recordRunOutcome('requires_action')`+ // moves the row off `started`, which is what frees its global capacity slot; the job+ // itself stays live awaiting approval, so its evidence is NOT released here. Without+ // this the row held a slot forever wherever no engine is armed, since a paused job is+ // not terminal and the dead-delivery path never looks at an identity-matched job.+ if (job.status === 'requires_action') {+ if (run.status !== 'started') {+ return;+ }+ await deps.methods.recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: 'requires_action',+ conversationId: run.conversationId,+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ // Every clustered replica runs this sweep, so N sweepers can observe the same+ // unprojected pause. The `run` above is a SNAPSHOT: once one sweeper projects the+ // pause, the owner's approval can claim a fresh slot, and a peer still holding the+ // pre-projection snapshot would pass hasResumeHandoffInFlight and unset that slot+ // and claim stamp under the running continuation. Fence it in the write itself —+ // on the SAME staleness bound as the caller's check above, so an abandoned claim+ // (worker died between claiming and resuming) still recovers rather than pinning+ // the row `started` forever with its approval unresumable.+ resumeClaimStaleBefore: new Date(now - RESUME_HANDOFF_STALE_MS),+ });+ return;+ }+ const terminal = TERMINAL_JOB_OUTCOMES[job.status];+ if (terminal == null) {+ // `running`: the owner is alive and owns the settlement.+ return;+ }+ // An `aborted` job flips the moment abortJob wins its status CAS — BEFORE its owner+ // unwinds and persists — so it is the abort fence, not this status, that decides. The+ // in-flight check in the caller already deferred those; reaching here means the owner+ // is past its presumed-alive window and this is the backstop.+ const intended =+ terminal === 'interrupted'+ ? { status: 'interrupted' as const, error: undefined }+ : retainedOutcome(job, terminal === 'error' ? 'error' : 'success');+ await deps.methods.recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: intended.status,+ // A pre-start abort reserved a conversationId but never created the conversation;+ // projecting it would point the card at a chat that does not exist.+ ...(terminal === 'interrupted' && job.createdEventEmitted !== true+ ? { clearConversationId: true }+ : { conversationId: run.conversationId }),+ error: intended.error,+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ });+ // AFTER the outcome write, never before: the retained job is the only surviving+ // evidence if that write fails, and a preserved job carries no `completedAt`, so the+ // store's finished-job sweep can never reap it on its own.+ await deps.clearReconciledJob(run.conversationId as string, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ });+}++/**+ * Converges the stranded reservations of LIVE (non-deleting) schedules, on POSITIVE+ * EVIDENCE ONLY.+ *+ * `fireSchedule` reserves the run and its global capacity slot before the delivery reaches+ * the chat route, and the clustered entrypoint arms no engine — so three states would+ * otherwise hold that slot indefinitely:+ *+ * - A RETAINED TERMINAL JOB. The generation finished, but its owner's Mongo outcome write+ * exhausted every retry (`recordScheduleOutcome` returns false) and left the preserved+ * job as the only surviving evidence. The armed engine's reconciler replays exactly+ * this; with no engine nothing did, so the run stayed `started` and its retained+ * generation evidence lived until store expiry.+ * - AN UNPROJECTED PAUSE. The generation paused for approval, but the owner's+ * `requires_action` projection failed every retry, so the row never left `started` even+ * though the pause itself is durable in the job.+ * - A DEAD DELIVERY. A pre-generation rejection (interactive limiter, PII, moderation)+ * dead-letters the delivery while the run stays `started` with no job at all.+ *+ * Both are safe in EVERY topology because both read positive, durable evidence instead of+ * inferring owner death from absence: an identity-matched job is authoritative wherever it+ * is observed (a shared store shows the real generation; a process-local store can only be+ * showing this process's own), and a `dead` delivery is shared state proving no generation+ * owns the reservation. This never claims, fires, or advances, and defers anything fenced+ * by an in-flight abort or resume hand-off. Auto-disable policy is deliberately NOT applied+ * (the armed engine owns that): the run settles and frees its slot, the streak is untouched.+ */+async function settleStrandedRuns(deps: ScheduleErasureDeps): Promise<void> {+ const runs = await deps.methods.getRunsForReconciliation(+ new Date(Date.now() - STRANDED_RUN_MIN_AGE_MS),+ SWEEP_BATCH,+ );+ const now = Date.now();+ for (const run of runs) {+ try {+ // `started` is the capacity-consuming state this pass exists to release. A paused+ // row holds no slot, and its approval-expiry path owns its own durable retry.+ if (run.status !== 'started') {+ continue;+ }+ if (hasAbortInFlight(run, now) || hasResumeHandoffInFlight(run, now)) {+ continue;+ }+ const job = run.conversationId+ ? await deps.getJobStatus(run.conversationId).then(+ (state) => ({ known: true, state }),+ () => ({ known: false, state: null }),+ )+ : { known: true, state: null };+ // Unknown is not gone: never settle on a failed job-store read.+ if (!job.known) {+ continue;+ }+ if (jobMatchesRun(job.state, run)) {+ await settleFromObservedJob(deps, run, job.state as JobState, now);+ continue;+ }+ // No job of THIS occurrence's identity, so the durable delivery is the authority.+ if (!run.deliveryKey) {+ continue;+ }+ const delivery = await deps.getTriggerDelivery(run.deliveryKey);+ if (delivery?.status !== 'dead') {+ continue;+ }+ // `dead` alone is NOT proof the request was rejected. The trigger host marks+ // response timeouts and invalid success responses `ambiguous`, and the engine+ // dead-letters those once retries are exhausted — so an ambiguous dead letter can+ // sit over a generation a peer accepted and is still running. Settle only on a+ // DEFINITE rejection, unless this process can observe generation absence+ // deployment-wide (safe topology), where the confirmed-absent job above is itself+ // authoritative evidence.+ if (delivery.lastError?.certainty !== 'definite' && !deps.canInferOwnerDeathFromMissingJob) {+ continue;+ }+ await deps.methods.recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: 'error',+ conversationId: run.conversationId,+ // `lastError` is an AgentTriggerDeliveryFailure, not a string: passing the object+ // into the String-typed run/schedule error fields fails the Mongoose cast, and the+ // per-row catch would swallow it while the run kept its capacity slot.+ error: delivery.lastError?.message ?? 'Scheduled delivery failed before running',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ });+ } catch (err) {+ logger.warn(`[schedules] stranded-run settle failed for ${run.scheduleId}:`, err);+ }+ }+}++/**+ * Erases soft-deleted schedules once they drain — and NOTHING else.+ *+ * A `deleting` row is normally erased by whichever actor first observes it drained: the+ * delete request, or the terminal outcome write (erase-on-settle). Both are single+ * best-effort attempts, so one transient failure — or a lease that outlived the delete —+ * leaves a hidden row holding the user's prompt, with no TTL and no way for the owner to+ * retry it (the row is hidden from their list). In the standard entrypoint the+ * reconciler retries it; the clustered entrypoint runs no engine, so nothing does.+ *+ * This is deliberately NOT the engine: it never claims, leases, fires, advances, or+ * reconciles a run, so running it in every replica of a clustered deployment is safe and+ * changes nothing about v1's single-process scheduling. It only re-drives+ * `eraseScheduleIfDrained`, which re-checks drained-ness itself (no active run, no live+ * lease) and is idempotent — concurrent sweepers race harmlessly.+ */+export function startScheduleErasureSweep(deps: ScheduleErasureDeps): ScheduleErasureSweep {+ let stopped = false;+ let timer: NodeJS.Timeout | undefined;++ async function sweep(): Promise<void> {+ try {+ await runAsSystem(async () => {+ const deleting = await deps.methods.getDeletingSchedules(SWEEP_BATCH);+ for (const schedule of deleting) {+ await settleAbandonedRuns(deps, schedule.id).catch((err) => {+ logger.warn(`[schedules] abandoned-run pass failed for ${schedule.id}:`, err);+ });+ await deps.methods.eraseScheduleIfDrained(schedule.id).catch((err) => {+ logger.warn(`[schedules] erasure sweep failed for ${schedule.id}:`, err);+ });+ }+ // Rotate the window (never-attempted first) so a batch of undrainable rows+ // cannot re-fill it every sweep and starve the rows behind them.+ await deps.methods+ .markEraseAttempted(deleting.map((schedule) => schedule.id))+ .catch((err) => logger.warn('[schedules] failed to stamp erase attempts:', err));+ // Live schedules too: converge reservations left stranded by a retained terminal+ // job or a dead delivery, so neither can hold a global capacity slot — and no+ // generation evidence can outlive its run — where no engine is armed.+ await settleStrandedRuns(deps).catch((err) =>+ logger.warn('[schedules] stranded-run convergence pass failed:', err),+ );+ });+ } catch (err) {+ logger.error('[schedules] erasure sweep failed:', err);+ }+ }++ function schedule(): void {+ if (stopped) {+ return;+ }+ // Jittered so replicas of a clustered deployment do not sweep in lockstep.+ const delay = SWEEP_MS + Math.floor(Math.random() * SWEEP_JITTER_MS);+ timer = setTimeout(() => {+ void sweep().finally(schedule);+ }, delay);+ timer.unref?.();+ }++ schedule();++ const engineSweep: ScheduleErasureSweep = {+ stop: () => {+ stopped = true;+ if (timer) {+ clearTimeout(timer);+ timer = undefined;+ }+ },+ };++ registerShutdownTask('schedule erasure sweep', () => engineSweep.stop(), { phase: 'pre-drain' });++ return engineSweep;+}diff --git a/packages/api/src/schedules/fire.spec.ts b/packages/api/src/schedules/fire.spec.tsnew file mode 100644index 0000000000..8284546901--- /dev/null+++ b/packages/api/src/schedules/fire.spec.ts@@ -0,0 +1,802 @@+import type { ScheduleEngineDeps, ScheduleLimits, ScheduleUserContext } from './types';+import type { FireableSchedule } from './types';+import { AgentTriggerServiceUnavailableError } from '../agents/triggers/service';+import { getAgentTriggerIdempotencyKey } from '../agents/triggers/envelope';+import { AgentTriggerDeliveryError } from '../agents/triggers/delivery';+import { buildFireClientRequestId, fireSchedule } from './fire';+import { withCapacitySlot } from './capacity';++const OWNER: ScheduleUserContext = { id: 'user-1', tenantId: 't1', role: 'USER' };+const LIMITS: ScheduleLimits = {+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+};++function makeSchedule(overrides: Partial<FireableSchedule> = {}): FireableSchedule {+ return {+ id: 'sched-1',+ user: 'user-1' as never,+ tenantId: 't1',+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ claimToken: 'ct-1',+ leaseBy: 'inst-1',+ runCount: 0,+ failureCount: 0,+ balanceSkipCount: 0,+ ...overrides,+ } as FireableSchedule;+}++/** In-memory run store exercising the real insert/count/delete/idempotency interplay. */+function makeMethods() {+ const runs = new Map<+ string,+ { status: string; conversationId?: string; capacitySlot?: number }+ >();+ const calls = {+ advance: 0,+ releaseLease: 0,+ disable: [] as string[],+ recordOutcome: [] as { status: string }[],+ skipped: [] as string[],+ setFireDetails: 0,+ };+ const key = (id: string, when: Date) => `${id}:${when.toISOString()}`;+ const methods = {+ releaseLease: jest.fn(async () => {+ calls.releaseLease += 1;+ return true;+ }),+ advanceSchedule: jest.fn(async () => {+ calls.advance += 1;+ return true;+ }),+ disableSchedule: jest.fn(async (_id: string, reason: string) => {+ calls.disable.push(reason);+ }),+ hasActiveRun: jest.fn(async (id: string) =>+ [...runs.entries()].some(([k, r]) => k.startsWith(`${id}:`) && r.status === 'started'),+ ),+ countActiveRuns: jest.fn(+ async () => [...runs.values()].filter((r) => r.status === 'started').length,+ ),+ insertScheduleRun: jest.fn(+ async (data: { scheduleId: string; scheduledFor: Date; conversationId?: string }) => {+ const k = key(data.scheduleId, data.scheduledFor);+ if (runs.has(k)) {+ return null; // unique {scheduleId, scheduledFor}+ }+ runs.set(k, { status: 'started', conversationId: data.conversationId });+ return { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor };+ },+ ),+ // Mirrors the partial-unique-index semantics: same-occurrence row => 'duplicate';+ // any OTHER started run for the schedule => 'overlap'; else reserve the slot.+ reserveStartedRun: jest.fn(+ async (data: {+ scheduleId: string;+ scheduledFor: Date;+ conversationId?: string;+ capacitySlot?: number;+ deliveryKey?: string;+ }) => {+ const k = key(data.scheduleId, data.scheduledFor);+ if (runs.has(k)) {+ // Mirrors the real method: a duplicate reports the EXISTING row's status so+ // the caller can tell "still running" from "already finished".+ return { conflict: 'duplicate' as const, existingStatus: runs.get(k)!.status };+ }+ // Mirrors the unique {capacitySlot} partial index (status:'started').+ if (+ data.capacitySlot != null &&+ [...runs.values()].some(+ (r) => r.status === 'started' && r.capacitySlot === data.capacitySlot,+ )+ ) {+ return { conflict: 'slot-taken' as const };+ }+ const overlap = [...runs.entries()].some(+ ([rk, r]) => rk.startsWith(`${data.scheduleId}:`) && r.status === 'started',+ );+ if (overlap) {+ return { conflict: 'overlap' as const };+ }+ runs.set(k, {+ status: 'started',+ conversationId: data.conversationId,+ capacitySlot: data.capacitySlot,+ });+ return { run: { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor } };+ },+ ),+ getCapacityOccupancy: jest.fn(async () => {+ const takenSlots: number[] = [];+ let unslotted = 0;+ for (const r of runs.values()) {+ if (r.status !== 'started') {+ continue;+ }+ if (typeof r.capacitySlot === 'number') {+ takenSlots.push(r.capacitySlot);+ } else {+ unslotted += 1;+ }+ }+ return { takenSlots, unslotted };+ }),+ revalidateClaim: jest.fn(async () => true),+ releaseLeaseByHolder: jest.fn(async () => undefined),+ deleteScheduleRun: jest.fn(+ async (id: string, when: Date, _status?: string, expectedConversationId?: string) => {+ const k = key(id, when);+ const row = runs.get(k) as { conversationId?: string } | undefined;+ // Mirrors the conversationId fence: a fire only deletes the row IT inserted.+ if (expectedConversationId != null && row?.conversationId !== expectedConversationId) {+ return;+ }+ runs.delete(k);+ },+ ),+ setRunFireDetails: jest.fn(async () => {+ calls.setFireDetails += 1;+ }),+ recordSkippedRun: jest.fn(async (data: { status: string }) => {+ calls.skipped.push(data.status);+ }),+ recordRunOutcome: jest.fn(+ async (data: { scheduleId: string; scheduledFor: Date; status: string }) => {+ const k = key(data.scheduleId, data.scheduledFor);+ if (runs.has(k)) {+ runs.set(k, { status: data.status });+ }+ calls.recordOutcome.push({ status: data.status });+ },+ ),+ };+ return { methods, runs, calls };+}++function makeDeps(+ methods: ReturnType<typeof makeMethods>['methods'],+ over: Partial<ScheduleEngineDeps> = {},+): ScheduleEngineDeps {+ return {+ methods: methods as unknown as ScheduleEngineDeps['methods'],+ getLimits: async () => LIMITS,+ getUserContext: async () => OWNER,+ isOutOfBalance: async () => false,+ agentAccess: async () => 'ok',+ hasScheduleAccess: async () => true,+ resolveFiles: async () => [],+ enqueueTrigger: jest.fn(async () => undefined),+ runInTenantContext: (_user, fn) => fn(),+ getJobStatus: async () => null,+ abortScheduledJob: async () => undefined,+ clearReconciledJob: async () => undefined,+ isOwnerDeleting: async () => false,+ isGloballyDisabled: async () => false,+ countActiveRunsGlobal: async () => methods.countActiveRuns(),+ withGlobalCapacitySlot: (cap: number, claim: (slot: number) => Promise<unknown>) =>+ withCapacitySlot(+ cap,+ () => methods.getCapacityOccupancy(),+ claim as Parameters<typeof withCapacitySlot>[2],+ ),+ ...over,+ } as ScheduleEngineDeps;+}++function mockFetch(impl: () => Promise<unknown> | never) {+ global.fetch = jest.fn(impl as never) as never;+}++const okResponse = (conversationId = 'convo-1') =>+ ({+ ok: true,+ status: 200,+ // The accept path answers with JSON; fire reads the body as text and JSON-parses it.+ text: async () => JSON.stringify({ conversationId }),+ }) as Response;++const dueAt = () => new Date(Date.now() - 60_000);++afterEach(() => jest.restoreAllMocks());++/**+ * The chat route validates `clientRequestId` against `/^[A-Za-z0-9:_-]{1,128}$/`+ * (CLIENT_REQUEST_ID_PATTERN in api/server/controllers/agents/request.js). An id+ * outside that charset makes the route answer 400 INVALID_CLIENT_REQUEST_ID, which+ * fails EVERY fire — so the encoding is a contract, not a formatting choice.+ */+describe('buildFireClientRequestId', () => {+ const ROUTE_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;++ it('stays within the charset the chat route accepts', () => {+ const id = buildFireClientRequestId(+ 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',+ new Date('2026-08-01T14:05:55.132Z'),+ );++ // A raw ISO instant carries a '.' in its milliseconds, which the route rejects.+ expect(id).not.toContain('.');+ expect(id).toMatch(ROUTE_PATTERN);+ });++ it('is deterministic per occurrence and distinct across occurrences', () => {+ const scheduleId = 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b';+ const first = new Date('2026-08-01T14:05:55.132Z');+ const second = new Date('2026-08-01T15:05:55.132Z');++ expect(buildFireClientRequestId(scheduleId, first)).toBe(+ buildFireClientRequestId(scheduleId, new Date(first.getTime())),+ );+ expect(buildFireClientRequestId(scheduleId, first)).not.toBe(+ buildFireClientRequestId(scheduleId, second),+ );+ });++ it('stays inside the 128-character cap', () => {+ expect(+ buildFireClientRequestId(+ 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',+ new Date('2026-08-01T14:05:55.132Z'),+ ).length,+ ).toBeLessThanOrEqual(128);+ });+});++describe('fireSchedule', () => {+ it('fires the happy path and records fire details', async () => {+ const { methods, runs } = makeMethods();+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.fired).toBe(true);+ // The conversation id is pre-generated and recorded on the run row up front+ // (so reconciliation can always find the occurrence's job), not read back from+ // the POST response.+ expect(result.conversationId).toMatch(/^[0-9a-f-]{36}$/);+ expect([...runs.values()][0].conversationId).toBe(result.conversationId);+ expect(methods.setRunFireDetails).toHaveBeenCalledTimes(1);+ expect([...runs.values()][0].status).toBe('started');+ });++ it('releases the old holder when a post-enqueue owner edit fences the advance', async () => {+ const { methods } = makeMethods();+ const enqueueTrigger = jest.fn(async () => undefined);+ methods.advanceSchedule.mockResolvedValueOnce(false);++ const result = await fireSchedule(+ makeDeps(methods, { enqueueTrigger }),+ makeSchedule({ leaseBy: 'claim-holder-1' }),+ LIMITS,+ dueAt(),+ );++ expect(result.fired).toBe(true);+ expect(enqueueTrigger).toHaveBeenCalledTimes(1);+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'claim-holder-1');+ });++ it('stamps the deterministic deliveryKey on the reservation (== the enqueued key)', async () => {+ const { methods } = makeMethods();+ const enqueueTrigger = jest.fn<+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,+ Parameters<ScheduleEngineDeps['enqueueTrigger']>+ >(async () => undefined);++ const result = await fireSchedule(+ makeDeps(methods, { enqueueTrigger }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );++ expect(result.fired).toBe(true);+ const reservation = (methods.reserveStartedRun as jest.Mock).mock.calls[0][0];+ const envelope = enqueueTrigger.mock.calls[0][0];+ // Stored BEFORE enqueue and equal to the enqueued delivery's idempotency key, so+ // reconciliation can read the durable delivery for THIS occurrence even if the enqueue+ // committed ambiguously.+ expect(reservation.deliveryKey).toEqual(expect.any(String));+ expect(reservation.deliveryKey).toBe(getAgentTriggerIdempotencyKey(envelope));+ });++ it('marks a Run Now trigger as manual for downstream limiter policy', async () => {+ const { methods } = makeMethods();+ const enqueueTrigger = jest.fn<+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,+ Parameters<ScheduleEngineDeps['enqueueTrigger']>+ >(async () => undefined);+ await fireSchedule(makeDeps(methods, { enqueueTrigger }), makeSchedule(), LIMITS, dueAt(), {+ manual: true,+ });+ const envelope = enqueueTrigger.mock.calls[0][0];+ expect(envelope).toMatchObject({+ mode: 'fire',+ principal: { userId: 'user-1', role: 'USER', tenantId: 't1' },+ event: {+ type: 'schedule.occurrence',+ source: { id: 'sched-1', type: 'schedule' },+ },+ run: { metadata: { manual: true } },+ });+ expect(enqueueTrigger).toHaveBeenCalledWith(envelope, { orderingKey: 'sched-1' });+ });++ it('marks an automatic occurrence as non-manual', async () => {+ const { methods } = makeMethods();+ const enqueueTrigger = jest.fn<+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,+ Parameters<ScheduleEngineDeps['enqueueTrigger']>+ >(async () => undefined);+ await fireSchedule(makeDeps(methods, { enqueueTrigger }), makeSchedule(), LIMITS, dueAt());++ expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({+ run: { metadata: { manual: false } },+ });+ });++ it('carries the claimed config revision and fire inputs on the durable trigger', async () => {+ const { methods } = makeMethods();+ const enqueueTrigger = jest.fn<+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,+ Parameters<ScheduleEngineDeps['enqueueTrigger']>+ >(async () => undefined);+ await fireSchedule(+ makeDeps(methods, {+ enqueueTrigger,+ resolveFiles: async () => [{ file_id: 'file-1' }],+ }),+ makeSchedule({ configRevision: 7, file_ids: ['file-1'] } as never),+ LIMITS,+ dueAt(),+ );+ expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({+ input: 'Summarize',+ target: { agentId: 'agent-1' },+ run: {+ timezone: 'America/New_York',+ files: [{ file_id: 'file-1' }],+ metadata: { manual: false, configRevision: 7 },+ },+ });+ });++ it('records a definite trigger admission rejection as an error', async () => {+ const { methods, calls } = makeMethods();+ const result = await fireSchedule(+ makeDeps(methods, {+ enqueueTrigger: async () => {+ throw new AgentTriggerServiceUnavailableError('not initialized');+ },+ }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );+ expect(result.fired).toBe(false);+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);+ });++ it('leaves an ambiguous persistence failure reconcilable', async () => {+ const { methods, runs, calls } = makeMethods();+ const result = await fireSchedule(+ makeDeps(methods, {+ enqueueTrigger: async () => {+ throw new Error('Mongo result unknown');+ },+ }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );+ expect(result.fired).toBe(false);+ expect(calls.recordOutcome).toHaveLength(0);+ expect([...runs.values()][0].status).toBe('started');+ expect(calls.advance).toBe(1);+ });++ it('records an invalid durable delivery as a definite error', async () => {+ const { methods, runs, calls } = makeMethods();+ const result = await fireSchedule(+ makeDeps(methods, {+ enqueueTrigger: async () => {+ throw new AgentTriggerDeliveryError('invalid delivery');+ },+ }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );+ expect(result.fired).toBe(false);+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);+ expect([...runs.values()][0].status).toBe('error');+ });++ it('does not orphan a run when file resolution fails', async () => {+ const { methods, runs, calls } = makeMethods();+ mockFetch(async () => okResponse());+ const deps = makeDeps(methods, {+ resolveFiles: async () => {+ throw new Error('db down');+ },+ });+ const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt());+ expect(result.fired).toBe(false);+ expect(runs.size).toBe(0); // no run row created+ // Automatic fire KEEPS the claim lease as a backoff (nextRunAt untouched → the+ // occurrence retries when the lease expires) so a transient file error can't get+ // this row re-claimed every tick and starve others.+ expect(calls.releaseLease).toBe(0);+ expect(global.fetch).not.toHaveBeenCalled();+ });++ it('releases the lease on a manual run-now when file resolution fails', async () => {+ const { methods, runs } = makeMethods();+ // An owner edit can rotate the token while the file lookup is in flight while+ // deliberately preserving this fire's unique lease holder.+ methods.releaseLease.mockResolvedValueOnce(false);+ mockFetch(async () => okResponse());+ const deps = makeDeps(methods, {+ resolveFiles: async () => {+ throw new Error('db down');+ },+ });+ const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt(), {+ manual: true,+ });+ expect(result.fired).toBe(false);+ expect(runs.size).toBe(0);+ // Run-now releases so the user can retry immediately (no misleading lease-held 409).+ expect(methods.releaseLease).toHaveBeenCalledWith('sched-1', 'ct-1');+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');+ expect(global.fetch).not.toHaveBeenCalled();+ });++ it('refuses the fire at the global capacity cap WITHOUT inserting a run', async () => {+ const { methods, runs } = makeMethods();+ // All 5 slots taken → the allocator finds no free slot and never inserts.+ for (let i = 0; i < 5; i++) {+ runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });+ }+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('capacity');+ expect(global.fetch).not.toHaveBeenCalled();+ // Slot-based capacity is decided BEFORE the write, so there is nothing to roll back.+ expect([...runs.values()].filter((r) => r.status === 'started')).toHaveLength(5);+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();+ expect(methods.deleteScheduleRun).not.toHaveBeenCalled();+ });++ it('does not let a principal override widen the global capacity cap', async () => {+ const { methods, runs } = makeMethods();+ // The single deployment-wide slot is already occupied.+ runs.set('other-0:x', { status: 'started', capacitySlot: 0 });+ mockFetch(async () => okResponse());+ const deps = makeDeps(methods, {+ // Base allows 1 concurrent scheduled generation; this owner's role/tenant+ // override raises their limit to 5.+ getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 5 : 1 }),+ });+ // Run Now resolves the OWNER's limits and skips the tick's base-config budget, so+ // without the clamp the override would hand out slots 1-4 and run five billed+ // generations against a deployment that advertises a cap of one.+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('capacity');+ expect(global.fetch).not.toHaveBeenCalled();+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();+ });++ it('still honors an owner override that is STRICTER than the deployment cap', async () => {+ const { methods, runs } = makeMethods();+ runs.set('other-0:x', { status: 'started', capacitySlot: 0 });+ mockFetch(async () => okResponse());+ const deps = makeDeps(methods, {+ getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 1 : 5 }),+ });+ // Only WIDENING is the defect; a tenant that wants less concurrency than the+ // deployment allows must still get it.+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('capacity');+ });++ it('claims a free slot and never exceeds the cap when slots collide', async () => {+ const { methods, runs } = makeMethods();+ // Slots 0 and 2 are taken; the allocator must land the fire on slot 1.+ runs.set('other-a:x', { status: 'started', capacitySlot: 0 });+ runs.set('other-b:x', { status: 'started', capacitySlot: 2 });+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.fired).toBe(true);+ const own = [...runs.entries()].find(([k]) => k.startsWith('sched-1:'));+ expect(own?.[1].capacitySlot).toBe(1);+ });++ it('re-fires cleanly next tick once capacity frees, exactly once', async () => {+ const { methods, runs } = makeMethods();+ for (let i = 0; i < 5; i++) {+ runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });+ }+ const enqueueTrigger = jest.fn(async () => undefined);+ const deps = makeDeps(methods, { enqueueTrigger });+ const schedule = makeSchedule();+ const when = dueAt();+ // Tick 1: every slot taken → refused before any insert.+ const first = await fireSchedule(deps, schedule, LIMITS, when);+ expect(first.skipped).toBe('capacity');+ // Capacity frees up before the next tick.+ runs.delete('other-0:x');+ // Tick 2: same occurrence re-claimed → now fires, exactly one live run.+ const second = await fireSchedule(deps, schedule, LIMITS, when);+ expect(second.fired).toBe(true);+ expect(methods.reserveStartedRun).toHaveBeenCalledTimes(1); // only the successful tick inserts+ expect(+ [...runs.entries()].filter(([k, r]) => k.startsWith('sched-1:') && r.status === 'started'),+ ).toHaveLength(1);+ expect(enqueueTrigger).toHaveBeenCalledTimes(1);+ });++ it('does not reserve a run when the claim already lapsed during preflight', async () => {+ const { methods, runs } = makeMethods();+ // The preflight (user/config/permission/balance/attachment queries) outlasted the+ // 5-minute lease and another worker re-claimed the occurrence.+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('superseded');+ // Reserving here would win the occurrence's unique row: the FRESH claimer would then+ // see `duplicate` and advance without firing, while this worker's own revalidation+ // fails and rollbackReservation deliberately retains the row (leaseBy changed). The+ // occurrence would be lost with its capacity slot held until the orphan sweep.+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);+ expect(global.fetch).not.toHaveBeenCalled();+ // The lease is handed back by holder so the fresh claimer is not left waiting.+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');+ });++ /**+ * A failed revalidation must NEVER advance. In every true supersession (takeover,+ * owner edit) the claim token rotated, so an advance would no-op anyway — the only+ * case where it can land is a PURE lease expiry with no takeover, where the token+ * never rotated. advanceSchedule checks no lease, so that advance moved nextRunAt+ * past an occurrence nothing had fired: a slow preflight silently DROPPED it+ * instead of leaving it due for the next claim to retry.+ */+ it('leaves the occurrence due when only the lease expired (no takeover)', async () => {+ const { methods } = makeMethods();+ // Same claim token, merely past leaseUntil: revalidateClaim fails on the lease+ // predicate alone while the token-fenced advance filter would still MATCH.+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('superseded');+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ expect(global.fetch).not.toHaveBeenCalled();+ });++ it('never advances from any superseded branch, including post-reserve', async () => {+ const { methods } = makeMethods();+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('superseded');+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ });++ /**+ * The lease can expire between the pre-reserve revalidation and the reservation+ * itself: the capacity allocator reads occupancy in between, and the preflight+ * before it can already have outlasted the 5-minute lease. Gating the rollback on+ * lease ownership stranded the row — the new holder's own reserve saw `duplicate`+ * and advanced past the occurrence without firing it, while this row held a global+ * capacity slot until the 30-minute orphan sweep.+ */+ it('deletes its own undispatched reservation even after a lease takeover', async () => {+ const { methods, runs } = makeMethods();+ // Valid at the pre-reserve check, superseded by the pre-POST one: that ordering IS+ // the scenario, since a claim already dead before reserving never reserves at all.+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('superseded');+ // Nothing was dispatched for this row, so it is unambiguously this fire's garbage.+ expect(methods.deleteScheduleRun).toHaveBeenCalledWith(+ 'sched-1',+ expect.any(Date),+ 'started',+ expect.any(String),+ );+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);+ expect(global.fetch).not.toHaveBeenCalled();+ });++ it('rechecks shutdown immediately before dispatch and rolls back the reservation', async () => {+ const { methods, runs } = makeMethods();+ // The coordinator flips AFTER the pre-reservation gate: the deployment-limit+ // read, capacity allocation, and claim revalidation all run in between and can+ // overlap SIGTERM. Without the recheck the POST lands on a closing listener —+ // booked as an error (or a refused connect that advances the occurrence).+ const isShuttingDown = jest.fn().mockReturnValueOnce(false).mockReturnValue(true);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(+ makeDeps(methods, { isShuttingDown }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );+ expect(result.skipped).toBe('superseded');+ expect(global.fetch).not.toHaveBeenCalled();+ // Rolled back, not advanced: the occurrence stays due for the restarted process.+ expect(methods.advanceSchedule).not.toHaveBeenCalled();+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);+ });++ it('deletes the reserved run when the schedule was hard-deleted mid-fire', async () => {+ const { methods, runs } = makeMethods();+ // Account deletion hard-deleted the schedule after this fire reserved its run.+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('superseded');+ // The orphaned reservation (no schedule left to own it) is deleted, not leaked.+ expect(methods.deleteScheduleRun).toHaveBeenCalledWith(+ 'sched-1',+ expect.any(Date),+ 'started',+ expect.any(String),+ );+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);+ });++ it('records an unavailable trigger service as a definite error', async () => {+ const { methods, runs, calls } = makeMethods();+ const result = await fireSchedule(+ makeDeps(methods, {+ enqueueTrigger: async () => {+ throw new AgentTriggerServiceUnavailableError('not ready');+ },+ }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );+ expect(result.fired).toBe(false);+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);+ expect([...runs.values()][0].status).toBe('error');+ });++ it('skips overlap when a prior run is still active', async () => {+ const { methods, runs, calls } = makeMethods();+ runs.set('sched-1:prior', { status: 'started' });+ mockFetch(async () => okResponse());+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('overlap');+ expect(calls.skipped).toEqual(['skipped_overlap']);+ expect(global.fetch).not.toHaveBeenCalled();+ });++ it('disables and stops firing when agent VIEW access was revoked', async () => {+ const { methods, calls } = makeMethods();+ const deps = makeDeps(methods, { agentAccess: async () => 'forbidden' });+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());+ expect(result.skipped).toBe('permission_revoked');+ expect(calls.disable).toEqual(['permission_revoked']);+ });++ it('skips a duplicate occurrence (idempotency claim already held)', async () => {+ const { methods, runs } = makeMethods();+ runs.set(`sched-1:${dueAt().toISOString()}`, { status: 'requires_action' });+ mockFetch(async () => okResponse());+ const when = new Date(dueAt().getTime());+ runs.set(`sched-1:${when.toISOString()}`, { status: 'requires_action' });+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);+ expect(result.skipped).toBe('duplicate');+ expect(global.fetch).not.toHaveBeenCalled();+ });++ /**+ * A balance skip is not a no-op: it stamps the card and walks the balance-skip streak+ * toward auto-disable. The preflight above it (user, config, permission and balance+ * lookups) can outlast the 5-minute lease, so writing it under a dead claim is a write+ * on behalf of a fire that no longer owns the occurrence.+ */+ it('does not record a balance skip under a superseded claim', async () => {+ const { methods, calls } = makeMethods();+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);+ mockFetch(async () => okResponse());++ const result = await fireSchedule(+ makeDeps(methods, { isOutOfBalance: async () => true }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );++ expect(result.skipped).toBe('superseded');+ expect(methods.recordSkippedRun).not.toHaveBeenCalled();+ expect(calls.skipped).toEqual([]);+ });++ it('records the balance skip normally while the claim is still valid', async () => {+ const { methods, calls } = makeMethods();+ mockFetch(async () => okResponse());++ const result = await fireSchedule(+ makeDeps(methods, { isOutOfBalance: async () => true }),+ makeSchedule(),+ LIMITS,+ dueAt(),+ );++ expect(result.skipped).toBe('balance');+ expect(calls.skipped).toEqual(['skipped_balance']);+ });++ /**+ * A `duplicate` means ANOTHER worker holds this occurrence's row — not that the+ * occurrence is done. Advancing past it hands the occurrence away: if that other+ * worker is a stale lease holder whose own revalidation then fails, it rolls its+ * undispatched row back and NOTHING ever fires the occurrence. Leaving nextRunAt+ * alone keeps it claimable; the worker that actually dispatches is the one that+ * advances, and the claim's lease provides the retry backoff.+ */+ /**+ * A settled-but-unadvanced occurrence (its fire was accepted, the post-accept advance+ * failed) leaves nextRunAt pointing at it. Refusing to advance on `duplicate` then+ * makes every future claim re-pick the same finished occurrence — a permanent stall.+ */+ it('advances past a duplicate whose run already settled', async () => {+ const { methods, runs, calls } = makeMethods();+ const when = new Date(dueAt().getTime());+ runs.set(`sched-1:${when.toISOString()}`, { status: 'success', conversationId: 'done' });+ mockFetch(async () => okResponse());++ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);++ expect(result.skipped).toBe('duplicate');+ expect(global.fetch).not.toHaveBeenCalled();+ expect(calls.advance).toBe(1);+ });++ it('does not advance past an occurrence another worker is holding', async () => {+ const { methods, runs, calls } = makeMethods();+ const when = new Date(dueAt().getTime());+ // A peer already reserved this occurrence's row.+ runs.set(`sched-1:${when.toISOString()}`, {+ status: 'started',+ conversationId: 'peer-convo',+ });+ mockFetch(async () => okResponse());++ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);++ expect(result.skipped).toBe('duplicate');+ expect(global.fetch).not.toHaveBeenCalled();+ expect(calls.advance).toBe(0);+ });++ it('releases the lease on a duplicate for run-now so the user can retry', async () => {+ const { methods, runs, calls } = makeMethods();+ const when = new Date(dueAt().getTime());+ runs.set(`sched-1:${when.toISOString()}`, { status: 'started' });+ mockFetch(async () => okResponse());++ await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when, { manual: true });++ // Automatic claims keep the lease as backoff; a manual click must not be told+ // "already in progress" for the full lease TTL.+ expect(calls.releaseLease).toBe(1);+ expect(calls.advance).toBe(0);+ });+});diff --git a/packages/api/src/schedules/fire.ts b/packages/api/src/schedules/fire.tsnew file mode 100644index 0000000000..de23f83455--- /dev/null+++ b/packages/api/src/schedules/fire.ts@@ -0,0 +1,604 @@+import { randomUUID } from 'node:crypto';+import { logger } from '@librechat/data-schemas';+import type { ScheduleEngineDeps, ScheduleLimits, FireResult, FireableSchedule } from './types';+import type { AgentTriggerEnvelope } from '../agents/triggers/envelope';+import type { JsonValue } from '../agents/json';+import {+ AgentTriggerEnvelopeError,+ createAgentTriggerEnvelope,+ getAgentTriggerIdempotencyKey,+} from '../agents/triggers/envelope';+import { AgentTriggerServiceUnavailableError } from '../agents/triggers/service';+import { AgentTriggerDeliveryError } from '../agents/triggers/delivery';+import { computeNextRunAt, cadenceIntervalMinutes } from './cadence';++/** Consecutive balance skips (pre-fire or mid-generation) before auto-disable. */+export const BALANCE_SKIP_DISABLE_THRESHOLD: number = 5;++/**+ * Stable source delivery identity for one scheduled occurrence. The trigger+ * service hashes this into the generation-compatible idempotency key.+ */+export function buildFireClientRequestId(scheduleId: string, scheduledFor: Date): string {+ return `sched:${scheduleId}:${scheduledFor.toISOString().replace('.', '-')}`;+}++/**+ * `ambiguous` means Mongo may have committed the idempotent trigger delivery even+ * though enqueue returned an error. Those must remain reconcilable rather than be+ * terminalized as a definite failure.+ */+class ScheduleFireError extends Error {+ constructor(+ message: string,+ readonly ambiguous: boolean,+ ) {+ super(message);+ }+}++/**+ * Builds the durable trigger envelope for one occurrence. Pure and deterministic in the+ * fields that {@link getAgentTriggerIdempotencyKey} hashes (only `requestId`/`receivedAt`+ * vary), so the `deliveryKey` derived from it is stable across retries and is stored on+ * the reservation BEFORE enqueue — that lets schedule reconciliation read the durable+ * delivery state even for an ambiguously-committed enqueue.+ */+function buildScheduleTriggerEnvelope(+ schedule: FireableSchedule,+ user: NonNullable<Awaited<ReturnType<ScheduleEngineDeps['getUserContext']>>>,+ scheduledFor: Date,+ files: Awaited<ReturnType<ScheduleEngineDeps['resolveFiles']>>,+ conversationId: string,+ manual: boolean,+): AgentTriggerEnvelope {+ const occurrenceId = buildFireClientRequestId(schedule.id, scheduledFor);+ const triggerFiles: JsonValue[] = files.map((file) => {+ const value: { [key: string]: JsonValue } = { file_id: file.file_id };+ if (file.filepath != null) value.filepath = file.filepath;+ if (file.filename != null) value.filename = file.filename;+ if (file.type != null) value.type = file.type;+ if (file.height != null) value.height = file.height;+ if (file.width != null) value.width = file.width;+ if (file.source != null) value.source = file.source;+ return value;+ });+ return createAgentTriggerEnvelope({+ mode: 'fire',+ requestId: randomUUID(),+ deliveryId: occurrenceId,+ receivedAt: Date.now(),+ principal: {+ id: user.id,+ ...(user.role != null && { role: user.role }),+ ...(user.tenantId != null && { tenantId: user.tenantId }),+ },+ target: { agentId: schedule.agent_id },+ event: {+ id: occurrenceId,+ type: 'schedule.occurrence',+ occurredAt: scheduledFor.getTime(),+ source: { id: schedule.id, type: 'schedule' },+ },+ input: schedule.prompt,+ run: {+ conversationId,+ timezone: schedule.timezone,+ ...(triggerFiles.length > 0 && { files: triggerFiles }),+ metadata: {+ manual,+ ...(typeof schedule.configRevision === 'number' && {+ configRevision: schedule.configRevision,+ }),+ },+ },+ });+}++async function enqueueScheduleTrigger(+ deps: ScheduleEngineDeps,+ envelope: AgentTriggerEnvelope,+ orderingKey: string,+): Promise<void> {+ try {+ await deps.enqueueTrigger(envelope, { orderingKey });+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ const definite =+ error instanceof AgentTriggerEnvelopeError ||+ error instanceof AgentTriggerDeliveryError ||+ error instanceof AgentTriggerServiceUnavailableError;+ // A storage error can be returned after Mongo committed the idempotent delivery.+ // Leave that occurrence reconcilable; a later worker may already own it.+ throw new ScheduleFireError(`Trigger enqueue failed: ${message}`, !definite);+ }+}++/**+ * Fires one claimed occurrence. The caller owns the lease; this function owns+ * the run-doc idempotency insert, the skip checks, durable trigger enqueue, and+ * advancing `nextRunAt` (every path advances so a schedule can never wedge).+ */+export async function fireSchedule(+ deps: ScheduleEngineDeps,+ schedule: FireableSchedule,+ limits: ScheduleLimits,+ scheduledFor: Date,+ options?: { manual?: boolean; dbNow?: Date },+): Promise<FireResult> {+ const { methods } = deps;+ // Compute the NEXT occurrence relative to the CLAIM's clock (the engine passes+ // the claim time derived from leaseUntil), keeping the advance self-consistent+ // with the claim that selected this occurrence. Falls back to the process clock+ // when no claim time is provided (e.g. manual run-now, which never reschedules+ // and so ignores the result anyway). Inter-replica skew shifts WHEN an occurrence+ // fires by at most the skew; it can never fire one twice (lease CAS + the unique+ // occurrence index) — see LEASE_SKEW_MARGIN_MS in data-schemas for the takeover+ // margin that protects the only cross-worker clock comparison.+ const now = options?.dbNow ?? new Date();+ const nextRunAt = computeNextRunAt({+ cadence: schedule.cadence,+ timezone: schedule.timezone,+ scheduleId: schedule.id,+ after: new Date(Math.max(now.getTime(), scheduledFor.getTime())),+ });+ // Every worker-side schedule write is fenced on the claim token so a stale+ // worker (lease expired + re-claimed, or the schedule edited/re-enabled/deleted+ // — all of which rotate the token) cannot clobber the newer authoritative state.+ const claimToken = schedule.claimToken;++ /**+ * Rolls back the `started` row this fire reserved. Fenced on the conversation id it+ * generated rather than on lease ownership: every call site here is a path where+ * NOTHING was dispatched, and the row can only carry this id if this fire inserted+ * it (the single-active partial index admits one). Lease state is the wrong fence —+ * the preflight and capacity allocator can outlast the 5-minute lease, and gating on+ * a takeover left the undispatched row (and its global capacity slot) stranded until+ * the 30-minute orphan sweep while the new holder saw `duplicate` and advanced past+ * the occurrence without firing it.+ */+ const rollbackReservation = (conversationId: string) =>+ methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', conversationId);++ /**+ * Steps aside from a superseded fire (owner edit/delete, or a lease-expiry re-claim).+ * `advance()` is fenced on the OLD claim token, which the edit rotated, so it no-ops+ * and would leave this worker's lease held until its TTL — reporting the edited+ * schedule / Run now as "already in progress" though no run was dispatched. Releasing+ * by HOLDER makes it immediately re-claimable, and correctly no-ops after a takeover+ * (leaseBy changed) so the new holder's lease is never stripped.+ */+ const releaseSupersededLease = async () => {+ if (schedule.leaseBy != null) {+ await methods.releaseLeaseByHolder(schedule.id, schedule.leaseBy);+ }+ };++ /**+ * Releases a manual run's lease without leaving its old holder behind when an owner+ * edit rotates the claim token during an asynchronous preflight. The token fence is+ * authoritative when it still matches; the unique holder fallback can only clear this+ * fire's lease and safely no-ops if another worker has taken it over.+ */+ const releaseManualLease = async () => {+ const released = await methods.releaseLease(schedule.id, claimToken);+ if (!released) {+ await releaseSupersededLease();+ }+ return released;+ };++ // A manual run-now must never reschedule the next automatic occurrence; it only+ // releases its serialization lease. Both writes are token-fenced, so an owner edit+ // racing after durable enqueue can make them miss while deliberately preserving the+ // old holder fields. Release by the unique OLD holder on a miss: an edit is unwedged+ // immediately, while a takeover changed leaseBy and remains untouched.+ const advance = async () => {+ if (options?.manual) {+ return releaseManualLease();+ }+ const advanced = await methods.advanceSchedule(+ schedule.id,+ nextRunAt,+ scheduledFor,+ claimToken,+ );+ if (!advanced) {+ await releaseSupersededLease();+ }+ return advanced;+ };++ /**+ * Steps aside after a failed revalidation WITHOUT advancing. `advance()` here is a+ * designed no-op in every true supersession (takeover, owner edit — both rotate the+ * claim token, so its token-fenced filter misses), which means the only case where it+ * actually LANDS is the one it must not: a PURE lease expiry with no takeover, where+ * the token never rotated. advanceSchedule checks no lease, so the advance moved+ * nextRunAt past an occurrence nothing had fired — a preflight outlasting the lease+ * silently dropped it instead of leaving it due for the next claim to retry.+ * Manual run-now still releases its serialization lease (release-only, no advance)+ * so a repeat click isn't met with a stale "already in progress".+ */+ const stepAsideSuperseded = async () => {+ if (options?.manual) {+ await releaseManualLease();+ } else {+ await releaseSupersededLease();+ }+ return { fired: false, skipped: 'superseded' as const };+ };++ if (nextRunAt == null) {+ await methods.disableSchedule(schedule.id, 'invalid_schedule', claimToken);+ await advance();+ return { fired: false, error: 'No next occurrence computable' };+ }++ const user = await deps.getUserContext(schedule.user);+ if (user == null) {+ await methods.disableSchedule(schedule.id, 'permission_revoked', claimToken);+ await advance();+ return { fired: false, skipped: 'user_missing' };+ }++ return deps.runInTenantContext(user, async () => {+ // Re-resolve limits for the OWNER (per-principal role/user + tenant config):+ // a tenant- or role-specific config (disabled schedules, different+ // auto-disable threshold) must win over the base config the engine read.+ const ownerLimits = await deps.getLimits(user);+ if (!ownerLimits.enabled) {+ await advance();+ return { fired: false, skipped: 'disabled' as const };+ }++ // Enforce a raised interval floor at fire time: create/update reject too-frequent+ // cadences, but an admin raising the floor later must also stop an already-enabled+ // schedule that now runs more often than policy allows.+ if (cadenceIntervalMinutes(schedule.cadence) < ownerLimits.minIntervalMinutes) {+ await methods.disableSchedule(schedule.id, 'invalid_schedule', claimToken);+ await advance();+ return { fired: false, skipped: 'disabled' as const };+ }++ // Account-deletion barrier, re-checked at the DISPATCH boundary. Admission (the+ // create/update/run-now handlers) is the primary gate, but there is always a window+ // between admission and persistence, so the owner is re-checked immediately before a+ // billed generation is dispatched. Skips silently: the deletion cascade owns the row.+ if (await deps.isOwnerDeleting(user.id)) {+ await advance();+ return { fired: false, skipped: 'user_deleting' as const };+ }++ // Re-check the owner's live schedule permission: a role that lost+ // SCHEDULES access after the schedule was created must stop firing.+ if (!(await deps.hasScheduleAccess(user))) {+ await methods.disableSchedule(schedule.id, 'permission_revoked', claimToken);+ await advance();+ return { fired: false, skipped: 'permission_revoked' as const };+ }++ // The owner-config generation this occurrence was CLAIMED under, stamped onto every+ // run row (fired or skipped) so bookkeeping can derive its fence from the row.+ const claimedRevision =+ typeof schedule.configRevision === 'number'+ ? { configRevision: schedule.configRevision }+ : {};++ const baseRun = {+ scheduleId: schedule.id,+ user: schedule.user,+ tenantId: schedule.tenantId,+ scheduledFor,+ };++ const agentAccess = await deps.agentAccess(schedule.agent_id, user);+ if (agentAccess !== 'ok') {+ // 'missing' → deleted; 'forbidden' → the owner's VIEW access was revoked.+ // Disable immediately instead of letting the loopback chat reject the run+ // and burn attempts toward the failure threshold.+ const reason = agentAccess === 'missing' ? 'agent_deleted' : 'permission_revoked';+ await methods.disableSchedule(schedule.id, reason, claimToken);+ await advance();+ return { fired: false, skipped: reason };+ }++ if (await deps.isOutOfBalance(user)) {+ // Revalidate BEFORE writing the skip. Everything above (user, config, permission+ // and balance lookups) can outlast the 5-minute lease, and a skip is not a no-op:+ // it stamps the card and walks the balance-skip streak toward auto-disable. Under+ // a dead claim — lease taken over, schedule deleted, account deletion begun — that+ // is a write on behalf of a fire that no longer owns this occurrence. The reserve+ // path below has its own revalidation for the same reason; this one guards the+ // branch that returns before ever reaching it.+ if (+ claimToken != null &&+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))+ ) {+ return stepAsideSuperseded();+ }+ // Skip rows carry no `bookkept:false` marker, so the reconciler has no path to+ // repair a half-applied skip. If the schedule-side bookkeeping throws, do NOT+ // advance: leave the occurrence due so the next claim retries it (the insert is+ // duplicate-guarded and the same-skip retry path is idempotent).+ try {+ await methods.recordSkippedRun(+ { ...baseRun, status: 'skipped_balance', ...claimedRevision },+ BALANCE_SKIP_DISABLE_THRESHOLD,+ );+ } catch (skipError) {+ logger.error(`[schedules] balance-skip bookkeeping failed for ${schedule.id}:`, skipError);+ return { fired: false, skipped: 'balance' as const };+ }+ await advance();+ return { fired: false, skipped: 'balance' as const };+ }++ // Resolve attachments BEFORE claiming the run row: a transient file-query+ // failure here must not orphan a `started` run that consumes capacity.+ const requestedFileIds = schedule.file_ids ?? [];+ let files: Awaited<ReturnType<ScheduleEngineDeps['resolveFiles']>>;+ try {+ files = requestedFileIds.length ? await deps.resolveFiles(requestedFileIds, user) : [];+ } catch (fileError) {+ logger.error(+ `[schedules] file resolution failed for ${schedule.id} (will retry):`,+ fileError,+ );+ // nextRunAt is untouched so the occurrence retries. Automatic claims keep the+ // lease as a backoff (releasing it would let the nextRunAt-sorted claimer+ // re-pick this failing row every tick and starve others / hammer the file+ // lookup); manual run-now releases so the user can retry immediately.+ if (options?.manual) {+ await releaseManualLease();+ }+ return { fired: false, error: 'File resolution failed' };+ }+ const droppedFileIds = requestedFileIds.filter(+ (id) => !files.some((file) => file.file_id === id),+ );++ // Revalidate BEFORE reserving, not only before the POST below. The preflight above+ // (user, config, permission, balance, attachment queries) can outlast the 5-minute+ // lease, and a stale worker that reserves anyway wins the occurrence's unique row:+ // the fresh claimer then sees `duplicate` and advances without firing, while this+ // worker's own revalidation fails and rollbackReservation deliberately RETAINS the+ // row (the lease takeover changed leaseBy). The occurrence is lost and its global+ // capacity slot stays held until the 30-minute orphan sweep. Nothing is reserved yet+ // here, so a superseded fire simply steps aside.+ if (+ claimToken != null &&+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))+ ) {+ return stepAsideSuperseded();+ }++ // SHUTDOWN gate at the dispatch boundary: the coordinator closes the listener+ // BEFORE the engine's pre-drain task runs, so a pass already past its preflight+ // would POST at a refusing socket — a definite connect failure that terminalizes+ // as `error` and walks a healthy schedule toward auto-disable for nothing more+ // than a restart. Nothing is reserved yet, so stepping aside leaves the+ // occurrence due for the restarted process (within the misfire grace).+ if (deps.isShuttingDown?.() === true) {+ logger.info(`[schedules] shutdown in progress; releasing claim on ${schedule.id}`);+ return stepAsideSuperseded();+ }++ // Pre-generate the conversation id and reserve the run row up front. The+ // loopback POST reuses it (streamId === conversationId), so reconciliation can+ // ALWAYS locate this occurrence's job — even if the post-accept detail write+ // fails — instead of mislabeling an accepted (or preserved) run as an orphan.+ // reserveStartedRun is the atomic overlap guard: the single-active partial index+ // rejects a second `started` run for the schedule, so a concurrent occurrence+ // surfaces as 'overlap' with no read-then-insert race.+ const conversationId = randomUUID();+ // Build the durable trigger envelope NOW so the deterministic deliveryKey can be+ // stored on the reservation below — before enqueue. That way schedule reconciliation+ // can read the delivery state (live vs dead) even when the enqueue commits ambiguously,+ // instead of orphaning a still-live delivery after the 30-minute cutoff. A schedule+ // whose envelope cannot be built can never fire, so skip WITHOUT reserving a phantom+ // `started` run (a definite, retry-invariant failure).+ let triggerEnvelope: AgentTriggerEnvelope;+ try {+ triggerEnvelope = buildScheduleTriggerEnvelope(+ schedule,+ user,+ scheduledFor,+ files,+ conversationId,+ options?.manual === true,+ );+ } catch (envelopeError) {+ logger.error(`[schedules] trigger envelope build failed for ${schedule.id}:`, envelopeError);+ if (options?.manual) {+ await releaseManualLease();+ } else {+ await advance();+ }+ return {+ fired: false,+ error: envelopeError instanceof Error ? envelopeError.message : String(envelopeError),+ };+ }+ const deliveryKey = getAgentTriggerIdempotencyKey(triggerEnvelope);+ // The GLOBAL fireConcurrency cap is enforced by claiming a unique capacity slot in+ // the SAME insert that reserves the run, so it is decided by the DB rather than by+ // a count read before the write. The allocator advances to the next free slot when+ // another admission wins one, and reports 'capacity' only when genuinely saturated.+ // Occupancy is read system-scoped so the cap stays global across tenants.+ // CLAMPED to the deployment-wide cap. The slots are global across every owner, so a+ // role/user/tenant override must never be able to WIDEN them: manual Run Now resolves+ // the owner's limits and bypasses the engine tick's base-config budget entirely, so+ // an override of 5 against a base of 1 would otherwise let concurrent clicks occupy+ // slots 0-4 and run five billed generations at once. Re-read without a principal for+ // the base value (the same read the tick budgets from); a STRICTER owner value still+ // applies, since only widening is the defect.+ const deploymentLimits = await deps.getLimits();+ const allocation = await deps.withGlobalCapacitySlot(+ Math.min(ownerLimits.fireConcurrency, deploymentLimits.fireConcurrency),+ async (capacitySlot) => {+ const attempt = await methods.reserveStartedRun({+ ...baseRun,+ conversationId,+ firedAt: new Date(),+ capacitySlot,+ deliveryKey,+ ...(typeof schedule.configRevision === 'number'+ ? { configRevision: schedule.configRevision }+ : {}),+ });+ return 'conflict' in attempt && attempt.conflict === 'slot-taken'+ ? 'slot-taken'+ : { claimed: attempt };+ },+ );+ if (allocation === 'capacity') {+ // Automatic claims keep the claim's lease as a backoff so the nextRunAt-sorted+ // claimer doesn't immediately re-pick this row and starve others; nextRunAt is+ // untouched, so the occurrence retries once the lease expires. A manual run-now+ // MUST release its lease, or repeated Run-now clicks hit a misleading "already+ // in progress" 409 for the full manual-lease TTL even after capacity frees.+ if (options?.manual) {+ await releaseManualLease();+ }+ return { fired: false, skipped: 'capacity' as const };+ }+ const reservation = allocation.claimed;+ if ('conflict' in reservation) {+ if (reservation.conflict === 'overlap') {+ // Another occurrence of this schedule is already active. Record the skip+ // (its own occurrence row) and advance past this one.+ try {+ await methods.recordSkippedRun({+ ...baseRun,+ status: 'skipped_overlap',+ ...claimedRevision,+ });+ } catch (skipError) {+ // Same reasoning as the balance skip: no reconciliation path repairs a+ // half-applied skip, so leave the occurrence due rather than advancing past it.+ logger.error(+ `[schedules] overlap-skip bookkeeping failed for ${schedule.id}:`,+ skipError,+ );+ return { fired: false, skipped: 'overlap' as const };+ }+ await advance();+ return { fired: false, skipped: 'overlap' as const };+ }+ // A duplicate means another fire already holds this occurrence's row — but+ // "still running it" and "already finished with it" need OPPOSITE handling.+ //+ // TERMINAL: the occurrence is done and merely never advanced past (its fire was+ // accepted but the post-accept advance failed). nextRunAt still points here, so+ // refusing to advance makes every future claim re-pick the same settled+ // occurrence — a permanent stall. Advance past it.+ //+ // ACTIVE: another worker owns it. Advancing hands the occurrence away — if that+ // worker is a stale lease holder whose own revalidation then fails, it rolls its+ // undispatched row back and nothing ever fires this occurrence. Leave nextRunAt+ // alone so it stays claimable; whichever worker actually dispatches is the one+ // that advances, and the claim's lease is the retry backoff (the same shape as+ // `capacity` above). A crashed holder's row is cleared by the orphan sweep, after+ // which the occurrence reserves cleanly.+ const settledAlready =+ reservation.existingStatus != null &&+ reservation.existingStatus !== 'started' &&+ reservation.existingStatus !== 'requires_action';+ if (settledAlready) {+ await advance();+ return { fired: false, skipped: 'duplicate' as const };+ }+ // Manual run-now still releases its lease so repeated clicks aren't met with a+ // stale "already in progress".+ if (options?.manual) {+ await releaseManualLease();+ }+ return { fired: false, skipped: 'duplicate' as const };+ }++ // Last check before the point of no return: re-verify this fire still holds an+ // authoritative claim (same claim token, lease unexpired, not deleting; and for+ // an automatic fire, still enabled). An owner delete/edit or a lease-expiry+ // re-claim landing AFTER the lease was taken but before here supersedes this+ // fire — roll the reservation back (status-fenced) and skip WITHOUT dispatching+ // a billed generation. Manual run-now still revalidates (a delete/edit can land+ // after acquireManualRunLease); it only relaxes the `enabled` check since the+ // user explicitly triggered it.+ if (+ claimToken != null &&+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))+ ) {+ await rollbackReservation(conversationId);+ return stepAsideSuperseded();+ }++ // SHUTDOWN recheck immediately before dispatch: the gate before the reservation+ // is not the last dispatch-boundary operation — the deployment-limit read,+ // capacity allocation, and claim revalidation all run after it and can overlap+ // SIGTERM, leaving the listener closing by the time the POST goes out. Roll the+ // reservation back (status-fenced) and step aside without advancing: the+ // occurrence stays due for the restarted process within the misfire grace.+ if (deps.isShuttingDown?.() === true) {+ logger.info(+ `[schedules] shutdown in progress; rolling back reserved dispatch of ${schedule.id}`,+ );+ await rollbackReservation(conversationId);+ return stepAsideSuperseded();+ }++ try {+ await enqueueScheduleTrigger(deps, triggerEnvelope, schedule.id);+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ const ambiguous = error instanceof ScheduleFireError && error.ambiguous;+ if (ambiguous) {+ // Mongo may have accepted the durable delivery, which can later start a+ // generation and call recordScheduleOutcome. Leave the run `started` (a+ // reconcilable, non-terminal state) so that completion can finalize it and+ // overlap/capacity keep seeing it; the orphan sweep settles it otherwise.+ // Do NOT terminalize here — `interrupted` would block the real outcome.+ logger.warn(+ `[schedules] trigger enqueue ambiguously failed for ${schedule.id} (left reconcilable):`,+ error,+ );+ await advance();+ return { fired: false, error: message };+ }+ // Definite enqueue rejection: no durable delivery exists and nothing started.+ logger.error(`[schedules] trigger enqueue rejected for ${schedule.id}:`, error);+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ error: message,+ autoDisableAfterFailures: ownerLimits.autoDisableAfterFailures,+ });+ await advance();+ return { fired: false, error: message };+ }++ // The trigger delivery is durable: the eventual chat reports its own terminal+ // outcome via the completion hook. Post-enqueue bookkeeping failures must NOT+ // flip the run to `error` (that would block the real completion, which only+ // matches started/paused).+ try {+ await advance();+ await methods.setRunFireDetails(schedule.id, scheduledFor, {+ conversationId,+ ...(droppedFileIds.length > 0 ? { droppedFileIds } : {}),+ });+ if (droppedFileIds.length > 0) {+ logger.warn(+ `[schedules] ${schedule.id} fired without ${droppedFileIds.length} missing attachment(s)`,+ );+ }+ } catch (bookkeepingError) {+ logger.error(+ `[schedules] post-enqueue bookkeeping failed for ${schedule.id} (run continues):`,+ bookkeepingError,+ );+ }+ return { fired: true, conversationId };+ });+}diff --git a/packages/api/src/schedules/gate.spec.ts b/packages/api/src/schedules/gate.spec.tsnew file mode 100644index 0000000000..6d2199707f--- /dev/null+++ b/packages/api/src/schedules/gate.spec.ts@@ -0,0 +1,162 @@+import type { SchedulesServiceDeps } from './service';+import { createSchedulesService } from './service';++/** Swappable per test: whether the stream store is SHARED across replicas. */+let mockIsRedis = false;++jest.mock('../stream/GenerationJobManager', () => ({+ GenerationJobManager: {+ getJobStore: () => null,+ abortJob: jest.fn(),+ get isRedis() {+ return mockIsRedis;+ },+ },+}));++type Cfg = { interfaceConfig?: { schedules?: unknown } };++function makeService(base: Cfg, merged: Cfg = base) {+ const deps = {+ methods: {+ countActiveRuns: jest.fn(async () => 0),+ getCapacityOccupancy: jest.fn(async () => ({ takenSlots: [], unslotted: 0 })),+ ensureScheduleIndexes: jest.fn(async () => undefined),+ acquireManualRunLease: jest.fn(async () => null),+ },+ getAppConfig: jest.fn(async (options?: { baseOnly?: boolean }) =>+ options?.baseOnly === true ? base : merged,+ ),+ findUserById: jest.fn(async () => null),+ findBalance: jest.fn(async () => null),+ upsertBalance: jest.fn(async () => null),+ initializeNullBalance: jest.fn(async () => null),+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),+ isUserDeleting: jest.fn(async () => false),+ enqueueAgentTrigger: jest.fn(async () => undefined),+ getTriggerDelivery: jest.fn(async () => null),+ } as unknown as SchedulesServiceDeps;+ return createSchedulesService(deps);+}++const schedule = { id: 's1', user: 'u1' } as never;+const limits = {+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+};++describe('v1 experimental gate, asserted at real entry points', () => {+ beforeEach(() => {+ // The harness mocks an in-memory job store, so every arming assertion below needs+ // the single-replica assertion the topology gate demands (see isTopologySafeToArm).+ process.env.SCHEDULES_SINGLE_PROCESS = 'true';+ });++ afterEach(() => {+ mockIsRedis = false;+ delete process.env.SCHEDULES_DISABLED;+ delete process.env.SCHEDULES_SINGLE_PROCESS;+ });++ it('REFUSES to arm on a process-local job store with no single-replica assertion', async () => {+ delete process.env.SCHEDULES_SINGLE_PROCESS;+ // The standard entrypoint arms the scheduler in EVERY replica. With a process-local+ // store a peer sees the globally visible `started` row but not its job, and after the+ // orphan cutoff marks a still-running generation interrupted and frees its capacity.+ // A process cannot count its own replicas, so unproven means refuse.+ const service = makeService({ interfaceConfig: { schedules: true } });+ expect(await service.initializeScheduleEngine()).toBeUndefined();+ });++ it('arms on a SHARED job store without any assertion, since replicas see each other', async () => {+ delete process.env.SCHEDULES_SINGLE_PROCESS;+ mockIsRedis = true;+ const service = makeService({ interfaceConfig: { schedules: true } });+ expect(await service.initializeScheduleEngine()).toBeDefined();+ });++ it('is OFF for limits when the admin never opted in', async () => {+ expect((await makeService({}).getLimits()).enabled).toBe(false);+ });++ it('STILL arms the engine when globally stopped, so reconciliation can settle prior state', async () => {+ // The engine owns firing AND reconciliation. Refusing to start would strand `started`+ // rows and preserved jobs left by a previous process until scheduling is re-enabled.+ // Firing is gated separately (runTick + getLimits), so nothing fires.+ const service = makeService({ interfaceConfig: { schedules: false } });+ expect(await service.initializeScheduleEngine()).toBeDefined();+ expect((await service.getLimits()).enabled).toBe(false);+ });++ it('reports limits DISABLED under the SCHEDULES_DISABLED lever, so writes and fires refuse', async () => {+ process.env.SCHEDULES_DISABLED = 'true';+ // Even with an explicit opt-in, the env stop must be visible wherever limits are+ // consulted — not only at the engine tick.+ const service = makeService({ interfaceConfig: { schedules: true } });+ expect((await service.getLimits()).enabled).toBe(false);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);+ });++ it('DOES arm the engine when the base is merely absent, so principal-scoped enables work', async () => {+ // Gating engine start on the base config would never start it for a role/user that+ // enables schedules via override, and would leave schedulesReady false so their+ // writes are rejected outright. Absent base is not a global stop; the fire path and+ // write handlers still refuse owners who do not have it enabled.+ const service = makeService({}, { interfaceConfig: { schedules: { use: true } } });+ expect(await service.initializeScheduleEngine()).toBeDefined();+ });++ it('stays OFF for an explicit false', async () => {+ expect((await makeService({ interfaceConfig: { schedules: false } }).getLimits()).enabled).toBe(+ false,+ );+ });++ it('turns ON for an explicit opt-in', async () => {+ expect((await makeService({ interfaceConfig: { schedules: true } }).getLimits()).enabled).toBe(+ true,+ );+ const tuned = makeService({ interfaceConfig: { schedules: { maxPerUser: 3 } } });+ const resolved = await tuned.getLimits();+ expect(resolved.enabled).toBe(true);+ expect(resolved.maxPerUser).toBe(3);+ });++ it('REFUSES a manual run-now while the global kill switch is on', async () => {+ // Run Now dispatches the same billed generation as an automatic fire, so gating only+ // the engine tick would leave this path open.+ process.env.SCHEDULES_DISABLED = 'true';+ const service = makeService({ interfaceConfig: { schedules: true } });+ const result = await service.fireScheduleNow(schedule, limits);+ expect(result).toEqual({ fired: false, skipped: 'disabled' });+ });++ it('refuses run-now when the BASE config disables it, even if a principal re-enables', async () => {+ const service = makeService(+ { interfaceConfig: { schedules: false } },+ { interfaceConfig: { schedules: true } },+ );+ const result = await service.fireScheduleNow(schedule, limits);+ expect(result).toEqual({ fired: false, skipped: 'disabled' });+ });++ it('constructing the service without a required dep fails LOUDLY at boot', () => {+ // The JS adapter is not typechecked against SchedulesServiceDeps, which is how the+ // deletion-barrier probe shipped unwired twice. A missing dep must not surface as a+ // cryptic per-fire "is not a function".+ expect(() =>+ createSchedulesService({+ methods: {},+ getAppConfig: jest.fn(),+ findUserById: jest.fn(),+ findBalance: jest.fn(),+ upsertBalance: jest.fn(),+ initializeNullBalance: jest.fn(),+ resolveAgentFireAccess: jest.fn(),+ } as unknown as SchedulesServiceDeps),+ ).toThrow(/isUserDeleting/);+ });+});diff --git a/packages/api/src/schedules/handlers.spec.ts b/packages/api/src/schedules/handlers.spec.tsnew file mode 100644index 0000000000..0c32909a1c--- /dev/null+++ b/packages/api/src/schedules/handlers.spec.ts@@ -0,0 +1,858 @@+import type { ISchedule } from '@librechat/data-schemas';+import type { Response } from 'express';+import type { SchedulesHandlersDeps } from './handlers';+import type { ServerRequest } from '~/types';+import { createSchedulesHandlers, toWireSchedule, computeCreateDigest } from './handlers';++/** A lean schedule doc carrying both public fields and internal bookkeeping. */+function fullScheduleDoc(): ISchedule {+ return {+ _id: 'mongo-id',+ __v: 0,+ id: 'sched-1',+ user: 'user-1',+ tenantId: 't1',+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ file_ids: ['file-1'],+ enabled: true,+ disabledReason: undefined,+ nextRunAt: new Date('2026-07-21T12:00:00Z'),+ lastRun: { conversationId: 'c1', status: 'success', firedAt: new Date() },+ runCount: 3,+ failureCount: 0,+ // Internal bookkeeping that must NEVER reach the browser.+ claimToken: 'ct-secret',+ leaseUntil: new Date(),+ leaseBy: 'inst-1',+ slot: 2,+ deleting: false,+ countedFor: [new Date()],+ balanceSkipCount: 1,+ bookkept: true,+ createdAt: new Date('2026-07-01T00:00:00Z'),+ updatedAt: new Date('2026-07-10T00:00:00Z'),+ } as unknown as ISchedule;+}++describe('toWireSchedule', () => {+ const INTERNAL_FIELDS = [+ '_id',+ '__v',+ 'tenantId',+ 'claimToken',+ 'leaseUntil',+ 'leaseBy',+ 'slot',+ 'deleting',+ 'countedFor',+ 'balanceSkipCount',+ 'bookkept',+ ];++ it('emits exactly the public TSchedule fields', () => {+ const wire = toWireSchedule(fullScheduleDoc());+ expect(Object.keys(wire).sort()).toEqual(+ [+ 'agent_id',+ 'cadence',+ // Public so the edit dialog can fence its PATCH on the revision it opened+ // with (updateSchedulePayloadSchema.expectedConfigRevision).+ 'configRevision',+ 'createdAt',+ 'disabledReason',+ 'enabled',+ 'failureCount',+ 'file_ids',+ 'id',+ 'lastRun',+ 'name',+ 'nextRunAt',+ 'prompt',+ 'runCount',+ 'target',+ 'timezone',+ 'updatedAt',+ 'user',+ ].sort(),+ );+ });++ it('leaks no internal bookkeeping field', () => {+ const wire = toWireSchedule(fullScheduleDoc()) as Record<string, unknown>;+ for (const field of INTERNAL_FIELDS) {+ expect(wire).not.toHaveProperty(field);+ }+ });++ it('preserves the public field values', () => {+ const wire = toWireSchedule(fullScheduleDoc());+ expect(wire.id).toBe('sched-1');+ expect(wire.name).toBe('Digest');+ expect(wire.runCount).toBe(3);+ expect(wire.cadence).toEqual({ frequency: 'daily', hour: 8, minute: 0 });+ });+});++/** Minimal Express double capturing the status/body the handler settled on. */+function makeRes() {+ const captured: { status?: number; body?: unknown; headers: Record<string, string> } = {+ headers: {},+ };+ const res = {+ status(code: number) {+ captured.status = code;+ return this;+ },+ json(payload: unknown) {+ captured.body = payload;+ return this;+ },+ set(name: string, value: string) {+ captured.headers[name] = value;+ return this;+ },+ };+ return { res: res as unknown as Response, captured };+}++const CREATE_BODY = {+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ cadence: { frequency: 'daily' as const, hour: 8, minute: 0 },+ timezone: 'America/New_York',+ clientRequestId: 'intent-1',+};++function makeCreateReq(): ServerRequest {+ return {+ body: { ...CREATE_BODY },+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ } as unknown as ServerRequest;+}++/** The digest a genuine retry of CREATE_BODY carries (zod defaults applied). */+function createBodyDigest(): string {+ return computeCreateDigest({ ...CREATE_BODY, target: 'new', enabled: true });+}++function makeCreateDeps(over: Partial<SchedulesHandlersDeps> = {}): SchedulesHandlersDeps {+ const methods = {+ countSchedulesByUser: jest.fn(async () => 0),+ createScheduleWithSlot: jest.fn(async (data: Partial<ISchedule>) => data as ISchedule),+ getScheduleByClientRequestId: jest.fn(async () => null),+ getScheduleById: jest.fn(async () => null),+ deleteScheduleById: jest.fn(async () => true),+ deleteUnarmedSchedule: jest.fn(async () => 'deleted'),+ markScheduleDeleting: jest.fn(async () => ({ id: 'sched-1' }) as ISchedule),+ updateScheduleById: jest.fn(async () => ({ id: 'sched-1' }) as ISchedule),+ armSchedule: jest.fn(async () => undefined),+ };+ return {+ methods: methods as unknown as SchedulesHandlersDeps['methods'],+ getLimits: async () => ({+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+ }),+ canViewAgent: async () => true,+ filterOwnedFileIds: async (ids: string[]) => ids,+ markFilesUsed: async () => undefined,+ fireNow: async () => null,+ deleteSchedule: async () => true,+ // Passes admission, then the barrier rises before the insert lands.+ isUserDeleting: jest.fn().mockResolvedValueOnce(false).mockResolvedValue(true),+ ...over,+ } as SchedulesHandlersDeps;+}++describe('createSchedule late-create compensation', () => {+ it('answers 410 when the guarded delete removes the unarmed revision', async () => {+ const deps = makeCreateDeps();+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);+ // Guarded to the exact unarmed revision this attempt inserted — never the+ // unconditional hard delete, which could erase a row a concurrent replay armed+ // (and the engine claimed) before the barrier re-check ran.+ expect(deps.methods.deleteUnarmedSchedule).toHaveBeenCalledWith(+ expect.any(String),+ 'user-1',+ 0,+ );+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();+ expect(captured.status).toBe(410);+ });++ it('soft-deletes instead when the row moved past the inserted revision', async () => {+ const deps = makeCreateDeps();+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('kept');+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);+ // An armed/claimed (or edited) row enters the ordinary drain-then-erase teardown+ // rather than being hard-deleted out from under a possible live fire.+ expect(deps.methods.markScheduleDeleting).toHaveBeenCalledWith(expect.any(String), 'user-1');+ expect(captured.status).toBe(410);+ });++ it('falls back to the durable soft-delete when the guarded delete fails', async () => {+ const deps = makeCreateDeps();+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);+ // Non-claimable at once, and erased by the reconciler's `deleting` sweep.+ expect(deps.methods.markScheduleDeleting).toHaveBeenCalledWith(expect.any(String), 'user-1');+ expect(captured.status).toBe(410);+ });++ it('refuses to report a clean 410 when no cleanup succeeded', async () => {+ const deps = makeCreateDeps();+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));+ (deps.methods.markScheduleDeleting as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);+ // 410 would claim the row is gone while the deleted user's prompt and+ // attachments stay in a live, non-TTL schedule.+ expect(captured.status).toBe(500);+ });++ /**+ * The compensation is best-effort by nature — when it runs, Mongo is usually the thing+ * failing, so no durable marker can be written either. Durability has to come from the+ * insert instead: an UNARMED row (no nextRunAt) is never claimed by the engine, so+ * even a total compensation failure cannot leave a schedule firing billed generations+ * for an account already being erased.+ */+ it('never inserts an armed row, so a failed compensation leaves an inert one', async () => {+ const deps = makeCreateDeps();+ (deps.methods.deleteScheduleById as jest.Mock).mockRejectedValue(new Error('mongo down'));+ (deps.methods.markScheduleDeleting as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const { res } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];+ expect(inserted.nextRunAt).toBeUndefined();+ // And it is never armed afterwards, because the barrier refused the create.+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ it('arms the schedule only after the barrier re-check clears', async () => {+ const armedRow = {+ ...CREATE_BODY,+ id: 'sched-fresh',+ target: 'new',+ enabled: true,+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ } as unknown as ISchedule;+ const deps = makeCreateDeps({+ isUserDeleting: jest.fn(async () => false),+ });+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(true);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armedRow);+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status).toBe(201);+ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];+ expect(inserted.nextRunAt).toBeUndefined();+ // The ONE shared arming CAS, fenced on the revision stamped at insert — never+ // updateScheduleById, whose token rotation would fence an engine claim off a row+ // a concurrent replay armed first.+ expect(deps.methods.armSchedule).toHaveBeenCalledWith(expect.any(String), expect.any(Date), 0);+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ it('stamps the immutable create digest on the inserted row', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const { res } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];+ expect(inserted.clientRequestDigest).toBe(createBodyDigest());+ });++ it('answers 201 with the current row when a concurrent PATCH wins the arming CAS', async () => {+ const patched = {+ ...CREATE_BODY,+ id: 'sched-fresh',+ target: 'new',+ enabled: true,+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ } as unknown as ISchedule;+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(false);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(patched);+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ // The revision moved: the PATCH's own arming governs, and this POST must not+ // overwrite it — it reports the row as the edit left it.+ expect(captured.status).toBe(201);+ expect((captured.body as { nextRunAt?: Date }).nextRunAt).toEqual(patched.nextRunAt);+ });++ /**+ * updateScheduleById filters out rows marked `deleting`, so a null arming result means+ * the deletion cascade claimed this row between the barrier re-check and the arming+ * write. Reporting the pre-delete snapshot as a 201 tells the client a schedule exists+ * that is already hidden and pending erasure.+ */+ it('does not report success when the arming write loses a delete race', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.updateScheduleById as jest.Mock).mockResolvedValue(null);+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status).toBe(410);+ expect(captured.status).not.toBe(201);+ });++ /**+ * A thrown arming write rolls the committed row back — but ONLY while it is still+ * the unarmed, unedited revision this attempt inserted: an ambiguously-committed+ * arm, a concurrent replay of this key that armed the row and already answered+ * 201, or a concurrent PATCH that edited it all leave a row that must survive.+ * deleteUnarmedSchedule carries exactly that guard (revision-fenced).+ */+ it('rolls back the committed row when the arming write throws', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status).toBe(500);+ expect(deps.methods.deleteUnarmedSchedule).toHaveBeenCalledWith(+ expect.any(String),+ 'user-1',+ 0,+ );+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();+ });++ it('answers 201 instead of deleting when the ambiguous arm actually committed', async () => {+ const armedRow = {+ ...CREATE_BODY,+ id: 'sched-fresh',+ target: 'new',+ enabled: true,+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ } as unknown as ISchedule;+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('socket reset'));+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('kept');+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armedRow);+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ // The write landed before the error surfaced: the schedule exists exactly as+ // requested, and deleting it would erase a row a concurrent replay may already+ // have confirmed with its own 201.+ expect(captured.status).toBe(201);+ });+});++describe('create idempotency', () => {+ /** The row the FIRST attempt committed for CREATE_BODY, digest-less (legacy shape). */+ const originalRow = (): ISchedule =>+ ({+ id: 'sched-original',+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ clientRequestId: 'intent-1',+ configRevision: 0,+ }) as unknown as ISchedule;++ /**+ * The retry is the whole point: the first attempt committed a row and then failed to+ * arm, so the client cannot know what persisted. Resolving to the ORIGINAL row is+ * what keeps one user intent from becoming two recurring schedules.+ */+ it('re-arms the row a previous attempt committed instead of creating another', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const original = originalRow();+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(original);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue({+ ...original,+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ });+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status ?? 201).toBe(201);+ // Nothing was inserted, and the arm is the GUARDED one keyed by the ORIGINAL row's+ // id (same deterministic jitter as the first attempt) — never updateScheduleById,+ // whose claim-token rotation would fence off an active run of this schedule and+ // whose revision bump would break a concurrent PATCH's CAS.+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();+ expect(deps.methods.armSchedule).toHaveBeenCalledWith('sched-original', expect.any(Date));+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ it('leaves an already-armed row completely untouched on replay', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(armed);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status ?? 201).toBe(201);+ expect(deps.methods.armSchedule).not.toHaveBeenCalled();+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ expect(deps.methods.deleteUnarmedSchedule).not.toHaveBeenCalled();+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();+ });++ /**+ * A retry can arrive while the user sits AT their schedule cap — with the first+ * attempt's row occupying the final slot. The key must resolve before any capacity+ * refusal, or the client is denied the very row it is trying to confirm.+ */+ it('resolves a retry occupying the final slot before the capacity pre-check', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.countSchedulesByUser as jest.Mock).mockResolvedValue(10);+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(armed);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status ?? 201).toBe(201);+ });++ it('resolves a concurrent retry that hits the allocator limit to the committed row', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };+ // Absent at the pre-insert lookup, present after the allocator reports 'limit':+ // the concurrent first attempt committed in between and took the last slot.+ (deps.methods.getScheduleByClientRequestId as jest.Mock)+ .mockResolvedValueOnce(null)+ .mockResolvedValue(armed);+ (deps.methods.createScheduleWithSlot as jest.Mock).mockResolvedValue('limit');+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status ?? 201).toBe(201);+ });++ it('refuses a key reused for a different schedule', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ // Same key, but the row it resolves to is a different intent entirely.+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue({+ id: 'sched-original',+ name: 'Something else',+ prompt: 'Different prompt',+ agent_id: 'agent-9',+ } as ISchedule);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ // A 201 here would describe a schedule the caller never asked for.+ expect(captured.status).toBe(409);+ expect(deps.methods.armSchedule).not.toHaveBeenCalled();+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ /**+ * The digest is the IMMUTABLE record of the create intent: a PATCH that reshaped the+ * row between the first attempt and its retry must not make a genuine replay read as+ * key reuse — the mutable row no longer matches, but the stamped digest still does.+ */+ it('matches a replay by digest even after a PATCH reshaped the row', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const patched = {+ ...originalRow(),+ name: 'Renamed by a PATCH',+ prompt: 'Edited prompt',+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ clientRequestDigest: createBodyDigest(),+ };+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(patched);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(patched);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status ?? 201).toBe(201);+ });++ it('refuses by digest when the key was reused for different content', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ // Field-identical row (a legacy comparison would pass) whose stamped digest says+ // the ORIGINAL create asked for something else — e.g. different attachments.+ const differentIntent = {+ ...originalRow(),+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ clientRequestDigest: 'not-this-request',+ };+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(differentIntent);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ expect(captured.status).toBe(409);+ });++ it('never compensates on the replay path, even when its arm fails', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(originalRow());+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ // The established row survives for the next retry; deleting it here is data loss.+ expect(captured.status).toBe(500);+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();+ expect(deps.methods.deleteUnarmedSchedule).not.toHaveBeenCalled();+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();+ });++ it('refuses a create without an idempotency key', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const req = makeCreateReq() as unknown as { body: Record<string, unknown> };+ delete req.body.clientRequestId;+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(req as unknown as ServerRequest, res);++ // Optional keys preserved the keyless duplicate path — the exact failure the key+ // exists to close — so the payload schema requires one.+ expect(captured.status).toBe(400);+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();+ });+});++describe('deferred deletion retry', () => {+ it('re-drives the full deletion of the caller’s soft-deleted schedules on list', async () => {+ const deps = makeCreateDeps({+ deleteSchedule: jest.fn(async () => 'deleted'),+ } as Partial<SchedulesHandlersDeps>);+ (deps.methods.getSchedulesByUser as jest.Mock) = jest.fn(async () => []);+ (deps.methods.getDeletingScheduleIds as jest.Mock) = jest.fn(async () => ['stranded-1']);+ (deps.methods.markEraseAttempted as jest.Mock) = jest.fn(async () => undefined);+ const { res } = makeRes();++ await createSchedulesHandlers(deps).listSchedules(+ { user: { id: 'user-1' } } as unknown as ServerRequest,+ res,+ );+ // Fire-and-forget, so let the microtask chain settle before asserting.+ await new Promise((resolve) => setImmediate(resolve));++ // The service delete (abort + settle + erase), not a bare erase probe: a schedule+ // stranded mid-drain with a still-active run needs the abort re-driven too.+ expect(deps.deleteSchedule).toHaveBeenCalledWith('stranded-1', 'user-1');+ // Stamped attempted so the bounded window rotates past rows that stay unconfirmed.+ expect(deps.methods.markEraseAttempted).toHaveBeenCalledWith(['stranded-1']);+ });++ it('keeps listing even when a stranded deletion re-drive rejects', async () => {+ const deps = makeCreateDeps({+ deleteSchedule: jest.fn(async () => {+ throw new Error('still draining');+ }),+ } as Partial<SchedulesHandlersDeps>);+ (deps.methods.getSchedulesByUser as jest.Mock) = jest.fn(async () => []);+ (deps.methods.getDeletingScheduleIds as jest.Mock) = jest.fn(async () => ['stranded-1']);+ (deps.methods.markEraseAttempted as jest.Mock) = jest.fn(async () => undefined);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).listSchedules(+ { user: { id: 'user-1' } } as unknown as ServerRequest,+ res,+ );+ await new Promise((resolve) => setImmediate(resolve));++ expect(captured.body).toEqual(expect.objectContaining({ schedules: [] }));+ });+});++describe('deleteSchedule result mapping', () => {+ function makeDeleteReq(): ServerRequest {+ return {+ params: { id: 'sched-1' },+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ } as unknown as ServerRequest;+ }++ const withResult = (result: string) =>+ makeCreateDeps({+ deleteSchedule: jest.fn(async () => result),+ } as Partial<SchedulesHandlersDeps>);++ it('404s when the schedule is not found', async () => {+ const { res, captured } = makeRes();+ await createSchedulesHandlers(withResult('not_found')).deleteSchedule(makeDeleteReq(), res);+ expect(captured.status).toBe(404);+ });++ it('answers 200 when drained and erased', async () => {+ const { res, captured } = makeRes();+ await createSchedulesHandlers(withResult('deleted')).deleteSchedule(makeDeleteReq(), res);+ expect(captured.status ?? 200).toBe(200);+ expect(captured.body).toEqual({ id: 'sched-1' });+ });++ it('answers 202 while a delivered abort is still settling', async () => {+ const { res, captured } = makeRes();+ await createSchedulesHandlers(withResult('draining')).deleteSchedule(makeDeleteReq(), res);+ expect(captured.status).toBe(202);+ expect(captured.body).toEqual({ id: 'sched-1' });+ });++ it('refuses honestly when the active run could not be confirmed stopped', async () => {+ // Reporting success would claim a possibly still-billing generation was stopped.+ const { res, captured } = makeRes();+ await createSchedulesHandlers(withResult('unconfirmed')).deleteSchedule(makeDeleteReq(), res);+ expect(captured.status).toBe(503);+ });+});++describe('capacity pre-check vs concurrent same-key insert', () => {+ it('re-checks the key before refusing an over-limit create', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.countSchedulesByUser as jest.Mock).mockResolvedValue(10);+ const armed = {+ ...CREATE_BODY,+ id: 'sched-original',+ target: 'new',+ enabled: true,+ configRevision: 0,+ nextRunAt: new Date('2026-07-22T12:00:00Z'),+ } as unknown as ISchedule;+ // Absent at the replay lookup, present by the capacity pre-check: the concurrent+ // first attempt committed in between — and may itself be the row at the cap.+ (deps.methods.getScheduleByClientRequestId as jest.Mock)+ .mockResolvedValueOnce(null)+ .mockResolvedValue(armed);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);++ // A 400 here would refuse the client the very row it is trying to confirm.+ expect(captured.status ?? 201).toBe(201);+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();+ });+});++describe('updateSchedule refuses field-less payloads', () => {+ it('rejects {} before touching claim-token or revision fencing', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).updateSchedule(+ {+ params: { id: 'sched-1' },+ body: {},+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ } as unknown as ServerRequest,+ res,+ );+ // An empty update still rotates claimToken and bumps configRevision, fencing a+ // legitimate in-flight occurrence for a request that changed nothing.+ expect(captured.status).toBe(400);+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });+});++describe('updateSchedule client revision fence', () => {+ const existingRow = () =>+ ({+ id: 'sched-1',+ enabled: true,+ agent_id: 'agent-1',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'UTC',+ nextRunAt: new Date('2026-07-31T09:00:00Z'),+ configRevision: 7,+ }) as unknown as ISchedule;++ const makePatchReq = (body: Record<string, unknown>) =>+ ({+ params: { id: 'sched-1' },+ body,+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ }) as unknown as ServerRequest;++ it('refuses a PATCH computed from a superseded revision', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(existingRow());+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(+ makePatchReq({ name: 'renamed', expectedConfigRevision: 6 }),+ res,+ );++ // The dialog rebuilds cadence whole from the snapshot it opened with, so the+ // server's fresh-read fence alone cannot see that another tab edited the row.+ expect(captured.status).toBe(409);+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ it('applies the PATCH and strips the fence field when the revision matches', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(existingRow());+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(+ makePatchReq({ name: 'renamed', expectedConfigRevision: 7 }),+ res,+ );++ expect(captured.status ?? 200).toBe(200);+ const [, , update] = (deps.methods.updateScheduleById as jest.Mock).mock.calls[0];+ // The fence input is not a schedule field; writing it would corrupt the row.+ expect(update).not.toHaveProperty('expectedConfigRevision');+ expect(update).toHaveProperty('name', 'renamed');+ });++ it('still refuses a PATCH that carries only the fence field', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(+ makePatchReq({ expectedConfigRevision: 7 }),+ res,+ );++ expect(captured.status).toBe(400);+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });+});++describe('attachment id deduplication', () => {+ it('accepts a create whose payload repeats an owned file id', async () => {+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(true);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue({+ ...CREATE_BODY,+ id: 'sched-1',+ file_ids: ['file-a'],+ } as unknown as ISchedule);+ const req = makeCreateReq() as unknown as { body: Record<string, unknown> };+ req.body.file_ids = ['file-a', 'file-a'];+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).createSchedule(req as unknown as ServerRequest, res);++ // The ownership query returns one doc per UNIQUE id, so an un-deduped list+ // read as a missing file and 400'd a valid request. The schema dedupes.+ expect(captured.status ?? 201).toBe(201);+ expect(deps.methods.createScheduleWithSlot).toHaveBeenCalledWith(+ expect.objectContaining({ file_ids: ['file-a'] }),+ expect.any(Number),+ );+ });+});++describe('updateSchedule re-enable attachment revalidation', () => {+ const disabledWithFiles = () =>+ ({+ id: 'sched-1',+ enabled: false,+ agent_id: 'agent-1',+ cadence: { type: 'daily', hour: 9, minute: 0 },+ timezone: 'UTC',+ nextRunAt: new Date('2026-07-31T09:00:00Z'),+ configRevision: 3,+ file_ids: ['file-a', 'file-b'],+ }) as unknown as ISchedule;++ const makeReEnableReq = () =>+ ({+ params: { id: 'sched-1' },+ body: { enabled: true },+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ }) as unknown as ServerRequest;++ it('refuses re-enabling when a stored attachment is no longer owned', async () => {+ const deps = makeCreateDeps({+ isUserDeleting: jest.fn(async () => false),+ filterOwnedFileIds: jest.fn(async () => ['file-a']),+ } as Partial<SchedulesHandlersDeps>);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(makeReEnableReq(), res);++ // The bounded upload hold only renews while the schedule fires, so a long-+ // disabled schedule can have lost its uploads; silently firing without them+ // is worse than telling the owner to replace the attachments.+ expect(captured.status).toBe(400);+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();+ });++ it('renews the retention hold on the stored attachments before committing', async () => {+ const markFilesUsed = jest.fn(async () => undefined);+ const deps = makeCreateDeps({+ isUserDeleting: jest.fn(async () => false),+ markFilesUsed,+ } as Partial<SchedulesHandlersDeps>);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(makeReEnableReq(), res);++ expect(markFilesUsed).toHaveBeenCalledWith(['file-a', 'file-b'], 'user-1');+ expect(captured.status ?? 200).toBe(200);+ expect(deps.methods.updateScheduleById).toHaveBeenCalled();+ });++ it('skips the stored-attachment recheck when the edit replaces file_ids', async () => {+ const filterOwnedFileIds = jest.fn(async (ids: string[]) => ids);+ const deps = makeCreateDeps({+ isUserDeleting: jest.fn(async () => false),+ filterOwnedFileIds,+ } as Partial<SchedulesHandlersDeps>);+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());+ const { res, captured } = makeRes();++ await createSchedulesHandlers(deps).updateSchedule(+ {+ params: { id: 'sched-1' },+ body: { enabled: true, file_ids: ['file-new'] },+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },+ } as unknown as ServerRequest,+ res,+ );++ // Supplied file_ids are validated by validatePayload; the stored list is+ // about to be overwritten, so rechecking it would refuse a valid replacement.+ expect(captured.status ?? 200).toBe(200);+ expect(filterOwnedFileIds).toHaveBeenCalledTimes(1);+ expect(filterOwnedFileIds).toHaveBeenCalledWith(['file-new'], 'user-1');+ });+});++describe('late-create compensation with a live manual run', () => {+ it('treats a draining rollback as compensated (the teardown owns it)', async () => {+ const deps = makeCreateDeps();+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('draining');+ const { res, captured } = makeRes();+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);+ // Soft-claimed and draining: hidden, unclaimable, erased once its live manual+ // run settles — never hard-deleted out from under that generation.+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();+ expect(captured.status).toBe(410);+ });+});diff --git a/packages/api/src/schedules/handlers.ts b/packages/api/src/schedules/handlers.tsnew file mode 100644index 0000000000..daec7c7304--- /dev/null+++ b/packages/api/src/schedules/handlers.ts@@ -0,0 +1,800 @@+import { logger } from '@librechat/data-schemas';+import { createHash, randomUUID } from 'node:crypto';+import { createSchedulePayloadSchema, updateSchedulePayloadSchema } from 'librechat-data-provider';+import type { TCreateSchedule, TUpdateSchedule } from 'librechat-data-provider';+import type { ScheduleMethods, ISchedule } from '@librechat/data-schemas';+import type { Response } from 'express';+import type {+ ScheduleDeleteResult,+ ScheduleUserContext,+ FireableSchedule,+ ScheduleLimits,+ FireResult,+} from './types';+import type { ServerRequest } from '~/types';+import { isValidTimezone, cadenceIntervalMinutes, computeNextRunAt } from './cadence';++export interface SchedulesHandlersDeps {+ methods: ScheduleMethods;+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;+ /** Agent existence + VIEW access for the requesting user. */+ canViewAgent: (agentId: string, req: ServerRequest) => Promise<boolean>;+ /** Filters to file ids owned by the user. */+ filterOwnedFileIds: (fileIds: string[], userId: string) => Promise<string[]>;+ /** Extends a bounded renewable upload hold on attached files so they survive to the+ * first fire, which consumes them permanently; a schedule that dies first lets the+ * hold lapse instead of retaining the upload forever. Throws when any file is gone. */+ markFilesUsed: (fileIds: string[], userId: string) => Promise<void>;+ /** Serialized manual fire (acquires the schedule lease); null if already leased. */+ fireNow: (schedule: FireableSchedule, limits: ScheduleLimits) => Promise<FireResult | null>;+ /**+ * Soft-deletes a schedule with quiescing: stops new claims, aborts in-flight+ * runs, and erases once drained. See ScheduleDeleteResult for the honest states.+ */+ deleteSchedule: (id: string, userId: string) => Promise<ScheduleDeleteResult>;+ /** Whether this user's account deletion has begun. Fail-closed (unknown == true). */+ isUserDeleting: (userId: string) => Promise<boolean>;+}++/**+ * Rolls back a create that raced past the account-deletion cascade. Hard-deletes only+ * the exact unarmed, unedited revision this attempt inserted: a concurrent same-key+ * replay can have ARMED the row — and the engine can have CLAIMED it — before the+ * post-insert barrier re-check ran, and hard deletion there would bypass run+ * quiescence and erase evidence out from under a live fire. Anything past that+ * revision is soft-deleted instead, which makes it non-claimable at once and hands it+ * to the ordinary drain-then-erase teardown. Returns false only when BOTH fail, which+ * the caller must surface rather than answering a clean 410.+ */+async function compensateLateCreate(+ deps: SchedulesHandlersDeps,+ id: string,+ userId: string,+ expectedConfigRevision: number,+): Promise<boolean> {+ const rolled = await deps.methods+ .deleteUnarmedSchedule(id, userId, expectedConfigRevision)+ .catch((err) => {+ logger.error(`[schedules] compensating delete failed for late create ${id}`, err);+ return null;+ });+ // 'draining' is compensated too: the row is soft-claimed (hidden, unclaimable) and+ // the ordinary erase-on-settle / sweep teardown finishes it once its live manual+ // run — the reason it could not erase immediately — settles.+ if (rolled === 'deleted' || rolled === 'missing' || rolled === 'draining') {+ return true;+ }+ const marked = await deps.methods.markScheduleDeleting(id, userId).catch((err) => {+ logger.error(`[schedules] compensating soft-delete failed for late create ${id}`, err);+ return null;+ });+ return marked != null;+}++/**+ * Refuses a scheduling WRITE once the owner's account deletion has begun. A one-shot+ * disable scan can never close this race (a create landing after the scan is simply not+ * in it), so admission consults the durable user-level barrier instead. Fail-closed.+ */+async function rejectIfUserDeleting(+ deps: SchedulesHandlersDeps,+ userId: string,+ res: Response,+): Promise<boolean> {+ if (!(await deps.isUserDeleting(userId))) {+ return false;+ }+ res.status(410).json({ error: 'This account is being deleted' });+ return true;+}++/** Bounded attempts to extend the upload hold on a schedule's attachments. */+const FILE_RETAIN_ATTEMPTS = 3;++/** Soft-deleted rows re-checked for erasure per list request. Small: it is a retry for+ * a rare stranded row, not a sweep, and it rides a user-facing read. */+const DEFERRED_ERASE_RETRY_LIMIT = 5;++/**+ * Digest of a create payload's FULL intent, stamped on the row at insert and never+ * edited. Replay matching compares this immutable record instead of mutable schedule+ * state, so a PATCH (or a policy auto-disable flipping `enabled`) landing between the+ * first attempt and its retry cannot make a genuine replay read as key reuse.+ */+export function computeCreateDigest(payload: TCreateSchedule): string {+ const canonical = JSON.stringify({+ name: payload.name,+ prompt: payload.prompt,+ agent_id: payload.agent_id,+ timezone: payload.timezone,+ target: payload.target,+ enabled: payload.enabled,+ cadence: {+ frequency: payload.cadence.frequency,+ hour: payload.cadence.hour,+ minute: payload.cadence.minute,+ daysOfWeek: payload.cadence.daysOfWeek ?? null,+ },+ file_ids: payload.file_ids ?? null,+ });+ return createHash('sha256').update(canonical).digest('hex');+}++function sameList<T>(left: T[] | undefined, right: T[] | undefined): boolean {+ if (left == null || right == null) {+ return (left?.length ?? 0) === (right?.length ?? 0);+ }+ return left.length === right.length && left.every((value, index) => value === right[index]);+}++/**+ * Legacy replay matching for rows stamped before `clientRequestDigest` existed.+ * Deliberately omits `enabled` — a policy auto-disable mutates it, and this+ * comparison exists precisely because mutable state makes a poor replay record.+ */+function matchesCreatedSchedule(existing: ISchedule, payload: TCreateSchedule): boolean {+ return (+ existing.name === payload.name &&+ existing.prompt === payload.prompt &&+ existing.agent_id === payload.agent_id &&+ existing.timezone === payload.timezone &&+ existing.target === payload.target &&+ existing.cadence?.frequency === payload.cadence.frequency &&+ existing.cadence?.hour === payload.cadence.hour &&+ existing.cadence?.minute === payload.cadence.minute &&+ sameList(existing.cadence?.daysOfWeek, payload.cadence.daysOfWeek) &&+ sameList(existing.file_ids, payload.file_ids)+ );+}++/** Whether an existing row is the schedule this create attempt is asking for: the+ * immutable digest when the row carries one, the legacy field comparison otherwise. */+function matchesCreateIntent(+ existing: ISchedule,+ payload: TCreateSchedule,+ digest: string,+): boolean {+ if (existing.clientRequestDigest != null) {+ return existing.clientRequestDigest === digest;+ }+ return matchesCreatedSchedule(existing, payload);+}++/** Public projection of a schedule — an allowlist of the `TSchedule` fields, so internal+ * bookkeeping (_id, tenantId, __v, claimToken, lease*, slot, deleting, countedFor,+ * balanceSkipCount, bookkept, ...) never reaches the browser. */+export type WireSchedule = Pick<+ ISchedule,+ | 'id'+ | 'user'+ | 'name'+ | 'prompt'+ | 'agent_id'+ | 'cadence'+ | 'timezone'+ | 'target'+ | 'file_ids'+ | 'enabled'+ | 'disabledReason'+ | 'nextRunAt'+ | 'lastRun'+ | 'runCount'+ | 'failureCount'+ | 'configRevision'+ | 'createdAt'+ | 'updatedAt'+>;++export function toWireSchedule(schedule: ISchedule): WireSchedule {+ return {+ id: schedule.id,+ user: schedule.user,+ name: schedule.name,+ prompt: schedule.prompt,+ agent_id: schedule.agent_id,+ cadence: schedule.cadence,+ timezone: schedule.timezone,+ target: schedule.target,+ file_ids: schedule.file_ids,+ enabled: schedule.enabled,+ disabledReason: schedule.disabledReason,+ nextRunAt: schedule.nextRunAt,+ lastRun: schedule.lastRun,+ runCount: schedule.runCount,+ failureCount: schedule.failureCount,+ configRevision: schedule.configRevision,+ createdAt: schedule.createdAt,+ updatedAt: schedule.updatedAt,+ };+}++function requestUser(req: ServerRequest): { id: string; tenantId?: string; role?: string } {+ const user = req.user as { id: string; tenantId?: string; role?: string };+ return { id: user.id, tenantId: user.tenantId, role: user.role };+}++type ScheduleHandler = (req: ServerRequest, res: Response) => Promise<void>;++export interface SchedulesHandlers {+ listSchedules: ScheduleHandler;+ getSchedule: ScheduleHandler;+ createSchedule: ScheduleHandler;+ updateSchedule: ScheduleHandler;+ deleteSchedule: ScheduleHandler;+ runScheduleNow: ScheduleHandler;+}++export function createSchedulesHandlers(deps: SchedulesHandlersDeps): SchedulesHandlers {+ async function validatePayload(+ req: ServerRequest,+ res: Response,+ payload: TCreateSchedule | TUpdateSchedule,+ limits: ScheduleLimits,+ ): Promise<boolean> {+ if (payload.timezone != null && !isValidTimezone(payload.timezone)) {+ res.status(400).json({ error: 'Invalid IANA timezone' });+ return false;+ }+ if (+ payload.cadence != null &&+ cadenceIntervalMinutes(payload.cadence) < limits.minIntervalMinutes+ ) {+ res.status(400).json({+ error: `Schedule interval must be at least ${limits.minIntervalMinutes} minutes`,+ });+ return false;+ }+ if (payload.agent_id != null && !(await deps.canViewAgent(payload.agent_id, req))) {+ res.status(400).json({ error: 'Agent not found or not accessible' });+ return false;+ }+ if (payload.file_ids != null && payload.file_ids.length > 0) {+ const owned = await deps.filterOwnedFileIds(payload.file_ids, requestUser(req).id);+ if (owned.length !== payload.file_ids.length) {+ res.status(400).json({ error: 'One or more attached files were not found' });+ return false;+ }+ }+ return true;+ }++ /**+ * Extends the bounded upload hold on attached files (so they survive to the first+ * fire, which consumes them permanently), with bounded retry. Returns false when it+ * exhausts retries — the caller then compensates (roll back the create / revert the+ * edit) so a persisted schedule never references files the upload sweep is about to+ * reap.+ */+ async function retainFiles(fileIds: string[], userId: string): Promise<boolean> {+ for (let attempt = 1; attempt <= FILE_RETAIN_ATTEMPTS; attempt++) {+ try {+ await deps.markFilesUsed(fileIds, userId);+ return true;+ } catch (err) {+ logger.error(+ `[schedules] attachment retention failed (attempt ${attempt}/${FILE_RETAIN_ATTEMPTS}):`,+ err,+ );+ }+ }+ return false;+ }++ /**+ * Re-drives the FULL deletion of the caller's soft-deleted schedules, off the+ * response path.+ *+ * A `deleting` row is settled and erased by the delete request itself, or by the+ * terminal outcome write (erase-on-settle). Both are best-effort, and the+ * reconciler that would otherwise retry does not exist in the clustered+ * entrypoint. Meanwhile the row is HIDDEN from the owner's list, so once a delete+ * answers 503-unconfirmed there is no UI/API-list handle left to retry the drain+ * with — the "please retry" the response asks for has nothing to click.+ *+ * A read the owner performs anyway is therefore the re-driver: bounded, scoped to+ * their own rows, a no-op when nothing is deleting. Re-driving the WHOLE delete+ * (abort, settle, erase) rather than only the erase is what un-strands a row whose+ * active run never settled; every step is idempotent and evidence-guarded, so+ * repeated polls race harmlessly.+ */+ function retryDeferredDeletions(userId: string): void {+ void deps.methods+ .getDeletingScheduleIds(userId, DEFERRED_ERASE_RETRY_LIMIT)+ .then(async (ids) => {+ if (ids.length === 0) {+ return;+ }+ // Stamp BEFORE attempting: the read window is least-recently-attempted+ // first, so rows this pass touches rotate to the back and an owner with+ // more stuck rows than the limit reaches all of them across successive+ // lists instead of re-driving the same unconfirmable few forever.+ await deps.methods.markEraseAttempted(ids);+ await Promise.all(+ ids.map((id) => deps.deleteSchedule(id, userId).catch(() => 'unconfirmed')),+ );+ })+ .catch((err) => logger.warn('[schedules] deferred deletion retry failed', err));+ }++ async function listSchedules(req: ServerRequest, res: Response): Promise<void> {+ const [schedules, limits] = await Promise.all([+ deps.methods.getSchedulesByUser(requestUser(req).id),+ deps.getLimits(requestUser(req)),+ ]);+ retryDeferredDeletions(requestUser(req).id);+ res.json({+ schedules: schedules.map(toWireSchedule),+ limits: { maxPerUser: limits.maxPerUser },+ });+ }++ async function getSchedule(req: ServerRequest, res: Response): Promise<void> {+ const { id } = req.params as { id: string };+ const schedule = await deps.methods.getScheduleById(id, requestUser(req).id);+ if (schedule == null) {+ res.status(404).json({ error: 'Schedule not found' });+ return;+ }+ res.json(toWireSchedule(schedule));+ }++ async function createSchedule(req: ServerRequest, res: Response): Promise<void> {+ const parsed = createSchedulePayloadSchema.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({ error: 'Invalid schedule payload', issues: parsed.error.issues });+ return;+ }+ const user = requestUser(req);+ if (await rejectIfUserDeleting(deps, user.id, res)) {+ return;+ }+ const limits = await deps.getLimits(user);+ if (!limits.enabled) {+ res.status(403).json({ error: 'Scheduled chats are disabled' });+ return;+ }+ if (!(await validatePayload(req, res, parsed.data, limits))) {+ return;+ }+ const digest = computeCreateDigest(parsed.data);++ /**+ * Answers a retry with the row its FIRST attempt committed — an ESTABLISHED+ * schedule this attempt must never reshape. Re-drives only the arm (guarded:+ * only-if-still-unarmed, jitter derived from the EXISTING row's id, no claim-token+ * rotation or revision bump, so an active run is never fenced off), and never+ * compensates: rollback belongs exclusively to the attempt that inserted a row.+ */+ const respondToReplay = async (existing: ISchedule): Promise<void> => {+ // A key identifies one create INTENT. Reusing it for different content is a+ // client bug, and silently returning the first row would hide it behind a 201+ // describing a schedule the caller did not ask for — so say so instead.+ if (!matchesCreateIntent(existing, parsed.data, digest)) {+ res.status(409).json({+ error: 'clientRequestId was already used to create a different schedule',+ });+ return;+ }+ // The key stays claimed while its row drains (the unique index spans deleting+ // rows), so a retry of a create whose schedule was since deleted resolves here.+ if (existing.deleting === true) {+ res.status(410).json({ error: 'Schedule no longer exists' });+ return;+ }+ if (await deps.isUserDeleting(user.id)) {+ res.status(410).json({ error: 'This account is being deleted' });+ return;+ }+ if (existing.enabled && existing.nextRunAt == null) {+ const next = computeNextRunAt({+ cadence: existing.cadence,+ timezone: existing.timezone,+ scheduleId: existing.id,+ });+ if (next != null) {+ try {+ await deps.methods.armSchedule(existing.id, next);+ } catch (armError) {+ logger.error(`[schedules] replay arming failed for ${existing.id}`, armError);+ res.status(500).json({ error: 'Failed to create schedule. Please retry.' });+ return;+ }+ }+ }+ const fresh = await deps.methods.getScheduleById(existing.id, user.id);+ if (fresh == null) {+ res.status(410).json({ error: 'Schedule no longer exists' });+ return;+ }+ logger.info(`[schedules] create retry resolved to ${existing.id} for user ${user.id}`);+ res.status(201).json(toWireSchedule(fresh));+ };++ // Resolve a retry BEFORE the capacity pre-check: a retry whose first attempt+ // already occupies the final slot is a replay of THAT row, not a request for a+ // new one — refusing it as over-limit would deny the client the very row it is+ // trying to confirm.+ const replayed = await deps.methods.getScheduleByClientRequestId(+ user.id,+ parsed.data.clientRequestId,+ );+ if (replayed != null) {+ await respondToReplay(replayed);+ return;+ }+ // Fail fast on an obvious over-limit BEFORE retaining attachments, so the common+ // case never clears an upload TTL it then can't use. The {user, slot} partial+ // unique index below is the atomic arbiter for the concurrent-create race.+ if ((await deps.methods.countSchedulesByUser(user.id)) >= limits.maxPerUser) {+ // A concurrent first attempt with this key can have landed between the replay+ // lookup above and this count — and can itself be what filled the last slot.+ // The retry must resolve to that row, not be refused for being at capacity.+ const raced = await deps.methods.getScheduleByClientRequestId(+ user.id,+ parsed.data.clientRequestId,+ );+ if (raced != null) {+ await respondToReplay(raced);+ return;+ }+ res.status(400).json({+ error: `Schedule limit reached (${limits.maxPerUser}). Delete a schedule to add another.`,+ });+ return;+ }+ // Retain attachments BEFORE creating, so a persisted (claimable) schedule never+ // references uploads still eligible for TTL expiry — there is no create-then-+ // retain window where a crash or a failed rollback leaves the two inconsistent.+ if (parsed.data.file_ids?.length && !(await retainFiles(parsed.data.file_ids, user.id))) {+ res.status(500).json({ error: 'Failed to retain schedule attachments' });+ return;+ }+ const id = `sched_${randomUUID()}`;+ const nextRunAt = parsed.data.enabled+ ? computeNextRunAt({+ cadence: parsed.data.cadence,+ timezone: parsed.data.timezone,+ scheduleId: id,+ })+ : undefined;+ // Atomic cap: createScheduleWithSlot claims a free per-user slot via the+ // {user, slot} partial unique index, so concurrent creates can never exceed+ // maxPerUser. 'limit' means a concurrent racer took the last slot after the+ // pre-check above; the just-retained files are then unreferenced (a rare, minor+ // leak of the user's own uploads) — acceptable vs. a partial/expiring commit.+ // Inserted WITHOUT nextRunAt regardless of `enabled`: the engine claims by+ // nextRunAt, so the row is inert until armed below. That is what makes the+ // barrier re-check durable — every failure mode leaves a row that cannot fire,+ // rather than one that fires for an account already being erased. The reconciler's+ // unarmed sweep later arms anything left this way, so an inert row is a delay, not+ // a permanent state.+ const created = await deps.methods.createScheduleWithSlot(+ {+ ...parsed.data,+ id,+ user: user.id as never,+ tenantId: user.tenantId,+ clientRequestDigest: digest,+ },+ limits.maxPerUser,+ );+ if (created === 'limit') {+ // A CONCURRENT first attempt with this key can be what filled the last slot;+ // resolve to it rather than refusing the retry for being at capacity.+ const raced = await deps.methods.getScheduleByClientRequestId(+ user.id,+ parsed.data.clientRequestId,+ );+ if (raced != null) {+ await respondToReplay(raced);+ return;+ }+ res.status(400).json({+ error: `Schedule limit reached (${limits.maxPerUser}). Delete a schedule to add another.`,+ });+ return;+ }+ if (created.id !== id) {+ // The allocator hit the idempotency index: a concurrent attempt with this key+ // committed first. That row is established — hand it to the replay path.+ await respondToReplay(created);+ return;+ }+ // FRESH insert (this attempt owns the row) from here down; compensation is safe.+ //+ // POST-INSERT barrier re-check. The admission check at the top of this handler+ // shrinks the window to roughly one request, but cannot close it: account deletion+ // can raise the barrier after we passed that check and before this insert landed,+ // and its one-shot disable scan would not have seen a row that did not exist yet.+ // Re-checking AFTER the write is what makes the barrier authoritative.+ if (await deps.isUserDeleting(user.id)) {+ // Best-effort tidy-up. Hard-deletes only the exact unarmed revision this+ // attempt inserted (a concurrent replay can have armed it, and the engine can+ // have claimed the armed row); anything past that is soft-deleted into the+ // ordinary drain-then-erase teardown. Its failure is reported but no longer+ // load-bearing for BILLING: even once the reconciler's sweep arms a residual+ // row, the fire path refuses it at the account-deletion barrier+ // (isOwnerDeleting). The residual is a retained row, not a billed generation.+ if (!(await compensateLateCreate(deps, id, user.id, created.configRevision ?? 0))) {+ res.status(500).json({ error: 'Failed to roll back schedule creation' });+ return;+ }+ res.status(410).json({ error: 'This account is being deleted' });+ return;+ }+ // ARM last, through the SAME unarmed-guarded CAS the replay path and the+ // reconciler's sweep use — one arming write per schedule, ever. Fenced on the+ // revision stamped at insert: a PATCH racing this arm bumps configRevision, and+ // this POST must not overwrite the nextRunAt that PATCH derived from newer+ // config. Never updateScheduleById here: its claim-token rotation would fence an+ // occurrence the engine may have claimed off a row a concurrent replay armed+ // first, and its revision bump would break a concurrent PATCH's CAS.+ if (nextRunAt) {+ let armedNow: boolean;+ try {+ armedNow = await deps.methods.armSchedule(id, nextRunAt, created.configRevision ?? 0);+ } catch (armError) {+ // Roll back ONLY the exact unarmed, unedited revision this attempt inserted.+ // An ambiguously-committed arm, a concurrent replay that armed the row and+ // already answered 201, or a concurrent PATCH that edited it (even while+ // leaving it unarmed) all leave a row that must survive.+ logger.error(`[schedules] arming failed for ${id}; rolling back the create`, armError);+ const rolledBack = await deps.methods+ .deleteUnarmedSchedule(id, user.id, created.configRevision ?? 0)+ .catch(() => null);+ if (rolledBack === 'kept') {+ const current = await deps.methods.getScheduleById(id, user.id);+ if (current != null) {+ res.status(201).json(toWireSchedule(current));+ return;+ }+ }+ // A retry is always safe: the same key collides on the idempotency index and+ // resolves to whatever this attempt left behind (or a clean re-insert).+ res.status(500).json({ error: 'Failed to create schedule. Please retry.' });+ return;+ }+ // Whether THIS call armed or a concurrent actor got there first (a replay's+ // arm, a PATCH whose own arming governs), the response reports the row as it+ // now stands; only a row the deletion cascade claimed answers 410.+ const current = await deps.methods.getScheduleById(id, user.id);+ if (current == null) {+ res.status(410).json({ error: 'Schedule no longer exists' });+ return;+ }+ if (!armedNow && current.nextRunAt == null && current.enabled) {+ // Unarmed with an unmoved revision would have matched the CAS; reaching here+ // means the revision moved to an edit that keeps it enabled but unarmed —+ // report it as-is, the reconciler's sweep recovers the arm.+ logger.warn(`[schedules] create for ${id} left unarmed after a concurrent edit`);+ }+ logger.info(`[schedules] created ${id} for user ${user.id}`);+ res.status(201).json(toWireSchedule(current));+ return;+ }+ logger.info(`[schedules] created ${id} for user ${user.id}`);+ res.status(201).json(toWireSchedule(created));+ }++ async function updateSchedule(req: ServerRequest, res: Response): Promise<void> {+ const parsed = updateSchedulePayloadSchema.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({ error: 'Invalid schedule payload', issues: parsed.error.issues });+ return;+ }+ // The fence input is not a schedule field: strip it before the emptiness check+ // and the update spread below, or it would count as an edit and be written to+ // the row.+ const { expectedConfigRevision, ...editedFields } = parsed.data;+ // A field-less PATCH is not a harmless no-op: updateScheduleById rotates the claim+ // token and bumps the config revision on every write, so an empty update would+ // fence a legitimate in-flight occurrence — its terminal bookkeeping revision-+ // fences to a no-op, and a fire in the POST-to-controller window is refused at+ // the admission boundary without ever running. Refuse before touching fencing.+ if (Object.keys(editedFields).length === 0) {+ res.status(400).json({ error: 'Schedule update must include at least one field' });+ return;+ }+ const { id } = req.params as { id: string };+ const user = requestUser(req);+ if (await rejectIfUserDeleting(deps, user.id, res)) {+ return;+ }+ const existing = await deps.methods.getScheduleById(id, user.id);+ if (existing == null) {+ res.status(404).json({ error: 'Schedule not found' });+ return;+ }+ // Client-side revision fence: the dialog rebuilds compound fields (cadence)+ // from the snapshot it opened with, so an edit from another tab is invisible+ // to the fresh-read fence below — the payload is internally consistent with a+ // row that no longer exists. Refuse before any side effect (file holds) when+ // the client says which revision it edited.+ if (expectedConfigRevision != null && existing.configRevision !== expectedConfigRevision) {+ res.status(409).json({ error: 'Schedule was modified concurrently. Please retry.' });+ return;+ }+ const limits = await deps.getLimits(user);+ // When the owner's config disables schedules, block edits that keep the+ // schedule enabled; still allow turning one OFF.+ if (!limits.enabled && (parsed.data.enabled ?? existing.enabled)) {+ res.status(403).json({ error: 'Scheduled chats are disabled' });+ return;+ }+ if (!(await validatePayload(req, res, parsed.data, limits))) {+ return;+ }+ const cadence = parsed.data.cadence ?? existing.cadence;+ const timezone = parsed.data.timezone ?? existing.timezone;+ const enabled = parsed.data.enabled ?? existing.enabled;+ // Re-validate the EFFECTIVE (possibly stored) cadence against the current+ // floor whenever this edit leaves the schedule enabled — otherwise a bare+ // {enabled:true} could re-enable an existing schedule that now runs too often.+ if (enabled && cadenceIntervalMinutes(cadence) < limits.minIntervalMinutes) {+ res.status(400).json({+ error: `Schedule interval must be at least ${limits.minIntervalMinutes} minutes`,+ });+ return;+ }+ // A supplied agent_id is validated in validatePayload; when an edit omits it+ // but leaves the schedule enabled (e.g. toggling `enabled` back on after an+ // agent_deleted/permission_revoked auto-disable), re-validate the STORED+ // agent too. Otherwise re-enabling clears disabledReason for a target the+ // next fire would immediately reject and disable again.+ if (+ enabled &&+ parsed.data.agent_id == null &&+ !(await deps.canViewAgent(existing.agent_id, req))+ ) {+ res.status(400).json({ error: 'Agent not found or not accessible' });+ return;+ }+ const cadenceChanged =+ parsed.data.cadence != null || parsed.data.timezone != null || parsed.data.enabled != null;+ const reEnabled = parsed.data.enabled === true && existing.enabled === false;+ // RECOVERY: an enabled schedule with no nextRunAt is inert — claimDueSchedule sorts+ // on nextRunAt and can never select it. Creation arms in a second write, so a crash+ // or a failed arm leaves exactly this state; re-arm on ANY edit rather than only a+ // cadence one, or a name/prompt edit would silently leave it dead.+ const needsArming = existing.nextRunAt == null;+ const update: Partial<ISchedule> = { ...editedFields } as Partial<ISchedule>;+ if (enabled && (cadenceChanged || needsArming)) {+ const nextRunAt = computeNextRunAt({ cadence, timezone, scheduleId: existing.id });+ if (nextRunAt == null) {+ res.status(400).json({ error: 'Schedule has no computable next run' });+ return;+ }+ update.nextRunAt = nextRunAt;+ }+ if (reEnabled) {+ update.failureCount = 0;+ update.balanceSkipCount = 0;+ }+ const unset = reEnabled ? { disabledReason: 1 as const } : undefined;+ // Retain the new attachments BEFORE committing the edit, so a retention failure+ // leaves the ENTIRE schedule unchanged rather than persisting prompt/cadence/+ // agent/enabled changes while only reverting file_ids. A file whose TTL was+ // cleared before the edit failed simply persists unreferenced (the user's own+ // upload) — a minor leak, not a partial config change future runs would use.+ if (parsed.data.file_ids?.length && !(await retainFiles(parsed.data.file_ids, user.id))) {+ res.status(500).json({ error: 'Failed to retain schedule attachments' });+ return;+ }+ // Re-enabling with STORED attachments: the bounded upload hold only renews while+ // the schedule fires, so a schedule that sat disabled past the hold can have lost+ // its uploads. Validate the effective list and renew its hold now, mirroring the+ // stored-agent recheck above — otherwise the re-enable succeeds and the next run+ // silently fires without the missing files instead of telling the user to+ // replace them.+ if (reEnabled && parsed.data.file_ids == null && existing.file_ids?.length) {+ // Deduped before the count comparison: the ownership query returns one doc+ // per unique id, so a legacy row holding the same id twice would otherwise+ // read as a missing file. New payloads are deduped at the schema.+ const storedIds = Array.from(new Set(existing.file_ids));+ const stillOwned = await deps.filterOwnedFileIds(storedIds, user.id);+ if (stillOwned.length !== storedIds.length) {+ res.status(400).json({+ error:+ 'One or more attached files are no longer available. Replace the attachments before re-enabling.',+ });+ return;+ }+ if (!(await retainFiles(storedIds, user.id))) {+ res.status(500).json({ error: 'Failed to retain schedule attachments' });+ return;+ }+ }+ // FENCED on the revision this edit was computed from. `nextRunAt` above is derived+ // from (cadence, timezone) resolved against the row read at the top of this handler,+ // so two overlapping edits — one changing cadence, one changing timezone — would+ // each persist a nextRunAt consistent with neither final state, and it would stay+ // wrong until the next edit (advanceSchedule recomputes from the row, so a fire+ // propagates the bad occurrence rather than repairing it). configRevision is already+ // the row's edit generation and is $inc'd inside the same atomic update, so the+ // loser simply retries against fresh state.+ const schedule = await deps.methods.updateScheduleById(existing.id, user.id, update, unset, {+ expectedConfigRevision: existing.configRevision,+ });+ if (schedule == null) {+ // Either the row is gone, or a concurrent edit moved the revision. Distinguish+ // them so the owner sees a retryable conflict rather than a phantom 404.+ const stillThere = await deps.methods.getScheduleById(existing.id, user.id);+ if (stillThere != null) {+ res.status(409).json({ error: 'Schedule was modified concurrently. Please retry.' });+ return;+ }+ res.status(404).json({ error: 'Schedule not found' });+ return;+ }+ res.json(toWireSchedule(schedule));+ }++ async function deleteSchedule(req: ServerRequest, res: Response): Promise<void> {+ const { id } = req.params as { id: string };+ // Quiesce-then-erase: disable + mark deleting (stops new claims, hides it),+ // abort in-flight loopback jobs, and erase once drained — so a live run's+ // evidence is never destroyed out from under it.+ const result = await deps.deleteSchedule(id, requestUser(req).id);+ if (result === 'not_found') {+ res.status(404).json({ error: 'Schedule not found' });+ return;+ }+ // HONEST failure: at least one active run could not be confirmed stopped, so its+ // generation may still be producing and billing. The schedule is already hidden+ // and fenced (no new claims), and the delete is idempotent — a retry re-runs the+ // drain. Reporting success here would claim the run was stopped.+ if (result === 'unconfirmed') {+ res.set('Retry-After', '30');+ res.status(503).json({+ error: 'Could not confirm the active run was stopped. Please retry shortly.',+ });+ return;+ }+ // 202 for `draining`: the aborts were delivered but a generation has not yet+ // recorded its terminal outcome. The schedule is hidden and erasure follows the+ // settlement (erase-on-settle), in any topology.+ res.status(result === 'draining' ? 202 : 200).json({ id });+ }++ async function runScheduleNow(req: ServerRequest, res: Response): Promise<void> {+ const { id } = req.params as { id: string };+ if (await rejectIfUserDeleting(deps, requestUser(req).id, res)) {+ return;+ }+ const schedule = await deps.methods.getScheduleById(id, requestUser(req).id);+ if (schedule == null) {+ res.status(404).json({ error: 'Schedule not found' });+ return;+ }+ const limits = await deps.getLimits(requestUser(req));+ const result = await deps.fireNow(schedule, limits);+ if (result == null) {+ res.status(409).json({ error: 'A run for this schedule is already in progress' });+ return;+ }+ if (!result.fired) {+ // A limiter refusal is the caller's own quota, not a conflicting schedule state,+ // so answer 429 rather than burying it in the generic 409.+ res.status(result.skipped === 'rate_limited' ? 429 : 409).json({+ error:+ result.skipped === 'rate_limited'+ ? 'Too many messages. Try running this schedule again shortly.'+ : (result.error ?? `Run skipped (${result.skipped ?? 'unknown'})`),+ skipped: result.skipped,+ });+ return;+ }+ res.json({ scheduleId: schedule.id, conversationId: result.conversationId, status: 'started' });+ }++ return {+ listSchedules,+ getSchedule,+ createSchedule,+ updateSchedule,+ deleteSchedule,+ runScheduleNow,+ };+}++export type { ScheduleUserContext };diff --git a/packages/api/src/schedules/index.ts b/packages/api/src/schedules/index.tsnew file mode 100644index 0000000000..4abce8246d--- /dev/null+++ b/packages/api/src/schedules/index.ts@@ -0,0 +1,8 @@+export * from './access';+export * from './cadence';+export * from './engine';+export * from './erasure';+export * from './fire';+export * from './handlers';+export * from './trigger';+export * from './types';diff --git a/packages/api/src/schedules/service.spec.ts b/packages/api/src/schedules/service.spec.tsnew file mode 100644index 0000000000..efc00f97b7--- /dev/null+++ b/packages/api/src/schedules/service.spec.ts@@ -0,0 +1,1628 @@+import { logger } from '@librechat/data-schemas';+import type { SchedulesServiceDeps } from './service';+import { isShutdownInProgress } from '../app/shutdown';+import { createSchedulesService } from './service';++/** Swappable per test: null keeps the no-job-store harness the drain tests rely on. */+let mockJobStore: { getJob: jest.Mock; deleteJob?: jest.Mock } | null = null;++jest.mock('../agents/checkpointer', () => ({+ deleteAgentCheckpoint: jest.fn(async () => undefined),+ // Non-empty by default so the scoped prune has something to delete in tests.+ captureAgentCheckpointGeneration: jest.fn(async (threadId: string) => ({+ threadId,+ checkpointIds: ['ck-1'],+ })),+}));+const checkpointerModule = jest.requireMock('../agents/checkpointer') as {+ deleteAgentCheckpoint: jest.Mock;+ captureAgentCheckpointGeneration: jest.Mock;+};++jest.mock('../stream/GenerationJobManager', () => ({+ GenerationJobManager: {+ // No configured job store by default: abortScheduledJob returns false, so the drain+ // loop is driven purely by getActiveRunsForUser (the run rows).+ getJobStore: () => mockJobStore,+ abortJob: jest.fn(),+ updateMetadata: jest.fn(async () => undefined),+ isRedis: false,+ },+}));++type ActiveRun = {+ scheduleId: string;+ scheduledFor: Date;+ conversationId?: string;+ status?: string;+};++let recordRunOutcome: jest.Mock;++function makeService(+ getActiveRunsForUser: jest.Mock<Promise<ActiveRun[]>, [string]>,+ getAppConfig?: SchedulesServiceDeps['getAppConfig'],+ enqueueAgentTrigger: SchedulesServiceDeps['enqueueAgentTrigger'] = jest.fn(async () => undefined),+): ReturnType<typeof createSchedulesService> {+ recordRunOutcome = jest.fn(async () => undefined);+ const methods = {+ suspendUserSchedulesForDeletion: jest.fn(async () => undefined),+ restoreUserSchedulesFromDeletion: jest.fn(async () => undefined),+ getActiveRunsForUser,+ countActiveRuns: jest.fn(async () => 0),+ requestRunAbort: jest.fn(async () => true),+ getScheduleRunAbortState: jest.fn(async () => null),+ markRunAbortPersisted: jest.fn(async () => undefined),+ recordRunOutcome,+ };+ const deps = {+ methods,+ getAppConfig: getAppConfig ?? jest.fn(async () => ({})),+ findUserById: jest.fn(async () => null),+ findBalance: jest.fn(async () => null),+ upsertBalance: jest.fn(async () => null),+ initializeNullBalance: jest.fn(async () => null),+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),+ isUserDeleting: jest.fn(async () => false),+ enqueueAgentTrigger,+ getTriggerDelivery: jest.fn(async () => null),+ } as unknown as SchedulesServiceDeps;+ // Short bounded waits so fail-closed drain paths resolve in test time.+ return createSchedulesService(deps, {+ drainTimeoutMs: 400,+ drainPollMs: 25,+ stopBarrierTimeoutMs: 400,+ stopBarrierPollMs: 25,+ });+}++const run = (): ActiveRun => ({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+});++describe('shutdown wiring', () => {+ it('carries the coordinator signal on the BASE deps so Run Now is gated too', async () => {+ const service = makeService(jest.fn(async (_userId: string) => []));+ // fireScheduleNow dispatches with engineDeps directly (not the engine's+ // per-pass wrapper), so the shutdown gate must live on the base deps or a+ // manual Run Now POSTs into a closing listener.+ expect(service.engineDeps.isShuttingDown).toBe(isShutdownInProgress);+ });+});++describe('manual Run Now lease cleanup', () => {+ it('releases the old holder when an owner edit fences cleanup after a preflight throw', async () => {+ const service = makeService(jest.fn(async (_userId: string) => []));+ const leased = {+ id: 's1',+ user: 'user-1',+ tenantId: 't1',+ name: 'Digest',+ prompt: 'Summarize',+ agent_id: 'agent-1',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ claimToken: 'manual-token',+ leaseBy: 'manual-holder',+ runCount: 0,+ failureCount: 0,+ balanceSkipCount: 0,+ } as never;+ const methods = service.engineDeps.methods as unknown as {+ acquireManualRunLease: jest.Mock;+ releaseLease: jest.Mock;+ releaseLeaseByHolder: jest.Mock;+ };+ methods.acquireManualRunLease = jest.fn(async () => leased);+ // Simulates an edit rotating claimToken while deliberately preserving leaseBy.+ methods.releaseLease = jest.fn(async () => false);+ methods.releaseLeaseByHolder = jest.fn(async () => undefined);+ service.engineDeps.getUserContext = jest.fn(async () => {+ throw new Error('user lookup failed');+ });++ await expect(+ service.fireScheduleNow(leased, {+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+ }),+ ).rejects.toThrow('user lookup failed');++ expect(methods.releaseLease).toHaveBeenCalledWith('s1', 'manual-token');+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'manual-holder');+ });+});++describe('balance initialization', () => {+ const balanceConfig = {+ interfaceConfig: {},+ balance: { enabled: true, startBalance: 20000 },+ } as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;++ function serviceWithBalance(+ existing: Record<string, unknown> | null,+ overrides: {+ initializeNullBalance?: jest.Mock;+ findBalance?: jest.Mock;+ } = {},+ ) {+ const upsertBalance = jest.fn(async () => ({ tokenCredits: 20000 }));+ const initializeNullBalance =+ overrides.initializeNullBalance ?? jest.fn(async () => ({ tokenCredits: 20000 }));+ const findBalance = overrides.findBalance ?? jest.fn(async () => existing);+ const service = createSchedulesService({+ methods: {} as unknown as SchedulesServiceDeps['methods'],+ getAppConfig: (async () => balanceConfig) as SchedulesServiceDeps['getAppConfig'],+ findUserById: jest.fn(async () => null),+ findBalance,+ upsertBalance,+ initializeNullBalance,+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),+ isUserDeleting: jest.fn(async () => false),+ enqueueAgentTrigger: jest.fn(async () => undefined),+ getTriggerDelivery: jest.fn(async () => null),+ } as unknown as SchedulesServiceDeps);+ return { service, upsertBalance, initializeNullBalance, findBalance };+ }++ const updateFrom = (spy: jest.Mock) =>+ (+ spy.mock.calls[0] as unknown as [+ string,+ { set: Record<string, unknown>; setOnInsert: Record<string, unknown> },+ ]+ )[1];++ const casUpdateFrom = (spy: jest.Mock) =>+ (+ spy.mock.calls[0] as unknown as [+ string,+ { tokenCredits: number; sync: Record<string, unknown> },+ ]+ )[1];++ /**+ * The balance READ and this write are separate statements. A concurrent charge that+ * creates the record in between would be overwritten by a blind `$set`, handing back+ * credits the user had already spent.+ */+ it('initializes the starting credit via setOnInsert, never $set', async () => {+ const { service, upsertBalance } = serviceWithBalance(null);++ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);++ expect(upsertBalance).toHaveBeenCalledTimes(1);+ const update = updateFrom(upsertBalance);+ expect(update.setOnInsert).toMatchObject({ tokenCredits: 20000 });+ expect(update.set).not.toHaveProperty('tokenCredits');+ });++ /**+ * The record was observed with a null credit, but the read and the write are separate+ * statements: a concurrent initializer/charge could have set and spent it in between.+ * The credit must go through a `{ tokenCredits: null }` CAS, never a blind `$set` that+ * would restore spent credits.+ */+ it('initializes an EXISTING null credit through the CAS, never a blind $set', async () => {+ const { service, upsertBalance, initializeNullBalance } = serviceWithBalance({+ autoRefillEnabled: false,+ });++ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);++ expect(initializeNullBalance).toHaveBeenCalledTimes(1);+ const cas = casUpdateFrom(initializeNullBalance);+ expect(cas.tokenCredits).toBe(20000);+ // The credit write goes ONLY through the CAS, never the unconditional upsert path.+ expect(upsertBalance).not.toHaveBeenCalled();+ });++ /**+ * When the CAS misses — a concurrent initializer/charge won — the preflight must re-read+ * the winner's balance rather than restore the starting credit.+ */+ it('re-reads the winner on a CAS miss and never restores spent credits', async () => {+ const initializeNullBalance = jest.fn(async () => null);+ // First read observes a null credit; the winner's re-read returns the spent balance.+ const findBalance = jest+ .fn()+ .mockResolvedValueOnce({ autoRefillEnabled: false, tokenCredits: null })+ .mockResolvedValueOnce({ tokenCredits: 5 });+ const { service } = serviceWithBalance(null, { initializeNullBalance, findBalance });++ const outOfBalance = await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);++ expect(initializeNullBalance).toHaveBeenCalledTimes(1);+ // Re-read after the miss (the two isOutOfBalance reads), and the small winner balance+ // wins: > 0 credits means the user is not pre-skipped for balance.+ expect(findBalance).toHaveBeenCalledTimes(2);+ expect(outOfBalance).toBe(false);+ });++ /**+ * A stale record whose credit is already set but whose refill config drifted still syncs+ * that config, and the sync must not widen into a credit write.+ */+ it('syncs refill config on a credited record without writing tokenCredits', async () => {+ const refillConfig = {+ interfaceConfig: {},+ balance: {+ enabled: true,+ startBalance: 20000,+ autoRefillEnabled: true,+ refillIntervalValue: 30,+ refillIntervalUnit: 'days',+ refillAmount: 10000,+ },+ } as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;+ const upsertBalance = jest.fn(async () => ({ tokenCredits: 100 }));+ const initializeNullBalance = jest.fn(async () => null);+ const service = createSchedulesService({+ methods: {} as unknown as SchedulesServiceDeps['methods'],+ getAppConfig: (async () => refillConfig) as SchedulesServiceDeps['getAppConfig'],+ findUserById: jest.fn(async () => null),+ findBalance: jest.fn(async () => ({+ tokenCredits: 100,+ autoRefillEnabled: false,+ lastRefill: new Date('2026-01-01T00:00:00.000Z'),+ })),+ upsertBalance,+ initializeNullBalance,+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),+ isUserDeleting: jest.fn(async () => false),+ enqueueAgentTrigger: jest.fn(async () => undefined),+ getTriggerDelivery: jest.fn(async () => null),+ } as unknown as SchedulesServiceDeps);++ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);++ // Credit is already set, so the CAS path is never taken.+ expect(initializeNullBalance).not.toHaveBeenCalled();+ expect(upsertBalance).toHaveBeenCalledTimes(1);+ const update = updateFrom(upsertBalance);+ expect(update.set).toMatchObject({ autoRefillEnabled: true });+ expect(update.set).not.toHaveProperty('tokenCredits');+ expect(update.setOnInsert).toEqual({});+ });+});++describe('deleteScheduleForOwner', () => {+ beforeEach(() => {+ // Module-level mock: without this it accumulates calls across tests and the+ // prune assertions below read a previous test's invocation.+ checkpointerModule.deleteAgentCheckpoint.mockClear();+ });++ /**+ * markScheduleDeleting runs FIRST and is one-shot: it matches only a not-yet-deleting+ * row, so a retry answers 404. Anything between it and the aborts that can throw+ * therefore strands the schedule — hidden and fenced, but with its paused job still+ * resumable and its checkpoint unpruned, until the 25-hour abandonment sweep.+ */+ it('still aborts a paused run when the checkpointer config cannot be resolved', async () => {+ const pausedRun = {+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'requires_action',+ };+ const service = makeService(+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]),+ // The checkpointer lookup resolves the owner's config; a transient failure here+ // must not cost the abort.+ jest.fn(async () => {+ throw new Error('config plane down');+ }) as unknown as SchedulesServiceDeps['getAppConfig'],+ );+ const methods = service.engineDeps.methods as unknown as {+ markScheduleDeleting: jest.Mock;+ getActiveRunsForSchedule: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ };+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));+ methods.getActiveRunsForSchedule = jest.fn(async () => [pausedRun]);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);+ // The owner must RESOLVE, or the checkpointer lookup short-circuits before it ever+ // reads the config and the throw under test never happens.+ (service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(+ async () => ({ id: 'user-1', tenantId: 't1', role: 'USER' }),+ );++ const abortJob = jest.fn(async () => ({ success: true }));+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'requires_action',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = abortJob;++ // A paused (not running) job is positive evidence: settled synchronously, erased.+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');+ expect(abortJob).toHaveBeenCalled();+ });++ /** Shared double set for the drain-discipline tests below. */+ function makeDeleteHarness(run: Partial<ActiveRun> & { scheduleId: string }) {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ markScheduleDeleting: jest.Mock;+ getActiveRunsForSchedule: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ recordRunOutcome: jest.Mock;+ getScheduleById: jest.Mock;+ };+ methods.markScheduleDeleting = jest.fn(async () => ({ id: run.scheduleId, user: 'user-1' }));+ methods.getActiveRunsForSchedule = jest.fn(async () => [run]);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);+ methods.recordRunOutcome = jest.fn(async () => undefined);+ methods.getScheduleById = jest.fn(async () => ({ user: 'user-1' }));+ return { service, methods };+ }++ it('settles a pause hand-off after the exact provider drain is confirmed', async () => {+ const { service, methods } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ });+ // The job can report requires_action before the run row records the pause. The+ // delete path must abort through #14925's provider-drain barrier before settling,+ // so no trailing controller write can race the outcome.+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'requires_action',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({+ success: true,+ signalDelivered: true,+ jobData: { status: 'requires_action' },+ }));++ await service.deleteScheduleForOwner('s1', 'user-1');++ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'interrupted', conversationId: 'c1' }),+ );+ });++ it('does not report success when the abort of a live run is not delivered', async () => {+ const { service, methods } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ });+ // A RUNNING identity-matched job: not settleable, must be aborted...+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ deleteJob: jest.fn(async () => true),+ } as unknown as typeof mockJobStore;+ // ...and the abort delivery FAILS (job store write rejected).+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => {+ throw new Error('store unreachable');+ });++ // The generation may still be producing and billing; claiming success would say+ // it was stopped. The schedule stays hidden and fenced; the delete is idempotent.+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');+ expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();+ });++ it('treats an unreadable job store as unknown, not absent', async () => {+ const { service, methods } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ });+ mockJobStore = {+ getJob: jest.fn(async () => {+ throw new Error('redis gone');+ }),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => {+ throw new Error('redis gone');+ });++ // With the store unreadable NOTHING is proven: the row must not be settled as+ // an orphan (its generation may be live) and the delete must not read as done.+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('settles a provably job-less run synchronously and erases without a reconciler', async () => {+ const { service, methods } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ });+ // Confirmed absence: the lookup SUCCEEDS and returns null (a crashed fire, or a+ // stale row from a previous topology). The clustered entrypoint has no reconciler,+ // so deferring this row to one retained the deleted schedule indefinitely there.+ mockJobStore = { getJob: jest.fn(async () => null) } as unknown as typeof mockJobStore;++ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),+ );+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');+ });++ /**+ * The durable checkpoint is keyed by conversationId ALONE, so pruning it when a+ * REPLACEMENT turn owns the conversation strips the resume state of a live generation+ * that has nothing to do with this schedule — the same hazard the interactive abort+ * route refuses with a 409.+ */+ it('does not prune the checkpoint when a replacement owns the conversation', async () => {+ const { service } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'requires_action',+ });+ // A job IS present, but it carries a DIFFERENT occurrence's identity: this+ // conversation now belongs to someone else's turn.+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 99,+ scheduleId: 'a-different-schedule',+ scheduledFor: new Date('2030-01-01T00:00:00.000Z').toISOString(),+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: true }));++ await service.deleteScheduleForOwner('s1', 'user-1');++ expect(checkpointerModule.deleteAgentCheckpoint).not.toHaveBeenCalled();+ });++ it('prunes the checkpoint of a paused run it does own', async () => {+ const { service } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'requires_action',+ });+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'requires_action',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: true }));++ await service.deleteScheduleForOwner('s1', 'user-1');++ // SCOPED to the checkpoint ids captured before the terminal transition, so a+ // replacement's later checkpoints can never be swept up by this prune.+ expect(checkpointerModule.deleteAgentCheckpoint).toHaveBeenCalledWith('c1', undefined, {+ threadId: 'c1',+ checkpointIds: ['ck-1'],+ });+ });++ it('reports draining when a live run was aborted but has not yet settled', async () => {+ const { service, methods } = makeDeleteHarness({+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ });+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ deleteJob: jest.fn(async () => true),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: true }));+ // The run row is still active, so the erase declines; settlement (and the+ // erase-on-settle it triggers) belongs to the aborted generation's outcome write.+ methods.eraseScheduleIfDrained = jest.fn(async () => false);+ // Public reads intentionally hide deleting rows. That absence must not turn a+ // declined erasure into a false `deleted` response.+ methods.getScheduleById.mockResolvedValue(null);++ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('draining');+ });++ afterEach(() => {+ mockJobStore = null;+ jest.restoreAllMocks();+ });+});++describe('erase-on-settle', () => {+ /**+ * Whichever process records a run's terminal outcome also attempts the deferred+ * erase of a deleting schedule. This is what makes a delete's `draining` state+ * converge in EVERY topology — the clustered entrypoint runs no reconciler, so+ * without it the hidden schedule (and its prompt, which has no TTL) survived its+ * last run indefinitely there.+ */+ it('attempts the deferred erase after recording a terminal outcome', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ getScheduleById: jest.Mock;+ recordRunOutcome: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ };+ methods.getScheduleById = jest.fn(async () => null);+ methods.recordRunOutcome = jest.fn(async () => undefined);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);++ await expect(+ service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'success',+ }),+ ).resolves.toBe(true);+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');+ });++ it('does not erase on a pause, which is not a settlement', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ getScheduleById: jest.Mock;+ recordRunOutcome: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ };+ methods.getScheduleById = jest.fn(async () => null);+ methods.recordRunOutcome = jest.fn(async () => undefined);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);++ await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'requires_action',+ });+ expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();+ });+});++describe('interactive Stop persistence barrier', () => {+ function outcomeService() {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ getScheduleById: jest.Mock;+ recordRunOutcome: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ getScheduleRunAbortState: jest.Mock;+ requestRunAbort: jest.Mock;+ markRunAbortPersisted: jest.Mock;+ };+ methods.getScheduleById = jest.fn(async () => null);+ methods.recordRunOutcome = jest.fn(async () => undefined);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);+ return { service, methods };+ }++ it('beginScheduledStop stamps a serialized stop; acknowledge marks persistence', async () => {+ const { service, methods } = outcomeService();+ methods.requestRunAbort = jest.fn(async () => 'in_progress' as const);+ methods.markRunAbortPersisted = jest.fn(async () => undefined);++ const stamp = await service.beginScheduledStop({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ });+ expect(stamp).toBe('in_progress');+ expect(methods.requestRunAbort).toHaveBeenCalledWith('s1', expect.any(Date), 'stop');++ await service.acknowledgeScheduledStopPersistence({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ });+ expect(methods.markRunAbortPersisted).toHaveBeenCalledWith('s1', expect.any(Date));+ });++ it('owner terminal settlement waits until the Stop persistence is acknowledged', async () => {+ const { service, methods } = outcomeService();+ const fresh = new Date();+ let reads = 0;+ // Unresolved fresh stop for the first two reads, then acknowledged.+ methods.getScheduleRunAbortState = jest.fn(async () => {+ reads += 1;+ return {+ status: 'started',+ abortSource: 'stop',+ abortRequestedAt: fresh,+ ...(reads > 2 ? { abortPersistedAt: new Date() } : {}),+ };+ });++ await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'success',+ });++ // Polled until the acknowledgement landed, and only THEN recorded the outcome.+ expect(reads).toBeGreaterThanOrEqual(3);+ expect(methods.recordRunOutcome).toHaveBeenCalled();+ expect(methods.getScheduleRunAbortState.mock.invocationCallOrder[0]).toBeLessThan(+ methods.recordRunOutcome.mock.invocationCallOrder[0],+ );+ });++ it('does not wait on a non-stop abort source', async () => {+ const { service, methods } = outcomeService();+ methods.getScheduleRunAbortState = jest.fn(async () => ({+ status: 'started',+ abortSource: 'deletion',+ abortRequestedAt: new Date(),+ }));++ await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'interrupted',+ });++ expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);+ expect(methods.recordRunOutcome).toHaveBeenCalled();+ });++ it('does not wait on a STALE stop stamp (route presumed dead)', async () => {+ const { service, methods } = outcomeService();+ // Older than ABORT_OWNER_PRESUMED_ALIVE_MS: the owner presumes the route dead.+ const stale = new Date(Date.now() - 31 * 60_000);+ methods.getScheduleRunAbortState = jest.fn(async () => ({+ status: 'started',+ abortSource: 'stop',+ abortRequestedAt: stale,+ }));++ await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'error',+ });++ expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);+ expect(methods.recordRunOutcome).toHaveBeenCalled();+ });++ /**+ * The poll budget expiring proves nothing about the Stop route's writes (slow checkpoint+ * cleanup looks identical), so the barrier must DEFER rather than terminalize the run and+ * release its capacity mid-persistence.+ */+ it('defers settlement when a fresh Stop never acknowledges within the poll budget', async () => {+ const { service, methods } = outcomeService();+ methods.getScheduleRunAbortState = jest.fn(async () => ({+ status: 'started',+ abortSource: 'stop',+ abortRequestedAt: new Date(),+ }));++ const settled = await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'interrupted',+ });++ // Reported NOT settled so a durable-retry caller re-drives it, and the run is left+ // active rather than terminalized while persistence may still be in flight.+ expect(settled).toBe(false);+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();+ });++ it('does not gate a pause (requires_action) on the Stop barrier', async () => {+ const { service, methods } = outcomeService();+ methods.getScheduleRunAbortState = jest.fn(async () => null);++ await service.recordScheduleOutcome({+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ status: 'requires_action',+ });++ // A pause is not a settlement, so the barrier is never consulted.+ expect(methods.getScheduleRunAbortState).not.toHaveBeenCalled();+ });+});++describe('attachment hold renewal', () => {+ it('renews the bounded upload hold at fire preflight, best-effort', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ extendFilesTTL: jest.Mock;+ getFiles: jest.Mock;+ };+ // The hold bridges upload -> first consumption; a renewal failure must not fail+ // the fire (the file resolves now; at worst the hold lapses later).+ methods.extendFilesTTL = jest.fn(async () => {+ throw new Error('mongo down');+ });+ methods.getFiles = jest.fn(async () => [+ { file_id: 'f1', filepath: '/f1', filename: 'a.png', type: 'image/png', source: 'local' },+ ]);++ const files = await service.engineDeps.resolveFiles(['f1'], { id: 'user-1', tenantId: 't1' });+ expect(methods.extendFilesTTL).toHaveBeenCalledWith(+ ['f1'],+ expect.objectContaining({ renewMs: expect.any(Number), maxLifetimeMs: expect.any(Number) }),+ { user: 'user-1', tenantId: 't1' },+ );+ expect(files).toHaveLength(1);+ });+});++describe('isScheduleLive policy recheck', () => {+ const liveRow = { id: 's1', user: 'u1', enabled: true } as never;++ it('refuses a resume while the operator kill switch is up', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as { getScheduleById: jest.Mock };+ methods.getScheduleById = jest.fn(async () => liveRow);+ process.env.SCHEDULES_DISABLED = 'true';+ try {+ // The row-level checks alone pass; only the policy recheck sees the switch.+ await expect(service.isScheduleLive('s1')).resolves.toBe(true);+ // A pause can sit for hours; an approval must not start a billed continuation+ // the operator believes is stopped.+ await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);+ } finally {+ delete process.env.SCHEDULES_DISABLED;+ }+ });++ it('refuses a resume when the owner lost SCHEDULES:USE', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ getScheduleById: jest.Mock;+ getRoleByName: jest.Mock;+ };+ methods.getScheduleById = jest.fn(async () => liveRow);+ methods.getRoleByName = jest.fn(async () => ({ permissions: { SCHEDULES: { USE: false } } }));+ (service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(+ async () => ({ id: 'u1', tenantId: 't1', role: 'USER' }),+ );++ await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);+ });+});++describe('quiesceUserSchedules drain wait', () => {+ afterEach(() => {+ mockJobStore = null;+ jest.useRealTimers();+ jest.restoreAllMocks();+ });++ it('polls the active runs until they drain, then returns', async () => {+ jest.useFakeTimers();+ const active = [run()];+ const getActive = jest+ .fn<Promise<ActiveRun[]>, [string]>()+ .mockResolvedValueOnce(active) // initial collection for abort+ .mockResolvedValueOnce(active) // first poll: still settling+ .mockResolvedValue([]); // subsequent polls: drained+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ // Each poll waits one interval; advance twice so the loop observes the drain.+ await jest.advanceTimersByTimeAsync(250);+ await jest.advanceTimersByTimeAsync(250);+ // The rows DRAINED, so the runs are genuinely settled even though this harness has+ // no job store and the aborts could not be confirmed delivered. The drain is the+ // authority; an undelivered abort whose run then settled must not defer deletion+ // forever.+ await expect(pending).resolves.toBe(true);++ // Initial read + at least one poll that observed a non-empty set + the empty one.+ expect(getActive.mock.calls.length).toBeGreaterThanOrEqual(3);+ });++ it('stops waiting at the bounded deadline when runs never drain', async () => {+ jest.useFakeTimers();+ const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([run()]); // never drains+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ // Advance past the full bounded timeout; the loop must give up, not hang, and must+ // report the drain as UNCONFIRMED so deletion defers rather than destroying.+ await jest.advanceTimersByTimeAsync(10_000);+ await expect(pending).resolves.toBe(false);++ // It polled repeatedly (bounded by the deadline) and surfaced the un-drained runs.+ expect(getActive.mock.calls.length).toBeGreaterThan(1);+ expect(warn).toHaveBeenCalled();+ });++ /**+ * A `started` row whose job is already terminal is the preserve-for-reconcile case:+ * the inline outcome write exhausted its retries, so the retained job is the ONLY+ * evidence the run finished. Account deletion deletes that job (nothing will ever+ * reconcile it once the rows are hard-deleted), so it has to project the job's+ * terminal state onto the row first — otherwise the row stays active, the bounded+ * drain can never confirm, and deletion defers until the 30-minute orphan cutoff.+ */+ it('does not settle a pause hand-off still in flight (started row, paused job)', async () => {+ jest.useFakeTimers();+ const started = { ...run(), status: 'started' };+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]);+ const service = makeService(getActive);+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'requires_action',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),+ })),+ } as unknown as typeof mockJobStore;++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ // The pause hand-off's writes are still in flight; confirming the drain here+ // let the destructive cascade run before they landed. Deferral is bounded: the+ // controller (or the paused-window reconciler) flips the row to requires_action+ // and the pending-deletion sweep retries.+ await expect(pending).resolves.toBe(false);++ expect(recordRunOutcome).not.toHaveBeenCalled();+ });++ it('settles a started run from its retained terminal job before dropping the evidence', async () => {+ jest.useFakeTimers();+ const started = { ...run(), status: 'started' };+ const getActive = jest+ .fn<Promise<ActiveRun[]>, [string]>()+ .mockResolvedValueOnce([started])+ .mockResolvedValue([]);+ const service = makeService(getActive);+ // deleteJob MUST be present: without it the store call throws, the delete never+ // happens, and this test would pass while proving nothing about the ordering.+ const deleteJob = jest.fn(async () => undefined);+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'complete',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),+ })),+ deleteJob,+ } as unknown as typeof mockJobStore;++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(500);+ await expect(pending).resolves.toBe(true);++ expect(recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'success' }),+ );+ // The retained job is the ONLY evidence this run finished, so it may not be deleted+ // until the outcome is durably recorded.+ expect(deleteJob).toHaveBeenCalled();+ expect(recordRunOutcome.mock.invocationCallOrder[0]).toBeLessThan(+ deleteJob.mock.invocationCallOrder[0],+ );+ });++ /**+ * `aborted` is the abort REQUEST landing, not the generation finishing: abortJob wins+ * its status CAS before the owner unwinds, and both owner paths settle LAST, after+ * saveMessage. Settling on that evidence confirms the drain while the partial write is+ * still in flight — account deletion then destroys the user's data and the owner+ * writes a message back for the account that no longer exists.+ */+ it('does not settle a run whose abort is still in flight', async () => {+ jest.useFakeTimers();+ const aborting = {+ ...run(),+ status: 'started',+ abortRequestedAt: new Date(Date.now() - 1000),+ };+ const service = makeService(+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([aborting]),+ );+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'aborted',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),+ })),+ deleteJob: jest.fn(async () => undefined),+ } as unknown as typeof mockJobStore;++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ // Unconfirmed, so the caller defers deletion (503 + Retry-After) instead of+ // destroying data the owner is still writing.+ await expect(pending).resolves.toBe(false);+ expect(recordRunOutcome).not.toHaveBeenCalled();+ });++ /**+ * The other direction: an abort whose owner never came back is presumed dead past the+ * grace, or the account could never be deleted at all.+ */+ it('settles a run whose abort request has outlived the owner grace', async () => {+ jest.useFakeTimers();+ const abandoned = {+ ...run(),+ status: 'started',+ abortRequestedAt: new Date(Date.now() - 45 * 60 * 1000),+ };+ const getActive = jest+ .fn<Promise<ActiveRun[]>, [string]>()+ .mockResolvedValueOnce([abandoned])+ .mockResolvedValue([]);+ const service = makeService(getActive);+ mockJobStore = {+ getJob: jest.fn(async () => null),+ deleteJob: jest.fn(async () => undefined),+ } as unknown as typeof mockJobStore;++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(500);+ await expect(pending).resolves.toBe(true);+ expect(recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),+ );+ });++ it('keeps the retained job when settling it fails', async () => {+ jest.useFakeTimers();+ const started = { ...run(), status: 'started' };+ const service = makeService(+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]),+ );+ const deleteJob = jest.fn(async () => undefined);+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'complete',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),+ })),+ deleteJob,+ } as unknown as typeof mockJobStore;+ recordRunOutcome.mockRejectedValue(new Error('mongo down'));++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(11_000);+ // The row never settles, so the drain cannot confirm and deletion defers.+ await expect(pending).resolves.toBe(false);++ // Evidence intact for the next pass.+ expect(deleteJob).not.toHaveBeenCalled();+ });++ it('does not wait when the user has no active runs', async () => {+ jest.useFakeTimers();+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);+ const service = makeService(getActive);++ // Nothing to abort and nothing to drain, so the quiesce is trivially CONFIRMED and+ // the deletion cascade may proceed to its destructive steps.+ await expect(service.quiesceUserSchedules('user-1', 'attempt-1')).resolves.toBe(true);+ // Only the initial collection read; the drain loop is skipped for an empty set.+ expect(getActive).toHaveBeenCalledTimes(1);+ });++ /** A `requires_action` row whose HITL approval was resumed through the chat UI. */+ const pausedRun = (): ActiveRun => ({ ...run(), status: 'requires_action' });++ it('does not terminalize a paused row whose resumed generation is still running', async () => {+ jest.useFakeTimers();+ // Backward-compatibility case from a pre-capacity resume (or rolling deploy):+ // the job is running while the durable row still reads paused.+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ createdAt: 1,+ })),+ };+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ // Terminalizing here would drop the row out of the active set, so the drain would+ // report nothing to wait for and the destructive cascade could delete messages the+ // resumed generation is still able to persist.+ await expect(pending).resolves.toBe(false);+ expect(recordRunOutcome).not.toHaveBeenCalled();+ });++ it('still terminalizes a genuinely paused row so deletion is not blocked forever', async () => {+ jest.useFakeTimers();+ // Paused for approval, no live generation behind it.+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'requires_action',+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ createdAt: 1,+ })),+ };+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ await pending;+ // An approval that will never be consumed must not keep the account undeletable.+ expect(recordRunOutcome).toHaveBeenCalledWith(+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),+ );+ });++ it('does not terminalize a paused row when the job lookup FAILS', async () => {+ jest.useFakeTimers();+ mockJobStore = {+ getJob: jest.fn(async () => {+ throw new Error('redis unavailable');+ }),+ };+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ // A thrown lookup is evidence of NOTHING. Reading it as "genuinely paused" would+ // terminalize a row whose resumed generation may still be running, and the drain+ // would then permit the destructive cascade.+ await expect(pending).resolves.toBe(false);+ expect(recordRunOutcome).not.toHaveBeenCalled();+ });++ it('terminalizes a paused row when a replacement turn took over the conversation', async () => {+ jest.useFakeTimers();+ // A replacement user turn reuses the conversationId but strips the scheduled+ // identity, so this running job is NOT this occurrence's generation.+ mockJobStore = {+ getJob: jest.fn(async () => ({ status: 'running', createdAt: 1 })),+ };+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);+ const service = makeService(getActive);++ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');+ await jest.advanceTimersByTimeAsync(10_000);+ await pending;+ expect(recordRunOutcome).toHaveBeenCalled();+ });+});++describe('global kill switch', () => {+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);++ afterEach(() => {+ delete process.env.SCHEDULES_DISABLED;+ });++ it('is off by default', async () => {+ const service = makeService(noRuns());+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);+ });++ it('trips on the SCHEDULES_DISABLED env lever without reading config', async () => {+ process.env.SCHEDULES_DISABLED = 'true';+ // Throwing getAppConfig proves the env lever works even when the config plane is+ // unhealthy — the case where a config-dependent kill switch would fail.+ const getAppConfig = jest.fn(async () => {+ throw new Error('config plane down');+ }) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);+ expect(getAppConfig).not.toHaveBeenCalled();+ });++ it('trips on `interface.schedules: false` read from the BASE config only', async () => {+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>+ options?.baseOnly === true+ ? { interfaceConfig: { schedules: false } }+ : // A principal-merged view that re-enables must NOT be consulted: the global+ // stop is base-only so no role/user/tenant override can widen past it.+ { interfaceConfig: { schedules: true } },+ ) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);+ expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });+ });++ it('does not trip when only a principal-merged config disables it', async () => {+ // Per-principal availability is NOT the global stop; the engine keeps claiming so+ // other principals still fire, and the fire path skips this owner's occurrences.+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>+ options?.baseOnly === true+ ? { interfaceConfig: { schedules: true } }+ : { interfaceConfig: { schedules: false } },+ ) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);+ });++ it('trips on the object form `{ use: false }` exactly like the boolean stop', async () => {+ // Both stop shapes must FREEZE occurrences (engine stops claiming, nothing+ // advances). A shape-blind gate left the engine claiming while getLimits refused+ // fires, so the disabled path ADVANCED each occurrence — a short maintenance stop+ // silently dropped everything it covered instead of leaving it due.+ const getAppConfig = jest.fn(async () => ({+ interfaceConfig: { schedules: { use: false, maxPerUser: 5 } },+ })) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);+ });++ it('does not trip on the object form while `use` stays enabled', async () => {+ const getAppConfig = jest.fn(async () => ({+ interfaceConfig: { schedules: { use: true, maxPerUser: 5 } },+ })) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);+ });+});++describe('scheduled resume capacity', () => {+ function makeResumeService(occupancy: { takenSlots: number[]; unslotted: number }) {+ const methods = {+ getScheduleById: jest.fn(async () => ({+ id: 's1',+ user: 'user-1',+ enabled: true,+ configRevision: 3,+ })),+ getRoleByName: jest.fn(async () => ({ permissions: { SCHEDULES: { USE: true } } })),+ getCapacityOccupancy: jest.fn(async () => occupancy),+ acquireResumeLease: jest.fn(async () => ({+ id: 's1',+ claimToken: 'resume-token',+ leaseBy: 'resume:resume-token',+ })),+ consumeResumeLease: jest.fn(async () => true),+ releaseLeaseByHolder: jest.fn(async () => undefined),+ markRunResumeClaimed: jest.fn(async (_id, _scheduledFor, capacitySlot) => ({+ capacitySlot,+ })),+ releaseRunResumeClaim: jest.fn(async () => true),+ };+ const service = createSchedulesService({+ methods: methods as unknown as SchedulesServiceDeps['methods'],+ getAppConfig: jest.fn(async () => ({+ interfaceConfig: {+ schedules: {+ use: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 1,+ },+ },+ })),+ findUserById: jest.fn(async () => ({ _id: 'user-1', role: 'USER' })),+ findBalance: jest.fn(async () => null),+ upsertBalance: jest.fn(async () => null),+ initializeNullBalance: jest.fn(async () => null),+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),+ isUserDeleting: jest.fn(async () => false),+ enqueueAgentTrigger: jest.fn(async () => undefined),+ getTriggerDelivery: jest.fn(async () => null),+ } as unknown as SchedulesServiceDeps);+ return { service, methods };+ }++ it('promotes a pause through the global slot allocator and releases by exact slot', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ const scheduledFor = '2026-08-17T12:00:00.000Z';++ await expect(service.claimScheduleResume('s1', scheduledFor)).resolves.toEqual({+ capacitySlot: 0,+ claimToken: 'resume-token',+ leaseBy: 'resume:resume-token',+ });+ expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', undefined, true, 60_000);+ expect(methods.markRunResumeClaimed).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);+ await expect(+ service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {+ expectedConfigRevision: 3,+ automatic: true,+ }),+ ).resolves.toBe(true);+ expect(methods.consumeResumeLease).toHaveBeenCalledWith(+ 's1',+ 'resume-token',+ 'resume:resume-token',+ true,+ 3,+ );+ await expect(service.releaseScheduleResumeClaim('s1', scheduledFor, 0)).resolves.toBe(true);+ expect(methods.releaseRunResumeClaim).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);+ });++ it('leaves the paused row untouched when deployment capacity is full', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [0], unslotted: 0 });++ await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({+ conflict: 'capacity',+ });+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');+ });++ it('normalizes an exhausted slot-collision retry to public capacity conflict', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ methods.markRunResumeClaimed.mockResolvedValue({ conflict: 'slot-taken' } as never);++ await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({+ conflict: 'capacity',+ });+ });++ it('refuses a resume whose schedule revision changed before the capacity handoff', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });++ await expect(+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {+ expectedConfigRevision: 2,+ automatic: true,+ }),+ ).resolves.toEqual({ conflict: 'inactive' });+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();+ });++ it('refuses an edit that wins during async policy checks at the final schedule CAS', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ methods.acquireResumeLease.mockResolvedValue(null as never);++ await expect(+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {+ expectedConfigRevision: 3,+ automatic: true,+ }),+ ).resolves.toEqual({ conflict: 'inactive' });+ expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', 3, true, 60_000);+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();+ });++ it('releases the old holder when the post-approval schedule handoff loses', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ methods.consumeResumeLease.mockResolvedValue(false);++ await expect(+ service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {+ expectedConfigRevision: 3,+ automatic: true,+ }),+ ).resolves.toBe(false);+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');+ });++ it('refuses a disabled automatic resume but preserves explicit Run Now semantics', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ methods.getScheduleById.mockResolvedValue({+ id: 's1',+ user: 'user-1',+ enabled: false,+ configRevision: 3,+ });++ await expect(+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {+ expectedConfigRevision: 3,+ automatic: true,+ }),+ ).resolves.toEqual({ conflict: 'inactive' });+ await expect(+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {+ expectedConfigRevision: 3,+ automatic: false,+ }),+ ).resolves.toEqual({+ capacitySlot: 0,+ claimToken: 'resume-token',+ leaseBy: 'resume:resume-token',+ });+ });++ it('refuses a resume when the owner lost schedule access before the capacity handoff', async () => {+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });+ methods.getRoleByName.mockResolvedValue({ permissions: { SCHEDULES: { USE: false } } });++ await expect(+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {+ expectedConfigRevision: 3,+ automatic: true,+ }),+ ).resolves.toEqual({ conflict: 'inactive' });+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();+ });+});++describe('deployment-wide limits', () => {+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);++ it('resolves a principal-less getLimits from the BASE config only', async () => {+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>+ options?.baseOnly === true+ ? { interfaceConfig: { schedules: { use: true, fireConcurrency: 1 } } }+ : // The principal/tenant-merged view. A bare getAppConfig() resolves THIS,+ // including whatever tenant the ALS context happens to carry.+ { interfaceConfig: { schedules: { use: true, fireConcurrency: 5 } } },+ ) as unknown as SchedulesServiceDeps['getAppConfig'];+ const service = makeService(noRuns(), getAppConfig);+ const limits = await service.getLimits();+ // NO principal must mean the DEPLOYMENT's config. Both callers of this form run+ // inside a tenant context (fireSchedule clamps the global capacity allocator from+ // within runInTenantContext(owner); the engine tick budgets from within+ // runAsSystem), so resolving the merged view would let a tenant override widen the+ // very global cap it is clamped against.+ expect(limits.fireConcurrency).toBe(1);+ expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });+ });+});++describe('durable trigger wiring', () => {+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);++ it('uses the shared trigger service instead of owning a second loopback transport', async () => {+ const enqueueAgentTrigger = jest.fn(async () => ({ status: 'queued' }));+ const service = makeService(noRuns(), undefined, enqueueAgentTrigger);+ const envelope = { version: 1, mode: 'fire' } as never;++ await service.engineDeps.enqueueTrigger(envelope, { orderingKey: 'schedule-1' });++ expect(enqueueAgentTrigger).toHaveBeenCalledWith(envelope, {+ orderingKey: 'schedule-1',+ });+ });+});++describe('admission revision fence', () => {+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);++ function serviceWithSchedule(schedule: { configRevision?: number; enabled?: boolean } | null) {+ const service = makeService(noRuns());+ (+ service.engineDeps.methods as unknown as {+ getScheduleById: jest.Mock;+ }+ ).getScheduleById = jest.fn(async () => schedule);+ return service;+ }++ it('admits when the claimed revision still matches', async () => {+ const service = serviceWithSchedule({ configRevision: 3 });+ expect(await service.isScheduleLive('sched-1', 3)).toBe(true);+ });++ it('REFUSES when an owner edit moved the revision on after the claim', async () => {+ // The fire was claimed under revision 3; the owner edited since (now 4). Persisting+ // would write the OLD prompt/agent into the edited schedule's history.+ const service = serviceWithSchedule({ configRevision: 4 });+ expect(await service.isScheduleLive('sched-1', 3)).toBe(false);+ });++ /**+ * A policy auto-disable flips `enabled` WITHOUT touching configRevision (an older+ * paused occurrence can resume, fail and cross the threshold while a newer occurrence+ * is already in the claim-to-controller window), so the revision fence cannot see it.+ */+ it('refuses an AUTOMATIC fire once the schedule was disabled', async () => {+ const service = serviceWithSchedule({ configRevision: 3, enabled: false });+ expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(false);+ });++ it('still admits Run Now on a disabled schedule', async () => {+ // An explicit user action, matching fireScheduleNow's own relaxation.+ const service = serviceWithSchedule({ configRevision: 3, enabled: false });+ expect(await service.isScheduleLive('sched-1', 3, { automatic: false })).toBe(true);+ expect(await service.isScheduleLive('sched-1', 3)).toBe(true);+ });++ it('admits an automatic fire while the schedule is still enabled', async () => {+ const service = serviceWithSchedule({ configRevision: 3, enabled: true });+ expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(true);+ });++ it('refuses a schedule that is gone regardless of revision', async () => {+ const service = serviceWithSchedule(null);+ expect(await service.isScheduleLive('sched-1', 3)).toBe(false);+ });++ it('stays permissive when either side has no revision (pre-existing rows)', async () => {+ expect(await serviceWithSchedule({}).isScheduleLive('sched-1', 3)).toBe(true);+ expect(await serviceWithSchedule({ configRevision: 4 }).isScheduleLive('sched-1')).toBe(true);+ });+});++describe('abort stamp is load-bearing (withheld abort on stamp failure)', () => {+ it('withholds the abort and reports unconfirmed when the stamp cannot be made durable', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ markScheduleDeleting: jest.Mock;+ getActiveRunsForSchedule: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ requestRunAbort: jest.Mock;+ };+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));+ methods.getActiveRunsForSchedule = jest.fn(async () => [+ {+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ },+ ]);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);+ // The stamp is what makes concurrent drains and the reconciler defer to the+ // owner's settle; signalling an abort without it re-opens the drain-mid-write+ // window the stamp exists to close.+ methods.requestRunAbort = jest.fn(async () => {+ throw new Error('mongo down');+ });+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: true }));++ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');+ expect(manager.abortJob).not.toHaveBeenCalled();+ });++ it('reports an exact generation that remains active as not stopped', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({+ success: false,+ failureReason: 'job_still_active',+ }));++ const delivered = await service.engineDeps.abortScheduledJob(+ 'c1',+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },+ { preserve: true },+ );+ expect(delivered).toBe(false);+ });+});++describe('deleteScheduleForOwner waits for the settle acknowledgement', () => {+ it('converges when an unconfirmed abort settles during the bounded drain', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const methods = service.engineDeps.methods as unknown as {+ markScheduleDeleting: jest.Mock;+ getActiveRunsForSchedule: jest.Mock;+ eraseScheduleIfDrained: jest.Mock;+ };+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));+ // Active at the abort pass, drained by the first drain poll: the owner settled.+ methods.getActiveRunsForSchedule = jest+ .fn()+ .mockResolvedValueOnce([+ {+ scheduleId: 's1',+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),+ conversationId: 'c1',+ status: 'started',+ },+ ])+ .mockResolvedValue([]);+ methods.eraseScheduleIfDrained = jest.fn(async () => true);+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'running',+ createdAt: 1,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ // Delivery cannot be locally proven (peer-owned generation): honest false...+ manager.abortJob = jest.fn(async () => ({ success: true, signalDelivered: false }));++ // ...but the run row leaving the active set is the durable acknowledgement, so+ // the delete converges instead of answering a spurious 503.+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');+ });+});++describe('provider-drained schedule aborts', () => {+ it('waits on the exact terminal generation instead of using the removed re-signal path', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'aborted',+ createdAt: 7,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: false }));++ const delivered = await service.engineDeps.abortScheduledJob(+ 'c1',+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },+ { preserve: true },+ );+ expect(manager.abortJob).toHaveBeenCalledWith('c1', {+ expectedCreatedAt: 7,+ awaitProviderDrain: true,+ });+ expect(delivered).toBe(true);+ });++ it('deletes terminal evidence only after the exact provider drain is confirmed', async () => {+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));+ const deleteJob = jest.fn(async () => true);+ mockJobStore = {+ getJob: jest.fn(async () => ({+ status: 'aborted',+ createdAt: 7,+ scheduleId: 's1',+ scheduledFor: '2026-01-01T00:00:00.000Z',+ })),+ deleteJob,+ } as unknown as typeof mockJobStore;+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;+ manager.abortJob = jest.fn(async () => ({ success: false }));++ const delivered = await service.engineDeps.abortScheduledJob(+ 'c1',+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },+ { preserve: false },+ );+ expect(delivered).toBe(true);+ expect(manager.abortJob).toHaveBeenCalledWith('c1', {+ expectedCreatedAt: 7,+ awaitProviderDrain: true,+ });+ expect(deleteJob).toHaveBeenCalledWith('c1', 7);+ });+});diff --git a/packages/api/src/schedules/service.ts b/packages/api/src/schedules/service.tsnew file mode 100644index 0000000000..393a724228--- /dev/null+++ b/packages/api/src/schedules/service.ts@@ -0,0 +1,1610 @@+import { logger, runAsSystem, tenantStorage, isRuntimeDisabled } from '@librechat/data-schemas';+import { getRefillEligibilityDate, Permissions, PermissionTypes } from 'librechat-data-provider';+import type { ScheduleMethods, AppConfig, IBalance } from '@librechat/data-schemas';+import type { TCheckpointerConfig } from 'librechat-data-provider';+import type { Types } from 'mongoose';+import type {+ ScheduleEngineDeps,+ ScheduleDeleteResult,+ ScheduleLimits,+ ScheduleUserContext,+ FireableSchedule,+ FireResult,+ JobIdentity,+} from './types';+import type { SerializableJobData } from '../stream/interfaces/IJobStore';+import type { BalanceUpdateFields } from '../types/balance';+import type { GetAppConfigOptions } from '../app/service';+import {+ DEFAULT_SCHEDULE_LIMITS,+ SCHEDULE_FILE_HOLD,+ hasResumeHandoffInFlight,+ hasAbortInFlight,+} from './types';+import { deleteAgentCheckpoint, captureAgentCheckpointGeneration } from '../agents/checkpointer';+import { fireSchedule, BALANCE_SKIP_DISABLE_THRESHOLD } from './fire';+import { GenerationJobManager } from '../stream/GenerationJobManager';+import { buildBalanceUpdateFields } from '../middleware/balance';+import { getAppConfigOptionsFromUser } from '../app/service';+import { isShutdownInProgress } from '../app/shutdown';+import { startScheduleErasureSweep } from './erasure';+import { getBalanceConfig } from '../app/config';+import { startScheduleEngine } from './engine';+import { withCapacitySlot } from './capacity';+import { isEnabled } from '../utils/common';++/** Recordable terminal/paused run outcome, as accepted by `recordRunOutcome`. */+type ScheduleRunOutcomeStatus = Parameters<ScheduleMethods['recordRunOutcome']>[0]['status'];++/** How a TERMINAL job status projects onto its run row. Mirrors the reconciler's+ * mapping, so a run settled from a retained job reads the same either way. */+const TERMINAL_JOB_OUTCOMES: Record<string, ScheduleRunOutcomeStatus | undefined> = {+ complete: 'success',+ error: 'error',+ aborted: 'interrupted',+};++/** Reason text recorded when account-deletion quiesce settles a run itself. */+const QUIESCE_SETTLE_ERRORS: Partial<Record<ScheduleRunOutcomeStatus, string>> = {+ interrupted: 'Account deleted while awaiting approval',+ error: 'Run ended in error',+};++/** Short schedule-document fence spanning capacity admission -> approval CAS. */+const SCHEDULE_RESUME_LEASE_MS = 60_000;++/**+ * Whether this process may arm the scheduler at all.+ *+ * v1 is single-process by design, but the standard entrypoint runs+ * `initializeScheduleEngine` in EVERY replica and nothing stops an operator from scaling+ * it. That is only safe when replicas can see each other's generations: a shared stream+ * store (Redis) gives every replica the same job view, so reconciliation and+ * deletion-time aborts reach the run's real owner. With the process-local store a peer+ * sees the globally visible `started` row but no job, and after the orphan cutoff it+ * marks a still-running generation interrupted and frees its capacity slot.+ *+ * A single replica with the in-memory store is perfectly safe, but a process cannot+ * observe its own replica count, so that case needs an explicit operator assertion.+ */+function isTopologySafeToArm(): boolean {+ return GenerationJobManager.isRedis || isEnabled(process.env.SCHEDULES_SINGLE_PROCESS);+}++/** Whether a persisted job still carries a given scheduled occurrence's identity. */+function jobMatchesIdentity(+ job: Pick<SerializableJobData, 'scheduleId' | 'scheduledFor'>,+ identity: JobIdentity,+): boolean {+ if (job.scheduleId !== identity.scheduleId || job.scheduledFor == null) {+ return false;+ }+ return new Date(job.scheduledFor).getTime() === new Date(identity.scheduledFor).getTime();+}++export interface RecordScheduleOutcomeInput {+ scheduleId?: string;+ scheduledFor?: string | Date;+ /** Exact generation whose terminal evidence is being persisted. */+ streamId?: string;+ jobCreatedAt?: number;+ status: ScheduleRunOutcomeStatus;+ conversationId?: string;+ /** Erase the row's reserved conversationId (pre-start abort: no conversation exists). */+ clearConversationId?: boolean;+ error?: string;+}++export type ScheduleResumeClaimResult =+ | { capacitySlot: number; claimToken: string; leaseBy: string }+ | { conflict: 'capacity' | 'overlap' | 'not-paused' | 'inactive' };++/**+ * Api-side dependencies the schedules service needs injected: model methods,+ * config/balance access, and the owner-scoped agent access check. Everything+ * else (job store and tenant context) lives in `@librechat/api` and is imported+ * directly.+ */+export interface SchedulesServiceDeps {+ methods: ScheduleMethods & {+ getRoleByName: (+ role?: string,+ ) => Promise<{ permissions?: Record<string, Record<string, boolean | undefined>> } | null>;+ getFiles: (+ filter: unknown,+ sort: unknown,+ select: unknown,+ ) => Promise<Array<{+ file_id: string;+ filepath: string;+ filename: string;+ type: string;+ height?: number;+ width?: number;+ source: string;+ }> | null>;+ /** Owner-scoped bounded TTL hold (`db.extendFilesTTL`-shaped). */+ extendFilesTTL: (+ fileIds: string[],+ hold: { renewMs: number; maxLifetimeMs: number },+ owner: { user: string; tenantId?: string | null },+ ) => Promise<number>;+ };+ getAppConfig: (options?: GetAppConfigOptions) => Promise<AppConfig | undefined>;+ findUserById: (+ userId: string | Types.ObjectId,+ ) => Promise<{ _id: Types.ObjectId; tenantId?: string; role?: string } | null>;+ findBalance: (userId: string) => Promise<IBalance | null>;+ /**+ * Upserts a balance record. `setOnInsert` carries fields that must ONLY apply to a+ * document this call creates — chiefly the starting credit — so a record created by a+ * concurrent charge is never overwritten with a fresh balance.+ */+ upsertBalance: (+ userId: string,+ update: { set: Partial<BalanceUpdateFields>; setOnInsert: Partial<BalanceUpdateFields> },+ ) => Promise<IBalance | null>;+ /**+ * Compare-and-set initialization for an EXISTING balance whose `tokenCredits` is still+ * null. Writes the starting credit (and any refill-config sync) only while+ * `{ user, tokenCredits: null }` still matches — never an upsert — so a concurrent+ * initializer/charge that already set credits between the read and this write is not+ * clobbered. Returns the winning document, or `null` when the CAS did not match (the+ * caller re-reads the winner rather than restoring credits).+ */+ initializeNullBalance: (+ userId: string,+ update: { tokenCredits: number; sync: Partial<BalanceUpdateFields> },+ ) => Promise<IBalance | null>;+ resolveAgentFireAccess: (+ agentId: string,+ user: ScheduleUserContext,+ ) => Promise<'ok' | 'missing' | 'forbidden'>;+ /** Whether this user's account deletion has begun. Fail-closed (unknown == true). */+ isUserDeleting: (userId: string) => Promise<boolean>;+ /** Shared durable trigger admission from the merged agent-trigger service. */+ enqueueAgentTrigger: ScheduleEngineDeps['enqueueTrigger'];+ /** Reads a durable trigger delivery by key from the merged agent-trigger service, for+ * reconciliation's live-vs-dead delivery check. */+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'];+}++export interface SchedulesService {+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;+ engineDeps: ScheduleEngineDeps;+ fireScheduleNow: (+ schedule: FireableSchedule,+ limits: ScheduleLimits,+ ) => Promise<FireResult | null>;+ recordScheduleOutcome: (input: RecordScheduleOutcomeInput) => Promise<boolean>;+ /**+ * Stamps a scheduled run's interactive Stop BEFORE the abort is signalled, so the owner+ * settlement barrier, reconciliation, and schedule/account deletion hold off settling or+ * erasing until {@link acknowledgeScheduledStopPersistence}. Serialized: `'in_progress'`+ * means a fresh Stop already owns the stamp and the caller must not signal a second abort;+ * `false` means there is no active run to stop.+ */+ beginScheduledStop: (input: {+ scheduleId: string;+ scheduledFor: string | Date;+ }) => Promise<boolean | 'in_progress'>;+ /**+ * Releases the Stop settlement barrier once the route's partial-message/checkpoint writes+ * have landed. Call ONLY after persistence succeeds — a failed persistence must leave the+ * barrier unresolved so the run stays preserved and recovers via the stale-owner timeout.+ */+ acknowledgeScheduledStopPersistence: (input: {+ scheduleId: string;+ scheduledFor: string | Date;+ /** Optional terminal outcome to re-drive once the barrier clears, so a settlement the+ * owner deferred past its poll budget still converges where no reconciler is armed. */+ settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };+ }) => Promise<void>;+ /** Re-enters a paused occurrence into the DB-enforced global/same-schedule+ * capacity set before its approval job is allowed to resume. */+ claimScheduleResume: (+ scheduleId: string,+ scheduledFor: string | Date,+ options?: { expectedConfigRevision?: number; automatic?: boolean },+ ) => Promise<ScheduleResumeClaimResult>;+ /** Guarded rollback when the approval CAS did not consume the paused action. */+ releaseScheduleResumeClaim: (+ scheduleId: string,+ scheduledFor: string | Date,+ capacitySlot: number,+ ) => Promise<boolean>;+ /** Atomically validates the live schedule generation after the approval CAS and+ * releases its short-lived document fence. This is the resume linearization point. */+ finalizeScheduleResumeClaim: (+ scheduleId: string,+ claimToken: string,+ leaseBy: string,+ options?: { expectedConfigRevision?: number; automatic?: boolean },+ ) => Promise<boolean>;+ /** Releases only the schedule-document fence after an unconsumed/ambiguous approval CAS. */+ releaseScheduleResumeFence: (scheduleId: string, leaseBy: string) => Promise<void>;+ /**+ * Whether a schedule is still live (exists and not soft-deleted). The loopback+ * chat controller calls this right after creating the generation job to re-fence+ * a fire against a delete/quiesce that landed in the claim -> POST window (when+ * the reservation row exists but the job did not yet, so the deletion's abort+ * missed it) — aborting before any messages are persisted.+ */+ isScheduleLive: (+ scheduleId: string,+ expectedConfigRevision?: number,+ options?: { automatic?: boolean; policy?: boolean },+ ) => Promise<boolean>;+ /** Soft-deletes an owner's schedule: stop claims, abort active runs, drain, erase. */+ deleteScheduleForOwner: (scheduleId: string, userId: string) => Promise<ScheduleDeleteResult>;+ /**+ * Quiesces all of a user's schedules ahead of account deletion (reversible suspension ++ * abort + drain). `token` identifies this deletion attempt and is what a later+ * {@link restoreUserSchedulesFromDeletion} restores against. Returns whether the drain was+ * CONFIRMED: false means at least one run could not be confirmed settled, and the caller+ * must NOT proceed to destructive deletion — the durable barrier keeps refusing new work+ * while a later pass finishes the cascade.+ */+ quiesceUserSchedules: (userId: string, token: string) => Promise<boolean>;+ /**+ * Reverses a quiesce whose account deletion was cancelled, re-enabling and re-arming only+ * the rows this exact attempt suspended. Safe to call even if quiesce never suspended a+ * row (no matching token → no-op).+ */+ restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise<void>;+ /** Arms the scheduler for THIS process. v1 is single-process only; the clustered+ * entrypoint does not start it. Returns undefined when index creation failed, or when+ * the topology cannot be shown safe (process-local job store with no single-process+ * assertion) — see isTopologySafeToArm. */+ initializeScheduleEngine: () => Promise<ReturnType<typeof startScheduleEngine> | undefined>;+ /** Starts erasure-ONLY maintenance for an entrypoint that never arms the engine (the+ * clustered worker). Idempotent per process and a no-op once the full engine is armed.+ * Arms nothing else — no claims, firing, cadence advancement, or absence-based+ * reconciliation — and refuses to infer owner death from a process-local missing job+ * (isTopologySafeToArm gates that). See startScheduleErasureSweep. */+ initializeScheduleErasureSweep: () => void;+}++/** Test-only overrides for the service's bounded waits (drains, barriers). */+export interface ScheduleServiceTimings {+ drainTimeoutMs?: number;+ drainPollMs?: number;+ stopBarrierTimeoutMs?: number;+ stopBarrierPollMs?: number;+}++/**+ * Builds the scheduler service around api-side dependencies. Each call owns its+ * own engine singleton and job-store-shared flag, so state never leaks between+ * instances.+ */+export function createSchedulesService(+ deps: SchedulesServiceDeps,+ timings?: ScheduleServiceTimings,+): SchedulesService {+ const { methods } = deps;++ // Fail LOUDLY at construction, not per-fire. The JS adapter (api/server/services/+ // Schedules) is not typechecked against SchedulesServiceDeps, so a missing dep would+ // otherwise surface only as a `deps.X is not a function` deep inside a live fire —+ // which is exactly how the deletion-barrier probe shipped unwired twice.+ const REQUIRED_DEPS: Array<keyof SchedulesServiceDeps> = [+ 'methods',+ 'getAppConfig',+ 'findUserById',+ 'findBalance',+ 'upsertBalance',+ 'initializeNullBalance',+ 'resolveAgentFireAccess',+ 'isUserDeleting',+ 'enqueueAgentTrigger',+ 'getTriggerDelivery',+ ];+ for (const key of REQUIRED_DEPS) {+ if (deps[key] == null) {+ throw new Error(`createSchedulesService: missing required dependency "${key}"`);+ }+ }++ /**+ * Resolves schedule limits, honoring per-principal (role/user) config overrides+ * when a user is supplied (routes pass req.user, the fire path passes the owner).+ */+ async function getLimits(user?: ScheduleUserContext): Promise<ScheduleLimits> {+ // The BASE `interface.schedules: false` is a global stop and must win over any+ // principal override. Without this a tenant/role/user override resolving to+ // enabled would let the sidebar and CRUD handlers admit schedules that+ // isGloballyDisabled() correctly refuses to ever fire.+ // isRuntimeDisabled, NOT `=== false`: the base stop has TWO shapes (`false` and+ // `{ use: false }`), and deepMerge turns a base `{ use: false, ... }` plus a principal+ // override of `true` into `{ use: true, ... }`. A literal-false check missed that, so+ // getLimits reported the feature enabled and Run Now dispatched straight through+ // fireSchedule — bypassing the operator's object-form emergency stop. Same predicate+ // the engine gate (isGloballyDisabled) already uses.+ if (+ user != null &&+ isRuntimeDisabled((await deps.getAppConfig({ baseOnly: true }))?.interfaceConfig?.schedules)+ ) {+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };+ }+ // NO principal means the DEPLOYMENT's config, so it must be read base-only. A bare+ // getAppConfig() still resolves principals and picks up the tenant from the ALS+ // context, and both callers of this form run inside one: fireSchedule clamps the+ // global capacity allocator from within runInTenantContext(owner), and the engine+ // tick budgets from within runAsSystem. Either would otherwise resolve a TENANT+ // override as if it were the deployment-wide value, which is exactly what the+ // global cap exists to prevent an override from widening.+ const appConfig = user+ ? await deps.getAppConfig(getAppConfigOptionsFromUser(user))+ : await deps.getAppConfig({ baseOnly: true });+ // The env kill switch is a GLOBAL stop and must be visible everywhere limits are+ // consulted (write handlers, fire path), not only at the engine tick.+ if (isEnabled(process.env.SCHEDULES_DISABLED)) {+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };+ }+ const config = appConfig?.interfaceConfig?.schedules;+ // EXPERIMENTAL, default-OFF (v1): scheduled chats are disabled unless an admin+ // explicitly enables them. Absence, null, or `false` all resolve to disabled, so a+ // deployment that never opts in never runs the scheduler. `true` uses the defaults;+ // an object opts in unless it sets `use: false`.+ if (config == null || config === false) {+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };+ }+ if (config === true) {+ return DEFAULT_SCHEDULE_LIMITS;+ }+ return {+ enabled: config.use !== false,+ maxPerUser: config.maxPerUser ?? DEFAULT_SCHEDULE_LIMITS.maxPerUser,+ minIntervalMinutes: config.minIntervalMinutes ?? DEFAULT_SCHEDULE_LIMITS.minIntervalMinutes,+ autoDisableAfterFailures:+ config.autoDisableAfterFailures ?? DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,+ fireConcurrency: config.fireConcurrency ?? DEFAULT_SCHEDULE_LIMITS.fireConcurrency,+ };+ }++ const MANUAL_RUN_LEASE_MS = 5 * 60 * 1000;+ // Bounded wait for aborted scheduled runs to settle during account-deletion quiesce,+ // before the message/conversation cascade runs. Long enough to cover a generation that+ // already returned from the model finishing its persistence; capped so account deletion+ // never blocks indefinitely on an unreachable peer-worker run.+ const QUIESCE_DRAIN_TIMEOUT_MS = timings?.drainTimeoutMs ?? 10 * 1000;+ const QUIESCE_DRAIN_POLL_MS = timings?.drainPollMs ?? 250;+ // The owner's terminal settlement waits (briefly, bounded) for an interactive Stop to+ // acknowledge its partial-message/checkpoint persistence before releasing capacity or+ // erasing evidence. Short: the Stop route's persistence is a couple of writes; if it+ // does not land in this window the stale-owner timeout (ABORT_OWNER_PRESUMED_ALIVE_MS)+ // takes over as the bounded recovery, so the settlement never blocks indefinitely.+ const STOP_BARRIER_TIMEOUT_MS = timings?.stopBarrierTimeoutMs ?? 5 * 1000;+ const STOP_BARRIER_POLL_MS = timings?.stopBarrierPollMs ?? 100;++ /**+ * Whether a refill would top up this zero-credit balance record right now,+ * mirroring the chat balance check's auto-refill eligibility (record-based).+ */+ function isRefillEligible(record: IBalance | null | undefined): boolean {+ if (record?.autoRefillEnabled !== true) {+ return false;+ }+ if (!(typeof record.refillAmount === 'number' && record.refillAmount > 0)) {+ return false;+ }+ const lastRefillDate = new Date(record.lastRefill ?? 0);+ if (Number.isNaN(lastRefillDate.getTime())) {+ return true;+ }+ // Mirror checkBalanceRecord's fallbacks exactly (interval 0 / 'days' when a+ // partially-synced record is missing them) so we never pre-skip a record the+ // interactive chat balance check would have refilled.+ return (+ new Date() >=+ getRefillEligibilityDate(+ lastRefillDate,+ record.refillIntervalValue ?? 0,+ record.refillIntervalUnit ?? 'days',+ )+ );+ }++ const engineDeps: ScheduleEngineDeps = {+ methods,+ getLimits,+ // On the BASE deps, not only the engine's per-pass wrapper: fireScheduleNow+ // (manual Run Now) dispatches with these deps directly, and its POST must be+ // gated by the same coordinator signal as the engine tick's.+ isShuttingDown: isShutdownInProgress,+ getUserContext: async (userId) => {+ const user = await deps.findUserById(userId);+ if (user == null) {+ return null;+ }+ return { id: user._id.toString(), tenantId: user.tenantId, role: user.role };+ },+ hasScheduleAccess: async (user) => {+ const role = await methods.getRoleByName(user.role);+ return role?.permissions?.[PermissionTypes.SCHEDULES]?.[Permissions.USE] === true;+ },+ isOutOfBalance: async (user) => {+ const appConfig = await deps.getAppConfig(getAppConfigOptionsFromUser(user));+ const balanceConfig = getBalanceConfig(appConfig);+ if (balanceConfig?.enabled !== true) {+ return false;+ }+ let record = await deps.findBalance(user.id);+ // Initialize/sync the record exactly as the chat's balance middleware would,+ // so a new user's startBalance is applied before we read it (avoids skipping+ // a schedule that an interactive chat would have allowed).+ if (balanceConfig.startBalance != null) {+ const updateFields = buildBalanceUpdateFields(balanceConfig, record, user.id);+ if (Object.keys(updateFields).length > 0) {+ // The read above and every write below are separate statements, and the credit+ // field is an INITIALIZATION value: a concurrent charge that set-and-spent the+ // record in between must never be handed back its starting balance. Each case+ // fences the credit write so it can only create, never restore, credits. The+ // refill-config fields are a genuine sync and never carry a credit.+ const { user: initUser, tokenCredits, ...syncFields } = updateFields;+ if (record == null) {+ // ABSENT record: upsert. The credit rides `$setOnInsert` so a document created+ // by a concurrent charge keeps its charged balance; the refill-config sync is+ // a legitimate `$set`.+ record = await deps.upsertBalance(user.id, {+ set: syncFields,+ setOnInsert: {+ ...(initUser != null ? { user: initUser } : {}),+ ...(tokenCredits != null ? { tokenCredits } : {}),+ },+ });+ } else if (tokenCredits != null) {+ // EXISTING record with a null credit: a blind `$set` would restore credits a+ // concurrent initializer/charge already set and spent between the read and+ // here. Initialize under a `{ tokenCredits: null }` CAS; on a miss re-read the+ // winner's balance instead of overwriting it.+ const initialized = await deps.initializeNullBalance(user.id, {+ tokenCredits,+ sync: syncFields,+ });+ record = initialized ?? (await deps.findBalance(user.id)) ?? record;+ } else if (Object.keys(syncFields).length > 0) {+ // EXISTING record with a real credit: only refill-config sync remains, which+ // never touches `tokenCredits` and is safe to `$set` directly.+ record = await deps.upsertBalance(user.id, { set: syncFields, setOnInsert: {} });+ }+ }+ }+ const credits = record?.tokenCredits ?? 0;+ if (credits > 0) {+ return false;+ }+ // At/below zero: an auto-refill user is only spared a pre-skip when a refill+ // would actually fire now (mirrors the chat balance check's eligibility). If+ // they aren't eligible yet, or the refill settings are incomplete, pre-skip as+ // a balance skip — otherwise the zero-credit fire reaches the chat, is rejected+ // there, and records a generic error that walks the schedule toward+ // too_many_failures instead of skipped_balance/insufficient_balance.+ if (balanceConfig.autoRefillEnabled === true && isRefillEligible(record)) {+ return false;+ }+ return true;+ },+ // Mirrors the loopback chat route's authorization (role AGENTS:USE + resource+ // VIEW with the manage:agents bypass); shared with the create/update precheck+ // so the two never diverge.+ agentAccess: (agentId, user) => deps.resolveAgentFireAccess(agentId, user),+ resolveFiles: async (fileIds, user) => {+ // Renew the bounded upload hold at every fire preflight, BEST-EFFORT: the hold+ // only has to bridge upload -> first consumption (a real send clears the TTL+ // permanently), and a failed renewal must not fail the fire — at worst the hold+ // lapses later and resolveFiles drops the reaped file (droppedFileIds records it).+ await methods+ .extendFilesTTL(fileIds, SCHEDULE_FILE_HOLD, { user: user.id, tenantId: user.tenantId })+ .catch((err) => logger.warn('[schedules] attachment hold renewal failed:', err));+ const files = await methods.getFiles(+ { file_id: { $in: fileIds }, user: user.id },+ null,+ '-text',+ );+ return (files ?? []).map((file) => ({+ file_id: file.file_id,+ filepath: file.filepath,+ filename: file.filename,+ type: file.type,+ height: file.height,+ width: file.width,+ source: file.source,+ }));+ },+ enqueueTrigger: deps.enqueueAgentTrigger,+ getTriggerDelivery: deps.getTriggerDelivery,+ runInTenantContext: (user, fn) =>+ tenantStorage.run({ tenantId: user.tenantId, userId: user.id }, fn),+ getJobStatus: async (conversationId) => {+ const job = await GenerationJobManager.getJobStore()?.getJob(conversationId);+ if (job == null) {+ return null;+ }+ return {+ status: job.status,+ createdAt: job.createdAt,+ scheduleId: job.scheduleId,+ scheduledFor: job.scheduledFor,+ createdEventEmitted: job.createdEventEmitted === true,+ preserveForScheduleReconcile: job.preserveForScheduleReconcile === true,+ ...(job.scheduleOutcome != null && { scheduleOutcome: job.scheduleOutcome }),+ ...(job.scheduleOutcomeError != null && {+ scheduleOutcomeError: job.scheduleOutcomeError,+ }),+ };+ },+ abortScheduledJob: async (conversationId, identity, options) => {+ const store = GenerationJobManager.getJobStore();+ if (store == null) {+ return false;+ }+ const job = await store.getJob(conversationId);+ // A null/identity-mismatched job is NOT reachable from this replica: it may be+ // a live generation on a peer worker's private in-memory store (unshared+ // topology). Report false so the caller knows the abort was NOT delivered.+ if (job == null || !jobMatchesIdentity(job, identity)) {+ return false;+ }+ if (options?.preserve !== false) {+ await GenerationJobManager.updateMetadata(+ conversationId,+ {+ preserveForScheduleReconcile: true,+ scheduleOutcome: 'interrupted',+ scheduleOutcomeError: 'Schedule deleted',+ },+ job.createdAt,+ );+ }++ // #14925's provider-drain contract is the authority here. It covers both+ // locally owned and cross-replica generations and does not return until the+ // exact provider segment can no longer persist user data. Calling abortJob on+ // an already-terminal job is intentional: awaitProviderDrain still waits for+ // trailing owner work even though no new abort transition is needed.+ const aborted = await GenerationJobManager.abortJob(conversationId, {+ expectedCreatedAt: job.createdAt,+ awaitProviderDrain: true,+ });+ if (+ aborted.failureReason === 'generation_replaced' ||+ aborted.failureReason === 'job_still_active'+ ) {+ return false;+ }+ if (options?.preserve === false) {+ await store.deleteJob(conversationId, job.createdAt);+ }+ return true;+ },+ clearReconciledJob: async (conversationId, identity) => {+ const store = GenerationJobManager.getJobStore();+ if (store == null) {+ return;+ }+ const job = await store.getJob(conversationId);+ // Only delete when the job still carries THIS run's identity, so a+ // replacement generation occupying the same conversationId is never destroyed.+ if (job == null || !jobMatchesIdentity(job, identity)) {+ return;+ }+ // CAS, not read-then-delete: the identity check above is a READ, and a replacement+ // generation can land between it and the delete. Passing the observed createdAt+ // makes the delete conditional on the job still being that exact generation, so+ // the store itself rejects the write if one did.+ await store.deleteJob(conversationId, job.createdAt);+ },+ // Counted in system scope so the cap is GLOBAL — a per-owner (tenant-scoped)+ // count would let multiple tenants collectively exceed fireConcurrency.+ countActiveRunsGlobal: () => runAsSystem(() => methods.countActiveRuns()),+ isOwnerDeleting: (userId) => deps.isUserDeleting(userId),+ isGloballyDisabled: async () => {+ // Env first: an incident lever that must work even if the DB/config plane is the+ // thing failing (a kill switch that needs a healthy DB is the one that fails when+ // you need it).+ if (isEnabled(process.env.SCHEDULES_DISABLED)) {+ return true;+ }+ // BASE config only: DB principal overrides can narrow availability but must never+ // widen past an operator's global stop, so `schedules: false` in librechat.yaml is+ // genuinely non-overridable rather than emergent from the override filters.+ // isRuntimeDisabled reads BOTH stop shapes (`false` and `{ use: false }`) — the+ // same predicate the override merge preserves base stops with. A shape-blind+ // check here made the object form disable getLimits but not this gate, so the+ // engine kept claiming and fireSchedule ADVANCED each occurrence: a short+ // maintenance stop silently dropped every occurrence it covered instead of+ // leaving them due.+ const base = await deps.getAppConfig({ baseOnly: true });+ return isRuntimeDisabled(base?.interfaceConfig?.schedules);+ },+ // Occupancy is read in SYSTEM scope so the cap is global across tenants (the+ // owner's tenant context would only see its own runs); the claim itself stays in+ // the caller's context so the inserted row keeps correct tenant ownership.+ withGlobalCapacitySlot: (cap, claim) =>+ withCapacitySlot(cap, () => runAsSystem(() => methods.getCapacityOccupancy()), claim),+ };++ let engine: ReturnType<typeof startScheduleEngine> | undefined;+ let erasureSweep: ReturnType<typeof startScheduleErasureSweep> | undefined;++ /**+ * Fallback cleanup for a process whose engine refused to arm (unsafe topology, or+ * index creation failed): DELETE stays open on this entrypoint, so soft-deleted rows+ * still accrue — and with no reconciler, an owner-death case (account deletion begun+ * elsewhere, or a failed one-shot erase) would retain the hidden prompt forever.+ * The engine's own reconcile pass covers this when armed, so never run both.+ */+ function startErasureFallback(): void {+ if (erasureSweep != null || engine != null) {+ return;+ }+ erasureSweep = startScheduleErasureSweep({+ methods,+ getJobStatus: engineDeps.getJobStatus,+ // Positive-evidence delivery convergence: a dead delivery settles its reservation+ // and frees the global capacity slot even where no engine is armed.+ getTriggerDelivery: engineDeps.getTriggerDelivery,+ // Releases a run's retained terminal job once its outcome is durable, so preserved+ // evidence (kept without `completedAt`, and therefore invisible to the store's+ // finished-job sweep) cannot outlive the run it belonged to.+ clearReconciledJob: engineDeps.clearReconciledJob,+ // If topology itself prevented arming, this process's missing job says+ // nothing about peer liveness. If only index creation failed, the topology+ // proof still holds and the existing owner-death backstop remains valid.+ canInferOwnerDeathFromMissingJob: isTopologySafeToArm(),+ });+ }++ async function initializeScheduleEngine(): Promise<+ ReturnType<typeof startScheduleEngine> | undefined+ > {+ if (engine != null) {+ return engine;+ }+ // Always arm. The engine owns BOTH firing and reconciliation, and reconciliation must+ // run even while firing is stopped: a previous process can leave `started` rows and+ // preserved terminal jobs that would otherwise never settle until scheduling is+ // re-enabled. Firing is gated separately — runTick refuses to claim while globally+ // disabled, and getLimits reports disabled to the write handlers and the fire path —+ // so "off" still means nothing fires, without stranding prior state.+ // Deliberately not gated on the base config either: schedules can be enabled for a+ // role/user/tenant while the base YAML stays default-off.+ // Explicitly build the Schedule/ScheduleRun indexes first — the unique+ // idempotency index and TTL retention index would otherwise never exist when+ // MONGO_AUTO_INDEX is disabled (the production default). If this fails the+ // unique {scheduleId, scheduledFor} guard may be absent, so leave the engine+ // DISABLED rather than firing without duplicate protection — the app still+ // runs; schedules simply don't fire until an operator resolves the index.+ if (!isTopologySafeToArm()) {+ logger.error(+ "[schedules] scheduler NOT started: this process cannot see other replicas' generations. " ++ 'The job store is process-local (Redis is off), but nothing here proves this is the only ' ++ 'replica — and the standard server arms the scheduler in EVERY replica. A peer would ' ++ 'reconcile runs whose jobs it cannot see, eventually marking a still-running generation ' ++ 'interrupted and releasing its capacity, and deletions routed elsewhere could not abort ' ++ 'the generation at all. Enable a shared stream store (USE_REDIS_STREAMS), or set ' ++ 'SCHEDULES_SINGLE_PROCESS=true to assert this deployment runs exactly one replica. ' ++ 'Schedule writes are refused (503) until then.',+ );+ startErasureFallback();+ return undefined;+ }+ try {+ await runAsSystem(() => methods.ensureScheduleIndexes());+ } catch (err) {+ logger.error(+ '[schedules] index creation failed — scheduler NOT started (fires need the unique idempotency index):',+ err,+ );+ startErasureFallback();+ return undefined;+ }+ engine = startScheduleEngine(engineDeps);+ if (erasureSweep != null) {+ // A later successful arm supersedes the fallback: the engine's reconcile pass+ // owns erasure from here.+ erasureSweep.stop();+ erasureSweep = undefined;+ }+ return engine;+ }++ /**+ * The clustered entrypoint's ONLY schedule maintenance. Exposes the same erasure sweep+ * the standard entrypoint falls back to, so a soft-deleted row whose delete/terminal+ * erase-on-settle attempts missed still drains instead of retaining the owner's prompt+ * forever. It shares startErasureFallback's idempotent startup guard, the sweep's own+ * shutdown registration, and the topology-fenced owner-death policy — and arms nothing+ * else, so running it in every clustered replica changes nothing about v1 scheduling.+ */+ function initializeScheduleErasureSweep(): void {+ startErasureFallback();+ }++ /**+ * Manual run-now fire. Acquires the schedule lease to serialize concurrent+ * run-now clicks (and to block against a background engine claim), then fires+ * in manual mode so the next automatic occurrence is left untouched. Returns+ * null when the lease is already held (a run is in progress).+ */+ async function fireScheduleNow(+ schedule: FireableSchedule,+ limits: ScheduleLimits,+ ): Promise<FireResult | null> {+ // The global stop means STOP: a manual run dispatches the same billed generation as+ // an automatic one, so gating only the engine tick would leave Run Now wide open.+ if (await engineDeps.isGloballyDisabled()) {+ return { fired: false, skipped: 'disabled' as const };+ }+ const leased = await methods.acquireManualRunLease(+ schedule.id,+ schedule.user,+ MANUAL_RUN_LEASE_MS,+ );+ if (leased == null) {+ return null;+ }+ const claimToken = leased.claimToken;+ try {+ // Fire the FRESH leased row (post-image with the new claim token), not the+ // snapshot the route read before the lease — an edit that committed in the+ // window in between is reflected, so a stale prompt/agent is never dispatched.+ return await fireSchedule(engineDeps, leased, limits, new Date(), { manual: true });+ } catch (err) {+ const released =+ claimToken != null+ ? await methods.releaseLease(schedule.id, claimToken).catch(() => false)+ : false;+ if (!released && leased.leaseBy != null) {+ // An owner edit during the failed preflight rotates the token while preserving+ // this unique holder. Clear only that holder; a takeover has a different value+ // and is therefore never stripped by this exception safety net.+ await methods.releaseLeaseByHolder(schedule.id, leased.leaseBy).catch(() => undefined);+ }+ throw err;+ }+ }++ const OUTCOME_RETRY_ATTEMPTS = 3;++ /**+ * Completion hook: called from the agents controller finalize paths when the+ * request carried a scheduleId. The caller deletes the job (`completeJob`) right+ * after, destroying the only evidence the reconciler could use — so a transient+ * Mongo failure here is RETRIED (bounded) before giving up, and the failure is+ * surfaced to the caller (returns false) so it can keep the job when it matters.+ */+ /**+ * OWNER-SIDE Stop barrier. An interactive Stop stamps `abortSource: 'stop'` before it+ * signals the abort, and only acknowledges (`abortPersistedAt`) once its partial-message+ * and checkpoint writes have landed. A terminal settlement here in between would release+ * the run's capacity — and let a concurrent schedule/account deletion erase the data —+ * while that write is still in flight. Wait (bounded) for the acknowledgement. A resolved+ * marker, a non-stop source, or a stamp gone stale (its route presumed dead) proceeds+ * immediately, so the stale-owner timeout remains the bounded recovery path.+ */+ async function waitForStopPersistence(scheduleId: string, scheduledFor: Date): Promise<boolean> {+ const deadline = Date.now() + STOP_BARRIER_TIMEOUT_MS;+ for (;;) {+ const state = await methods.getScheduleRunAbortState(scheduleId, scheduledFor);+ if (+ state == null ||+ state.abortSource !== 'stop' ||+ state.abortPersistedAt != null ||+ !hasAbortInFlight(state, Date.now())+ ) {+ // Cleared to settle: no Stop owns the run, it acknowledged, or its owner is past+ // the stale cutoff and is presumed dead (the bounded recovery path).+ return true;+ }+ if (Date.now() >= deadline) {+ // Still an UNACKNOWLEDGED, FRESH Stop. The poll budget expiring proves nothing about+ // the route's writes — slow checkpoint cleanup looks exactly like this — so treating+ // the barrier as satisfied would terminalize the run and release its capacity (and+ // its deletion/erasure barriers) while beforePublish may still be writing. DEFER+ // instead: the acknowledgement settles it, or the stale-owner cutoff authorizes a+ // later attempt.+ return false;+ }+ await new Promise((resolve) => setTimeout(resolve, STOP_BARRIER_POLL_MS));+ }+ }++ async function recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt,+ status,+ conversationId,+ clearConversationId,+ error,+ }: RecordScheduleOutcomeInput): Promise<boolean> {+ if (!scheduleId || !scheduledFor) {+ return true;+ }+ const terminal = status !== 'requires_action';+ if (terminal) {+ // Honor an in-flight interactive Stop's persistence before terminalizing. A deferral+ // is NOT a failure to record — the run is deliberately left active/preserved — but it+ // must report "not settled" so callers with durable retry (the approval-expiry host+ // action, reconciliation) re-drive it rather than assuming the outcome landed.+ if (!(await waitForStopPersistence(scheduleId, new Date(scheduledFor)))) {+ logger.info(+ `[schedules] deferring terminal settlement for ${scheduleId}: interactive Stop persistence is still unacknowledged`,+ );+ return false;+ }+ }+ if (terminal && streamId && jobCreatedAt != null) {+ try {+ await GenerationJobManager.updateMetadata(+ streamId,+ {+ preserveForScheduleReconcile: true,+ scheduleOutcome:+ status === 'success' ||+ status === 'error' ||+ status === 'interrupted' ||+ status === 'skipped_balance'+ ? status+ : 'error',+ ...(error ? { scheduleOutcomeError: error } : {}),+ },+ jobCreatedAt,+ );+ } catch (err) {+ logger.error('[schedules] failed to retain terminal outcome evidence:', err);+ return false;+ }+ }+ for (let attempt = 1; attempt <= OUTCOME_RETRY_ATTEMPTS; attempt++) {+ try {+ // Resolve the owner's limits so auto-disable uses the same per-principal+ // threshold as the fire path (not the global default).+ const schedule = await methods.getScheduleById(scheduleId);+ const owner = schedule ? await engineDeps.getUserContext(schedule.user) : null;+ const limits = await getLimits(owner ?? undefined);+ await methods.recordRunOutcome({+ scheduleId,+ scheduledFor: new Date(scheduledFor),+ status,+ clearConversationId,+ conversationId,+ error,+ autoDisableAfterFailures: limits.autoDisableAfterFailures,+ balanceSkipDisableThreshold: BALANCE_SKIP_DISABLE_THRESHOLD,+ });+ // ERASE-ON-SETTLE: whichever process records a run's terminal outcome also+ // attempts the deferred erase of a deleting schedule. This is what makes a+ // delete's `draining` state converge in EVERY topology — the clustered+ // entrypoint runs no reconciler, so without this the hidden schedule (and its+ // prompt, which has no TTL) survived its last run indefinitely there. A cheap+ // guarded no-op for live schedules (the erase filters on `deleting: true`).+ if (status !== 'requires_action') {+ await methods.eraseScheduleIfDrained(scheduleId).catch((err) => {+ logger.warn(`[schedules] erase-on-settle failed for ${scheduleId}:`, err);+ });+ }+ if (terminal && streamId && jobCreatedAt != null) {+ await GenerationJobManager.updateMetadata(+ streamId,+ { preserveForScheduleReconcile: false },+ jobCreatedAt,+ ).catch((err) => {+ logger.warn('[schedules] failed to release terminal outcome evidence:', err);+ });++ // A paused occurrence can be aborted while its provider is already drained.+ // In that case the manager's normal drain-time cleanup has already passed;+ // reap the exact terminal job after the Mongo outcome becomes durable.+ const store = GenerationJobManager.getJobStore();+ const job = await store?.getJob(streamId).catch(() => null);+ if (+ job?.createdAt === jobCreatedAt &&+ job.providerDrained !== false &&+ job.terminalPersistencePending !== true &&+ job.status !== 'running' &&+ job.status !== 'requires_action' &&+ job.preserveForScheduleReconcile !== true+ ) {+ await store?.deleteJob(streamId, jobCreatedAt).catch((err) => {+ logger.warn('[schedules] failed to clear settled generation evidence:', err);+ });+ }+ }+ return true;+ } catch (err) {+ logger.error(+ `[schedules] failed to record run outcome (attempt ${attempt}/${OUTCOME_RETRY_ATTEMPTS}):`,+ err,+ );+ }+ }+ return false;+ }++ async function beginScheduledStop({+ scheduleId,+ scheduledFor,+ }: {+ scheduleId: string;+ scheduledFor: string | Date;+ }): Promise<boolean | 'in_progress'> {+ if (!scheduleId || !scheduledFor) {+ return false;+ }+ return methods.requestRunAbort(scheduleId, new Date(scheduledFor), 'stop');+ }++ async function acknowledgeScheduledStopPersistence({+ scheduleId,+ scheduledFor,+ settle,+ }: {+ scheduleId: string;+ scheduledFor: string | Date;+ /** Terminal outcome to (re-)drive once the barrier clears. Supplied by the Stop route+ * so a settlement its owner already DEFERRED past the poll budget still converges. */+ settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };+ }): Promise<void> {+ if (!scheduleId || !scheduledFor) {+ return;+ }+ await methods.markRunAbortPersisted(scheduleId, new Date(scheduledFor));+ if (settle == null) {+ return;+ }+ // The owner calls recordScheduleOutcome ONCE. If its Stop barrier deferred (slow+ // beforePublish), nothing would re-drive it where no schedule reconciler is armed —+ // the run would stay `started`, its job preserved, and its global capacity slot held.+ // Now that the barrier is acknowledged, settle from here; recordRunOutcome is+ // match-guarded and idempotent, so a run the owner already settled is a no-op.+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ status: settle.status,+ conversationId: settle.conversationId,+ error: settle.error,+ }).catch((err) => logger.warn('[schedules] post-acknowledgement settlement failed:', err));+ }++ async function isScheduleLive(+ scheduleId: string,+ expectedConfigRevision?: number,+ options?: { automatic?: boolean; policy?: boolean },+ ): Promise<boolean> {+ if (!scheduleId) {+ return false;+ }+ const schedule = await methods.getScheduleById(scheduleId);+ if (schedule == null) {+ return false;+ }+ // An AUTOMATIC fire must still be wanted. A policy auto-disable (too many failures,+ // insufficient balance) flips `enabled` WITHOUT touching configRevision, so the+ // revision fence below cannot see it — an occurrence already in the claim-to-+ // controller window would otherwise start a billed generation for a schedule that+ // has just been switched off. Run Now is an explicit user action and stays allowed+ // on a disabled schedule, matching fireScheduleNow.+ if (options?.automatic === true && schedule.enabled === false) {+ return false;+ }+ // REVISION FENCE at the admission boundary. Existence alone is not enough: an owner+ // edit landing between the claim and this point means the dispatched prompt/agent+ // came from a config the owner has since replaced, and nothing downstream would+ // catch it because the run persists under the NEW schedule. Refuse before any+ // message is written. Absent on either side disables the fence, so pre-existing+ // schedules and older fires keep working.+ if (+ expectedConfigRevision != null &&+ typeof schedule.configRevision === 'number' &&+ schedule.configRevision !== expectedConfigRevision+ ) {+ return false;+ }+ // LIVE dispatch policy, re-applied exactly as the fire path applies it. A pause+ // can sit unanswered for hours, and an operator's global kill switch, a narrowed+ // `interface.schedules` availability, or a revoked SCHEDULES:USE permission+ // landing in that window touches neither the row nor its revision — none of the+ // checks above can see it, and approving the pause would start a fresh billed+ // continuation the operator believes is stopped. Applies to manual runs too: an+ // emergency stop must stop those approvals as well.+ if (options?.policy === true) {+ if (await engineDeps.isGloballyDisabled()) {+ return false;+ }+ const owner = await engineDeps.getUserContext(schedule.user);+ if (owner == null) {+ return false;+ }+ const limits = await engineDeps.runInTenantContext(owner, () => getLimits(owner));+ if (!limits.enabled) {+ return false;+ }+ if (!(await engineDeps.hasScheduleAccess(owner))) {+ return false;+ }+ }+ return true;+ }++ async function claimScheduleResume(+ scheduleId: string,+ scheduledFor: string | Date,+ options?: { expectedConfigRevision?: number; automatic?: boolean },+ ): Promise<ScheduleResumeClaimResult> {+ const schedule = await methods.getScheduleById(scheduleId);+ if (schedule == null) {+ return { conflict: 'inactive' };+ }+ if (options?.automatic !== false && schedule.enabled === false) {+ return { conflict: 'inactive' };+ }+ if (+ options?.expectedConfigRevision != null &&+ typeof schedule.configRevision === 'number' &&+ schedule.configRevision !== options.expectedConfigRevision+ ) {+ return { conflict: 'inactive' };+ }+ const owner = await engineDeps.getUserContext(schedule.user);+ if (owner == null) {+ return { conflict: 'inactive' };+ }+ return engineDeps.runInTenantContext(owner, async () => {+ const [ownerLimits, deploymentLimits, globallyDisabled, hasAccess] = await Promise.all([+ getLimits(owner),+ getLimits(),+ engineDeps.isGloballyDisabled(),+ engineDeps.hasScheduleAccess(owner),+ ]);+ if (!ownerLimits.enabled || globallyDisabled || !hasAccess) {+ return { conflict: 'inactive' };+ }+ // FINAL schedule-side admission fence. Everything above is asynchronous and an+ // owner edit/disable can land while it runs. Claim the schedule document under+ // the expected config generation now, carry this lease through the approval CAS,+ // and atomically consume it before provider execution begins.+ const resumeLease = await methods.acquireResumeLease(+ scheduleId,+ options?.expectedConfigRevision,+ options?.automatic !== false,+ SCHEDULE_RESUME_LEASE_MS,+ );+ if (resumeLease?.claimToken == null || resumeLease.leaseBy == null) {+ return { conflict: 'inactive' };+ }+ let retainResumeLease = false;+ try {+ const allocation = await engineDeps.withGlobalCapacitySlot(+ Math.min(ownerLimits.fireConcurrency, deploymentLimits.fireConcurrency),+ async (capacitySlot) => {+ const attempt = await methods.markRunResumeClaimed(+ scheduleId,+ new Date(scheduledFor),+ capacitySlot,+ );+ return 'conflict' in attempt && attempt.conflict === 'slot-taken'+ ? 'slot-taken'+ : { claimed: attempt };+ },+ );+ if (allocation === 'capacity') {+ return { conflict: 'capacity' };+ }+ const claimed = allocation.claimed;+ if ('conflict' in claimed) {+ // withCapacitySlot consumes this internal collision sentinel by retrying the+ // next free slot. Normalize defensively at the public boundary as well so a+ // custom/test allocator can never leak an implementation-only conflict.+ return {+ conflict: claimed.conflict === 'slot-taken' ? 'capacity' : claimed.conflict,+ };+ }+ retainResumeLease = true;+ return {+ capacitySlot: claimed.capacitySlot,+ claimToken: resumeLease.claimToken,+ leaseBy: resumeLease.leaseBy,+ };+ } finally {+ if (!retainResumeLease) {+ await methods.releaseLeaseByHolder(scheduleId, resumeLease.leaseBy);+ }+ }+ });+ }++ function releaseScheduleResumeClaim(+ scheduleId: string,+ scheduledFor: string | Date,+ capacitySlot: number,+ ): Promise<boolean> {+ return methods.releaseRunResumeClaim(scheduleId, new Date(scheduledFor), capacitySlot);+ }++ async function finalizeScheduleResumeClaim(+ scheduleId: string,+ claimToken: string,+ leaseBy: string,+ options?: { expectedConfigRevision?: number; automatic?: boolean },+ ): Promise<boolean> {+ const consumed = await methods.consumeResumeLease(+ scheduleId,+ claimToken,+ leaseBy,+ options?.automatic !== false,+ options?.expectedConfigRevision,+ );+ if (!consumed) {+ // An owner edit rotates claimToken but deliberately leaves the old holder's+ // lease in place. Clear only that holder so the edited schedule is immediately+ // usable; a takeover changed leaseBy and is therefore untouched.+ await methods.releaseLeaseByHolder(scheduleId, leaseBy);+ }+ return consumed;+ }++ function releaseScheduleResumeFence(scheduleId: string, leaseBy: string): Promise<void> {+ return methods.releaseLeaseByHolder(scheduleId, leaseBy);+ }++ /** Aborts an active run's loopback job (identity-guarded). Returns whether the+ * abort was delivered (false when the job wasn't reachable — e.g. a peer worker's+ * private store, or a transient error). */+ async function abortActiveRun(+ run: { scheduleId: string; scheduledFor: Date; conversationId?: string },+ preserve: boolean,+ options?: { stampRenewal?: boolean; settleAfterAbort?: boolean },+ ): Promise<boolean> {+ if (!run.conversationId) {+ return false;+ }+ // Record the abort REQUEST before signalling. This keeps the run holding its global+ // capacity slot until its generation owner writes a terminal outcome (settlement),+ // so an abort that has been asked for but not yet honored cannot free capacity for a+ // new run while the old generation is still alive. The stamp is LOAD-BEARING: it is+ // what makes a concurrent drain (and the reconciler) defer to the owner's settle,+ // so if it cannot be made durable the abort must NOT be signalled — report+ // undelivered and let the caller's unconfirmed/retry path re-drive both.+ //+ // `stampRenewal: false` skips the stamp entirely: callers pass it for CLEANUP+ // aborts of a job they just observed terminal/absent (nothing live will be+ // signalled). Renewing there would re-arm the 30-minute owner-death fence on+ // every retry against a dead owner, so the run could never age into the+ // reconciler's recovery.+ if (options?.stampRenewal !== false) {+ try {+ await methods.requestRunAbort(run.scheduleId, run.scheduledFor, 'deletion');+ } catch (err) {+ logger.warn('[schedules] failed to record abort request; abort withheld:', err);+ return false;+ }+ }+ const stopped = await engineDeps+ .abortScheduledJob(+ run.conversationId,+ { scheduleId: run.scheduleId, scheduledFor: run.scheduledFor },+ { preserve },+ )+ .catch((err) => {+ logger.warn('[schedules] failed to abort run job on quiesce:', err);+ return false;+ });+ if (!stopped || options?.settleAfterAbort === false) {+ return stopped;+ }++ // awaitProviderDrain in abortScheduledJob is the positive persistence fence:+ // the exact provider owner has unwound, so this row can now leave the active+ // set without racing a late message write.+ const recorded = await recordScheduleOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: 'interrupted',+ conversationId: run.conversationId,+ error: 'Schedule deleted while the run was active',+ });+ if (!recorded) {+ return false;+ }+ await engineDeps+ .clearReconciledJob(run.conversationId, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ })+ .catch((err) => logger.warn('[schedules] failed to clear deleted run evidence:', err));+ return true;+ }++ /** Resolve the owner's durable-checkpointer config (in their tenant context) so a+ * paused-run checkpoint can be pruned on delete, mirroring the interactive abort. */+ async function resolveOwnerCheckpointer(+ ownerId: string | Types.ObjectId,+ ): Promise<TCheckpointerConfig | undefined> {+ const owner = await engineDeps.getUserContext(ownerId);+ if (owner == null) {+ return undefined;+ }+ const appConfig = await engineDeps.runInTenantContext(owner, () =>+ deps.getAppConfig(getAppConfigOptionsFromUser(owner)),+ );+ return appConfig?.endpoints?.agents?.checkpointer;+ }++ /**+ * Soft-deletes a schedule for its owner: disables + marks it `deleting` (so the+ * engine can no longer claim it and it disappears from the owner's list), rotates+ * the claim token to fence any in-flight worker, then DRAINS with the same evidence+ * discipline as account-deletion quiesce — a run whose job is provably absent or+ * settled is recorded and erased here, synchronously; a live generation is aborted+ * and settles through its own outcome write (which erases on settle, so no+ * reconciler is required in any topology). The result is honest: `unconfirmed`+ * means at least one run could not be shown stopped, and the caller must not+ * claim it was.+ */+ async function deleteScheduleForOwner(+ scheduleId: string,+ userId: string,+ ): Promise<ScheduleDeleteResult> {+ const schedule = await methods.markScheduleDeleting(scheduleId, userId);+ if (schedule == null) {+ return 'not_found';+ }+ const active = await methods.getActiveRunsForSchedule(scheduleId);+ // Resolve the checkpointer config once (only when a paused run needs pruning) in+ // the owner's tenant context, matching the interactive abort endpoint's prune.+ const hasPausedRun = active.some(+ (run) => run.status === 'requires_action' && run.conversationId != null,+ );+ // BEST-EFFORT: the prune this feeds is itself best-effort, and a lookup failure+ // must not cost the aborts below.+ const checkpointer = hasPausedRun+ ? await resolveOwnerCheckpointer(schedule.user).catch((err) => {+ logger.warn(`[schedules] checkpointer lookup failed for delete ${scheduleId}:`, err);+ return undefined;+ })+ : undefined;+ let unconfirmed = 0;+ for (const run of active) {+ // UNKNOWN is not ABSENT — the same distinction the quiesce path draws. A lookup+ // that succeeded and returned null is positive evidence no generation holds this+ // conversation; a lookup that THREW is evidence of nothing.+ const live = run.conversationId+ ? await engineDeps.getJobStatus(run.conversationId).then(+ (job) => ({ known: true, job }),+ () => ({ known: false, job: null }),+ )+ : { known: true, job: null };+ const isThisGeneration =+ live.job != null &&+ jobMatchesIdentity(live.job, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ });+ // Capture the paused run's checkpoint ids BEFORE any terminal transition below:+ // the prune afterwards is scoped to exactly this set, so checkpoints a+ // replacement turn writes after this point can never be swept up by it.+ const checkpointGeneration =+ run.status === 'requires_action' && run.conversationId+ ? await captureAgentCheckpointGeneration(run.conversationId, checkpointer)+ : undefined;+ // Same abort-in-flight deferral as quiesce: post-abort job state (status `aborted`,+ // or absence once the abort deleted the job) appears before the owner has persisted+ // and settled, so it is not evidence that the generation is done.+ //+ // A paused job whose run row is still `started` is a pause HAND-OFF in flight:+ // the job reports requires_action the instant the run interrupts, while the+ // controller's pause branch is still flushing this segment's writes and records+ // the pause on the row only after them. Settling on the job state alone let the+ // cascade complete before those writes landed, which then recreated the paused+ // response for the deleted schedule. The controller's record (or the paused-+ // window reconciler) flips the row to requires_action, so the deferral is+ // bounded — a later pass settles it.+ const pauseHandoffInFlight =+ isThisGeneration &&+ live.job?.status === 'requires_action' &&+ (run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));+ const settleable =+ live.known &&+ !hasAbortInFlight(run, Date.now()) &&+ !pauseHandoffInFlight &&+ !(isThisGeneration && live.job?.status === 'running');+ if (settleable) {+ // Positive evidence nothing is generating: settle the row HERE so the erase+ // below can proceed without any reconciler — the clustered entrypoint has+ // none, and deferring these rows to it retained the deleted schedule (and its+ // prompt) indefinitely in that topology. Settle BEFORE aborting: a retained+ // job is the only evidence of a finished run whose outcome write failed.+ const retainedOutcome = isThisGeneration+ ? TERMINAL_JOB_OUTCOMES[live.job!.status]+ : undefined;+ const settledStatus = retainedOutcome ?? 'interrupted';+ const settled = await methods+ .recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: settledStatus,+ conversationId: run.conversationId,+ ...(settledStatus === 'interrupted' ? { error: 'Schedule deleted' } : {}),+ autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,+ })+ .then(+ () => true,+ (err) => {+ logger.warn(`[schedules] failed to settle run on delete ${scheduleId}:`, err);+ return false;+ },+ );+ if (settled) {+ // Cleanup abort of a job just observed terminal/absent: nothing live gets+ // signalled, so the stamp is NOT renewed — re-arming the owner-death fence+ // on every delete retry would keep a dead owner's row from ever aging into+ // the reconciler's recovery.+ await abortActiveRun(run, false, {+ stampRenewal: false,+ settleAfterAbort: false,+ });+ } else {+ unconfirmed += 1;+ }+ } else {+ // A live generation (or an unknown one): abort it, preserving the job so its+ // outcome survives, and require the DELIVERY to be confirmed. An abort that+ // was not delivered leaves a generation that keeps producing and billing —+ // reporting this delete as a success would claim otherwise.+ const aborted = await abortActiveRun(run, true);+ if (!aborted) {+ unconfirmed += 1;+ }+ }+ // HITL: prune the durable checkpoints of a run aborted while paused so a new turn+ // in this conversation can't rehydrate the stale interrupt before the Mongo TTL+ // reclaims it (thread_id === conversationId). Idempotent / no-op otherwise.+ //+ // Two guards against a REPLACEMENT owning the conversation: ownership is re-read+ // fresh (the settle/abort awaits above are a window a replacement can claim the+ // conversationId in), and the deletion is SCOPED to the checkpoint ids captured+ // before the terminal transition — so even a replacement that slips in between+ // this read and the delete only ever loses checkpoints that predate the abort,+ // never its own. An unreadable store proves nothing, so it is left alone.+ if (+ run.status === 'requires_action' &&+ run.conversationId &&+ checkpointGeneration != null &&+ checkpointGeneration.checkpointIds.length > 0+ ) {+ const fresh = await engineDeps.getJobStatus(run.conversationId).then(+ (job) => ({ known: true, job }),+ () => ({ known: false, job: null }),+ );+ const freshIsThisGeneration =+ fresh.job != null &&+ jobMatchesIdentity(fresh.job, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ });+ const ownsConversation = fresh.known && (fresh.job == null || freshIsThisGeneration);+ if (ownsConversation) {+ await deleteAgentCheckpoint(run.conversationId, checkpointer, checkpointGeneration).catch(+ () => undefined,+ );+ }+ }+ }+ if (unconfirmed > 0) {+ // The run row leaving the active set is the durable acknowledgement that its+ // generation stopped and persisted (the owner settles LAST) — wait for it,+ // bounded, before answering. Without this, every cross-replica delete whose+ // abort delivery cannot be locally proven would answer 503 even though the+ // owner settles moments later.+ const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;+ let remaining = unconfirmed;+ while (Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));+ remaining = (await methods.getActiveRunsForSchedule(scheduleId)).length;+ if (remaining === 0) {+ break;+ }+ }+ if (remaining > 0) {+ return 'unconfirmed';+ }+ }+ const erased = await methods.eraseScheduleIfDrained(scheduleId).catch((err) => {+ logger.warn(`[schedules] erase failed for ${scheduleId}:`, err);+ return false;+ });+ if (erased) {+ return 'deleted';+ }+ // eraseScheduleIfDrained already returns true when the row was concurrently+ // erased or is absent. A false result therefore means the deleting row is still+ // protected by a live lease/run, or the erase could not be confirmed.+ return 'draining';+ }++ /**+ * Quiesces every schedule of a user ahead of account deletion: REVERSIBLY suspends them+ * under the attempt `token` (non-claimable, so no new occurrence fires while the cascade+ * runs) and aborts the loopback jobs of any in-flight runs, so a scheduled generation+ * cannot keep persisting messages after the account's messages/conversations are deleted.+ * The suspension is reversible so a cancelled deletion can restore it (see+ * restoreUserSchedulesFromDeletion) rather than stranding a live user with erased rows.+ */+ async function quiesceUserSchedules(userId: string, token: string): Promise<boolean> {+ await methods.suspendUserSchedulesForDeletion(userId, token);+ const active = await methods.getActiveRunsForUser(userId);+ const unconfirmed: string[] = [];+ for (const run of active) {+ // Current resumes promote the row back to `started` with a capacity slot, but a+ // rolling deploy or crash-era row may still be `requires_action` while its job is+ // already `running`. So the row status alone cannot prove a genuine pause. Read+ // the live job BEFORE aborting, which settles or deletes it and would erase this+ // evidence.+ // UNKNOWN is not ABSENT. A lookup that THREW is evidence of nothing, while one+ // that succeeded and returned null is positive evidence that no generation holds+ // this conversation. Collapsing the two (a bare `.catch(() => null)`) would let a+ // transient store failure read as "genuinely paused" and terminalize a row whose+ // resumed generation is still running — the exact hazard this check exists for.+ const live = run.conversationId+ ? await engineDeps.getJobStatus(run.conversationId).then(+ (job) => ({ known: true, job }),+ () => ({ known: false, job: null }),+ )+ : { known: true, job: null };+ const isThisGeneration =+ live.job != null &&+ jobMatchesIdentity(live.job, {+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ });+ // Settle only on POSITIVE evidence that nothing is generating: an identity-matched+ // job that is not running, an identity MISMATCH (a replacement turn owns the+ // conversation, so this occurrence's generation is already gone), or a confirmed+ // absence. Anything unknown falls through and the drain waits for it.+ //+ // An abort IN FLIGHT is not such evidence, and this is the subtle case: `aborted`+ // and post-abort ABSENCE both appear the instant abortJob wins its CAS, while the+ // owner has yet to persist. Deferring to the owner's settle (see+ // ABORT_SETTLE_GRACE_MS) is what stops a drain from being confirmed mid-write.+ const abortInFlight = hasAbortInFlight(run, Date.now());+ // Same pause hand-off deferral as the schedule delete path: a paused job whose+ // row is still `started` has the controller's pause-branch writes in flight,+ // and settling on the job state alone confirms the drain before they land.+ const pauseHandoffInFlight =+ isThisGeneration &&+ live.job?.status === 'requires_action' &&+ (run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));+ const settleable =+ live.known &&+ !abortInFlight &&+ !pauseHandoffInFlight &&+ !(isThisGeneration && live.job?.status === 'running');+ // Aborts here never preserve for reconcile: account deletion hard-deletes these+ // run rows, so no reconcile pass would ever finalize or clear a retained job.+ const retainedOutcome = isThisGeneration+ ? TERMINAL_JOB_OUTCOMES[live.job!.status]+ : undefined;+ // `settleable` ALREADY means positive evidence that nothing is generating: an+ // identity-matched job that is not running, an identity MISMATCH (a replacement+ // turn owns the conversation), or a confirmed absence. Any extra condition on top+ // of it strands exactly the cases it was computed to cover — a `started` row whose+ // process died before creating its job reads as a confirmed absence, and the old+ // clause left it active through the whole drain, so account deletion answered 503+ // on every attempt until the 30-minute orphan sweep.+ if (settleable) {+ // SETTLE BEFORE ABORTING. The abort deletes the retained job, and for a run+ // whose inline outcome write exhausted its retries that job is the ONLY evidence+ // it finished — reading its status into a local is not the same as durably+ // recording it. If this write fails, the abort below has not yet run, so the+ // evidence survives for the next pass; the drain simply does not confirm and+ // deletion defers, which is the safe direction.+ // Either the retained job's own outcome, or — for a genuinely PAUSED run whose+ // approval will never be consumed for a deleted account — `interrupted`. Without+ // this a single paused run blocked the account's deletion permanently.+ const settledStatus = retainedOutcome ?? 'interrupted';+ const settledError = QUIESCE_SETTLE_ERRORS[settledStatus];+ const settled = await methods+ .recordRunOutcome({+ scheduleId: run.scheduleId,+ scheduledFor: run.scheduledFor,+ status: settledStatus,+ conversationId: run.conversationId,+ ...(settledError ? { error: settledError } : {}),+ autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,+ })+ .then(() => true)+ .catch((err) => {+ logger.warn('[schedules] failed to settle run on quiesce:', err);+ return false;+ });+ // Only now is the job disposable. On a failed settle, leave it: the row is still+ // active, so the drain reports unconfirmed and a later pass retries with its+ // evidence intact. Cleanup abort of an observed-settled job — no stamp renewal,+ // or every quiesce retry would re-arm the owner-death fence and a dead owner's+ // row could never age into recovery.+ if (settled) {+ await abortActiveRun(run, false, {+ stampRenewal: false,+ settleAfterAbort: false,+ });+ }+ continue;+ }+ // Not settleable here (a live generation, or an unknown job): abort it and let the+ // bounded drain below wait for its own terminal outcome to land.+ const aborted = await abortActiveRun(run, false);+ if (!aborted && run.conversationId) {+ unconfirmed.push(run.conversationId);+ }+ }+ // WAIT (bounded) for the aborted generations to actually settle before the+ // account-deletion cascade deletes messages/conversations: a run that already+ // returned from the model can observe the abort but still persist its messages,+ // which would otherwise resurrect data for the deleted account after the cascade+ // ran. Poll the run rows (they leave the active set once their outcome is+ // recorded) until drained or the deadline.+ const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;+ let remaining = active.length;+ while (remaining > 0 && Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));+ remaining = (await methods.getActiveRunsForUser(userId)).length;+ }+ // Surface anything that did not drain / could not be confirmed so the deletion+ // cascade defers rather than destroying while a generation may still persist.+ // Re-evaluate rather than trusting the initial abort-delivery result: an abort can+ // report false because the job was briefly unreachable, yet that generation then+ // finishes and records a terminal outcome during the drain poll. The run is genuinely+ // settled at that point, so keeping its id in `unconfirmed` would defer account+ // deletion forever. The DRAIN is the authority; delivery is only a hint.+ const confirmed = remaining === 0;+ if (confirmed && unconfirmed.length > 0) {+ logger.info(+ `[schedules] ${unconfirmed.length} abort(s) were not confirmed delivered but their ` ++ 'runs settled during the drain; treating the quiesce as complete.',+ );+ }+ if (!confirmed) {+ logger.warn(+ `[schedules] account-deletion quiesce did not confirm ${Math.max(remaining, unconfirmed.length)} ` ++ `in-flight scheduled run(s) settled${unconfirmed.length ? ` [${unconfirmed.join(', ')}]` : ''} ` ++ '— a peer worker generation may still persist data. Guaranteed quiescing requires a ' ++ 'shared stream store (USE_REDIS_STREAMS).',+ );+ }+ return confirmed;+ }++ /**+ * Reverses a quiesce whose account deletion was cancelled (a controller failure that+ * released the user-deletion fence). Re-enables and re-arms only the rows this exact+ * attempt suspended; a schedule the owner independently deleted, or one a newer attempt+ * re-suspended, is left as-is.+ */+ async function restoreUserSchedulesFromDeletion(userId: string, token: string): Promise<void> {+ await methods.restoreUserSchedulesFromDeletion(userId, token);+ }++ return {+ getLimits,+ engineDeps,+ fireScheduleNow,+ recordScheduleOutcome,+ beginScheduledStop,+ acknowledgeScheduledStopPersistence,+ claimScheduleResume,+ releaseScheduleResumeClaim,+ finalizeScheduleResumeClaim,+ releaseScheduleResumeFence,+ isScheduleLive,+ deleteScheduleForOwner,+ quiesceUserSchedules,+ restoreUserSchedulesFromDeletion,+ initializeScheduleEngine,+ initializeScheduleErasureSweep,+ };+}diff --git a/packages/api/src/schedules/trigger.spec.ts b/packages/api/src/schedules/trigger.spec.tsnew file mode 100644index 0000000000..249d05ca32--- /dev/null+++ b/packages/api/src/schedules/trigger.spec.ts@@ -0,0 +1,84 @@+import {+ captureScheduleFireContext,+ exemptFromConcurrencyLimiter,+ exemptFromUserLimiter,+ isScheduleFireRequest,+ readScheduleFireContext,+} from './trigger';++function request(manual = false) {+ return {+ _isAgentTrigger: true,+ body: {+ scheduleId: 'spoofed',+ agentTrigger: {+ version: 1,+ deliveryId: 'delivery-1',+ event: {+ id: 'occurrence-1',+ type: 'schedule.occurrence',+ occurredAt: 1_725_000_000_000,+ source: { id: 'schedule-1', type: 'schedule' },+ },+ metadata: { manual, configRevision: 7 },+ },+ },+ };+}++describe('schedule trigger context', () => {+ it('projects a verified generic trigger into schedule lifecycle fields', () => {+ const req = request();++ expect(captureScheduleFireContext(req)).toEqual({+ scheduleId: 'schedule-1',+ scheduledFor: new Date(1_725_000_000_000).toISOString(),+ manual: false,+ configRevision: 7,+ });+ expect(req.body).toMatchObject({+ scheduleId: 'schedule-1',+ scheduledFor: new Date(1_725_000_000_000).toISOString(),+ scheduleConfigRevision: 7,+ });+ expect(isScheduleFireRequest(req)).toBe(true);+ expect(exemptFromUserLimiter(req)).toBe(true);+ expect(exemptFromConcurrencyLimiter(req)).toBe(true);+ });++ it('keeps manual runs on the interactive user and concurrency limiters', () => {+ const req = request(true);+ captureScheduleFireContext(req);++ expect(exemptFromUserLimiter(req)).toBe(false);+ expect(exemptFromConcurrencyLimiter(req)).toBe(false);+ });++ it('rejects copied metadata without a verified agent-trigger identity', () => {+ const req = request();+ req._isAgentTrigger = false;++ expect(readScheduleFireContext(req)).toBeUndefined();+ expect(captureScheduleFireContext(req)).toBeUndefined();+ expect(req.body.scheduleId).toBe('spoofed');+ expect(isScheduleFireRequest(req)).toBe(false);+ });++ it.each([+ { event: { type: 'schedule.occurrence', occurredAt: 1, source: { id: 's', type: 'webhook' } } },+ {+ event: { type: 'schedule.occurrence', occurredAt: -1, source: { id: 's', type: 'schedule' } },+ },+ {+ event: { type: 'schedule.occurrence', occurredAt: 1, source: { id: 's', type: 'schedule' } },+ metadata: { manual: 'yes' },+ },+ ])('fails closed for malformed schedule trigger metadata', (agentTrigger) => {+ expect(+ readScheduleFireContext({+ _isAgentTrigger: true,+ body: { agentTrigger: { version: 1, ...agentTrigger } },+ }),+ ).toBeUndefined();+ });+});diff --git a/packages/api/src/schedules/trigger.ts b/packages/api/src/schedules/trigger.tsnew file mode 100644index 0000000000..16f7b020d8--- /dev/null+++ b/packages/api/src/schedules/trigger.ts@@ -0,0 +1,113 @@+interface ScheduleTriggerRequest {+ _isAgentTrigger?: boolean;+ _isScheduledFire?: boolean;+ _isManualScheduledFire?: boolean;+ body?: Record<string, unknown>;+}++interface TriggerRecord {+ [key: string]: unknown;+}++export interface ScheduleFireContext {+ scheduleId: string;+ scheduledFor: string;+ manual: boolean;+ configRevision?: number;+}++function record(value: unknown): TriggerRecord | undefined {+ return value != null && typeof value === 'object' && !Array.isArray(value)+ ? (value as TriggerRecord)+ : undefined;+}++/**+ * Reads the schedule identity carried by the trusted trigger host. The generic+ * trigger token is the trust boundary; ordinary chat requests cannot opt into+ * schedule bookkeeping or limiter exemptions by copying these body fields.+ */+export function readScheduleFireContext(+ req?: ScheduleTriggerRequest,+): ScheduleFireContext | undefined {+ if (req?._isAgentTrigger !== true) {+ return undefined;+ }+ const trigger = record(req.body?.agentTrigger);+ const event = record(trigger?.event);+ const source = record(event?.source);+ const metadata = record(trigger?.metadata);+ if (+ trigger?.version !== 1 ||+ event?.type !== 'schedule.occurrence' ||+ source?.type !== 'schedule' ||+ typeof source.id !== 'string' ||+ source.id.trim().length === 0 ||+ typeof event.occurredAt !== 'number' ||+ !Number.isSafeInteger(event.occurredAt) ||+ event.occurredAt < 0+ ) {+ return undefined;+ }+ if (metadata?.manual != null && typeof metadata.manual !== 'boolean') {+ return undefined;+ }+ if (+ metadata?.configRevision != null &&+ (typeof metadata.configRevision !== 'number' ||+ !Number.isSafeInteger(metadata.configRevision) ||+ metadata.configRevision < 0)+ ) {+ return undefined;+ }+ return {+ scheduleId: source.id,+ scheduledFor: new Date(event.occurredAt).toISOString(),+ manual: metadata?.manual === true,+ ...(typeof metadata?.configRevision === 'number' && {+ configRevision: metadata.configRevision,+ }),+ };+}++/**+ * Captures the verified trigger classification once, then projects the minimum+ * schedule fields expected by the existing generation lifecycle hooks.+ */+export function captureScheduleFireContext(+ req: ScheduleTriggerRequest,+): ScheduleFireContext | undefined {+ const context = readScheduleFireContext(req);+ req._isScheduledFire = context != null;+ req._isManualScheduledFire = context?.manual === true;+ if (context != null && req.body != null) {+ req.body.scheduleId = context.scheduleId;+ req.body.scheduledFor = context.scheduledFor;+ if (context.configRevision == null) {+ delete req.body.scheduleConfigRevision;+ } else {+ req.body.scheduleConfigRevision = context.configRevision;+ }+ }+ return context;+}++export function isScheduleFireRequest(req?: ScheduleTriggerRequest): boolean {+ return typeof req?._isScheduledFire === 'boolean'+ ? req._isScheduledFire+ : readScheduleFireContext(req) != null;+}++/** Automatic occurrences are bounded by schedule cadence and global capacity. */+export function exemptFromUserLimiter(req?: ScheduleTriggerRequest): boolean {+ if (typeof req?._isScheduledFire === 'boolean') {+ return req._isScheduledFire && req._isManualScheduledFire !== true;+ }+ const context = readScheduleFireContext(req);+ return context != null && !context.manual;+}++/** Acquire/release sites share this exact predicate to keep counters balanced. */+export function exemptFromConcurrencyLimiter(req?: ScheduleTriggerRequest): boolean {+ return exemptFromUserLimiter(req);+}diff --git a/packages/api/src/schedules/types.ts b/packages/api/src/schedules/types.tsnew file mode 100644index 0000000000..52c5febb42--- /dev/null+++ b/packages/api/src/schedules/types.ts@@ -0,0 +1,282 @@+import type {+ ScheduleMethods,+ ISchedule,+ AgentTriggerDeliveryStatus,+ AgentTriggerDeliveryFailure,+} from '@librechat/data-schemas';+import type { Types } from 'mongoose';+import type { AgentTriggerEnqueueOptions, AgentTriggerEnvelope } from '../agents/triggers';+import type { SlotClaimResult } from './capacity';++export interface ScheduleLimits {+ /** Feature-level switch: when false the engine claims/fires nothing. */+ enabled: boolean;+ maxPerUser: number;+ minIntervalMinutes: number;+ autoDisableAfterFailures: number;+ fireConcurrency: number;+}++export const DEFAULT_SCHEDULE_LIMITS: ScheduleLimits = {+ enabled: true,+ maxPerUser: 10,+ minIntervalMinutes: 60,+ autoDisableAfterFailures: 5,+ fireConcurrency: 5,+};++export interface ScheduleUserContext {+ id: string;+ tenantId?: string;+ role?: string;+}++/**+ * Outcome of an owner-initiated schedule delete.+ * - `deleted`: drained and erased.+ * - `draining`: every active run's abort was DELIVERED; erasure follows once the+ * generation records its terminal outcome (erase-on-settle), in any topology.+ * - `unconfirmed`: at least one run could not be confirmed stopped (job store+ * unreachable, or the abort was not delivered). The schedule stays hidden and+ * fenced, but its generation may still be producing — callers must not report+ * the run as stopped.+ */+export type ScheduleDeleteResult = 'not_found' | 'deleted' | 'draining' | 'unconfirmed';++/**+ * How long an abort's OWNER is presumed alive and obliged to settle the run itself.+ *+ * The rule this enforces: **job state is never persistence acknowledgement.** A job+ * reads `aborted` — or vanishes, or carries `completedAt` — the moment `abortJob` wins+ * its status CAS, which is BEFORE the owner unwinds and writes anything. Both owner+ * paths deliberately settle LAST, after `saveMessage`, so the ONLY acknowledgement that+ * all persistence-producing work finished is the owner's own terminal outcome write+ * (i.e. the run row leaving the active set). Reading post-abort job state as "nothing is+ * generating" confirms a drain mid-write: account deletion destroys the user's data and+ * the owner then writes a message back for the deleted account.+ *+ * So while an abort is in flight, nothing but the owner settles — deletion drains+ * defer (503 + Retry-After) and the reconciler leaves the row alone. Past the window+ * the owner is presumed dead, and the reconciler's orphan handling takes over.+ *+ * Deliberately the reconciler's ORPHAN cutoff rather than a shorter local grace: that+ * presumption already exists and is already the thing that would eventually clear the+ * row. One rule, not two.+ */+export const ABORT_OWNER_PRESUMED_ALIVE_MS: number = 30 * 60_000;++/**+ * Whether an abort was requested for this run recently enough that its owner is still+ * expected to persist and settle. `abortRequestedAt` is stamped BEFORE any abort is+ * signalled (see `abortActiveRun` and the interactive abort route), which is what makes+ * it usable as evidence that post-abort job state is not yet a settled generation.+ */+/** How long a resume-claim stamp fences quiesce settling before it is presumed+ * crashed. A resume's own generation shows `running` (already unsettleable);+ * this bound only covers the re-pause hand-off window plus crash recovery. */+export const RESUME_HANDOFF_STALE_MS: number = 10 * 60_000;++/** Whether a paused run's RESUME hand-off is still in flight: its approval was+ * consumed and the continuation's re-pause writes may still be landing, so the+ * paused job state is not settleable evidence yet. */+export function hasResumeHandoffInFlight(+ run: { resumeClaimedAt?: Date | null },+ now: number,+): boolean {+ return (+ run.resumeClaimedAt != null && now - run.resumeClaimedAt.getTime() < RESUME_HANDOFF_STALE_MS+ );+}++/**+ * The outcome the generation owner intended for a RETAINED `complete` job, or `success`+ * when it left none (a fire from before the stamp, or a store that never carried it).+ * An unrecognized value is treated as `success` rather than forwarded: it crossed a+ * serialization boundary, and `recordRunOutcome` would reject a status outside its+ * union — losing a rare refinement beats failing the recovery write outright.+ */+export function retainedOutcome(+ jobState: JobState | null,+ fallback: 'success' | 'error',+): {+ status: 'success' | 'error' | 'interrupted' | 'skipped_balance';+ error?: string;+} {+ const stamped = jobState?.scheduleOutcome;+ if (stamped === 'skipped_balance') {+ return { status: 'skipped_balance' };+ }+ if (stamped === 'interrupted') {+ return { status: 'interrupted', error: jobState?.scheduleOutcomeError };+ }+ if (stamped === 'error' || fallback === 'error') {+ return { status: 'error', error: jobState?.scheduleOutcomeError ?? 'Run ended in error' };+ }+ return { status: fallback };+}++export function hasAbortInFlight(run: { abortRequestedAt?: Date }, now: number): boolean {+ if (run.abortRequestedAt == null) {+ return false;+ }+ return now - new Date(run.abortRequestedAt).getTime() < ABORT_OWNER_PRESUMED_ALIVE_MS;+}++/**+ * Renewable upload hold for schedule attachments (extendFilesTTL-shaped), replacing+ * the earlier permanent TTL removal, which leaked the upload forever when the schedule+ * was deleted before its first run, its file_ids were replaced, or creation failed.+ * Touched at create/edit and at every fire preflight; the FIRST fire that actually+ * sends the file clears its TTL permanently through the ordinary consumption path, so+ * the hold only has to bridge upload -> first consumption. `renewMs` covers the longest+ * cadence gap (weekly) twice over; `maxLifetimeMs` bounds a schedule that never manages+ * to consume (auto-disable stops its renewals long before this ceiling).+ */+export const SCHEDULE_FILE_HOLD: { renewMs: number; maxLifetimeMs: number } = {+ renewMs: 14 * 24 * 60 * 60 * 1000,+ maxLifetimeMs: 90 * 24 * 60 * 60 * 1000,+};++export interface ScheduleFileRef {+ file_id: string;+ filepath?: string;+ filename?: string;+ type?: string;+ height?: number;+ width?: number;+ source?: string;+}++export interface ScheduleEngineDeps {+ methods: ScheduleMethods;+ /** Resolves interface.schedules limits, per-principal when a user is given. */+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;+ /** Loads the owning user (id + tenant) or null when deleted/disabled. */+ getUserContext: (userId: string | Types.ObjectId) => Promise<ScheduleUserContext | null>;+ /** Whether the balance feature gates this user and they are out of credits. */+ isOutOfBalance: (user: ScheduleUserContext) => Promise<boolean>;+ /** Live agent access for the owner: 'missing' (deleted) vs 'forbidden' (ACL revoked). */+ agentAccess: (+ agentId: string,+ user: ScheduleUserContext,+ ) => Promise<'ok' | 'missing' | 'forbidden'>;+ /** Whether the owning user's current role still grants SCHEDULES access. */+ hasScheduleAccess: (user: ScheduleUserContext) => Promise<boolean>;+ /** Re-resolves stored file_ids to attachment payloads; missing files are simply absent. */+ resolveFiles: (fileIds: string[], user: ScheduleUserContext) => Promise<ScheduleFileRef[]>;+ /** Persists a trusted trigger delivery before this occurrence releases its claim. */+ enqueueTrigger: (+ envelope: AgentTriggerEnvelope,+ options?: AgentTriggerEnqueueOptions,+ ) => Promise<unknown>;+ /**+ * Reads the durable trigger delivery for a reservation's `deliveryKey`, so+ * reconciliation can tell a still-live admission (`staging`/`pending`/`leased`, which a+ * `Retry-After` can defer up to 24h) or a dead-letter (`dead`, with its `lastError`)+ * apart from a genuinely orphaned jobless run. Null when no delivery record exists.+ */+ getTriggerDelivery: (deliveryKey: string) => Promise<{+ status: AgentTriggerDeliveryStatus;+ lastError?: AgentTriggerDeliveryFailure;+ } | null>;+ /** Runs fn inside the owner's tenant ALS context. */+ runInTenantContext: <T>(user: ScheduleUserContext, fn: () => Promise<T>) => Promise<T>;+ /**+ * Job-store state for a run's conversation, or null when the job is gone. Carries+ * the job's scheduled identity so reconciliation can verify the job at this+ * conversationId is THIS occurrence's generation (a replacement user turn reuses+ * the conversationId but strips scheduleId/scheduledFor) before trusting the status.+ */+ getJobStatus: (conversationId: string) => Promise<JobState | null>;+ /** True once graceful shutdown began; fires step aside instead of dispatching. */+ isShuttingDown?: () => boolean;+ /**+ * Aborts the loopback generation for a scheduled occurrence, identity-guarded so+ * it never signals/clobbers a replacement turn that reused the conversationId.+ * Used by deletion quiescing to stop an in-flight run before its evidence is+ * erased. `preserve` keeps the terminal job for the reconciler (per-schedule+ * delete, whose run row survives to drive cleanup); account deletion passes false+ * since it hard-deletes the run rows, so a preserved job would leak in the store.+ */+ abortScheduledJob: (+ conversationId: string,+ identity: JobIdentity,+ options?: { preserve?: boolean },+ ) => Promise<boolean>;+ /**+ * Deletes a retained terminal job after the reconciler has finalized its run.+ * Identity-guarded: only deletes when the job still carries this run's+ * scheduleId/scheduledFor, so a replacement generation occupying the same+ * conversationId is never destroyed. Gives `preserveForReconcile` jobs (kept+ * without `completedAt` so the finished-job sweep can't reap them early) a+ * definitive cleanup path.+ */+ clearReconciledJob: (conversationId: string, identity: JobIdentity) => Promise<void>;+ /** Global in-flight scheduled-run count (system tenant scope) for the fire cap. */+ countActiveRunsGlobal: () => Promise<number>;+ /**+ * The GLOBAL kill switch, deliberately distinct from per-principal availability.+ * True when scheduling is stopped for the whole deployment: the SCHEDULES_DISABLED+ * env lever (works even when the config plane is unhealthy), or `interface.schedules:+ * false` in the BASE config — read base-only so no role/user/tenant override can+ * re-enable it. Checked once per engine tick, so the uncached read is negligible.+ */+ isGloballyDisabled: () => Promise<boolean>;+ /** Whether the run owner's account deletion has begun. Fail-closed (unknown == true). */+ isOwnerDeleting: (userId: string) => Promise<boolean>;+ /**+ * Runs `claim` against the lowest free GLOBAL capacity slot, retrying the next slot+ * when the unique partial index rejects a collision. Enforces fireConcurrency in the+ * database instead of via a read-then-compare count, so concurrent admissions of+ * different schedules cannot both pass a cap-1 check. Occupancy is read in system+ * scope so the cap stays global across tenants.+ */+ withGlobalCapacitySlot: <T>(+ cap: number,+ claim: (slot: number) => Promise<SlotClaimResult<T>>,+ ) => Promise<{ claimed: T } | 'capacity'>;+}++/** The immutable scheduled identity of a generation job, for reconcile/abort fencing. */+export interface JobIdentity {+ scheduleId: string;+ scheduledFor: string | Date;+}++/** Job-store state plus the job's scheduled identity (absent on a replacement turn). */+export interface JobState {+ status: string;+ createdAt?: number;+ scheduleId?: string;+ scheduledFor?: string;+ /** Whether the generation ever emitted its created event — false means a+ * pre-start abort whose conversation never came to exist. */+ createdEventEmitted?: boolean;+ /** The outcome the generation owner intended to record, stamped on a retained+ * terminal job. A generic `complete` cannot distinguish a clean run from a balance+ * refusal or a swallowed provider failure, so reconciliation prefers this when the+ * owner left it. See SerializableJobData.scheduleOutcome. */+ scheduleOutcome?: string;+ scheduleOutcomeError?: string;+ preserveForScheduleReconcile?: boolean;+}++export interface FireResult {+ fired: boolean;+ conversationId?: string;+ skipped?:+ | 'overlap'+ | 'balance'+ | 'capacity'+ | 'duplicate'+ | 'superseded'+ | 'agent_deleted'+ | 'user_missing'+ | 'user_deleting'+ | 'permission_revoked'+ | 'rate_limited'+ | 'disabled';+ error?: string;+}++export type FireableSchedule = ISchedule;diff --git a/packages/api/src/stream/ApprovalLifecycle.ts b/packages/api/src/stream/ApprovalLifecycle.tsindex 29797a09cf..ac6961af51 100644--- a/packages/api/src/stream/ApprovalLifecycle.ts+++ b/packages/api/src/stream/ApprovalLifecycle.ts@@ -1,6 +1,11 @@import { logger } from '@librechat/data-schemas';import type { Agents } from 'librechat-data-provider';-import type { IJobStoreV2, JobMetadataPatch, SteerQueueItem } from '~/stream/interfaces/IJobStore';+import type {+ IJobStoreV2,+ JobMetadataPatch,+ SerializableJobData,+ SteerQueueItem,+} from '~/stream/interfaces/IJobStore';import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';import {isPendingActionExpired,@@ -377,15 +382,17 @@ export class ApprovalLifecycle {}/**- * Expires the observed approval and returns the winning job identity. Callers- * that need to notify runtime-local subscribers can use the identity to avoid- * delivering the predecessor's terminal event to a replacement generation.+ * Expires the observed approval and returns the winning pre-transition job.+ * Callers that need to notify runtime-local subscribers or run host lifecycle+ * hooks get both the exact generation identity and its trusted metadata without+ * a racy post-transition read (the terminal TTL may already remove the hash).*/async expireWithIdentity(streamId: string,expectedActionId?: string,expectedCreatedAt?: number,- ): Promise<number | null> {+ options?: { markHostActionPending?: boolean },+ ): Promise<SerializableJobData | null> {const job = await this.waitForPausePersistence(streamId, expectedCreatedAt);if (!job ||@@ -394,14 +401,25 @@ export class ApprovalLifecycle {) {return null;}+ // Built-in in-memory stores may mutate the returned job object in place during+ // transitionStatus. Snapshot it before the CAS so the host hook receives the+ // observed generation metadata even when terminal TTL removes or rewrites it.+ const expiredJob = { ...job };const createdAt = job.createdAt;const ok = await this.store.transitionStatus(streamId, {from: 'requires_action',to: 'aborted',clear: ['pendingAction', 'pendingActionId'],// completedAt lets the stores' terminal-cleanup reclaim the job; without- // it an expired approval lingers in the in-memory map indefinitely.- patch: { error: 'Approval expired before a decision was made', completedAt: Date.now() },+ // it an expired approval lingers in the in-memory map indefinitely. When a host+ // action is owed, `terminalHostActionPending` retains the job (and keeps it+ // enumerable) until the adapter acknowledges — set ATOMICALLY here so a crash right+ // after this CAS still leaves the durable retry evidence.+ patch: {+ error: 'Approval expired before a decision was made',+ completedAt: Date.now(),+ ...(options?.markHostActionPending === true && { terminalHostActionPending: true }),+ },expectActionId: expectedActionId,expectCreatedAt: createdAt,});@@ -409,6 +427,6 @@ export class ApprovalLifecycle {this.callbacks.onExpired?.(streamId, createdAt);logger.debug(`[ApprovalLifecycle] expired pending review: ${streamId}`);}- return ok ? createdAt : null;+ return ok ? expiredJob : null;}}diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex b4a859d896..4b40b6c205 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -449,6 +449,13 @@ export interface GenerationJobManagerOptions {cleanupOnComplete?: boolean;}+/** Host-owned lifecycle seam for durable work layered on agent generations.+ * Delivery is at-least-once: implementations must be idempotent by generation. */+export type ApprovalExpiredHandler = (+ streamId: string,+ job: SerializableJobData,+) => void | Promise<void>;+export interface CreateGenerationJobOptions {startupTelemetry?: AgentStartupTelemetry;initialMetadata?: Partial<t.GenerationJobMetadata>;@@ -695,6 +702,10 @@ class GenerationJobManagerClass {/** Whether to cleanup event transport immediately on job completion */private _cleanupOnComplete = true;+ /** Optional host hook; the generic stream runtime does not know what external+ * durable work (scheduled chats, webhooks, etc.) a generation represents. */+ private approvalExpiredHandler: ApprovalExpiredHandler | undefined;+constructor(options?: GenerationJobManagerOptions) {const jobStore =options?.jobStore ?? new InMemoryJobStore({ ttlAfterComplete: 0, maxJobs: 1000 });@@ -823,6 +834,12 @@ class GenerationJobManagerClass {return this._isRedis;}+ /** Installs the application-owned approval-expiry hook without coupling the+ * stream package to any particular trigger/scheduler implementation. */+ setApprovalExpiredHandler(handler?: ApprovalExpiredHandler): void {+ this.approvalExpiredHandler = handler;+ }+private get storeLabel(): GenerationJobStore {return this._isRedis ? 'redis' : 'memory';}@@ -2499,6 +2516,13 @@ class GenerationJobManagerClass {agent_id: jobData.agent_id,// Surface whether the turn was temporary so a resume keeps it non-persisted.isTemporary: jobData.isTemporary,+ scheduleId: jobData.scheduleId,+ scheduledFor: jobData.scheduledFor,+ scheduleConfigRevision: jobData.scheduleConfigRevision,+ scheduleManual: jobData.scheduleManual,+ scheduleOutcome: jobData.scheduleOutcome,+ scheduleOutcomeError: jobData.scheduleOutcomeError,+ preserveForScheduleReconcile: jobData.preserveForScheduleReconcile,// Surface deferred tools discovered before the pause so the resume route can// replay them into createRun (the rebuilt graph passes `messages: []`).discoveredTools: jobData.discoveredTools,@@ -3503,6 +3527,7 @@ class GenerationJobManagerClass {if (!retainForTerminalReplay &&terminalJob?.providerDrained !== false &&+ terminalJob?.preserveForScheduleReconcile !== true &&(terminalJob?.createdAt !== createdAt || terminalJob.terminalPersistencePending !== true)) {// A same-stream replacement created after the claim makes this a safe@@ -6359,6 +6384,7 @@ class GenerationJobManagerClass {job?.createdAt === expectedCreatedAt &&job.providerExecutionId === providerExecutionId &&job.providerDrained === true &&+ job.preserveForScheduleReconcile !== true &&job.terminalPersistencePending !== true &&(job.status === 'complete' || job.status === 'aborted')) {@@ -7004,19 +7030,42 @@ class GenerationJobManagerClass {);}}- const expiredCreatedAt = await this._approvals.expireWithIdentity(+ const expiredJob = await this._approvals.expireWithIdentity(streamId,actionId,expectedCreatedAt ?? observedJob?.createdAt,+ // Only retain a durable host-action marker when a host adapter is installed to+ // consume it — a store with no handler owes no action and accumulates nothing.+ { markHostActionPending: this.approvalExpiredHandler != null },);- if (expiredCreatedAt == null) {+ if (expiredJob == null) {return false;}- await this.notifyApprovalExpiredRuntime(streamId, expiredCreatedAt, observedRuntime);+ await this.runApprovalExpiredHandler(streamId, expiredJob);+ await this.notifyApprovalExpiredRuntime(streamId, expiredJob.createdAt, observedRuntime);return true;}+ private async runApprovalExpiredHandler(+ streamId: string,+ job: SerializableJobData,+ ): Promise<void> {+ try {+ await this.approvalExpiredHandler?.(streamId, job);+ // Success: the durable host action is settled. Clear its pending marker, fenced to+ // this exact generation so a replacement at the same streamId keeps its own state.+ // A no-op handler (or none) clears harmlessly, so non-scheduled jobs never linger.+ await this.jobStore.clearTerminalHostAction?.(streamId, job.createdAt);+ } catch (err) {+ // Expiry itself already won its exact CAS. Keep terminal notification moving; the+ // `terminalHostActionPending` marker stays set so a later cleanup pass (this or+ // another replica, across restarts) re-enumerates the job and retries this+ // idempotent hook until it acknowledges.+ logger.error(`[GenerationJobManager] Approval-expiry host hook failed: ${streamId}`, err);+ }+ }+private async notifyApprovalExpiredRuntime(streamId: string,createdAt: number,@@ -7058,10 +7107,37 @@ class GenerationJobManagerClass {private async expireStaleApprovals(): Promise<void> {let changed = false;- for (const streamId of this.runtimeState.keys()) {+ // Scan durable pauses as well as local runtimes. A process can restart while an+ // approval waits; if expiry were limited to runtimeState, store cleanup would+ // terminalize that ownerless job without crossing the host lifecycle hook.+ const candidates = new Map<string, SerializableJobData>();+ if (this.jobStore.getRequiresActionJobs) {+ try {+ for (const job of await this.jobStore.getRequiresActionJobs()) {+ candidates.set(job.streamId, job);+ }+ } catch (err) {+ logger.error('[GenerationJobManager] Failed to enumerate pending approvals', err);+ }+ }+ // Also scan terminal jobs that still owe a host lifecycle hook. An expired approval is+ // no longer in the requires_action index, so without this a failed host hook (e.g. the+ // schedule outcome write) would never be retried on a replica or after a restart that+ // has no local runtime — the exact clustered-entrypoint gap, which runs no reconciler.+ if (this.jobStore.getTerminalHostActionJobs) {+ try {+ for (const job of await this.jobStore.getTerminalHostActionJobs()) {+ candidates.set(job.streamId, job);+ }+ } catch (err) {+ logger.error('[GenerationJobManager] Failed to enumerate pending host actions', err);+ }+ }+ const streamIds = new Set([...this.runtimeState.keys(), ...candidates.keys()]);+ for (const streamId of streamIds) {let job: SerializableJobData | null;try {- job = await this.jobStore.getJob(streamId);+ job = candidates.get(streamId) ?? (await this.jobStore.getJob(streamId));} catch (err) {logger.error(`[GenerationJobManager] Failed to read job during approval expiry sweep: ${streamId}`,@@ -7078,6 +7154,13 @@ class GenerationJobManagerClass {// The `errorEvent` flag (set by emitError) keeps this idempotent vs the win path.const runtime = this.runtimeState.get(streamId);if (job?.status === 'aborted' && job.error === APPROVAL_EXPIRED_ERROR) {+ // Retry the durable host hook ONLY while its marker is unacknowledged, so a+ // successful ack prevents duplicate work. The terminal SSE relay below is+ // separately idempotent (emitError's errorEvent flag) and always runs so a+ // loser-replica subscriber still gets a terminal event.+ if (job.terminalHostActionPending === true) {+ await this.runApprovalExpiredHandler(streamId, job);+ }await this.notifyApprovalExpiredRuntime(streamId, job.createdAt, runtime);changed = this.releaseJobOwnership(streamId, job.createdAt) || changed;continue;diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.tsindex 8506f13f5a..ff3be25270 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts@@ -333,6 +333,13 @@ describe('RedisJobStore', () => {model: 'test-model',agent_id: 'agent-1',isTemporary: false,+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleConfigRevision: 4,+ scheduleManual: false,+ scheduleOutcome: 'interrupted',+ scheduleOutcomeError: 'Schedule deleted',+ preserveForScheduleReconcile: true,promptTokens: 0,discoveredTools: [],preemptCapable: true,@@ -388,6 +395,13 @@ describe('RedisJobStore', () => {model: 'test-model',agent_id: 'agent-1',isTemporary: false,+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleConfigRevision: 4,+ scheduleManual: false,+ scheduleOutcome: 'interrupted',+ scheduleOutcomeError: 'Schedule deleted',+ preserveForScheduleReconcile: true,promptTokens: 0,discoveredTools: [],resolvedAskUserQuestions: [@@ -411,6 +425,13 @@ describe('RedisJobStore', () => {responseMessageId: 'response-1',agent_id: 'agent-1',isTemporary: '0',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ scheduleConfigRevision: '4',+ scheduleManual: '0',+ scheduleOutcome: 'interrupted',+ scheduleOutcomeError: 'Schedule deleted',+ preserveForScheduleReconcile: '1',promptTokens: '0',discoveredTools: '[]',resolvedAskUserQuestions: JSON.stringify([@@ -644,6 +665,7 @@ describe('RedisJobStore', () => {const evalResult = createDeferred<number>();const runningMembership = createDeferred<number>();const requiresActionRemoval = createDeferred<number>();+ const terminalHostActionRemoval = createDeferred<number>();const userMembership = createDeferred<number>();const userExpiry = createDeferred<number>();const started: string[] = [];@@ -667,9 +689,13 @@ describe('RedisJobStore', () => {started.push('user');return userMembership.promise;}),- srem: jest.fn(() => {- started.push('requires_action');- return requiresActionRemoval.promise;+ srem: jest.fn((key: string) => {+ if (key === 'stream:requires_action') {+ started.push('requires_action');+ return requiresActionRemoval.promise;+ }+ started.push('terminal_host_action');+ return terminalHostActionRemoval.promise;}),hgetall: jest.fn(() => jobHashFromCreationCall(evalJobCreation.mock.calls[0])),expire,@@ -684,16 +710,23 @@ describe('RedisJobStore', () => {expect(started).toEqual(['job']);evalResult.resolve(1);- await waitFor(() => started.length === 4);+ await waitFor(() => started.length === 5);- expect(started).toEqual(['job', 'running', 'requires_action', 'user']);+ expect(started).toEqual(['job', 'running', 'requires_action', 'terminal_host_action', 'user']);expect(settled).toBe(false);expect(expire).not.toHaveBeenCalled();userMembership.resolve(1);await waitFor(() => expire.mock.calls.length === 1);- expect(started).toEqual(['job', 'running', 'requires_action', 'user', 'user_expiry']);+ expect(started).toEqual([+ 'job',+ 'running',+ 'requires_action',+ 'terminal_host_action',+ 'user',+ 'user_expiry',+ ]);expect(expire).toHaveBeenCalledWith('stream:user:{user-1}:jobs', 60);expect(settled).toBe(false);@@ -705,6 +738,10 @@ describe('RedisJobStore', () => {await Promise.resolve();expect(settled).toBe(false);+ terminalHostActionRemoval.resolve(1);+ await Promise.resolve();+ expect(settled).toBe(false);+requiresActionRemoval.resolve(1);await expect(creating).resolves.toMatchObject({streamId: 'stream-1',diff --git a/packages/api/src/stream/__tests__/pendingAction.spec.ts b/packages/api/src/stream/__tests__/pendingAction.spec.tsindex 551e68f60f..cdca10c1e1 100644--- a/packages/api/src/stream/__tests__/pendingAction.spec.ts+++ b/packages/api/src/stream/__tests__/pendingAction.spec.ts@@ -782,6 +782,56 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()});describe('expireApproval notification', () => {+ test('delivers the exact expired generation to the host lifecycle hook', async () => {+ const streamId = 'stream-expire-host-hook';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.updateMetadata(streamId, {+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ });+ await manager.approvals.pause(streamId, buildAction(streamId));+ const onApprovalExpired = jest.fn(async () => undefined);+ manager.setApprovalExpiredHandler(onApprovalExpired);++ expect(await manager.expireApproval(streamId)).toBe(true);++ expect(onApprovalExpired).toHaveBeenCalledWith(+ streamId,+ expect.objectContaining({+ createdAt: job.createdAt,+ status: 'requires_action',+ scheduleId: 'schedule-1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ }),+ );+ });++ test('expires a durable paused job even when no process-local runtime survived', async () => {+ const streamId = 'stream-expire-ownerless-host-hook';+ const job = await jobStore.createJob(streamId, 'user-1', streamId, undefined, {+ scheduleId: 'schedule-ownerless',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ });+ const lifecycle = new ApprovalLifecycle(jobStore);+ await lifecycle.pause(streamId, buildAction(streamId, { expiresAt: Date.now() - 1 }));+ const onApprovalExpired = jest.fn(async () => undefined);+ manager.setApprovalExpiredHandler(onApprovalExpired);++ await (+ manager as unknown as { expireStaleApprovals: () => Promise<void> }+ ).expireStaleApprovals();++ expect(onApprovalExpired).toHaveBeenCalledWith(+ streamId,+ expect.objectContaining({+ createdAt: job.createdAt,+ scheduleId: 'schedule-ownerless',+ status: 'requires_action',+ }),+ );+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({ status: 'aborted' });+ });+test('does not expire a replacement paused on the same action id', async () => {const streamId = 'stream-expire-epoch-fence';const now = jest.spyOn(Date, 'now').mockReturnValue(1000);@@ -928,6 +978,154 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()});});+ describe('durable approval-expiry host action', () => {+ const sweep = (m: GenerationJobManagerClass) =>+ (m as unknown as { expireStaleApprovals: () => Promise<void> }).expireStaleApprovals();++ async function pauseScheduled(streamId: string) {+ const job = await manager.createJob(streamId, 'user-1');+ await manager.updateMetadata(streamId, {+ scheduleId: 's1',+ scheduledFor: '2026-08-17T12:00:00.000Z',+ });+ await manager.approvals.pause(streamId, buildAction(streamId));+ return job;+ }++ test('retains the marker on hook failure and retries it on a later cleanup pass', async () => {+ const streamId = 'stream-host-retry';+ await pauseScheduled(streamId);+ const handler = jest+ .fn<Promise<void>, [string, { scheduleId?: string }]>()+ .mockRejectedValueOnce(new Error('mongo down'))+ .mockResolvedValue(undefined);+ manager.setApprovalExpiredHandler(handler);++ // First expiry wins its CAS, but the host hook FAILS: the durable marker is retained.+ expect(await manager.expireApproval(streamId)).toBe(true);+ expect(handler).toHaveBeenCalledTimes(1);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(1);++ // A later cleanup pass re-enumerates the aborted job and retries; it now succeeds+ // and clears the marker.+ await sweep(manager);+ expect(handler).toHaveBeenCalledTimes(2);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);+ });++ test('a fresh manager (restart / other replica) retries the pending host action', async () => {+ const streamId = 'stream-host-restart';+ const job = await pauseScheduled(streamId);+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));+ expect(await manager.expireApproval(streamId)).toBe(true);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);++ // Another replica / a restarted process: a fresh manager over the SAME durable store,+ // with NO local runtime for this stream.+ const other = new GenerationJobManagerClass();+ other.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ other.initialize();+ const succeeding = jest.fn().mockResolvedValue(undefined);+ other.setApprovalExpiredHandler(succeeding);++ await sweep(other);++ expect(succeeding).toHaveBeenCalledWith(+ streamId,+ expect.objectContaining({ createdAt: job.createdAt, scheduleId: 's1' }),+ );+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();+ await other.destroy();+ });++ test('a successful acknowledgement prevents duplicate work on later sweeps', async () => {+ const streamId = 'stream-host-ack';+ await pauseScheduled(streamId);+ const handler = jest.fn().mockResolvedValue(undefined);+ manager.setApprovalExpiredHandler(handler);++ expect(await manager.expireApproval(streamId)).toBe(true);+ expect(handler).toHaveBeenCalledTimes(1);+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);++ await sweep(manager);+ expect(handler).toHaveBeenCalledTimes(1); // no duplicate invocation+ });++ test('clearTerminalHostAction is identity-fenced against a replacement generation', async () => {+ const streamId = 'stream-host-fence';+ const job = await pauseScheduled(streamId);+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));+ await manager.expireApproval(streamId);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);++ // A clear fenced to a DIFFERENT generation must not clear this one's action.+ await jobStore.clearTerminalHostAction(streamId, job.createdAt + 1);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);++ // The exact generation clears it.+ await jobStore.clearTerminalHostAction(streamId, job.createdAt);+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();+ });++ test('terminalizes and delivers the terminal state even when the first hook fails', async () => {+ const streamId = 'stream-host-notify';+ await pauseScheduled(streamId);+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));++ // The CAS + terminal notification proceed regardless of the host hook outcome.+ expect(await manager.expireApproval(streamId)).toBe(true);+ expect(await manager.getJobStatus(streamId)).toBe('aborted');+ });++ test('retention is refreshed on each retry so a long host outage keeps the evidence', async () => {+ const streamId = 'stream-host-refresh';+ await pauseScheduled(streamId);+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('mongo down')));+ expect(await manager.expireApproval(streamId)).toBe(true);++ const before = (await jobStore.getJob(streamId))?.terminalHostActionRefreshedAt;++ // A later cleanup pass re-enumerates it for retry; the hook still fails.+ await sweep(manager);++ const after = await jobStore.getJob(streamId);+ // Still pending, and its retention basis moved forward — so the bounded retention+ // window measures from the LAST retry, not from when the approval expired.+ expect(after?.terminalHostActionPending).toBe(true);+ expect(after?.terminalHostActionRefreshedAt).toEqual(expect.any(Number));+ if (before != null) {+ expect(after!.terminalHostActionRefreshedAt!).toBeGreaterThanOrEqual(before);+ }+ });++ test('a non-scheduled job with a no-op host hook accumulates no marker', async () => {+ const streamId = 'stream-host-nonsched';+ await manager.createJob(streamId, 'user-1'); // no schedule metadata+ await manager.approvals.pause(streamId, buildAction(streamId));+ // The schedule adapter is installed but owns no action for a non-scheduled job.+ manager.setApprovalExpiredHandler(+ jest.fn(async (_streamId: string, job: { scheduleId?: string }) => {+ if (!job.scheduleId) {+ return;+ }+ }),+ );++ expect(await manager.expireApproval(streamId)).toBe(true);+ // Marked atomically then cleared on the no-op success — nothing lingers.+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);+ });+ });+describe('facade integration', () => {test('requires_action drops the running count but keeps the user-active set', async () => {const streamId = 'stream-counts';@@ -997,9 +1195,9 @@ describe('InMemoryJobStore — approval expiry cleanup', () => {expect((await store.getJob('epoch-guard'))?.status).toBe('error');});- test('cleanup() finalizes and reclaims a past-expiry pending-approval job', async () => {+ test('cleanup() finalizes a past-expiry approval, retaining it for host-hook retry', async () => {const store = new InMemoryJobStore({ ttlAfterComplete: 0 });- await store.createJob('s1', 'u1');+ const job = await store.createJob('s1', 'u1');const action = buildPendingAction(buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),@@ -1011,7 +1209,17 @@ describe('InMemoryJobStore — approval expiry cleanup', () => {patch: { pendingAction: action, pendingActionId: action.actionId },});- // A past-expiry approval must be finalized + reclaimed, not left resident.+ // A past-expiry approval is finalized (aborted) but RETAINED with a pending host-action+ // marker so the manager relay can still run its lifecycle hook (the store cannot know+ // whether one is owed); it is enumerable for that retry.+ await store.cleanup();+ const aborted = await store.getJob('s1');+ expect(aborted?.status).toBe('aborted');+ expect(aborted?.terminalHostActionPending).toBe(true);+ expect(await store.getTerminalHostActionJobs()).toHaveLength(1);++ // Once the host action acknowledges, the marker clears and the next cleanup reclaims it.+ await store.clearTerminalHostAction('s1', job.createdAt);await store.cleanup();expect(await store.getJob('s1')).toBeNull();});diff --git a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts b/packages/api/src/stream/__tests__/staleJobReaping.spec.tsindex d6982ba4f8..fca8fff107 100644--- a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts+++ b/packages/api/src/stream/__tests__/staleJobReaping.spec.ts@@ -34,6 +34,35 @@ describe('InMemoryJobStore - stale running-job failsafe', () => {await store.destroy();});+ it('retains a terminal schedule job until reconciliation releases it', async () => {+ const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });++ const job = await store.createJob('scheduled-run', 'u1', 'scheduled-run', undefined, {+ scheduleId: 'schedule-1',+ preserveForScheduleReconcile: true,+ });+ await store.updateJob(+ job.streamId,+ {+ status: 'complete',+ completedAt: Date.now(),+ scheduleOutcome: 'success',+ },+ job.createdAt,+ );++ expect(await store.cleanup()).toBe(0);+ expect(await store.hasJob(job.streamId)).toBe(true);++ await store.updateJob(job.streamId, { preserveForScheduleReconcile: false }, job.createdAt);++ expect(await store.cleanup()).toBe(1);+ expect(await store.hasJob(job.streamId)).toBe(false);++ await store.destroy();+ });+it('does not reap a running job with recent activity even if created long ago', async () => {const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');const store = new InMemoryJobStore({ staleJobTimeout: 1000 });diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.tsindex 9b9653d00c..2ae4ad2d18 100644--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts@@ -46,6 +46,10 @@ const STEER_RECEIPT_MAX_PER_STREAM: number = 100;const CREATE_REPLACEMENT_RECEIPT_MAX = 32;const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;const TERMINAL_PERSISTENCE_RETENTION_MS = 5 * 60 * 1000;+/** Bounded window a terminal job owing a host lifecycle hook is retained for retry before+ * it is reaped even if unacknowledged (a permanently-failing hook cannot leak forever).+ * Mirrors the Redis store's 24h host-action TTL. */+const HOST_ACTION_RETENTION_MS = 24 * 60 * 60 * 1000;function assertCreateIdempotencyArguments(claimKey?: string,@@ -1026,6 +1030,40 @@ export class InMemoryJobStore implements IJobStoreV2 {return running;}+ async getRequiresActionJobs(): Promise<SerializableJobData[]> {+ const paused: SerializableJobData[] = [];+ for (const job of this.jobs.values()) {+ if (job.status === 'requires_action') {+ paused.push(job);+ }+ }+ return paused;+ }++ async getTerminalHostActionJobs(): Promise<SerializableJobData[]> {+ const pending: SerializableJobData[] = [];+ const now = Date.now();+ for (const job of this.jobs.values()) {+ if (job.terminalHostActionPending === true) {+ // Enumerating IS the retry attempt: refresh retention so evidence outlives a host+ // dependency that is unreachable for longer than the retention window.+ job.terminalHostActionRefreshedAt = now;+ pending.push(job);+ }+ }+ return pending;+ }++ async clearTerminalHostAction(streamId: string, expectedCreatedAt?: number): Promise<void> {+ const job = this.jobs.get(streamId);+ // Identity-fenced: a replacement generation at this streamId (a newer createdAt) must+ // not have a predecessor's marker cleared on its behalf.+ if (job == null || (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)) {+ return;+ }+ delete job.terminalHostActionPending;+ }+async cleanup(): Promise<number> {const now = Date.now();const toDelete: Array<{ streamId: string; createdAt: number }> = [];@@ -1089,7 +1127,19 @@ export class InMemoryJobStore implements IJobStoreV2 {}const isFinished = ['complete', 'error', 'aborted'].includes(job.status);- if (isFinished && job.completedAt) {+ // A terminal job that still owes a host lifecycle hook is retained (like a+ // reconcile-preserved job) so a later cleanup pass can enumerate and retry it —+ // but only within a bounded window, so a permanently-failing hook cannot leak.+ const hostActionHeld =+ job.terminalHostActionPending === true &&+ now - (job.terminalHostActionRefreshedAt ?? job.completedAt ?? 0) <=+ HOST_ACTION_RETENTION_MS;+ if (+ isFinished &&+ job.completedAt &&+ job.preserveForScheduleReconcile !== true &&+ !hostActionHeld+ ) {// A pending final event is a persistence barrier, not an ordinary// completed row. Give its owner/recovery path a bounded window even// when normal terminal retention is configured to zero.@@ -1117,11 +1167,13 @@ export class InMemoryJobStore implements IJobStoreV2 {job.status = 'aborted';job.completedAt = now;job.error = 'Approval expired before a decision was made';+ // Store-won expiry (the manager's own sweep did not win the CAS): mark the host+ // action pending so the manager relay still runs its lifecycle hook, and RETAIN the+ // job (even under zero terminal retention) so a sweep can enumerate it — cleared and+ // reaped once the hook acknowledges.+ job.terminalHostActionPending = true;delete job.pendingAction;delete job.pendingActionId;- if (this.ttlAfterComplete === 0) {- toDelete.push({ streamId, createdAt: job.createdAt });- }} else if (this.staleJobTimeout > 0 && job.status === 'running') {// Failsafe: reap jobs stuck in "running" with no generation activity for// longer than the stale timeout. These are crashed/hung generations thatdiff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex b218d9d4ac..ea46b4a4e1 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -1540,6 +1540,9 @@ const KEYS = {runningJobs: 'stream:running',/** Jobs paused for human review (global set - single slot) */requiresActionJobs: 'stream:requires_action',+ /** Terminal jobs that still owe a durable host lifecycle hook (global set). Retains+ * the aborted approval-expiry job for cross-replica / post-restart hook retry. */+ terminalHostActionJobs: 'stream:terminal_host_action',/** User's active jobs set, tenant-qualified when tenantId is available */userJobs: (userId: string, tenantId?: string) =>tenantId ? `stream:user:{${tenantId}:${userId}}:jobs` : `stream:user:{${userId}}:jobs`,@@ -2246,6 +2249,11 @@ export class RedisJobStore implements IJobStoreV2 {statusKey === KEYS.requiresActionJobs? this.redis.sadd(KEYS.requiresActionJobs, streamId): this.redis.srem(KEYS.requiresActionJobs, streamId),+ // Terminal host-action membership follows the durable hash field, not status, so+ // an aborted approval-expiry job stays enumerable for hook retry until acked.+ job?.terminalHostActionPending === true+ ? this.redis.sadd(KEYS.terminalHostActionJobs, streamId)+ : this.redis.srem(KEYS.terminalHostActionJobs, streamId),];for (const userJobsKey of observedUserKeys) {if (userJobsKey !== activeUserKey) {@@ -2277,6 +2285,11 @@ export class RedisJobStore implements IJobStoreV2 {} else {pipeline.srem(KEYS.requiresActionJobs, streamId);}+ if (job?.terminalHostActionPending === true) {+ pipeline.sadd(KEYS.terminalHostActionJobs, streamId);+ } else {+ pipeline.srem(KEYS.terminalHostActionJobs, streamId);+ }for (const userJobsKey of observedUserKeys) {if (userJobsKey !== activeUserKey) {pipeline.srem(userJobsKey, streamId);@@ -2438,6 +2451,12 @@ export class RedisJobStore implements IJobStoreV2 {if (terminal && patch?.terminalPersistencePending === true) {ttl = Math.max(ttl, TERMINAL_PERSISTENCE_RETENTION_TTL_S);}+ if (terminal && patch?.terminalHostActionPending === true) {+ // A terminal job owing a host hook must outlive the normal completed TTL so cleanup+ // can still enumerate and retry it across restarts; the pause backstop (24h) bounds+ // the retry window. Cleared to the completed TTL on acknowledgement.+ ttl = Math.max(ttl, this.ttl.requiresAction);+ }if (to === 'requires_action') {// A paused job must outlive its approval window, even when that window is// longer than the running TTL — otherwise Redis evicts it before a@@ -2681,6 +2700,70 @@ export class RedisJobStore implements IJobStoreV2 {return jobs;}+ async getRequiresActionJobs(): Promise<SerializableJobData[]> {+ const streamIds = await this.redis.smembers(KEYS.requiresActionJobs);+ if (streamIds.length === 0) {+ return [];+ }+ const jobs = await Promise.all(streamIds.map((streamId) => this.getJob(streamId)));+ return jobs.filter(+ (job): job is SerializableJobData => job != null && job.status === 'requires_action',+ );+ }++ async getTerminalHostActionJobs(): Promise<SerializableJobData[]> {+ const streamIds = await this.redis.smembers(KEYS.terminalHostActionJobs);+ if (streamIds.length === 0) {+ return [];+ }+ const jobs = await Promise.all(streamIds.map((streamId) => this.getJob(streamId)));+ // The durable hash field is the source of truth; a stale set entry (job reaped, or the+ // marker already cleared) is filtered out and self-heals via reconcileJobMembership.+ const stale: string[] = [];+ const pending: SerializableJobData[] = [];+ for (let i = 0; i < streamIds.length; i++) {+ const job = jobs[i];+ if (job != null && job.terminalHostActionPending === true) {+ pending.push(job);+ } else {+ stale.push(streamIds[i]);+ }+ }+ if (stale.length > 0) {+ await this.redis.srem(KEYS.terminalHostActionJobs, ...stale).catch(() => undefined);+ }+ // Enumerating IS the retry attempt: extend each pending job's TTL so unacknowledged+ // host-action evidence outlives a host dependency (e.g. Mongo) that stays unreachable+ // longer than the retention window. A deployment that stops sweeping lets it age out.+ if (pending.length > 0 && this.ttl.requiresAction > 0) {+ await Promise.all(+ pending.map((job) =>+ this.redis.expire(KEYS.job(job.streamId), this.ttl.requiresAction).catch(() => undefined),+ ),+ );+ }+ return pending;+ }++ async clearTerminalHostAction(streamId: string, expectedCreatedAt?: number): Promise<void> {+ // Identity-fenced: only clear when the hash still holds this exact generation, so a+ // replacement at the same streamId is never cleared through its predecessor. The HDEL+ // and the completed-TTL reset happen atomically; membership is then reconciled to SREM.+ const cleared = (await this.redis.eval(+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'redis.call("HDEL", KEYS[1], "terminalHostActionPending") ' ++ 'if tonumber(ARGV[2]) > 0 then redis.call("EXPIRE", KEYS[1], ARGV[2]) end ' ++ 'return 1',+ 1,+ KEYS.job(streamId),+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',+ String(this.ttl.completed),+ )) as number;+ if (cleared === 1) {+ await this.redis.srem(KEYS.terminalHostActionJobs, streamId).catch(() => undefined);+ }+ }+async cleanup(): Promise<number> {const now = Date.now();const streamIds = await this.redis.smembers(KEYS.runningJobs);@@ -2848,6 +2931,10 @@ export class RedisJobStore implements IJobStoreV2 {patch: {error: 'Approval expired before a decision was made',completedAt: Date.now(),+ // Store-won expiry: mark the host action pending (and extend retention via+ // the TTL rule above) so the manager relay still runs its lifecycle hook on+ // a replica that owns the runtime; cleared once that hook acknowledges.+ terminalHostActionPending: true,},// Scope the CAS to the action we observed as stale: if the user resolved it// and the run re-paused on a fresh action between the read and here, the@@ -4477,6 +4564,26 @@ export class RedisJobStore implements IJobStoreV2 {promptTokens: data.promptTokens ? parseInt(data.promptTokens, 10) : undefined,agent_id: data.agent_id || undefined,isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,+ scheduleId: data.scheduleId || undefined,+ scheduledFor: data.scheduledFor || undefined,+ scheduleConfigRevision: data.scheduleConfigRevision+ ? parseInt(data.scheduleConfigRevision, 10)+ : undefined,+ scheduleManual: data.scheduleManual != null ? data.scheduleManual === '1' : undefined,+ scheduleOutcome:+ data.scheduleOutcome === 'success' ||+ data.scheduleOutcome === 'error' ||+ data.scheduleOutcome === 'interrupted' ||+ data.scheduleOutcome === 'skipped_balance'+ ? data.scheduleOutcome+ : undefined,+ scheduleOutcomeError: data.scheduleOutcomeError || undefined,+ preserveForScheduleReconcile:+ data.preserveForScheduleReconcile != null+ ? data.preserveForScheduleReconcile === '1'+ : undefined,+ terminalHostActionPending:+ data.terminalHostActionPending != null ? data.terminalHostActionPending === '1' : undefined,// Deferred tools discovered before a HITL pause; replayed into createRun on resume.discoveredTools: data.discoveredTools ? JSON.parse(data.discoveredTools) : undefined,activityPhaseSnapshot: data.activityPhaseSnapshotdiff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 10143c62cd..50cb847ebc 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -123,6 +123,31 @@ export interface SerializableJobData {/** Whether sync has been sent to a client */syncSent: boolean;+ /** Trusted schedule identity copied atomically into the generation job. */+ scheduleId?: string;+ scheduledFor?: string;+ scheduleConfigRevision?: number;+ scheduleManual?: boolean;+ /** Terminal outcome evidence retained when the schedule row could not be updated. */+ scheduleOutcome?: 'success' | 'error' | 'interrupted' | 'skipped_balance';+ scheduleOutcomeError?: string;+ preserveForScheduleReconcile?: boolean;+ /**+ * A terminal transition (currently approval expiry) still owes a durable host+ * lifecycle hook. Set atomically with that transition and cleared only once the host+ * adapter acknowledges success, so the job is retained (not reaped) and enumerable by+ * cleanup across restarts and replicas until the hook completes. Generic: a host with+ * no action clears it immediately on its no-op success, so nothing accumulates.+ */+ terminalHostActionPending?: boolean;+ /**+ * Last time a cleanup pass enumerated this pending host action for retry. Retention is+ * measured from this rather than `completedAt`, so evidence survives as long as some+ * replica is still actively retrying the hook (e.g. Mongo unreachable for days), while a+ * deployment that stops retrying entirely still lets it age out instead of leaking.+ */+ terminalHostActionRefreshedAt?: number;+/** Serialized final event for replay */finalEvent?: string;@@ -277,6 +302,13 @@ export type JobMetadataPatch = Partial<| 'model'| 'agent_id'| 'isTemporary'+ | 'scheduleId'+ | 'scheduledFor'+ | 'scheduleConfigRevision'+ | 'scheduleManual'+ | 'scheduleOutcome'+ | 'scheduleOutcomeError'+ | 'preserveForScheduleReconcile'| 'promptTokens'| 'discoveredTools'| 'activityPhaseSnapshot'@@ -661,6 +693,18 @@ export interface IJobStore {deleteJob(streamId: string, expectedCreatedAt?: number): Promise<boolean>;hasJob(streamId: string): Promise<boolean>;getRunningJobs(): Promise<SerializableJobData[]>;+ /** Optional durable paused-job enumeration. Built-in stores implement it so+ * the manager can own approval expiry even after the original runtime died. */+ getRequiresActionJobs?(): Promise<SerializableJobData[]>;+ /** Optional durable enumeration of terminal jobs that still owe a host lifecycle+ * hook (see `terminalHostActionPending`). Built-in stores implement it so cleanup can+ * retry the host adapter after a restart / on another replica, even though the job is+ * no longer in the requires_action index. */+ getTerminalHostActionJobs?(): Promise<SerializableJobData[]>;+ /** Clears the pending-host-action marker once the adapter acknowledges success.+ * Identity-fenced on `expectedCreatedAt` so a replacement generation at the same+ * streamId is never cleared through its predecessor. */+ clearTerminalHostAction?(streamId: string, expectedCreatedAt?: number): Promise<void>;cleanup(): Promise<number>;recordActivity?(streamId: string, expectedCreatedAt?: number): void;getJobCount(): Promise<number>;@@ -902,6 +946,9 @@ export interface IJobStoreV2 extends IJobStore {/** Get all running jobs (for cleanup) */getRunningJobs(): Promise<SerializableJobData[]>;+ /** Get durable paused jobs so approval expiry is not process-runtime-dependent. */+ getRequiresActionJobs?(): Promise<SerializableJobData[]>;+/** Cleanup expired jobs */cleanup(): Promise<number>;diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.tsindex 1a4b2e3376..b334412267 100644--- a/packages/api/src/stream/metadata.ts+++ b/packages/api/src/stream/metadata.ts@@ -30,6 +30,27 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): Jif (metadata.isTemporary !== undefined) {patch.isTemporary = metadata.isTemporary;}+ if (metadata.scheduleId) {+ patch.scheduleId = metadata.scheduleId;+ }+ if (metadata.scheduledFor) {+ patch.scheduledFor = metadata.scheduledFor;+ }+ if (metadata.scheduleConfigRevision !== undefined) {+ patch.scheduleConfigRevision = metadata.scheduleConfigRevision;+ }+ if (metadata.scheduleManual !== undefined) {+ patch.scheduleManual = metadata.scheduleManual;+ }+ if (metadata.scheduleOutcome !== undefined) {+ patch.scheduleOutcome = metadata.scheduleOutcome;+ }+ if (metadata.scheduleOutcomeError !== undefined) {+ patch.scheduleOutcomeError = metadata.scheduleOutcomeError;+ }+ if (metadata.preserveForScheduleReconcile !== undefined) {+ patch.preserveForScheduleReconcile = metadata.preserveForScheduleReconcile;+ }if (metadata.promptTokens !== undefined) {patch.promptTokens = metadata.promptTokens;}diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex 145df044b8..b6727ebfbf 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -31,6 +31,20 @@ export interface GenerationJobMetadata {agent_id?: string;/** Whether the originating turn was a temporary chat; a HITL resume keeps it so. */isTemporary?: boolean;+ /** Trusted scheduled-occurrence identity. These fields are accepted only from a+ * verified agent-trigger request and let pause/resume/reconciliation keep the+ * occurrence attached to the exact generation that owns it. */+ scheduleId?: string;+ scheduledFor?: string;+ scheduleConfigRevision?: number;+ scheduleManual?: boolean;+ /** Intended terminal classification retained when Mongo outcome persistence+ * fails. The scheduler reconciler consumes this evidence before clearing the job. */+ scheduleOutcome?: 'success' | 'error' | 'interrupted' | 'skipped_balance';+ scheduleOutcomeError?: string;+ /** Prevent normal terminal cleanup until schedule reconciliation has consumed+ * the retained outcome evidence. */+ preserveForScheduleReconcile?: boolean;/*** Deferred-tool names discovered (via `tool_search`) before a HITL pause. A resume* replays these into `createRun` because the rebuilt graph uses `messages: []`, sodiff --git a/packages/client/src/components/ControlCombobox.tsx b/packages/client/src/components/ControlCombobox.tsxindex 0233501cf6..05fc1a0bde 100644--- a/packages/client/src/components/ControlCombobox.tsx+++ b/packages/client/src/components/ControlCombobox.tsx@@ -5,6 +5,7 @@ import { Search, ChevronDown } from 'lucide-react';import { SelectRenderer } from '@ariakit/react-components/select/select-renderer';import type { OptionWithIcon } from '~/common';import { usePopoverZIndex } from './OriginalDialog';+import { fieldControl } from './Field';import './AnimatePopover.css';import { JSX } from 'react/jsx-runtime';import { cn } from '~/utils';@@ -14,7 +15,10 @@ interface ControlComboboxProps {displayValue?: string;items: OptionWithIcon[];setValue: (value: string) => void;+ onBlur?: React.FocusEventHandler<HTMLButtonElement>;ariaLabel: string;+ ariaInvalid?: boolean;+ ariaDescribedBy?: string;searchPlaceholder?: string;selectPlaceholder?: string;isCollapsed: boolean;@@ -29,6 +33,8 @@ interface ControlComboboxProps {placement?: Ariakit.SelectStoreProps['placement'];popoverClassName?: string;matchTriggerWidth?: boolean;+ /** `field` matches the `Input` primitive so this can sit in a form row. */+ variant?: 'default' | 'field';gutter?: number;/*** Radix dialogs trap focus, so a portaled popover rendered outside the dialog@@ -46,7 +52,10 @@ function ControlCombobox({displayValue,items,setValue,+ onBlur,ariaLabel,+ ariaInvalid,+ ariaDescribedBy,searchPlaceholder,selectPlaceholder,containerClassName,@@ -61,6 +70,7 @@ function ControlCombobox({placement,popoverClassName,matchTriggerWidth = true,+ variant = 'default',gutter = 4,portal = true,}: ControlComboboxProps): JSX.Element {@@ -136,7 +146,13 @@ function ControlCombobox({);return (- <div className={cn('flex w-full items-center justify-center px-1', containerClassName)}>+ <div+ className={cn(+ 'flex w-full items-center justify-center px-1',+ variant === 'field' && 'px-0',+ containerClassName,+ )}+ ><Ariakit.SelectLabel store={select} className="sr-only">{ariaLabel}</Ariakit.SelectLabel>@@ -145,11 +161,15 @@ function ControlCombobox({store={select}id={selectId}disabled={disabled}+ onBlur={onBlur}+ aria-invalid={ariaInvalid || undefined}+ aria-describedby={ariaDescribedBy}className={cn('flex items-center justify-center gap-2 rounded-full bg-surface-secondary','text-text-primary hover:bg-surface-tertiary','border border-border-light',isCollapsed ? 'h-9 w-9' : 'h-9 w-full rounded-xl px-3 py-2 text-sm',+ variant === 'field' && cn(fieldControl, 'justify-start hover:bg-surface-hover'),className,)}>diff --git a/packages/client/src/components/Dropdown.tsx b/packages/client/src/components/Dropdown.tsxindex c18ddb9bd3..c4ba9a1ae9 100644--- a/packages/client/src/components/Dropdown.tsx+++ b/packages/client/src/components/Dropdown.tsx@@ -4,6 +4,7 @@ import { matchSorter } from 'match-sorter';import * as Select from '@ariakit/react/select';import * as Combobox from '@ariakit/react/combobox';import type { Option } from '~/common';+import { fieldControl } from './Field';import { cn } from '~/utils/';import './Dropdown.css';@@ -25,6 +26,8 @@ interface DropdownProps {ariaLabel?: string;'aria-labelledby'?: string;portal?: boolean;+ /** `field` matches the `Input` primitive so this can sit in a form row. */+ variant?: 'default' | 'field';/** Renders the popover into this element instead of document.body */portalElement?: ((element: HTMLElement) => HTMLElement | null) | HTMLElement | null;disabled?: boolean;@@ -57,6 +60,7 @@ const Dropdown: React.FC<DropdownProps> = ({ariaLabel,'aria-labelledby': ariaLabelledBy,portal = true,+ variant = 'default',portalElement,disabled = false,searchable = false,@@ -140,7 +144,7 @@ const Dropdown: React.FC<DropdownProps> = ({);return (- <div className={cn('relative', className)}>+ <div className={cn('relative', variant === 'field' && 'w-full', className)}><Select.Selectstore={selectProps}disabled={disabled}@@ -149,6 +153,7 @@ const Dropdown: React.FC<DropdownProps> = ({'disabled:cursor-not-allowed disabled:opacity-50 disabled:hover:bg-transparent disabled:hover:text-text-primary',/** Horizontal padding would squeeze the icon, which flex-shrinks to fit */iconOnly ? 'size-10 justify-center px-0' : 'w-fit gap-2 px-3',+ variant === 'field' && fieldControl,triggerClassName,)}data-testid={testId}diff --git a/packages/client/src/components/Field.ts b/packages/client/src/components/Field.tsnew file mode 100644index 0000000000..5c09d95a6b--- /dev/null+++ b/packages/client/src/components/Field.ts@@ -0,0 +1,11 @@+/**+ * The shared appearance of a form control — border, radius, type scale and focus+ * treatment. Owned here so `Input`, `Textarea`, and the select/combobox triggers+ * that have to sit beside them in a form cannot drift apart as the theme evolves.+ * Callers compose a variant rather than restating these classes locally.+ */+export const fieldBase: string =+ 'lc-field flex w-full rounded-lg border border-border-light px-3 py-2 text-sm text-text-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50';++/** A single-line control sized to sit in a form row, matching `Input`. */+export const fieldControl: string = `${fieldBase} h-10 bg-transparent`;diff --git a/packages/client/src/components/Input.tsx b/packages/client/src/components/Input.tsxindex a8eae5c54f..0db5998670 100644--- a/packages/client/src/components/Input.tsx+++ b/packages/client/src/components/Input.tsx@@ -1,4 +1,5 @@import * as React from 'react';+import { fieldControl } from './Field';import { cn } from '~/utils';import './Field.css';@@ -8,10 +9,7 @@ const Input: React.ForwardRefExoticComponent<InputProps & React.RefAttributes<HTReact.forwardRef<HTMLInputElement, InputProps>(({ className, ...props }, ref) => {return (<input- className={cn(- 'lc-field flex h-10 w-full rounded-lg border border-border-light bg-transparent px-3 py-2 text-sm text-text-primary ring-offset-surface-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50',- className ?? '',- )}+ className={cn(fieldControl, 'ring-offset-surface-primary', className ?? '')}ref={ref}{...props}/>diff --git a/packages/client/src/components/Textarea.tsx b/packages/client/src/components/Textarea.tsxindex 8c9a2ee1cc..19259faa18 100644--- a/packages/client/src/components/Textarea.tsx+++ b/packages/client/src/components/Textarea.tsx@@ -1,6 +1,7 @@/* eslint-disable */import * as React from 'react';import TextareaAutosize from 'react-textarea-autosize';+import { fieldBase } from './Field';import { cn } from '~/utils';import './Field.css';@@ -11,10 +12,7 @@ const Textarea: React.ForwardRefExoticComponent<> = React.forwardRef<HTMLTextAreaElement, TextareaProps>(({ className = '', ...props }, ref) => {return (<textarea- className={cn(- 'lc-field flex min-h-20 w-full resize-none rounded-lg border border-border-light bg-surface-secondary px-3 py-2 text-sm text-text-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50',- className,- )}+ className={cn(fieldBase, 'min-h-20 resize-none bg-surface-secondary', className)}ref={ref}{...props}/>diff --git a/packages/client/src/components/index.ts b/packages/client/src/components/index.tsindex 8181ae945b..b03669d4ad 100644--- a/packages/client/src/components/index.ts+++ b/packages/client/src/components/index.ts@@ -9,6 +9,7 @@ export * from './Checkbox';export * from './Dialog';export * from './DropdownMenu';export * from './HoverCard';+export * from './Field';export * from './Input';export * from './InputNumber';export * from './SecretInput';diff --git a/packages/client/src/theme/semanticTokens.spec.ts b/packages/client/src/theme/semanticTokens.spec.tsindex 66c763a82f..0b9cc25a19 100644--- a/packages/client/src/theme/semanticTokens.spec.ts+++ b/packages/client/src/theme/semanticTokens.spec.ts@@ -112,12 +112,23 @@ describe('light brand text', () => {describe('shared field and dropdown interaction styles', () => {it('keeps pointer focus stable and keyboard focus visible on text fields', () => {- ['Input.tsx', 'Textarea.tsx'].forEach((component) => {+ /** The focus treatment lives in the shared field module, so guard it there and+ * assert the primitives still compose it rather than restating the classes. */+ const field = readFileSync(join(__dirname, '..', 'components', 'Field.ts'), 'utf8');++ expect(field).toMatch(/focus-visible:border-border-medium/);+ expect(field).toMatch(/focus-visible:ring-2/);+ expect(field).toMatch(/focus-visible:ring-text-primary/);++ const composers: Array<[string, RegExp]> = [+ ['Input.tsx', /\bfieldControl\b/],+ ['Textarea.tsx', /\bfieldBase\b/],+ ['Dropdown.tsx', /\bfieldControl\b/],+ ['ControlCombobox.tsx', /\bfieldControl\b/],+ ];+ composers.forEach(([component, token]) => {const source = readFileSync(join(__dirname, '..', 'components', component), 'utf8');-- expect(source).toMatch(/focus-visible:border-border-medium/);- expect(source).toMatch(/focus-visible:ring-2/);- expect(source).toMatch(/focus-visible:ring-text-primary/);+ expect(source).toMatch(token);});const secretInput = readFileSync(diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.tsindex e20e9729a6..80acb4990a 100644--- a/packages/data-provider/src/api-endpoints.ts+++ b/packages/data-provider/src/api-endpoints.ts@@ -392,6 +392,11 @@ export const getCategories = () => `${BASE_URL}/api/categories`;export const getAllPromptGroups = () => `${prompts()}/all`;+/* Scheduled chats */+export const schedules = () => `${BASE_URL}/api/schedules`;+export const schedule = (id: string) => `${schedules()}/${encodeURIComponent(id)}`;+export const runSchedule = (id: string) => `${schedule(id)}/run`;+/* Skills */export const skills = () => `${BASE_URL}/api/skills`;export const importSkill = () => `${skills()}/import`;diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex bc97dcd1a1..9384f8ff1c 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1563,6 +1563,19 @@ export const interfaceSchema = z}),]).optional(),+ schedules: z+ .union([+ z.boolean(),+ z.object({+ use: z.boolean().optional(),+ create: z.boolean().optional(),+ maxPerUser: z.number().int().min(0).optional(),+ minIntervalMinutes: z.number().int().min(1).optional(),+ autoDisableAfterFailures: z.number().int().min(1).optional(),+ fireConcurrency: z.number().int().min(1).optional(),+ }),+ ])+ .optional(),}).default({modelSelect: true,@@ -1625,6 +1638,11 @@ export const interfaceSchema = zpublic: true,snapshotFiles: true,},+ // `schedules` is deliberately ABSENT from this default. It is experimental and+ // default-off in v1, and zod applies this whole object when `interface` is omitted+ // from librechat.yaml — including it would silently enable the feature (and permit+ // billable scheduled runs) on every deployment that never opted in. The PERMISSION+ // defaults live in updateInterfacePermissions, which is a separate concern.});export type TInterfaceConfig = z.infer<typeof interfaceSchema>;diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.tsindex 3b94c80b6b..89383f070b 100644--- a/packages/data-provider/src/data-service.ts+++ b/packages/data-provider/src/data-service.ts@@ -6,6 +6,7 @@ import * as permissions from './accessPermissions';import * as endpoints from './api-endpoints';import { uploadEventStream } from './upload';import * as mcp from './types/mcpServers';+import * as sch from './types/schedules';import * as a from './types/assistants';import * as m from './types/mutations';import * as ag from './types/agents';@@ -1078,6 +1079,30 @@ export function listSkills(params?: sk.TSkillListRequest): Promise<sk.TSkillListreturn request.get(endpoints.listSkillsWithFilters(params ?? {}));}+export function getSchedules(): Promise<sch.TSchedulesResponse> {+ return request.get(endpoints.schedules());+}++export function getSchedule(id: string): Promise<sch.TSchedule> {+ return request.get(endpoints.schedule(id));+}++export function createSchedule(payload: sch.TCreateSchedule): Promise<sch.TSchedule> {+ return request.post(endpoints.schedules(), payload);+}++export function updateSchedule(id: string, payload: sch.TUpdateSchedule): Promise<sch.TSchedule> {+ return request.patch(endpoints.schedule(id), payload);+}++export function deleteSchedule(id: string): Promise<{ id: string }> {+ return request.delete(endpoints.schedule(id));+}++export function runScheduleNow(id: string): Promise<sch.TScheduleRunNowResponse> {+ return request.post(endpoints.runSchedule(id), {});+}+export function getSkill(id: string): Promise<sk.TSkill> {return request.get(endpoints.getSkill(id));}diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.tsindex 8c98967cf3..dbe8b51764 100644--- a/packages/data-provider/src/index.ts+++ b/packages/data-provider/src/index.ts@@ -28,6 +28,7 @@ export * from './types/files';export * from './types/mcpServers';export * from './types/mutations';export * from './types/queries';+export * from './types/schedules';export * from './types/skills';export * from './types/runs';export * from './types/web';diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.tsindex 7a485996ef..8f7b8fa2e3 100644--- a/packages/data-provider/src/keys.ts+++ b/packages/data-provider/src/keys.ts@@ -90,6 +90,9 @@ export enum QueryKeys {skillStates = 'skillStates',/* General user favorites */favorites = 'favorites',+ /* Scheduled chats */+ schedules = 'schedules',+ schedule = 'schedule',}// Dynamic query keys that require parameters@@ -138,4 +141,8 @@ export enum MutationKeys {updateSkillNodeContent = 'updateSkillNodeContent',convoPin = 'convoPin',archiveAllConversations = 'archiveAllConversations',+ createSchedule = 'createSchedule',+ updateSchedule = 'updateSchedule',+ deleteSchedule = 'deleteSchedule',+ runSchedule = 'runSchedule',}diff --git a/packages/data-provider/src/permissions.ts b/packages/data-provider/src/permissions.tsindex 08f2c7953d..deeedd57f4 100644--- a/packages/data-provider/src/permissions.ts+++ b/packages/data-provider/src/permissions.ts@@ -68,6 +68,10 @@ export enum PermissionTypes {* Type for Shared Link Permissions*/SHARED_LINKS = 'SHARED_LINKS',+ /**+ * Type for Scheduled Chats Permissions+ */+ SCHEDULES = 'SCHEDULES',}/**@@ -92,11 +96,21 @@ export const PERMISSION_TYPE_INTERFACE_FIELDS: Record<PermissionTypes, string> =[PermissionTypes.REMOTE_AGENTS]: 'remoteAgents',[PermissionTypes.SKILLS]: 'skills',[PermissionTypes.SHARED_LINKS]: 'sharedLinks',+ [PermissionTypes.SCHEDULES]: 'schedules',};/** Set of interface config field names that correspond to role permissions. */export const INTERFACE_PERMISSION_FIELDS = new Set(Object.values(PERMISSION_TYPE_INTERFACE_FIELDS));+/**+ * Interface fields that seed a permission (use/create) BUT also carry runtime+ * config that must survive DB/tenant/user overrides. For these, override+ * sanitizers strip only the permission sub-keys (use/create) from the object+ * form and PRESERVE the boolean form — for `schedules`, `interface.schedules: false`+ * is the runtime feature disable that `getLimits` reads, not a permission toggle.+ */+export const RUNTIME_CONFIG_INTERFACE_FIELDS = new Set<string>(['schedules']);+/*** YAML sub-keys within composite interface permission fields that map to permission bits.* When an interface permission field is an object, only these sub-keys are stripped from@@ -245,6 +259,12 @@ export const skillPermissionsSchema = z.object({});export type TSkillPermissions = z.infer<typeof skillPermissionsSchema>;+export const schedulesPermissionsSchema = z.object({+ [Permissions.USE]: z.boolean().default(true),+ [Permissions.CREATE]: z.boolean().default(true),+});+export type TSchedulesPermissions = z.infer<typeof schedulesPermissionsSchema>;+export const sharedLinksPermissionsSchema = z.object({[Permissions.CREATE]: z.boolean().default(true),[Permissions.SHARE]: z.boolean().default(true),@@ -270,4 +290,5 @@ export const permissionsSchema = z.object({[PermissionTypes.REMOTE_AGENTS]: remoteAgentsPermissionsSchema,[PermissionTypes.SKILLS]: skillPermissionsSchema,[PermissionTypes.SHARED_LINKS]: sharedLinksPermissionsSchema,+ [PermissionTypes.SCHEDULES]: schedulesPermissionsSchema,});diff --git a/packages/data-provider/src/roles.spec.ts b/packages/data-provider/src/roles.spec.tsindex 3cc1740e47..f3b1068d0c 100644--- a/packages/data-provider/src/roles.spec.ts+++ b/packages/data-provider/src/roles.spec.ts@@ -90,7 +90,8 @@ describe('roleDefaults', () => {permType === PermissionTypes.PROMPTS ||permType === PermissionTypes.AGENTS ||permType === PermissionTypes.SKILLS ||- permType === PermissionTypes.SHARED_LINKS;+ permType === PermissionTypes.SHARED_LINKS ||+ permType === PermissionTypes.SCHEDULES;expect({permType,diff --git a/packages/data-provider/src/roles.ts b/packages/data-provider/src/roles.tsindex 42ce28ce1f..a4eda4eb7f 100644--- a/packages/data-provider/src/roles.ts+++ b/packages/data-provider/src/roles.ts@@ -13,6 +13,7 @@ import {fileSearchPermissionsSchema,multiConvoPermissionsSchema,mcpServersPermissionsSchema,+ schedulesPermissionsSchema,sharedLinksPermissionsSchema,peoplePickerPermissionsSchema,remoteAgentsPermissionsSchema,@@ -117,6 +118,10 @@ const defaultRolesSchema = z.object({[Permissions.SHARE]: z.boolean().default(true),[Permissions.SHARE_PUBLIC]: z.boolean().default(true),}),+ [PermissionTypes.SCHEDULES]: schedulesPermissionsSchema.extend({+ [Permissions.USE]: z.boolean().default(true),+ [Permissions.CREATE]: z.boolean().default(true),+ }),}),}),[SystemRoles.USER]: roleSchema.extend({@@ -211,6 +216,10 @@ export const roleDefaults = defaultRolesSchema.parse({[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },},},[SystemRoles.USER]: {@@ -268,6 +277,10 @@ export const roleDefaults = defaultRolesSchema.parse({[Permissions.SHARE]: true,[Permissions.SHARE_PUBLIC]: true,},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: true,+ [Permissions.CREATE]: true,+ },},},});diff --git a/packages/data-provider/src/types/schedules.ts b/packages/data-provider/src/types/schedules.tsnew file mode 100644index 0000000000..9e50b6ac83--- /dev/null+++ b/packages/data-provider/src/types/schedules.ts@@ -0,0 +1,127 @@+import { z } from 'zod';++export const scheduleFrequencies = ['hourly', 'daily', 'weekdays', 'weekly'] as const;+export type ScheduleFrequency = (typeof scheduleFrequencies)[number];++export const scheduleTargets = ['new'] as const;+export type ScheduleTarget = (typeof scheduleTargets)[number];++export type ScheduleDisabledReason =+ | 'too_many_failures'+ | 'agent_deleted'+ | 'invalid_schedule'+ | 'permission_revoked'+ | 'insufficient_balance';++export type ScheduleRunStatus =+ | 'started'+ | 'requires_action'+ | 'success'+ | 'error'+ | 'interrupted'+ | 'skipped_overlap'+ | 'skipped_balance';++export const scheduleCadenceSchema = z.object({+ frequency: z.enum(scheduleFrequencies),+ hour: z.number().int().min(0).max(23),+ minute: z.number().int().min(0).max(59),+ daysOfWeek: z+ .array(z.number().int().min(0).max(6))+ .min(1)+ .max(7)+ .transform((days) => Array.from(new Set(days)))+ .optional(),+});+export type TScheduleCadence = z.infer<typeof scheduleCadenceSchema>;++export const createSchedulePayloadSchema = z.object({+ name: z.string().trim().min(1).max(256),+ prompt: z.string().trim().min(1).max(32000),+ agent_id: z.string().trim().min(1),+ cadence: scheduleCadenceSchema,+ timezone: z.string().min(1),+ target: z.enum(scheduleTargets).default('new'),+ file_ids: z+ .array(z.string())+ .max(10)+ .transform((ids) => Array.from(new Set(ids)))+ .optional(),+ enabled: z.boolean().default(true),+ /**+ * Client-generated key making creation idempotent across retries. Creation commits+ * the row and arms it in two writes, so a failure between them leaves the client+ * unable to tell whether anything persisted; retrying blind can produce two recurring+ * schedules. A retry carrying the same key resolves to the original row instead.+ * REQUIRED: an optional key preserves the keyless duplicate path for any client+ * that omits it, which is exactly the failure the key exists to close.+ */+ clientRequestId: z.string().trim().min(1).max(128),+});+export type TCreateSchedule = z.infer<typeof createSchedulePayloadSchema>;++/** Idempotency is a property of the CREATE attempt, not of the schedule's config. */+export const updateSchedulePayloadSchema = createSchedulePayloadSchema+ .omit({ clientRequestId: true })+ .partial()+ .extend({+ /**+ * The configRevision the client's edit was computed from (captured when the+ * dialog opened). The server fences the update on it, so a concurrent edit+ * from another tab answers 409 instead of being silently overwritten by a+ * payload rebuilt from a stale snapshot (cadence is sent whole, so the+ * server-side fresh-read fence alone cannot detect this).+ */+ expectedConfigRevision: z.number().int().min(0).optional(),+ });+export type TUpdateSchedule = z.infer<typeof updateSchedulePayloadSchema>;++export type TScheduleLastRun = {+ conversationId?: string;+ status: ScheduleRunStatus;+ error?: string;+ firedAt: string;+};++export type TSchedule = {+ id: string;+ user: string;+ name: string;+ prompt: string;+ agent_id: string;+ cadence: TScheduleCadence;+ timezone: string;+ target: ScheduleTarget;+ file_ids?: string[];+ enabled: boolean;+ disabledReason?: ScheduleDisabledReason;+ nextRunAt?: string;+ lastRun?: TScheduleLastRun;+ runCount: number;+ failureCount: number;+ configRevision?: number;+ createdAt: string;+ updatedAt: string;+};++export type TScheduleRun = {+ scheduleId: string;+ scheduledFor: string;+ firedAt?: string;+ conversationId?: string;+ status: ScheduleRunStatus;+ error?: string;+ droppedFileIds?: string[];+ durationMs?: number;+};++export type TSchedulesResponse = {+ schedules: TSchedule[];+ limits: { maxPerUser: number };+};++export type TScheduleRunNowResponse = {+ scheduleId: string;+ conversationId: string;+ status: 'started';+};diff --git a/packages/data-schemas/jest.config.mjs b/packages/data-schemas/jest.config.mjsindex 739c2eecf9..de247fbc11 100644--- a/packages/data-schemas/jest.config.mjs+++ b/packages/data-schemas/jest.config.mjs@@ -19,6 +19,9 @@ export default {// lines: 57,// },// },+ // Download the in-memory MongoDB binary once, before workers fork: on a cold+ // cache the parallel downloads race their final rename and fail whole suites.+ globalSetup: '<rootDir>/jest.globalSetup.mjs',maxWorkers: '50%',restoreMocks: true,testTimeout: 15000,diff --git a/packages/data-schemas/jest.globalSetup.mjs b/packages/data-schemas/jest.globalSetup.mjsnew file mode 100644index 0000000000..1c892c4129--- /dev/null+++ b/packages/data-schemas/jest.globalSetup.mjs@@ -0,0 +1,27 @@+import { MongoBinary } from 'mongodb-memory-server-core';++/**+ * Download the in-memory MongoDB binary ONCE, before jest forks its workers.+ *+ * Dozens of suites here spin up `MongoMemoryServer`, and on a cold binary cache+ * every worker starts its own download of the same archive. They then race on the+ * final `rename('<file>.tgz.downloading', '<file>.tgz')`, and whichever workers+ * lose that race fail their whole suite with+ * `ENOENT ... rename '.../mongodb-linux-...tgz.downloading'` — a beforeAll timeout+ * plus an unrelated-looking error, most visibly on CI where the cache starts empty.+ *+ * Pre-warming serially makes the download happen exactly once; every worker then+ * finds the extracted binary already present. Failures are swallowed: if the+ * download cannot happen here it will be retried by the first suite that needs it,+ * and a pre-warm problem must not fail the run on its own.+ */+export default async function globalSetup() {+ try {+ await MongoBinary.getPath({});+ } catch (error) {+ console.warn(+ '[data-schemas jest] Could not pre-warm the mongodb-memory-server binary:',+ error instanceof Error ? error.message : error,+ );+ }+}diff --git a/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.tsindex 9f73ef3bcb..33ea021d07 100644--- a/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts+++ b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts@@ -20,6 +20,7 @@ import {import { decrementTagCounts } from '~/methods/conversationTag';import { tenantStorage } from '~/config/tenantContext';import { supportsTransactions } from '~/utils/transactions';+import { createScheduleMethods } from '~/methods/schedule';import { createUserMethods } from '~/methods/user';import { createFileMethods } from '~/methods/file';import { createModels } from '~/models';@@ -353,4 +354,80 @@ describeLive('Amazon DocumentDB live compatibility', () => {expect(typeof disabled).toBe('boolean');});});++ /**+ * The scheduler's rewritten write shapes (classic operators, worker clock — the+ * DocumentDB-portable replacements for its original pipeline/$$NOW CAS forms),+ * exercised through the real methods against the live engine.+ */+ describe('scheduler write shapes', () => {+ const scheduleMethods = () => createScheduleMethods(mongoose);+ const scheduleData = (overrides: Record<string, unknown> = {}) => ({+ id: `sched_compat_${runId}_${randomUUID().slice(0, 8)}`,+ user: new mongoose.Types.ObjectId(),+ name: `compat ${runId}`,+ prompt: 'compat probe',+ agent_id: 'agent_compat',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ nextRunAt: new Date(Date.now() - 60_000),+ ...overrides,+ });++ afterAll(async () => {+ await mongoose.models.Schedule.deleteMany({ name: `compat ${runId}` });+ await mongoose.models.ScheduleRun.deleteMany({ scheduleId: { $regex: runId } });+ });++ it('claims a due schedule via the classic-operator lease CAS', async () => {+ const methods = scheduleMethods();+ const schedule = await methods.createSchedule(scheduleData() as never);+ const claimed = await methods.claimDueSchedule({+ instanceId: `compat-${runId}`,+ leaseMs: 60_000,+ });+ expect(claimed?.leaseUntil).toBeInstanceOf(Date);+ // Held lease: a second claim must not steal it.+ const contender = await methods.claimDueSchedule({+ instanceId: `compat2-${runId}`,+ leaseMs: 60_000,+ });+ expect(contender?.id).not.toBe(schedule.id);+ capabilities['scheduler lease CAS'] = 'supported';+ });++ it('stamps and resolves an abort request with guarded classic updates', async () => {+ const methods = scheduleMethods();+ const schedule = await methods.createSchedule(scheduleData() as never);+ const scheduledFor = new Date('2026-07-20T12:00:00Z');+ await mongoose.models.ScheduleRun.create({+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor,+ status: 'started',+ firedAt: new Date(),+ });+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(state?.abortSource).toBe('stop');+ expect(state?.abortPersistedAt).toBeInstanceOf(Date);+ capabilities['scheduler abort stamps'] = 'supported';+ });++ it('arms only an unarmed row through the shared arming CAS', async () => {+ const methods = scheduleMethods();+ const schedule = await methods.createSchedule(+ scheduleData({ nextRunAt: undefined }) as never,+ );+ const armAt = new Date(Date.now() + HOUR);+ expect(await methods.armSchedule(schedule.id, armAt, 0)).toBe(true);+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 2 * HOUR), 0)).toBe(+ false,+ );+ capabilities['scheduler arming CAS'] = 'supported';+ });+ });});diff --git a/packages/data-schemas/src/app/interface.ts b/packages/data-schemas/src/app/interface.tsindex 143e62bc5f..ec26054139 100644--- a/packages/data-schemas/src/app/interface.ts+++ b/packages/data-schemas/src/app/interface.ts@@ -64,6 +64,7 @@ export async function loadDefaultInterface({remoteAgents: interfaceConfig?.remoteAgents,skills: interfaceConfig?.skills,sharedLinks: interfaceConfig?.sharedLinks,+ schedules: interfaceConfig?.schedules,});return loadedInterface;diff --git a/packages/data-schemas/src/app/resolution.spec.ts b/packages/data-schemas/src/app/resolution.spec.tsindex de47138f39..41d0f7aec2 100644--- a/packages/data-schemas/src/app/resolution.spec.ts+++ b/packages/data-schemas/src/app/resolution.spec.ts@@ -77,6 +77,121 @@ describe('mergeConfigOverrides', () => {expect(iface.parameters).toBe(true);});+ it('folds a boolean schedules override onto inherited limits (does not collapse them)', () => {+ const base = {+ interfaceConfig: {+ schedules: { use: true, maxPerUser: 50, minIntervalMinutes: 5 },+ },+ } as unknown as AppConfig;+ // Enabling the feature for a role via the natural boolean toggle must PRESERVE+ // the deployment's configured limits rather than reverting them to defaults.+ const enabled = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: true } }, 10),+ ]) as unknown as Record<string, unknown>;+ const iface = enabled.interfaceConfig as Record<string, Record<string, unknown>>;+ expect(iface.schedules).toEqual({ use: true, maxPerUser: 50, minIntervalMinutes: 5 });+ // A boolean-false override disables while still preserving the limit values.+ const disabled = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: false } }, 10),+ ]) as unknown as Record<string, unknown>;+ const dIface = disabled.interfaceConfig as Record<string, Record<string, unknown>>;+ expect(dIface.schedules).toEqual({ use: false, maxPerUser: 50, minIntervalMinutes: 5 });+ });++ it('does not let a boolean override re-enable a globally-disabled feature', () => {+ // Both values are booleans, so neither fold above applies and the plain fallback+ // used to replace the base `false`. The service reads the BASE value and keeps+ // refusing every write, so the client would render a panel whose create, edit and+ // run actions all fail. A global stop may be narrowed, never widened.+ const base = { interfaceConfig: { schedules: false } } as unknown as AppConfig;+ const merged = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: true } }, 10),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(merged.interfaceConfig.schedules).toBe(false);++ // An enabled base still honours a boolean override in both directions.+ const enabledBase = { interfaceConfig: { schedules: true } } as unknown as AppConfig;+ expect(+ (+ mergeConfigOverrides(enabledBase, [+ fakeConfig({ interface: { schedules: false } }, 10),+ ]) as unknown as Record<string, Record<string, unknown>>+ ).interfaceConfig.schedules,+ ).toBe(false);+ });++ /**+ * Overrides are folded in priority order into an ACCUMULATED value, so a stop applied+ * INSIDE the merge compares against whatever the previous override left — which let a+ * low-priority `false` outrank a high-priority `true`. The base stop is the only+ * non-negotiable one; overrides still order normally among themselves.+ */+ it('lets a higher-priority override win over a lower-priority one', () => {+ const base = { interfaceConfig: { schedules: true } } as unknown as AppConfig;+ const merged = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: false } }, 10),+ fakeConfig({ interface: { schedules: true } }, 20),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(merged.interfaceConfig.schedules).toBe(true);++ // And the reverse ordering still disables.+ const disabled = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: true } }, 10),+ fakeConfig({ interface: { schedules: false } }, 20),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(disabled.interfaceConfig.schedules).toBe(false);+ });++ it('keeps the base stop non-negotiable even for the highest-priority override', () => {+ const base = { interfaceConfig: { schedules: false } } as unknown as AppConfig;+ const merged = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: true } }, 10),+ fakeConfig({ interface: { schedules: true } }, 99),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(merged.interfaceConfig.schedules).toBe(false);+ });++ it('applies the base stop through the object form too', () => {+ const base = {+ interfaceConfig: { schedules: { use: false, maxPerUser: 5 } },+ } as unknown as AppConfig;+ const merged = mergeConfigOverrides(base, [+ fakeConfig({ interface: { schedules: { maxPerUser: 50 } } }, 10),+ ]) as unknown as Record<string, Record<string, Record<string, unknown>>>;+ // The limit tuning lands, but the stop survives it.+ expect(merged.interfaceConfig.schedules).toMatchObject({ use: false, maxPerUser: 50 });+ });++ it('folds an object schedules override onto a boolean base, inheriting the enable state', () => {+ // Enabled base, object override that only TUNES a limit (no `use`): the enable+ // state must be inherited from the boolean base, not silently dropped.+ const enabledBase = {+ interfaceConfig: { schedules: true },+ } as unknown as AppConfig;+ const tuned = mergeConfigOverrides(enabledBase, [+ fakeConfig({ interface: { schedules: { maxPerUser: 3 } } }, 10),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(tuned.interfaceConfig.schedules).toEqual({ use: true, maxPerUser: 3 });++ // Disabled base, same object override: tuning a limit must NOT re-enable the+ // globally-disabled feature.+ const disabledBase = {+ interfaceConfig: { schedules: false },+ } as unknown as AppConfig;+ const stillOff = mergeConfigOverrides(disabledBase, [+ fakeConfig({ interface: { schedules: { maxPerUser: 3 } } }, 10),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(stillOff.interfaceConfig.schedules).toEqual({ use: false, maxPerUser: 3 });++ // A config override cannot flip the enable state: `use` is a permission sub-key,+ // stripped from interface overrides before merge (enable is permission-managed),+ // so an override attempting `use: true` on a disabled base stays disabled.+ const cannotReEnable = mergeConfigOverrides(disabledBase, [+ fakeConfig({ interface: { schedules: { use: true, maxPerUser: 3 } } }, 10),+ ]) as unknown as Record<string, Record<string, unknown>>;+ expect(cannotReEnable.interfaceConfig.schedules).toEqual({ use: false, maxPerUser: 3 });+ });+it('sorts by priority — higher priority wins', () => {const configs = [fakeConfig({ registration: { enabled: false } }, 100),diff --git a/packages/data-schemas/src/app/resolution.ts b/packages/data-schemas/src/app/resolution.tsindex 1f79163df9..cdc77cf328 100644--- a/packages/data-schemas/src/app/resolution.ts+++ b/packages/data-schemas/src/app/resolution.ts@@ -2,6 +2,7 @@ import {BASE_PRINCIPAL_CONFIG_SECTIONS,BASE_ONLY_CONFIG_SECTIONS,INTERFACE_PERMISSION_FIELDS,+ RUNTIME_CONFIG_INTERFACE_FIELDS,PERMISSION_SUB_KEYS,isProcessMCPServerConfig,} from 'librechat-data-provider';@@ -201,6 +202,30 @@ function deepMerge<T extends AnyObject>(target: T, source: AnyObject, depth = 0,depth,currentPath,);+ } else if (+ typeof sourceVal === 'boolean' &&+ targetVal != null &&+ typeof targetVal === 'object' &&+ !Array.isArray(targetVal) &&+ RUNTIME_CONFIG_INTERFACE_FIELDS.has(key)+ ) {+ // A runtime-config interface field (e.g. schedules) toggled by a boolean+ // override is a runtime enable/disable, not a replacement of its config. Fold+ // the boolean into the `use` flag so inherited object-form limits (maxPerUser,+ // minIntervalMinutes, ...) survive instead of collapsing to global defaults.+ result[key] = { ...(targetVal as AnyObject), use: sourceVal };+ } else if (+ typeof targetVal === 'boolean' &&+ sourceVal != null &&+ typeof sourceVal === 'object' &&+ !Array.isArray(sourceVal) &&+ RUNTIME_CONFIG_INTERFACE_FIELDS.has(key)+ ) {+ // Symmetric case: an OBJECT override (e.g. tuning maxPerUser) on top of a+ // BOOLEAN base must inherit the base's enable state unless it sets `use`+ // explicitly — otherwise setting a limit on a globally-disabled feature+ // (`schedules: false`) would silently re-enable it.+ result[key] = { use: targetVal, ...(sourceVal as AnyObject) };} else {result[key] = sourceVal;}@@ -313,8 +338,12 @@ export function mergeConfigOverrides(baseConfig: AppConfig, configs: IConfig[]):if (Object.keys(uiOnly).length > 0) {filtered[field] = uiOnly;}+ } else if (RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {+ // Dual-purpose field: the boolean form is a runtime disable, not a+ // permission toggle, so preserve it (e.g. schedules: false).+ filtered[field] = fieldVal;}- // boolean permission fields (e.g. runCode: false) are fully stripped+ // other boolean permission fields (e.g. runCode: false) are fully stripped}if (Object.keys(filtered).length > 0) {remapped[mappedKey] = filtered;@@ -327,5 +356,59 @@ export function mergeConfigOverrides(baseConfig: AppConfig, configs: IConfig[]):}}- return merged;+ return preserveRuntimeStops(baseConfig, merged);+}++/** Whether a runtime-config interface field reads as OFF in either of its two shapes:+ * boolean `false`, or the object form with `use: false`. Exported so runtime gates+ * (e.g. the schedule engine's global stop) apply the same semantics as the merge. */+export function isRuntimeDisabled(value: unknown): boolean {+ if (value === false) {+ return true;+ }+ return (+ value != null &&+ typeof value === 'object' &&+ !Array.isArray(value) &&+ (value as AnyObject).use === false+ );+}++/**+ * Re-applies a BASE-level runtime stop after all overrides are merged.+ *+ * A deployment that turns a runtime feature off is not making a preference an override+ * can outrank: the service reads the base value and keeps refusing writes, so an+ * override that only flips the client's view produces a panel whose every action fails.+ *+ * Applied here rather than inside the merge deliberately. The merge folds overrides in+ * priority order into an ACCUMULATED value, so a guard there compares against whatever+ * the previous override left — which let a LOW-priority `false` block a HIGH-priority+ * `true` and broke highest-priority-wins. Overrides still order normally among+ * themselves; only the base stop is non-negotiable.+ */+function preserveRuntimeStops<T extends AppConfig>(baseConfig: AppConfig, merged: T): T {+ const baseInterface = (baseConfig as unknown as AnyObject).interfaceConfig as+ | AnyObject+ | undefined;+ const mergedInterface = (merged as unknown as AnyObject).interfaceConfig as AnyObject | undefined;+ if (baseInterface == null || mergedInterface == null) {+ return merged;+ }+ let patched: AnyObject | undefined;+ for (const key of RUNTIME_CONFIG_INTERFACE_FIELDS) {+ if (!isRuntimeDisabled(baseInterface[key]) || isRuntimeDisabled(mergedInterface[key])) {+ continue;+ }+ const current = mergedInterface[key];+ patched ??= { ...mergedInterface };+ patched[key] =+ current != null && typeof current === 'object' && !Array.isArray(current)+ ? { ...(current as AnyObject), use: false }+ : false;+ }+ if (patched == null) {+ return merged;+ }+ return { ...(merged as unknown as AnyObject), interfaceConfig: patched } as unknown as T;}diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.tsindex d9f16f327b..1370343de3 100644--- a/packages/data-schemas/src/methods/index.ts+++ b/packages/data-schemas/src/methods/index.ts@@ -98,6 +98,7 @@ import {type UpdateSkillResult,type ValidationIssue,} from './skill';+import { createScheduleMethods, type ScheduleMethods } from './schedule';import {createAgentTriggerDeliveryMethods,AgentTriggerDeliveryConflictError,@@ -196,6 +197,7 @@ export type AllMethods = UserMethods &SkillMethods &SkillSyncMethods &AgentTriggerDeliveryMethods &+ ScheduleMethods &AgentMethods &ConfigMethods &MCPAuthorityMethods &@@ -334,6 +336,7 @@ export function createMethods(...skillMethods,...createSkillSyncMethods(mongoose),...createAgentTriggerDeliveryMethods(mongoose),+ ...createScheduleMethods(mongoose),/* Tier 5 */...agentMethods,/* Config */@@ -395,6 +398,7 @@ export type {UpsertSkillSyncCredentialInput,SkillSyncMethods,AgentTriggerDeliveryMethods,+ ScheduleMethods,AgentMethods,ConfigMethods,MCPAuthorityMethods,diff --git a/packages/data-schemas/src/methods/schedule.methods.spec.ts b/packages/data-schemas/src/methods/schedule.methods.spec.tsnew file mode 100644index 0000000000..057ffcf2d2--- /dev/null+++ b/packages/data-schemas/src/methods/schedule.methods.spec.ts@@ -0,0 +1,2759 @@+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import type { Model } from 'mongoose';+import type {+ ISchedule,+ IScheduleRun,+ IScheduleDocument,+ IScheduleRunDocument,+} from '~/types/schedule';+import type { ScheduleMethods } from './schedule';+import { createScheduleMethods } from './schedule';+import { createModels } from '../models';++jest.mock('~/config/winston', () => ({+ error: jest.fn(),+ warn: jest.fn(),+ info: jest.fn(),+ debug: jest.fn(),+}));++let mongoServer: MongoMemoryServer;+let Schedule: Model<IScheduleDocument>;+let ScheduleRun: Model<IScheduleRunDocument>;+let methods: ScheduleMethods;++/** Standing up a real in-memory MongoDB plus index builds routinely outruns the+ * 15s default hook timeout when the suite runs alongside the rest of the package+ * (and on a cold binary cache), which failed all 115 tests on an unrelated-looking+ * beforeAll timeout. The work itself takes ~2s idle; this is headroom, not a+ * slow expectation. */+const DB_SETUP_TIMEOUT_MS = 60_000;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+ createModels(mongoose);+ Schedule = mongoose.models.Schedule as Model<IScheduleDocument>;+ ScheduleRun = mongoose.models.ScheduleRun as Model<IScheduleRunDocument>;+ await Schedule.init();+ await ScheduleRun.init();+ methods = createScheduleMethods(mongoose);+}, DB_SETUP_TIMEOUT_MS);++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+}, DB_SETUP_TIMEOUT_MS);++beforeEach(async () => {+ await Schedule.deleteMany({});+ await ScheduleRun.deleteMany({});+});++let scheduleCounter = 0;++function scheduleData(overrides: Partial<ISchedule> = {}): Partial<ISchedule> {+ scheduleCounter += 1;+ return {+ id: `sched_${scheduleCounter}`,+ user: new mongoose.Types.ObjectId(),+ name: 'Morning digest',+ prompt: 'Summarize my inbox',+ agent_id: 'agent_test',+ cadence: { frequency: 'daily', hour: 8, minute: 0 },+ timezone: 'America/New_York',+ target: 'new',+ enabled: true,+ nextRunAt: new Date(Date.now() - 60_000),+ ...overrides,+ };+}++function runData(+ schedule: Pick<ISchedule, 'id' | 'user'>,+ overrides: Partial<IScheduleRun> = {},+): Partial<IScheduleRun> {+ return {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: new Date('2026-07-20T12:00:00Z'),+ status: 'started',+ firedAt: new Date(),+ ...overrides,+ };+}++async function getSchedule(id: string): Promise<ISchedule> {+ const schedule = await methods.getScheduleById(id);+ if (schedule == null) {+ throw new Error(`schedule ${id} not found`);+ }+ return schedule;+}++async function getRun(scheduleId: string, scheduledFor: Date): Promise<IScheduleRun> {+ const run = await ScheduleRun.findOne({ scheduleId, scheduledFor }).lean<IScheduleRun>();+ if (run == null) {+ throw new Error(`run ${scheduleId} @ ${scheduledFor.toISOString()} not found`);+ }+ return run;+}++describe('claimDueSchedule', () => {+ it('grants exactly one winner across 8 concurrent claims', async () => {+ await methods.createSchedule(scheduleData());+ const results = await Promise.all(+ Array.from({ length: 8 }, (_, i) =>+ methods.claimDueSchedule({ instanceId: `inst-${i}`, leaseMs: 60_000 }),+ ),+ );+ const winners = results.filter((result) => result != null);+ expect(winners).toHaveLength(1);+ expect(winners[0]?.leaseBy).toMatch(/^inst-\d$/);+ expect(winners[0]?.leaseUntil).toBeInstanceOf(Date);+ });++ it('blocks a second claim while the lease is live', async () => {+ const created = await methods.createSchedule(scheduleData());+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ expect(first?.id).toBe(created.id);+ const second = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });+ expect(second).toBeNull();+ });++ it('allows re-claim after the lease expires past the skew margin', async () => {+ const created = await methods.createSchedule(scheduleData());+ // Takeover requires expiry to be at least LEASE_SKEW_MARGIN_MS in the past+ // (worker-clock comparisons tolerate inter-replica skew), so lease a NEGATIVE+ // duration deep enough to clear the margin instead of sleeping it out.+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: -31_000 });+ expect(first?.id).toBe(created.id);+ const second = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });+ expect(second?.id).toBe(created.id);+ expect(second?.leaseBy).toBe('inst-b');+ });++ it('does not take over a lease expired less than the skew margin ago', async () => {+ await methods.createSchedule(scheduleData());+ // Expired 1s ago by this clock — within the margin, so a skew-behind holder may+ // still legitimately consider it live.+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: -1_000 });+ expect(first).not.toBeNull();+ const contender = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });+ expect(contender).toBeNull();+ });++ it('claims due schedules in nextRunAt order', async () => {+ const now = Date.now();+ const later = await methods.createSchedule(scheduleData({ nextRunAt: new Date(now - 60_000) }));+ const earlier = await methods.createSchedule(+ scheduleData({ nextRunAt: new Date(now - 120_000) }),+ );+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ const second = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ const third = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ expect(first?.id).toBe(earlier.id);+ expect(second?.id).toBe(later.id);+ expect(third).toBeNull();+ });++ it('never claims disabled, future, or never-scheduled schedules', async () => {+ const now = Date.now();+ await methods.createSchedule(+ scheduleData({ enabled: false, nextRunAt: new Date(now - 60_000) }),+ );+ await methods.createSchedule(scheduleData({ nextRunAt: new Date(now + 3_600_000) }));+ await methods.createSchedule(scheduleData({ nextRunAt: undefined }));+ const claimed = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ expect(claimed).toBeNull();+ });+});++describe('insertScheduleRun idempotency', () => {+ it('returns null on a duplicate {scheduleId, scheduledFor} claim', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-20T12:00:00Z');+ const first = await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ const second = await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ expect(first?.status).toBe('started');+ expect(second).toBeNull();+ expect(await ScheduleRun.countDocuments({ scheduleId: schedule.id })).toBe(1);+ });++ it('inserts distinct occurrences for different scheduledFor values', async () => {+ // Terminal status so the single-active partial index (one `started` per+ // schedule) doesn't interfere — this asserts occurrence-level distinctness.+ const schedule = await methods.createSchedule(scheduleData());+ const first = await methods.insertScheduleRun(+ runData(schedule, { scheduledFor: new Date('2026-07-20T12:00:00Z'), status: 'success' }),+ );+ const second = await methods.insertScheduleRun(+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z'), status: 'success' }),+ );+ expect(first).not.toBeNull();+ expect(second).not.toBeNull();+ expect(await ScheduleRun.countDocuments({ scheduleId: schedule.id })).toBe(2);+ });+});++describe('recordSkippedRun duplicate-occurrence guard', () => {+ it('does not overwrite lastRun/counters when the occurrence already fired', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const when = new Date('2026-07-20T12:00:00Z');+ // The occurrence already ran to success (e.g. a claim retry after the POST was+ // accepted but the lease advance failed).+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: when, status: 'success' }));+ // A stale balance preflight must NOT relabel that occurrence as a balance skip.+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: when,+ status: 'skipped_balance',+ },+ 5,+ );+ const after = await getSchedule(schedule.id);+ expect(after.lastRun).toBeUndefined();+ expect(after.balanceSkipCount).toBe(0);+ expect(await getRun(schedule.id, when)).toMatchObject({ status: 'success' });+ });+});++describe('settledAt retention marker', () => {+ it('stamps settledAt on terminal outcomes but never on a pause', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const when = new Date('2026-07-20T12:00:00Z');+ await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));++ // The retention TTL index expires on `settledAt`, so a PAUSE must not carry it:+ // an approval window longer than the retention period would otherwise let Mongo+ // reap a still-live requires_action row out from under its retained job.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: when,+ status: 'requires_action',+ autoDisableAfterFailures: 5,+ });+ expect((await getRun(schedule.id, when)).settledAt).toBeUndefined();++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: when,+ status: 'success',+ autoDisableAfterFailures: 5,+ });+ expect((await getRun(schedule.id, when)).settledAt).toBeInstanceOf(Date);+ });++ it('stamps settledAt on skip rows inserted terminal', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const when = new Date('2026-07-20T12:00:00Z');+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: when,+ status: 'skipped_overlap',+ },+ 5,+ );+ expect((await getRun(schedule.id, when)).settledAt).toBeInstanceOf(Date);+ });+});++describe('reserveStartedRun (single-active overlap guard)', () => {+ it('reserves the slot, then rejects a concurrent occurrence as overlap', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const first = await methods.reserveStartedRun(+ runData(schedule, { scheduledFor: new Date('2026-07-20T12:00:00Z') }),+ );+ const second = await methods.reserveStartedRun(+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z') }),+ );+ expect('run' in first).toBe(true);+ expect(second).toEqual({ conflict: 'overlap' });+ // Once the first terminalizes it leaves the active index, so the next occurrence reserves.+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor: new Date('2026-07-20T12:00:00Z') },+ { $set: { status: 'success' } },+ );+ const third = await methods.reserveStartedRun(+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z') }),+ );+ expect('run' in third).toBe(true);+ });++ it('reports a re-fired same occurrence as duplicate', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const when = new Date('2026-07-20T12:00:00Z');+ await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor: when },+ { $set: { status: 'success' } },+ );+ const again = await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));+ // The status travels with the conflict: this exact case (settled row, occurrence+ // never advanced past) is the one the caller must ADVANCE past rather than treat as+ // owned by a live worker.+ expect(again).toEqual({ conflict: 'duplicate', existingStatus: 'success' });+ });+});++describe('markRunResumeClaimed (paused resume capacity)', () => {+ const when = new Date('2026-07-20T12:00:00Z');++ it('atomically promotes a paused run and claims a global capacity slot', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { scheduledFor: when, status: 'requires_action' }));++ await expect(methods.markRunResumeClaimed(schedule.id, when, 0)).resolves.toEqual({+ capacitySlot: 0,+ });+ await expect(getRun(schedule.id, when)).resolves.toMatchObject({+ status: 'started',+ capacitySlot: 0,+ resumeClaimedAt: expect.any(Date),+ });+ });++ it('reports global-slot and same-schedule overlap conflicts without changing the pause', async () => {+ const first = await methods.createSchedule(scheduleData());+ const second = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(first, { capacitySlot: 0 }));+ await ScheduleRun.create(runData(second, { scheduledFor: when, status: 'requires_action' }));++ await expect(methods.markRunResumeClaimed(second.id, when, 0)).resolves.toEqual({+ conflict: 'slot-taken',+ });++ const later = new Date('2026-07-21T12:00:00Z');+ await ScheduleRun.create(runData(first, { scheduledFor: later, status: 'requires_action' }));+ await expect(methods.markRunResumeClaimed(first.id, later, 1)).resolves.toEqual({+ conflict: 'overlap',+ });+ await expect(getRun(second.id, when)).resolves.toMatchObject({ status: 'requires_action' });+ await expect(getRun(first.id, later)).resolves.toMatchObject({ status: 'requires_action' });+ });++ it('rolls back only the exact resume reservation', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { scheduledFor: when, status: 'requires_action' }));+ await methods.markRunResumeClaimed(schedule.id, when, 2);++ await expect(methods.releaseRunResumeClaim(schedule.id, when, 1)).resolves.toBe(false);+ await expect(methods.releaseRunResumeClaim(schedule.id, when, 2)).resolves.toBe(true);+ await expect(getRun(schedule.id, when)).resolves.toMatchObject({ status: 'requires_action' });+ expect((await getRun(schedule.id, when)).capacitySlot).toBeUndefined();+ });+});++describe('createScheduleWithSlot (atomic per-user cap)', () => {+ it('allows exactly maxPerUser and reports limit under concurrency', async () => {+ const user = new mongoose.Types.ObjectId();+ const results = await Promise.all(+ Array.from({ length: 8 }, () => methods.createScheduleWithSlot(scheduleData({ user }), 3)),+ );+ const created = results.filter((r) => r !== 'limit');+ expect(created).toHaveLength(3);+ expect(results.filter((r) => r === 'limit')).toHaveLength(5);+ // Distinct slots [0,3) were assigned — the partial unique index is the arbiter.+ const slots = new Set(+ (created as ISchedule[]).map((s) => s.slot).filter((s) => typeof s === 'number'),+ );+ expect(slots.size).toBe(3);+ expect(await methods.countSchedulesByUser(user)).toBe(3);+ });++ it('frees a slot on delete so a new create can take it', async () => {+ const user = new mongoose.Types.ObjectId();+ const a = (await methods.createScheduleWithSlot(scheduleData({ user }), 2)) as ISchedule;+ await methods.createScheduleWithSlot(scheduleData({ user }), 2);+ expect(await methods.createScheduleWithSlot(scheduleData({ user }), 2)).toBe('limit');+ await methods.deleteScheduleById(a.id, user);+ const c = await methods.createScheduleWithSlot(scheduleData({ user }), 2);+ expect(c).not.toBe('limit');+ expect(await methods.countSchedulesByUser(user)).toBe(2);+ });+});++describe('recordRunOutcome', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('success finalizes the run, increments runCount, and resets failure state', async () => {+ const schedule = await methods.createSchedule(+ scheduleData({+ failureCount: 2,+ balanceSkipCount: 3,+ disabledReason: 'insufficient_balance',+ }),+ );+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-1',+ durationMs: 1234,+ autoDisableAfterFailures: 3,+ });+ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('success');+ expect(run.conversationId).toBe('convo-1');+ expect(run.durationMs).toBe(1234);+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(1);+ expect(updated.failureCount).toBe(0);+ expect(updated.balanceSkipCount).toBe(0);+ expect(updated.disabledReason).toBeUndefined();+ expect(updated.enabled).toBe(true);+ expect(updated.lastRun?.status).toBe('success');+ expect(updated.lastRun?.conversationId).toBe('convo-1');+ expect(updated.lastRun?.firedAt).toBeInstanceOf(Date);+ });++ it('error increments failureCount without disabling below the threshold', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ error: 'provider exploded',+ autoDisableAfterFailures: 3,+ });+ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('error');+ expect(run.error).toBe('provider exploded');+ const updated = await getSchedule(schedule.id);+ expect(updated.failureCount).toBe(1);+ expect(updated.runCount).toBe(0);+ expect(updated.enabled).toBe(true);+ expect(updated.disabledReason).toBeUndefined();+ expect(updated.lastRun?.status).toBe('error');+ expect(updated.lastRun?.error).toBe('provider exploded');+ });++ it('error at the threshold disables the schedule and releases the lease', async () => {+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 2 }));+ const claimed = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });+ expect(claimed?.id).toBe(schedule.id);+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ error: 'provider exploded',+ autoDisableAfterFailures: 3,+ });+ const updated = await getSchedule(schedule.id);+ expect(updated.failureCount).toBe(3);+ expect(updated.enabled).toBe(false);+ expect(updated.disabledReason).toBe('too_many_failures');+ expect(updated.leaseUntil).toBeUndefined();+ expect(updated.leaseBy).toBeUndefined();+ });++ it('requires_action surfaces lastRun for the card without touching counters', async () => {+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 1 }));+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-paused',+ autoDisableAfterFailures: 3,+ });+ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('requires_action');+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(0);+ expect(updated.failureCount).toBe(1);+ // lastRun now reflects the pause so the card can show "Needs approval".+ expect(updated.lastRun?.status).toBe('requires_action');+ expect(updated.lastRun?.conversationId).toBe('convo-paused');+ });++ it('clearConversationId erases the reserved id so replays cannot restore a dead link', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor, conversationId: 'reserved-but-never-created' }),+ );++ // Pre-start abort: an id was reserved but no conversation ever came to exist.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'interrupted',+ clearConversationId: true,+ autoDisableAfterFailures: 3,+ });++ const run = await getRun(schedule.id, scheduledFor);+ expect(run.conversationId).toBeUndefined();+ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun?.conversationId).toBeUndefined();++ // The crash-retry replay reads the ROW; with the id erased it has nothing+ // to project, so the dead link cannot be restored.+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor },+ { $set: { bookkept: false } },+ );+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'interrupted',+ conversationId: run.conversationId,+ autoDisableAfterFailures: 3,+ });+ const replayed = await getSchedule(schedule.id);+ expect(replayed.lastRun?.conversationId).toBeUndefined();+ });++ it('re-affirmed pauses keep the ORIGINAL fire time and do not churn updatedAt', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const originalFiredAt = new Date('2026-07-20T12:00:02Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, firedAt: originalFiredAt }));+ const pause = () =>+ methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-paused',+ autoDisableAfterFailures: 3,+ });++ await pause();+ const afterFirst = await getSchedule(schedule.id);+ // Reconciliation re-affirms a long-lived pause every pass; a fresh stamp per+ // pass walked the card's timestamp forward and reordered updated-time listings+ // for as long as the approval sat waiting.+ await pause();+ const afterSecond = await getSchedule(schedule.id);++ expect(afterSecond.lastRun?.firedAt?.toISOString()).toBe(originalFiredAt.toISOString());+ expect(afterSecond.updatedAt?.toISOString()).toBe(afterFirst.updatedAt?.toISOString());+ });++ it('does not write a pause card when no active run matches (spoof guard)', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ // No run row for this occurrence: the card is written only after a matching active+ // run is confirmed, so a spoofed scheduleId can never stamp a pause onto a schedule.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-spoof',+ autoDisableAfterFailures: 3,+ });+ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun).toBeUndefined();+ });++ it('re-affirms the pause card on a retried requires_action (already-paused row)', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });+ // The row is already `requires_action`; a crash-replay retry must still write the+ // card (keyed on existence, not modification) so the pause never goes invisible.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-2',+ autoDisableAfterFailures: 3,+ });+ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('requires_action');+ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun?.status).toBe('requires_action');+ expect(updated.lastRun?.conversationId).toBe('convo-2');+ });++ it('does not pin the card to Needs approval when a resume terminalizes first', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ // The user answers instantly: the resumed generation settles the run BEFORE the+ // pause write lands. The pause must lose, or the card shows "Needs approval" for a+ // job that already finished (the run row is terminal, so nothing corrects it).+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });++ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('success');+ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun?.status).toBe('success');+ });++ it('does not let a delayed older outcome overwrite a newer run on the card', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const older = new Date('2026-07-26T10:00:00.000Z');+ const newer = new Date('2026-07-26T11:00:00.000Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: older }));+ // Pausing the older occurrence is what frees the single-active-`started` slot and+ // lets a later occurrence run at all — the interleaving under test.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'requires_action',+ conversationId: 'convo-older',+ autoDisableAfterFailures: 3,+ });+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: newer }));++ // The newer run finishes while the older one is still parked at its approval.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: newer,+ status: 'success',+ conversationId: 'convo-newer',+ autoDisableAfterFailures: 3,+ });+ // The older one settles afterwards (resumed pause / reconciler replay).+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'error',+ error: 'stale',+ conversationId: 'convo-older',+ autoDisableAfterFailures: 3,+ });++ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun?.status).toBe('success');+ expect(updated.lastRun?.conversationId).toBe('convo-newer');+ // The streak counters are order-fenced too, not just the card. This assertion used+ // to expect 1 on the reasoning that counters are order-insensitive — but a streak+ // is not: letting the stale failure land rebuilds one the newer success had+ // cleared, and at `autoDisableAfterFailures: 1` that alone disables the schedule.+ expect(updated.failureCount).toBe(0);+ // The row still settles; only the schedule-level streak ignores the late outcome.+ expect(await getRun(schedule.id, older)).toMatchObject({ status: 'error' });+ });+});++describe('recordSkippedRun', () => {+ it('skipped_balance increments the counter and disables at the threshold', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: new Date('2026-07-20T12:00:00Z'),+ status: 'skipped_balance',+ },+ 2,+ );+ let updated = await getSchedule(schedule.id);+ expect(updated.balanceSkipCount).toBe(1);+ expect(updated.enabled).toBe(true);+ const firstRun = await getRun(schedule.id, new Date('2026-07-20T12:00:00Z'));+ expect(firstRun.status).toBe('skipped_balance');+ expect(firstRun.firedAt).toBeInstanceOf(Date);+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: new Date('2026-07-21T12:00:00Z'),+ status: 'skipped_balance',+ },+ 2,+ );+ updated = await getSchedule(schedule.id);+ expect(updated.balanceSkipCount).toBe(2);+ expect(updated.enabled).toBe(false);+ expect(updated.disabledReason).toBe('insufficient_balance');+ });++ it('is retryable: a duplicate skipped_balance occurrence does not double-count the streak', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-20T12:00:00Z');+ const data = {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor,+ status: 'skipped_balance' as const,+ };+ // First attempt inserts the row + increments the streak.+ await methods.recordSkippedRun(data, 3);+ // Retry of the SAME occurrence (a crash before the counter landed on the first+ // try): the row already exists, but the bookkeeping must still run — exactly once.+ await methods.recordSkippedRun(data, 3);+ const updated = await getSchedule(schedule.id);+ expect(updated.balanceSkipCount).toBe(1);+ expect(updated.lastRun?.status).toBe('skipped_balance');+ });++ it('skipped_overlap records the run without counter changes', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-20T12:00:00Z');+ await methods.recordSkippedRun(+ { scheduleId: schedule.id, user: schedule.user, scheduledFor, status: 'skipped_overlap' },+ 2,+ );+ const run = await getRun(schedule.id, scheduledFor);+ expect(run.status).toBe('skipped_overlap');+ const updated = await getSchedule(schedule.id);+ expect(updated.balanceSkipCount).toBe(0);+ expect(updated.enabled).toBe(true);+ // the skip surfaces on the card via lastRun+ expect(updated.lastRun?.status).toBe('skipped_overlap');+ });++ it('a subsequent success resets balanceSkipCount', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: new Date('2026-07-20T12:00:00Z'),+ status: 'skipped_balance',+ },+ 3,+ );+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(1);+ const scheduledFor = new Date('2026-07-21T12:00:00Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-2',+ autoDisableAfterFailures: 3,+ });+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);+ });+});++describe('countActiveRuns', () => {+ it('counts started runs across all schedules for the fire cap', async () => {+ expect(await methods.countActiveRuns()).toBe(0);+ const a = await methods.createSchedule(scheduleData());+ const b = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(a, { scheduledFor: new Date('2026-07-20T12:00:00Z') }));+ await methods.insertScheduleRun(runData(b, { scheduledFor: new Date('2026-07-20T13:00:00Z') }));+ expect(await methods.countActiveRuns()).toBe(2);+ // terminal + skipped runs don't count as active+ await methods.recordRunOutcome({+ scheduleId: a.id,+ scheduledFor: new Date('2026-07-20T12:00:00Z'),+ status: 'success',+ autoDisableAfterFailures: 3,+ });+ expect(await methods.countActiveRuns()).toBe(1);+ });+});++describe('recordRunOutcome — reconciled completions and no-match guard', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('finalizes a resumed run from requires_action with full bookkeeping', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, status: 'requires_action' }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-resumed',+ autoDisableAfterFailures: 3,+ });+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('success');+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(1);+ expect(updated.lastRun?.status).toBe('success');+ expect(updated.lastRun?.conversationId).toBe('convo-resumed');+ });++ it('interrupted records lastRun without touching counters', async () => {+ const schedule = await methods.createSchedule(scheduleData({ runCount: 4, failureCount: 1 }));+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'interrupted',+ autoDisableAfterFailures: 3,+ });+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('interrupted');+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(4);+ expect(updated.failureCount).toBe(1);+ expect(updated.lastRun?.status).toBe('interrupted');+ });++ it('does not touch schedule bookkeeping when no matching run row exists', async () => {+ const schedule = await methods.createSchedule(scheduleData({ runCount: 7 }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'spoofed',+ autoDisableAfterFailures: 3,+ });+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(7);+ expect(updated.lastRun).toBeUndefined();+ });+});++describe('recordRunOutcome idempotency + crash-retry (bookkeeping)', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('counts an occurrence at most once across repeated invocations', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ const outcome = {+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success' as const,+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ };+ await methods.recordRunOutcome(outcome);+ // A second call (e.g. reconciler racing the inline finish) must not re-count.+ await methods.recordRunOutcome(outcome);+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(1);+ expect(+ (updated as { countedFor?: Date[] }).countedFor?.map((d) => new Date(d).toISOString()),+ ).toContain(scheduledFor.toISOString());+ expect((await getRun(schedule.id, scheduledFor)).bookkept).toBe(true);+ });++ it('does not double-count a crashed later occurrence when an earlier one interleaves', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const t1 = new Date('2026-07-20T10:00:00Z');+ const t2 = new Date('2026-07-20T11:00:00Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: t1 }));+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: t2 }));++ // Later occurrence B (t2) finishes + counts, then crashes before bookkept:true.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: t2,+ status: 'success',+ autoDisableAfterFailures: 3,+ });+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor: t2 },+ { $set: { bookkept: false } },+ );++ // Earlier occurrence A (t1) then counts — a single scalar guard would move to+ // t1 here, so replaying B would look uncounted again.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: t1,+ status: 'success',+ autoDisableAfterFailures: 3,+ });++ // Reconciler replays the still-unbookkept B: the per-occurrence guard blocks a re-count.+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor: t2,+ status: 'success',+ autoDisableAfterFailures: 3,+ });++ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(2); // A + B, each counted exactly once+ });++ it('finalizeBookkeeping recovers a terminalized-but-uncounted run (crash between writes)', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ // Simulate a crash after the run row was terminalized but before bookkeeping.+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor },+ { $set: { status: 'success', bookkept: false } },+ );+ expect((await getSchedule(schedule.id)).runCount).toBe(0);++ const unbookkept = await methods.getUnbookkeptRuns(new Date(Date.now() + 1000), 100);+ expect(unbookkept.map((r) => r.scheduleId)).toContain(schedule.id);+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ autoDisableAfterFailures: 3,+ });+ const updated = await getSchedule(schedule.id);+ expect(updated.runCount).toBe(1);+ expect((await getRun(schedule.id, scheduledFor)).bookkept).toBe(true);+ // No longer surfaced as needing bookkeeping.+ expect(await methods.getUnbookkeptRuns(new Date(Date.now() + 1000), 100)).toHaveLength(0);+ });++ it('rotates the unbookkept window past rows whose replay keeps failing', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const early = new Date('2026-07-20T10:00:00Z');+ const late = new Date('2026-07-20T11:00:00Z');+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor: early, firedAt: early, status: 'success' }),+ );+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor: late, firedAt: late, status: 'success' }),+ );+ await ScheduleRun.updateMany({ scheduleId: schedule.id }, { $set: { bookkept: false } });++ const cutoff = new Date(Date.now() + 1000);+ const first = await methods.getUnbookkeptRuns(cutoff, 1);+ expect(first).toHaveLength(1);+ expect(new Date(first[0].firedAt!).toISOString()).toBe(early.toISOString());++ // The engine stamps every row it attempted (success or failure); the next+ // bounded read must then surface the row BEHIND it, not the same one again.+ await methods.markRunsReconciled(first);+ const second = await methods.getUnbookkeptRuns(cutoff, 1);+ expect(new Date(second[0].firedAt!).toISOString()).toBe(late.toISOString());+ });++ it('replayed bookkeeping projects the ORIGINAL firedAt, not the replay time', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const originalFiredAt = new Date('2026-07-20T12:00:03Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, firedAt: originalFiredAt }));+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor },+ { $set: { status: 'success', bookkept: false } },+ );++ // The reconciler replays minutes (or several retries) later — lastRun must+ // still show when the run actually fired, not when recovery caught up.+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ autoDisableAfterFailures: 3,+ });++ const updated = await getSchedule(schedule.id);+ expect(updated.lastRun?.firedAt?.toISOString()).toBe(originalFiredAt.toISOString());+ });+});++describe('bookkeeping policy is crash-retryable / streak-correct', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('re-applies auto-disable on a bookkeeping retry after the count already landed', async () => {+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 2 }));+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ // First terminal error: increments to threshold (3) and disables.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ autoDisableAfterFailures: 3,+ });+ expect((await getSchedule(schedule.id)).enabled).toBe(false);+ // Simulate a crash where the count landed (countedFor set) but the disable+ // was lost: re-enable and replay via finalizeBookkeeping. The count must NOT+ // double (guard), but the disable policy must re-apply.+ await Schedule.updateOne({ id: schedule.id }, { $set: { enabled: true } });+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ autoDisableAfterFailures: 3,+ });+ const updated = await getSchedule(schedule.id);+ expect(updated.failureCount).toBe(3); // not double-counted+ expect(updated.enabled).toBe(false); // disable re-applied+ });++ it('resets the balance-skip streak on an intervening error, then interrupt, then overlap', async () => {+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 4 }));+ const outcomes: Array<[Date, 'error' | 'interrupted']> = [+ [new Date('2026-07-20T01:00:00Z'), 'error'],+ [new Date('2026-07-20T02:00:00Z'), 'interrupted'],+ ];+ for (const [when, status] of outcomes) {+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: when }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: when,+ status,+ autoDisableAfterFailures: 10,+ });+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);+ }+ // Also an overlap skip breaks the streak.+ await Schedule.updateOne({ id: schedule.id }, { $set: { balanceSkipCount: 3 } });+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor: new Date('2026-07-20T03:00:00Z'),+ status: 'skipped_overlap',+ },+ 5,+ );+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);+ });+});++describe('getRunsForReconciliation fairness', () => {+ it('always includes started runs even behind a backlog of paused rows', async () => {+ const s = await methods.createSchedule(scheduleData());+ // Older paused rows + a newer started row; started must still be returned.+ for (let i = 0; i < 3; i++) {+ await methods.insertScheduleRun(+ runData(s, {+ scheduledFor: new Date(`2026-07-1${i}T12:00:00Z`),+ status: 'requires_action',+ firedAt: new Date('2020-01-01T00:00:00Z'),+ }),+ );+ }+ await methods.insertScheduleRun(+ runData(s, {+ scheduledFor: new Date('2026-07-25T12:00:00Z'),+ status: 'started',+ firedAt: new Date('2020-06-01T00:00:00Z'),+ }),+ );+ const runs = await methods.getRunsForReconciliation(new Date('2026-01-01T00:00:00Z'), 2);+ expect(runs.some((r) => r.status === 'started')).toBe(true);+ });+});++describe('deleteSchedulesByUser', () => {+ it('removes the user’s schedules and their runs', async () => {+ const userId = new mongoose.Types.ObjectId();+ const a = await methods.createSchedule(scheduleData({ user: userId }));+ await methods.createSchedule(scheduleData({ user: userId }));+ const other = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(a, { scheduledFor: new Date('2026-07-20T12:00:00Z') }));+ await methods.insertScheduleRun(+ runData(other, { scheduledFor: new Date('2026-07-20T12:00:00Z') }),+ );++ await methods.deleteSchedulesByUser(userId);++ expect(await methods.getSchedulesByUser(userId)).toHaveLength(0);+ expect(await getSchedule(other.id)).not.toBeNull();+ // The other user's run survives; the deleted user's runs are gone.+ expect(await ScheduleRun.countDocuments({ scheduleId: a.id })).toBe(0);+ expect(await ScheduleRun.countDocuments({ scheduleId: other.id })).toBe(1);+ });+});++describe('reversible account-deletion suspension', () => {+ async function rawSchedule(id: string): Promise<ISchedule | null> {+ return Schedule.findOne({ id }).lean<ISchedule>();+ }++ it('suspends under a token, fencing firing while snapshotting the prior state', async () => {+ const armedAt = new Date('2030-01-01T00:00:00Z');+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));+ const before = await getSchedule(schedule.id);++ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ const after = await getSchedule(schedule.id);+ expect(after.enabled).toBe(false);+ expect(after.nextRunAt).toBeUndefined();+ expect(after.claimToken).not.toBe(before.claimToken);+ // Reversible, NOT destructive: never marked for erasure.+ expect(after.deleting).not.toBe(true);+ expect(after.deletionSuspension?.token).toBe('attempt-1');+ expect(after.deletionSuspension?.enabled).toBe(true);+ expect(after.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());+ });++ it('restores enabled + next-run from the snapshot and clears the suspension', async () => {+ const armedAt = new Date('2030-01-01T00:00:00Z');+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');++ const restored = await getSchedule(schedule.id);+ expect(restored.enabled).toBe(true);+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());+ expect(restored.deletionSuspension).toBeUndefined();+ });++ it('restores a previously DISABLED schedule as disabled with no next run', async () => {+ const schedule = await methods.createSchedule(scheduleData({ enabled: false }));+ await Schedule.updateOne({ id: schedule.id }, { $unset: { nextRunAt: 1 } });+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');++ const restored = await getSchedule(schedule.id);+ expect(restored.enabled).toBe(false);+ expect(restored.nextRunAt).toBeUndefined();+ expect(restored.deletionSuspension).toBeUndefined();+ });++ it('restore is fenced to the exact attempt token', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ // A different attempt's restore must not touch this suspension.+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-2');++ const after = await getSchedule(schedule.id);+ expect(after.enabled).toBe(false);+ expect(after.deletionSuspension?.token).toBe('attempt-1');+ });++ it('cannot resurrect a schedule the owner independently deleted mid-suspension', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');+ // The owner deletes this one schedule while the account deletion is in flight.+ await methods.markScheduleDeleting(schedule.id, schedule.user);++ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');++ const after = await rawSchedule(schedule.id);+ expect(after?.deleting).toBe(true);+ expect(after?.enabled).toBe(false); // NOT re-enabled by restore+ });++ it('is idempotent per token — a retry does not overwrite the snapshot', async () => {+ const armedAt = new Date('2030-01-01T00:00:00Z');+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');+ // A retry of the SAME attempt must not re-snapshot the now-disabled state.+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ const after = await getSchedule(schedule.id);+ expect(after.deletionSuspension?.enabled).toBe(true);+ expect(after.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());+ });++ /**+ * A crash between suspend and restore leaves an ABANDONED suspension. A later attempt+ * must adopt that attempt's snapshot rather than re-snapshotting the row's current+ * (already-suspended) state — otherwise the true pre-suspension state is lost and the+ * schedule stays disabled forever once the second attempt restores.+ */+ it('a second attempt adopts the abandoned snapshot instead of capturing suspended state', async () => {+ const armedAt = new Date('2030-01-01T00:00:00Z');+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ // attempt-1's process died without restoring; attempt-2 now suspends the same rows.+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-2');++ const suspended = await getSchedule(schedule.id);+ expect(suspended.deletionSuspension?.token).toBe('attempt-2');+ // The ORIGINAL pre-suspension state, not the suspended state attempt-2 observed.+ expect(suspended.deletionSuspension?.enabled).toBe(true);+ expect(suspended.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());++ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-2');+ const restored = await getSchedule(schedule.id);+ expect(restored.enabled).toBe(true);+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());+ expect(restored.deletionSuspension).toBeUndefined();+ });++ /**+ * The restore's callers cannot retry it later: a cancelled deletion restores while the+ * user-deletion fence is still armed and then releases that fence, after which nothing+ * re-drives this. A single transient write failure would therefore leave a LIVE account+ * holding silently disabled, next-run-less schedules.+ */+ it('retries a transient write failure instead of stranding the rows disabled', async () => {+ const armedAt = new Date('2030-01-01T00:00:00Z');+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');++ const bulkWrite = jest.spyOn(Schedule, 'bulkWrite');+ bulkWrite.mockImplementationOnce(() => {+ throw new Error('transient primary stepdown');+ });++ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');+ bulkWrite.mockRestore();++ const restored = await getSchedule(schedule.id);+ expect(restored.enabled).toBe(true);+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());+ expect(restored.deletionSuspension).toBeUndefined();+ });++ /** Retrying is only safe because each attempt re-reads the rows STILL carrying the+ * token: a partially-applied unordered write converges on exactly the stragglers, and+ * an already-restored row is simply no longer in the set. */+ it('converges on the stragglers of a partially applied restore', async () => {+ const userId = new mongoose.Types.ObjectId();+ const first = await methods.createSchedule(scheduleData({ user: userId }));+ const second = await methods.createSchedule(scheduleData({ user: userId }));+ await methods.suspendUserSchedulesForDeletion(userId, 'attempt-1');++ // Exactly what a partially-applied unordered bulkWrite leaves behind: one row already+ // restored and out of the token set, one still suspended.+ await methods.restoreUserSchedulesFromDeletion(userId, 'attempt-1');+ await Schedule.updateOne(+ { id: second.id },+ {+ $set: {+ enabled: false,+ deletionSuspension: { token: 'attempt-1', enabled: true },+ },+ $unset: { nextRunAt: 1 },+ },+ );++ await methods.restoreUserSchedulesFromDeletion(userId, 'attempt-1');++ for (const id of [first.id, second.id]) {+ const restored = await getSchedule(id);+ expect(restored.enabled).toBe(true);+ expect(restored.deletionSuspension).toBeUndefined();+ }+ });++ it('a successful deletion removes suspended rows and their snapshots', async () => {+ const userId = new mongoose.Types.ObjectId();+ const schedule = await methods.createSchedule(scheduleData({ user: userId }));+ await methods.suspendUserSchedulesForDeletion(userId, 'attempt-1');++ await methods.deleteSchedulesByUser(userId);++ expect(await rawSchedule(schedule.id)).toBeNull();+ });+});++describe('acquireManualRunLease / releaseLease', () => {+ it('serializes concurrent run-now attempts and can be released without advancing', async () => {+ const schedule = await methods.createSchedule(+ scheduleData({ nextRunAt: new Date('2026-08-01T12:00:00Z') }),+ );+ const results = await Promise.all(+ Array.from({ length: 6 }, () =>+ methods.acquireManualRunLease(schedule.id, schedule.user, 60_000),+ ),+ );+ expect(results.filter(Boolean)).toHaveLength(1);++ await methods.releaseLease(schedule.id);+ const afterRelease = await getSchedule(schedule.id);+ expect(afterRelease.leaseUntil).toBeUndefined();+ expect(afterRelease.leaseBy).toBeUndefined();+ // release must NOT reschedule the next automatic occurrence+ expect(afterRelease.nextRunAt?.toISOString()).toBe('2026-08-01T12:00:00.000Z');++ const reacquired = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);+ // Returns the fresh claim token (truthy) rather than a boolean.+ expect(reacquired).toBeTruthy();+ });++ it('returns the FRESH row (reflecting an edit) with a new claim token', async () => {+ const schedule = await methods.createSchedule(scheduleData({ name: 'original' }));+ // An edit commits after a caller read the schedule but before the lease is taken.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });+ const leased = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);+ // The lease returns the post-image, so a manual fire uses the edited snapshot.+ expect(leased?.name).toBe('edited');+ expect(leased?.claimToken).toBeTruthy();+ });++ it('blocks against a held engine lease and rejects a non-owner', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const claimed = await methods.claimDueSchedule({ instanceId: 'engine-1', leaseMs: 60_000 });+ expect(claimed?.id).toBe(schedule.id);+ expect(await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000)).toBeNull();+ expect(+ await methods.acquireManualRunLease(schedule.id, new mongoose.Types.ObjectId(), 60_000),+ ).toBeNull();+ });++ it('uses a unique per-lease holder so a stale run-now cannot strip a replacement lease', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const first = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);+ expect(first?.leaseBy).toMatch(/^manual:/);+ // Release and re-acquire (a fresh run-now): the replacement lease has a DIFFERENT holder.+ await methods.releaseLease(schedule.id);+ const second = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);+ expect(second?.leaseBy).toMatch(/^manual:/);+ expect(second?.leaseBy).not.toBe(first?.leaseBy);+ // A stalled fire from the FIRST lease releasing by its (old) holder must NOT clear the+ // second lease — the unique holder makes releaseLeaseByHolder no-op there.+ await methods.releaseLeaseByHolder(schedule.id, first!.leaseBy!);+ const afterStale = await getSchedule(schedule.id);+ expect(afterStale.leaseBy).toBe(second?.leaseBy);+ expect(afterStale.leaseUntil).toBeDefined();+ // Releasing by the CURRENT holder does clear it.+ await methods.releaseLeaseByHolder(schedule.id, second!.leaseBy!);+ const cleared = await getSchedule(schedule.id);+ expect(cleared.leaseBy).toBeUndefined();+ });+});++describe('scheduled resume lease fencing', () => {+ it('atomically consumes the same live config generation and releases its lease', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const fence = await methods.acquireResumeLease(schedule.id, 0, true, 60_000);++ expect(fence?.claimToken).toBeTruthy();+ expect(fence?.leaseBy).toMatch(/^resume:/);+ await expect(+ methods.consumeResumeLease(schedule.id, fence!.claimToken!, fence!.leaseBy!, true, 0),+ ).resolves.toBe(true);+ const consumed = await getSchedule(schedule.id);+ expect(consumed.leaseUntil).toBeUndefined();+ expect(consumed.leaseBy).toBeUndefined();+ });++ it('rejects an owner edit that wins before the final resume handoff', async () => {+ const schedule = await methods.createSchedule(scheduleData({ name: 'before' }));+ const fence = await methods.acquireResumeLease(schedule.id, 0, true, 60_000);+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'after' });++ await expect(+ methods.consumeResumeLease(schedule.id, fence!.claimToken!, fence!.leaseBy!, true, 0),+ ).resolves.toBe(false);+ // Owner edits intentionally leave the in-flight holder fields intact. The stale+ // resume can release only its unique holder without touching any newer takeover.+ expect((await getSchedule(schedule.id)).leaseBy).toBe(fence!.leaseBy);+ await methods.releaseLeaseByHolder(schedule.id, fence!.leaseBy!);+ expect((await getSchedule(schedule.id)).leaseBy).toBeUndefined();+ });+});++describe('claim-token fencing (stale worker cannot mutate an edited/deleted schedule)', () => {+ it('an owner edit rotates the token so a stale disable/advance no-ops and revalidate fails', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 });+ const staleToken = claim!.claimToken!;+ expect(staleToken).toBeTruthy();+ // The stale worker still holds a valid claim until the owner edits.+ expect(await methods.revalidateClaim(schedule.id, staleToken)).toBe(true);++ // Owner edits (re-enable / rename) — updateScheduleById rotates the claim token.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });++ // The stale worker's fenced writes now match nothing.+ await methods.disableSchedule(schedule.id, 'too_many_failures', staleToken);+ await methods.advanceSchedule(+ schedule.id,+ new Date('2030-01-01T00:00:00Z'),+ claim!.nextRunAt,+ staleToken,+ );+ const after = await methods.getScheduleById(schedule.id);+ expect(after!.enabled).toBe(true); // stale disable no-op'd+ expect(after!.disabledReason).toBeUndefined();+ expect(after!.nextRunAt?.getTime()).toBe(claim!.nextRunAt?.getTime()); // stale advance no-op'd+ expect(await methods.revalidateClaim(schedule.id, staleToken)).toBe(false);+ });++ it('revalidateClaim fails once the lease expires (a re-claim could be imminent)', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: -1 }); // already expired+ expect(await methods.revalidateClaim(schedule.id, claim!.claimToken!)).toBe(false);+ });+});++describe('owner edit clears a dead paused card', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ /** Projects a `requires_action` card whose run captured revision 0, matching a real+ * fire: the reservation stamps the config generation on the run row. */+ async function pauseUnderRevisionZero(): Promise<ISchedule> {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-paused',+ autoDisableAfterFailures: 3,+ });+ const paused = await getSchedule(schedule.id);+ expect(paused.lastRun?.status).toBe('requires_action');+ return schedule;+ }++ it('a rename drops the pause the stale approval can no longer clear', async () => {+ const schedule = await pauseUnderRevisionZero();+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {+ name: 'renamed',+ });+ expect(edited?.name).toBe('renamed');+ expect(edited?.configRevision).toBe(1);+ expect(edited?.lastRun).toBeUndefined();+ });++ it('a disabling edit drops the pause (no future occurrence would repair it)', async () => {+ const schedule = await pauseUnderRevisionZero();+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, { enabled: false });+ expect(edited?.enabled).toBe(false);+ expect(edited?.lastRun).toBeUndefined();+ });++ it('preserves a terminal success card across an edit', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-ok',+ autoDisableAfterFailures: 3,+ });+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {+ name: 'renamed',+ });+ expect(edited?.lastRun?.status).toBe('success');+ expect(edited?.lastRun?.conversationId).toBe('convo-ok');+ });++ /** A resume terminalizes the paused occurrence between the handler's read and the+ * edit; the clear is fenced on the card STILL being the pause, so the settled result+ * survives rather than being cleared on a stale pre-read. */+ it('preserves a terminal outcome that races the edit ahead of it', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });+ // The pause settles to interrupted before the edit lands.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'interrupted',+ error: 'stopped',+ autoDisableAfterFailures: 3,+ });+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {+ name: 'renamed',+ });+ expect(edited?.lastRun?.status).toBe('interrupted');+ });++ it('a stale-revision pause cannot resurrect the cleared card', async () => {+ const schedule = await pauseUnderRevisionZero();+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });+ // The run row is still `requires_action` under revision 0; a reconciliation re-affirm+ // of the pause must not re-stamp the card now that the schedule advanced to revision 1.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-paused',+ autoDisableAfterFailures: 3,+ });+ const after = await getSchedule(schedule.id);+ expect(after.lastRun).toBeUndefined();+ });++ it('a stale old-revision terminal outcome cannot overwrite the edited card or counters', async () => {+ const schedule = await pauseUnderRevisionZero();+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });+ // The dead occurrence (revision 0) finally settles; its bookkeeping is revision-fenced,+ // so it neither re-projects the card nor advances counters on the revision-1 schedule.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'convo-paused',+ autoDisableAfterFailures: 3,+ });+ const after = await getSchedule(schedule.id);+ expect(after.lastRun).toBeUndefined();+ expect(after.runCount).toBe(0);+ });+});++describe('schedule deletion is retryable', () => {+ /**+ * The mark is the FIRST of several teardown steps (read active runs, abort their+ * jobs, prune checkpoints, erase when drained). A one-shot mark made every retry+ * answer 404 after any of those threw, stranding the schedule with its job and+ * checkpoint alive until the background expiry windows.+ */+ it('re-marks an already-deleting schedule so the caller can re-drive teardown', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const first = await methods.markScheduleDeleting(schedule.id, schedule.user);+ expect(first).not.toBeNull();++ const retry = await methods.markScheduleDeleting(schedule.id, schedule.user);+ expect(retry).not.toBeNull();+ expect(retry!.deleting).toBe(true);+ expect(retry!.enabled).toBe(false);+ // The token rotates again, which only re-fences stale workers.+ expect(retry!.claimToken).not.toBe(first!.claimToken);+ });++ it('still refuses a schedule owned by someone else', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const other = new mongoose.Types.ObjectId();+ expect(await methods.markScheduleDeleting(schedule.id, other)).toBeNull();+ });+});++describe('unarmed-schedule recovery', () => {+ /**+ * Creation arms in a second write, so a crash or a failed arm leaves a row that looks+ * enabled to its owner and occupies a slot while claimDueSchedule — which sorts on+ * nextRunAt — can never select it.+ */+ it('finds enabled schedules with no nextRunAt and arms them', async () => {+ const schedule = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });+ expect((await getSchedule(schedule.id)).nextRunAt).toBeUndefined();++ const unarmed = await methods.getUnarmedSchedules(10);+ expect(unarmed.map((s) => s.id)).toContain(schedule.id);++ const when = new Date('2030-01-01T00:00:00.000Z');+ await methods.armSchedule(schedule.id, when);+ expect((await getSchedule(schedule.id)).nextRunAt?.getTime()).toBe(when.getTime());+ // Now armed, so it drops out of the sweep.+ expect((await methods.getUnarmedSchedules(10)).map((s) => s.id)).not.toContain(schedule.id);+ });++ it('never disturbs a schedule that armed itself in the meantime', async () => {+ const schedule = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });+ const real = new Date('2030-06-01T00:00:00.000Z');+ await methods.armSchedule(schedule.id, real);++ // A concurrent sweep pass computed its own value from a stale read.+ await methods.armSchedule(schedule.id, new Date('2030-01-01T00:00:00.000Z'));++ expect((await getSchedule(schedule.id)).nextRunAt?.getTime()).toBe(real.getTime());+ });++ it('excludes disabled and deleting schedules from the sweep', async () => {+ const disabled = await methods.createSchedule({+ ...scheduleData(),+ nextRunAt: undefined,+ enabled: false,+ });+ const deleting = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });+ await methods.markScheduleDeleting(deleting.id, deleting.user);++ const ids = (await methods.getUnarmedSchedules(10)).map((s) => s.id);+ expect(ids).not.toContain(disabled.id);+ expect(ids).not.toContain(deleting.id);+ });+});++describe('card projection fences (regressions in the ordered projection)', () => {+ /**+ * The terminal path passes the row's revision to projectLastRun; the pause path did+ * not. An owner edit landing between fire and approval therefore got the OLD config's+ * pause stamped on it, and because the later terminal write IS revision-fenced it+ * could never replace that status — the card stuck on "Needs approval".+ */+ it('does not stamp a pause onto a schedule edited after the run started', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');+ const runRevision = (await getSchedule(schedule.id)).configRevision;+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor, configRevision: runRevision }),+ );+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });++ // The row still pauses (the generation really is waiting) but the edited+ // schedule's card is not stamped by the superseded config.+ expect(await getRun(schedule.id, scheduledFor)).toMatchObject({ status: 'requires_action' });+ expect((await getSchedule(schedule.id)).lastRun).toBeUndefined();+ });++ /**+ * A skip wrote `lastRun` directly, dropping the `scheduledFor` ordering marker that+ * projectLastRun keys off — so the NEXT projection saw an absent marker and let an+ * older occurrence's result overwrite the newer skip.+ */+ /**+ * The pause wins its row transition and THEN projects, so a resume that terminalizes+ * in between leaves exactly this state: row still active, card already settled for the+ * SAME occurrence. Occurrence ordering alone permits the pause (equal `scheduledFor`),+ * which would pin the card to "Needs approval" for a run that had already finished —+ * and nothing corrects it until the next occurrence runs.+ *+ * The interleaving needs two concurrent writers, but the state it produces does not:+ * constructing it directly tests the guard on exactly the input it exists for.+ */+ it('refuses to walk an occurrence card back from a settled status to a pause', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));+ // The resumed generation already settled this occurrence's card.+ await mongoose.models.Schedule.updateOne(+ { id: schedule.id },+ {+ $set: {+ lastRun: {+ status: 'success',+ firedAt: new Date(),+ scheduledFor,+ conversationId: 'convo-1',+ },+ },+ },+ );++ // The in-flight pause now lands for that same occurrence.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });++ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('success');+ });++ it('still projects a pause when the occurrence card is not yet settled', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-1',+ autoDisableAfterFailures: 3,+ });++ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('requires_action');+ });++ it('keeps the occurrence marker when recording a skip', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const older = new Date('2026-07-26T10:00:00.000Z');+ const newer = new Date('2026-07-26T11:00:00.000Z');+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: older }));+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'requires_action',+ conversationId: 'convo-older',+ autoDisableAfterFailures: 3,+ });++ // The newer occurrence is skipped for balance while the older one sits paused.+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ scheduledFor: newer,+ user: schedule.user,+ status: 'skipped_balance',+ firedAt: new Date(),+ },+ 5,+ );+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_balance');++ // The older occurrence then resumes and finishes. It must NOT roll the card back.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'success',+ conversationId: 'convo-older',+ autoDisableAfterFailures: 3,+ });++ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_balance');+ });+});++describe('auto-disable carries the run config generation', () => {+ /**+ * The auto-disable policy re-reads the schedule and then writes, so the decision and+ * the write are not atomic. Carrying the run's config generation on the write closes+ * that window: an owner edit landing inside it rotates configRevision, and the stale+ * decision no longer applies.+ *+ * NOTE the ordinary re-enable path is NOT this scenario — `updateSchedule` zeroes+ * failureCount/balanceSkipCount when re-enabling, so a re-enabled schedule cannot be+ * sitting at the threshold. The state below is reached only through the read→write+ * window itself, which is why it is constructed directly.+ */+ it('does not disable when the run config generation is stale', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const runRevision = (await getSchedule(schedule.id)).configRevision;+ // A plain rename — NOT a re-enable, so the counters are deliberately left alone.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });+ expect((await getSchedule(schedule.id)).configRevision).toBeGreaterThan(runRevision!);+ // At threshold and still enabled: what a decision made just before that edit saw.+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });++ const stale = new Date('2026-07-26T11:00:00.000Z');+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor: stale, configRevision: runRevision }),+ );+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: stale,+ status: 'error',+ error: 'stale decision',+ autoDisableAfterFailures: 1,+ });++ expect((await getSchedule(schedule.id)).enabled).toBe(true);+ });++ /**+ * The threshold read and the disable write are separate statements. A concurrent+ * success resets the failure streak in between, so a revision-only fence would still+ * let the stale decision disable a schedule whose streak had just been cleared.+ */+ it('does not disable once a concurrent success reset the failure streak', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const current = await getSchedule(schedule.id);+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });++ // The decision was made on failureCount >= 5; a success then zeroed it.+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 0 } });+ await methods.disableSchedule(+ schedule.id,+ 'too_many_failures',+ undefined,+ current.configRevision,+ {+ failureCount: { $gte: 5 },+ },+ );++ expect((await getSchedule(schedule.id)).enabled).toBe(true);+ });++ it('still disables when the streak is intact', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const current = await getSchedule(schedule.id);+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });++ await methods.disableSchedule(+ schedule.id,+ 'too_many_failures',+ undefined,+ current.configRevision,+ {+ failureCount: { $gte: 5 },+ },+ );++ expect(await getSchedule(schedule.id)).toMatchObject({+ enabled: false,+ disabledReason: 'too_many_failures',+ });+ });++ it('still auto-disables when the run matches the current config generation', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');+ const current = await getSchedule(schedule.id);+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor, configRevision: current.configRevision }),+ );++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ error: 'boom',+ autoDisableAfterFailures: 1,+ });++ expect(await getSchedule(schedule.id)).toMatchObject({+ enabled: false,+ disabledReason: 'too_many_failures',+ });+ });+});++describe('reserveStartedRun duplicate reporting', () => {+ /**+ * The caller has to tell "another worker is still running this" from "this occurrence+ * already finished and was merely never advanced past" — they need opposite handling,+ * and a bare `duplicate` cannot distinguish them.+ */+ it('reports the existing row status on a duplicate', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');+ const base = {+ scheduleId: schedule.id,+ scheduledFor,+ user: schedule.user,+ firedAt: new Date(),+ };+ await methods.reserveStartedRun({ ...base, conversationId: 'first' });++ const active = await methods.reserveStartedRun({ ...base, conversationId: 'second' });+ expect(active).toMatchObject({ conflict: 'duplicate', existingStatus: 'started' });++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ autoDisableAfterFailures: 3,+ });++ const settled = await methods.reserveStartedRun({ ...base, conversationId: 'third' });+ expect(settled).toMatchObject({ conflict: 'duplicate', existingStatus: 'success' });+ });+});++describe('deleteScheduleRun conversation fence', () => {+ it('deletes only the reservation the caller inserted', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T12:00:00.000Z');+ await methods.reserveStartedRun({+ scheduleId: schedule.id,+ scheduledFor,+ user: schedule.user,+ conversationId: 'conv-mine',+ firedAt: new Date(),+ });++ // Another fire's id must not match: this is what makes the rollback safe without+ // consulting lease ownership, which the preflight can outlive.+ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-theirs');+ expect(await methods.getActiveRunsForSchedule(schedule.id)).toHaveLength(1);++ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-mine');+ expect(await methods.getActiveRunsForSchedule(schedule.id)).toHaveLength(0);+ });++ it('still honors the status fence so a settled run is never rolled back', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2026-07-26T13:00:00.000Z');+ await methods.reserveStartedRun({+ scheduleId: schedule.id,+ scheduledFor,+ user: schedule.user,+ conversationId: 'conv-mine',+ firedAt: new Date(),+ });+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ autoDisableAfterFailures: 5,+ });++ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-mine');++ const runs = await methods.getRunsForReconciliation(new Date(Date.now() + 1000), 10);+ expect(runs).toHaveLength(0);+ expect(await methods.getScheduleById(schedule.id)).toMatchObject({+ lastRun: expect.objectContaining({ status: 'success' }),+ });+ });+});++describe('erasure leaves an idempotency tombstone', () => {+ it('retains the create key so a late retry cannot resurrect deleted work', async () => {+ const user = new mongoose.Types.ObjectId();+ const schedule = await methods.createSchedule(+ scheduleData({+ user,+ deleting: true,+ nextRunAt: undefined,+ clientRequestId: 'late-retry-key',+ clientRequestDigest: 'digest-1',+ }),+ );++ await expect(methods.eraseScheduleIfDrained(schedule.id)).resolves.toBe(true);++ // The content is gone, the identity is not: a create retry with this key must+ // resolve to the tombstone (the handler answers 410 on deleting rows) instead+ // of missing entirely and recreating the recurring work the owner deleted.+ const tombstone = await methods.getScheduleByClientRequestId(user, 'late-retry-key');+ expect(tombstone).not.toBeNull();+ expect(tombstone?.erased).toBe(true);+ expect(tombstone?.deleting).toBe(true);+ expect(tombstone?.clientRequestDigest).toBe('digest-1');+ // ALLOWLIST semantics: everything except the replay-detection identity is gone,+ // derived from the live schema paths so fields added later cannot leak either.+ const contentKeys = Object.keys(tombstone ?? {}).filter(+ (key) =>+ ![+ '_id',+ '__v',+ 'id',+ 'user',+ 'tenantId',+ 'clientRequestId',+ 'clientRequestDigest',+ 'deleting',+ 'erased',+ 'erasedAt',+ 'enabled',+ 'createdAt',+ 'updatedAt',+ ].includes(key),+ );+ expect(contentKeys).toEqual([]);+ expect(tombstone?.prompt).toBeUndefined();+ expect(tombstone?.name).toBeUndefined();+ expect(tombstone?.cadence).toBeUndefined();+ expect(tombstone?.timezone).toBeUndefined();++ // Tombstones are inert to every sweep: re-erasing or re-deleting them forever+ // would pin the bounded windows.+ const sweepRows = await methods.getDeletingSchedules(50);+ expect(sweepRows.some((row) => row.id === schedule.id)).toBe(false);+ const retryIds = await methods.getDeletingScheduleIds(user, 50);+ expect(retryIds).not.toContain(schedule.id);+ });++ it('hard-deletes rows that never carried a create key', async () => {+ const user = new mongoose.Types.ObjectId();+ const schedule = await methods.createSchedule(+ scheduleData({ user, deleting: true, nextRunAt: undefined }),+ );++ await expect(methods.eraseScheduleIfDrained(schedule.id)).resolves.toBe(true);+ const gone = await mongoose.models.Schedule.findOne({ id: schedule.id }).lean();+ expect(gone).toBeNull();+ });+});++describe('deletion quiescing (soft-delete, drain, erase)', () => {+ it('markScheduleDeleting hides + un-claims; erase waits for active runs to drain', async () => {+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);+ const sched = schedule as ISchedule;+ const when = new Date('2026-07-20T12:00:00Z');+ await methods.reserveStartedRun(runData(sched, { scheduledFor: when }));++ const marked = await methods.markScheduleDeleting(sched.id, sched.user);+ expect(marked!.deleting).toBe(true);+ expect(marked!.enabled).toBe(false);+ // Hidden from the owner and no longer claimable by the engine.+ expect(await methods.getSchedulesByUser(sched.user)).toHaveLength(0);+ expect(await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 })).toBeNull();++ // Erase is blocked while a run is still active (evidence preserved). The raw+ // doc still exists (getScheduleById hides `deleting` schedules by design).+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(false);+ expect(await Schedule.findOne({ id: sched.id }).lean()).not.toBeNull();++ // Once the run terminalizes, the schedule and its runs erase.+ await ScheduleRun.updateOne(+ { scheduleId: sched.id, scheduledFor: when },+ { $set: { status: 'interrupted' } },+ );+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(true);+ expect(await Schedule.findOne({ id: sched.id }).lean()).toBeNull();+ expect(await ScheduleRun.countDocuments({ scheduleId: sched.id })).toBe(0);+ // Its slot is freed for reuse.+ const replacement = await methods.createScheduleWithSlot(+ scheduleData({ user: sched.user }),+ 10,+ );+ expect(replacement).not.toBe('limit');+ });++ it('a deleting schedule is hidden, non-mutable, and non-claimable (no re-enable)', async () => {+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);+ const sched = schedule as ISchedule;+ await methods.markScheduleDeleting(sched.id, sched.user);+ // Hidden and unreadable through the owner-facing read.+ expect(await methods.getScheduleById(sched.id)).toBeNull();+ expect(await methods.getScheduleById(sched.id, sched.user)).toBeNull();+ // A stale/concurrent re-enable PATCH is refused (returns null -> handler 404).+ const reEnabled = await methods.updateScheduleById(sched.id, sched.user, {+ enabled: true,+ nextRunAt: new Date(Date.now() - 60_000),+ });+ expect(reEnabled).toBeNull();+ // Even if a nextRunAt/enabled somehow lingered, the engine won't claim it.+ expect(await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 })).toBeNull();+ });++ it('keeps a live lease and does not erase until it is released (worker mid-claim)', async () => {+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);+ const sched = schedule as ISchedule;+ // A worker has CLAIMED the schedule (live lease) but not yet inserted a run row.+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 });+ expect(claim?.id).toBe(sched.id);+ const marked = await methods.markScheduleDeleting(sched.id, sched.user);+ // The lease is PRESERVED so the worker can prove ownership on its rollback.+ expect(marked?.leaseBy).toBe('w1');+ // Erase is blocked while the lease is live, even with no active run row.+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(false);+ expect(await Schedule.findOne({ id: sched.id }).lean()).not.toBeNull();+ // Once the worker releases its own lease (by holder), it drains and erases.+ await methods.releaseLeaseByHolder(sched.id, 'w1');+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(true);+ expect(await Schedule.findOne({ id: sched.id }).lean()).toBeNull();+ });++ it('frees the slot immediately on soft-delete so a new create can take it under the cap', async () => {+ const user = new mongoose.Types.ObjectId();+ const a = (await methods.createScheduleWithSlot(scheduleData({ user }), 1)) as ISchedule;+ expect(await methods.createScheduleWithSlot(scheduleData({ user }), 1)).toBe('limit');+ await methods.markScheduleDeleting(a.id, user);+ // The deleting schedule no longer occupies the cap, so a replacement fits.+ const b = await methods.createScheduleWithSlot(scheduleData({ user }), 1);+ expect(b).not.toBe('limit');+ });+});++describe('config fence is DERIVED at the seam, not passed by callers', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('skips bookkeeping when the owner edited the schedule after the run started', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ // The run captured the config generation it started under.+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor, configRevision: schedule.configRevision ?? 0 }),+ );+ // Owner edits -> configRevision moves on.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });++ // NOTE: no fence token is passed — recordRunOutcome derives it from the run row.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ error: 'boom',+ autoDisableAfterFailures: 3,+ });++ // The run still terminalizes (evidence is preserved)...+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('error');+ // ...but it must NOT count a failure against the schedule the owner just edited.+ const after = await getSchedule(schedule.id);+ expect(after.failureCount).toBe(0);+ expect(after.lastRun).toBeUndefined();+ });++ it('applies bookkeeping normally when the revision still matches', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(+ runData(schedule, { scheduledFor, configRevision: schedule.configRevision ?? 0 }),+ );+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'success',+ conversationId: 'c1',+ autoDisableAfterFailures: 3,+ });+ const after = await getSchedule(schedule.id);+ expect(after.runCount).toBe(1);+ expect(after.lastRun?.status).toBe('success');+ });++ it('fences the crash-retry path (finalizeBookkeeping) the same way', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.insertScheduleRun(+ runData(schedule, {+ scheduledFor,+ status: 'error',+ bookkept: false,+ configRevision: schedule.configRevision ?? 0,+ }),+ );+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });++ // The reconciler replays bookkeeping for an unbookkept run; it must honor the same+ // fence as the inline path, or a crash would let it apply what inline refused.+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'error',+ autoDisableAfterFailures: 3,+ });+ expect((await getSchedule(schedule.id)).failureCount).toBe(0);+ });+});++describe('skip bookkeeping is fenced by the same derived seam', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('does not stamp the card when the owner edited after the occurrence was claimed', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ // Owner edits AFTER the worker claimed at the old revision.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });++ await methods.recordSkippedRun({+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor,+ status: 'skipped_overlap',+ configRevision: schedule.configRevision ?? 0, // the CLAIMED revision, now stale+ });++ // The skip row still exists as evidence, but the schedule card is untouched.+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('skipped_overlap');+ expect((await getSchedule(schedule.id)).lastRun).toBeUndefined();+ });++ it('stamps the card normally when the revision still matches', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.recordSkippedRun({+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor,+ status: 'skipped_overlap',+ configRevision: schedule.configRevision ?? 0,+ });+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_overlap');+ });++ it('does not walk the balance streak toward auto-disable under a stale revision', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });+ await methods.recordSkippedRun(+ {+ scheduleId: schedule.id,+ user: schedule.user,+ scheduledFor,+ status: 'skipped_balance',+ configRevision: schedule.configRevision ?? 0,+ },+ 1, // threshold of 1 would disable immediately if the fence did not hold+ );+ const after = await getSchedule(schedule.id);+ expect(after.balanceSkipCount).toBe(0);+ expect(after.enabled).toBe(true);+ });+});++describe('streak counters are order-fenced', () => {+ const older = new Date('2026-07-20T08:00:00Z');+ const newer = new Date('2026-07-20T09:00:00Z');++ /**+ * A `requires_action` run holds no capacity slot and does not block its schedule's+ * later occurrences, so an older occurrence routinely settles AFTER a newer one — a+ * resumed pause, or a reconciler replay. Unfenced, its late `error` rebuilds a streak+ * the newer success had cleared.+ */+ it('does not let an older failure rebuild a streak a newer success cleared', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: newer,+ status: 'success',+ autoDisableAfterFailures: 5,+ });+ expect((await getSchedule(schedule.id)).failureCount).toBe(0);++ // The older occurrence settles late, in error.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'error',+ error: 'late failure',+ autoDisableAfterFailures: 5,+ });++ const after = await getSchedule(schedule.id);+ expect(after.failureCount).toBe(0);+ });++ /**+ * `autoDisableAfterFailures` may be configured as low as 1, where a single stale+ * failure is the difference between a healthy schedule and a disabled one.+ */+ it('does not auto-disable a healthy schedule on a stale failure at threshold 1', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: newer,+ status: 'success',+ autoDisableAfterFailures: 1,+ });+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'error',+ autoDisableAfterFailures: 1,+ });++ const after = await getSchedule(schedule.id);+ expect(after.enabled).toBe(true);+ expect(after.disabledReason).toBeUndefined();+ });++ it('still counts occurrences that settle in order', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ // The older row is `requires_action`: the single-active partial index admits only+ // one `started` run per schedule, which is exactly why pauses can pile up.+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: older,+ status: 'error',+ autoDisableAfterFailures: 5,+ });+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor: newer,+ status: 'error',+ autoDisableAfterFailures: 5,+ });++ expect((await getSchedule(schedule.id)).failureCount).toBe(2);+ });+});++describe('updateScheduleById revision fence', () => {+ it('refuses an edit computed from a superseded revision', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const staleRevision = schedule.configRevision ?? 0;+ // A concurrent edit lands first and moves the revision on.+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'winner' });++ const loser = await methods.updateScheduleById(+ schedule.id,+ schedule.user,+ { name: 'loser' },+ undefined,+ { expectedConfigRevision: staleRevision },+ );++ // Otherwise the loser would persist a nextRunAt derived from a (cadence, timezone)+ // pair that no longer describes the row.+ expect(loser).toBeNull();+ expect((await getSchedule(schedule.id)).name).toBe('winner');+ });++ it('applies an edit whose revision is still current', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const updated = await methods.updateScheduleById(+ schedule.id,+ schedule.user,+ { name: 'renamed' },+ undefined,+ { expectedConfigRevision: schedule.configRevision ?? 0 },+ );+ expect(updated?.name).toBe('renamed');+ });+});++describe('pause replay is fenced against a concurrent resume claim', () => {+ /**+ * Every clustered replica runs the erasure sweep, so several can observe the same+ * unprojected pause. The first projects it; the owner's approval then claims a fresh+ * slot; a peer still holding the PRE-projection snapshot passes its in-memory+ * hand-off check and replays the pause — unsetting `capacitySlot` and+ * `resumeClaimedAt` under a continuation that is already running. The fence has to+ * live in the write, not in the caller's snapshot.+ */+ it('refuses to replay a pause onto a row a resume has claimed', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2030-03-01T00:00:00Z');+ await methods.reserveStartedRun(+ runData(schedule, { scheduledFor, conversationId: 'convo-race', capacitySlot: 0 }),+ );+ // Sweeper A wins: the row leaves `started` and frees its slot.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-race',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),+ });+ // The approval now claims a FRESH slot; this is the state a stale peer must not touch.+ const claimed = await methods.markRunResumeClaimed(schedule.id, scheduledFor, 3);+ expect(claimed).toEqual({ capacitySlot: 3 });++ // Sweeper B replays from its pre-projection snapshot.+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-race',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),+ });++ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();+ expect(run?.status).toBe('started');+ expect(run?.capacitySlot).toBe(3);+ expect(run?.resumeClaimedAt).toBeDefined();+ });++ /** The fence must not block the recovery it exists to enable: a genuinely stuck+ * `started` row carries no resume claim, because markRunResumeClaimed only matches+ * `requires_action` — the stuck row blocks its own approval until this replay runs. */+ it('still recovers a stuck row that carries no resume claim', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2030-03-02T00:00:00Z');+ await methods.reserveStartedRun(+ runData(schedule, { scheduledFor, conversationId: 'convo-stuck', capacitySlot: 1 }),+ );++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-stuck',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),+ });++ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();+ expect(run?.status).toBe('requires_action');+ expect(run?.capacitySlot).toBeUndefined();+ });++ /**+ * The fence is a staleness CUTOFF, not an existence check, and this is why: a worker+ * that dies between markRunResumeClaimed and either resuming or rolling back leaves+ * `resumeClaimedAt` set forever. Rejecting on existence alone would pin the row+ * `started` for good — holding its global capacity slot, and with its approval+ * permanently unresumable, since markRunResumeClaimed only matches `requires_action`.+ * That is precisely the stuck state this replay exists to clear.+ */+ it('recovers a row whose resume claim was abandoned by a dead worker', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const scheduledFor = new Date('2030-03-03T00:00:00Z');+ await methods.reserveStartedRun(+ runData(schedule, { scheduledFor, conversationId: 'convo-abandoned', capacitySlot: 2 }),+ );+ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-abandoned',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),+ });+ await methods.markRunResumeClaimed(schedule.id, scheduledFor, 4);+ // The claiming worker died here: the stamp is now well past the hand-off bound.+ await ScheduleRun.updateOne(+ { scheduleId: schedule.id, scheduledFor },+ { $set: { resumeClaimedAt: new Date(Date.now() - 60 * 60_000) } },+ { timestamps: false },+ );++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'requires_action',+ conversationId: 'convo-abandoned',+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),+ });++ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();+ expect(run?.status).toBe('requires_action');+ expect(run?.capacitySlot).toBeUndefined();+ expect(run?.resumeClaimedAt).toBeUndefined();+ });+});++describe('reconciliation rotates the paused window', () => {+ /**+ * Paused runs accumulate without bound (no capacity slot, no occurrence block) and+ * the approval TTL that ends them has no ceiling, so a full window of still-LIVE+ * pauses can sit at the front of an oldest-first ordering forever. Ordering by when+ * each row was last examined is what guarantees every row eventually gets a turn.+ */+ it('serves rows behind a full window once the leaders have been examined', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ const base = Date.parse('2026-07-01T00:00:00Z');+ const olderThan = new Date(base + 10_000_000);+ for (let i = 0; i < 100; i++) {+ await ScheduleRun.create(+ runData(schedule, {+ scheduledFor: new Date(base + i * 60_000),+ status: 'requires_action',+ firedAt: new Date(base + i * 60_000),+ }),+ );+ }+ // Newer than the 100 leaders, but still inside the age filter — the point is that+ // ORDERING keeps it out of the first window, not that it is too young to qualify.+ const queuedAt = new Date(base + 120 * 60_000);+ await ScheduleRun.create(+ runData(schedule, {+ scheduledFor: queuedAt,+ status: 'requires_action',+ firedAt: queuedAt,+ }),+ );++ const first = await methods.getRunsForReconciliation(olderThan, 100);+ // The newest row is behind a full batch, so this pass cannot reach it.+ expect(first.some((run) => run.scheduledFor?.getTime() === queuedAt.getTime())).toBe(false);++ // Examining the batch rotates it to the back.+ await methods.markRunsReconciled(first);++ const second = await methods.getRunsForReconciliation(olderThan, 100);+ expect(second.some((run) => run.scheduledFor?.getTime() === queuedAt.getTime())).toBe(true);+ });+});++describe('requestRunAbort (renewable, source-aware stamp)', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('renews freshness on a deletion retry without stealing an unresolved stop source', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));++ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);+ const first = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(first?.abortRequestedAt).toBeInstanceOf(Date);+ expect(first?.abortSource).toBe('stop');++ // A racing deletion must not steal the source (the settlement barrier reads it to+ // know the Stop route still has writes in flight) but MUST renew the freshness+ // stamp — its abort is a new signal the fences have to cover.+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);+ const second = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(second?.abortSource).toBe('stop');+ expect(second?.abortRequestedAt!.getTime()).toBeGreaterThanOrEqual(+ first!.abortRequestedAt!.getTime(),+ );+ });++ it('lets a stop attempt take over a deletion-stamped run and re-open its persistence', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));++ // Deletion stamps first; the user's Stop then wins the abort — the owner's+ // barrier must now wait on the ROUTE's persistence, so the stop takes the source.+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(state?.abortSource).toBe('stop');+ expect(state?.abortPersistedAt).toBeUndefined();+ });++ it('lets deletion claim the source once the stop attempt resolved', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));++ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(state?.abortSource).toBe('deletion');+ });++ it('re-opens the persistence marker on a fresh stop attempt', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));++ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);+ // A SECOND stop attempt persists anew; the resolved marker of the first must not+ // release the owner's barrier while the new attempt's writes are in flight.+ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);+ expect(state?.abortPersistedAt).toBeUndefined();+ });++ it('reports false when no active run holds the occurrence', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { status: 'success' }));+ expect(+ await methods.requestRunAbort(schedule.id, new Date('2026-07-20T12:00:00Z'), 'stop'),+ ).toBe(false);+ });++ it('serializes concurrent stop attempts: the second answers in_progress', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));++ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);+ // A second Stop while the first is unresolved must neither re-open the+ // persistence marker nor be told to proceed — its lost-CAS resolution would+ // release the settlement barrier while the winner is still persisting.+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe('in_progress');+ });++ it('marks abort persistence for the settlement barrier', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule));+ await methods.requestRunAbort(schedule.id, new Date('2026-07-20T12:00:00Z'), 'stop');+ await methods.markRunAbortPersisted(schedule.id, new Date('2026-07-20T12:00:00Z'));+ const state = await methods.getScheduleRunAbortState(+ schedule.id,+ new Date('2026-07-20T12:00:00Z'),+ );+ expect(state?.abortPersistedAt).toBeInstanceOf(Date);+ });+});++describe('recordRunOutcome settles a mid-generation balance refusal as skipped_balance', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('walks the balance streak instead of the failure streak and frees capacity', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0, conversationId: 'c1' }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'skipped_balance',+ conversationId: 'c1',+ autoDisableAfterFailures: 5,+ balanceSkipDisableThreshold: 5,+ });++ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();+ expect(run?.status).toBe('skipped_balance');+ expect(run?.capacitySlot).toBeUndefined();+ expect(run?.bookkept).toBe(true);++ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.balanceSkipCount).toBe(1);+ expect(row?.failureCount).toBe(0);+ expect(row?.enabled).toBe(true);+ expect(row?.lastRun?.status).toBe('skipped_balance');+ });++ it('auto-disables with insufficient_balance at the threshold, not too_many_failures', async () => {+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 4 }));+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0 }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'skipped_balance',+ autoDisableAfterFailures: 5,+ balanceSkipDisableThreshold: 5,+ });++ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.enabled).toBe(false);+ expect(row?.disabledReason).toBe('insufficient_balance');+ });++ it('is idempotent across the crash-retry replay (finalizeBookkeeping)', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0 }));++ await methods.recordRunOutcome({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'skipped_balance',+ autoDisableAfterFailures: 5,+ balanceSkipDisableThreshold: 5,+ });+ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor,+ status: 'skipped_balance',+ autoDisableAfterFailures: 5,+ balanceSkipDisableThreshold: 5,+ });++ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.balanceSkipCount).toBe(1);+ });++ it('surfaces an unbookkept skipped_balance run to the reconciler', async () => {+ const schedule = await methods.createSchedule(scheduleData());+ await ScheduleRun.create(+ runData(schedule, {+ status: 'skipped_balance',+ bookkept: false,+ firedAt: new Date(Date.now() - 10 * 60_000),+ }),+ );+ const rows = await methods.getUnbookkeptRuns(new Date(Date.now() - 60_000), 10);+ expect(rows.map((run) => run.scheduleId)).toContain(schedule.id);+ });+});++describe('skip bookkeeping is crash-recoverable', () => {+ const scheduledFor = new Date('2026-07-20T12:00:00Z');++ it('marks skip rows bookkept only after the schedule-side writes land', async () => {+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 3 }));+ await methods.recordSkippedRun({+ scheduleId: schedule.id,+ scheduledFor,+ user: schedule.user,+ status: 'skipped_overlap',+ });+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();+ expect(run?.bookkept).toBe(true);+ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.lastRun?.status).toBe('skipped_overlap');+ expect(row?.balanceSkipCount).toBe(0);+ });++ it('replays a half-applied overlap skip through the reconciler', async () => {+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 3 }));+ // Simulate the crash: the row exists but no schedule-side write landed.+ await ScheduleRun.create(+ runData(schedule, {+ status: 'skipped_overlap',+ bookkept: false,+ firedAt: new Date(Date.now() - 10 * 60_000),+ }),+ );+ const rows = await methods.getUnbookkeptRuns(new Date(Date.now() - 60_000), 10);+ expect(rows.map((run) => run.scheduleId)).toContain(schedule.id);++ await methods.finalizeBookkeeping({+ scheduleId: schedule.id,+ scheduledFor: new Date('2026-07-20T12:00:00Z'),+ status: 'skipped_overlap',+ autoDisableAfterFailures: 5,+ });+ const row = await Schedule.findOne({ id: schedule.id }).lean();+ // Without the replay, the missed reset left a stale balance streak that a later+ // balance skip would push over the auto-disable threshold.+ expect(row?.balanceSkipCount).toBe(0);+ expect(row?.lastRun?.status).toBe('skipped_overlap');+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();+ expect(run?.bookkept).toBe(true);+ });+});++describe('deleteUnarmedSchedule (guarded create rollback)', () => {+ it('erases a drained unarmed row and its settled runs', async () => {+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: undefined }));+ await ScheduleRun.create(runData(schedule, { status: 'success' }));++ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('deleted');+ expect(await Schedule.findOne({ id: schedule.id }).lean()).toBeNull();+ expect(await ScheduleRun.findOne({ scheduleId: schedule.id }).lean()).toBeNull();+ });++ it('defers to the drain when a live manual run holds the row', async () => {+ // An unarmed row is visible and Run Now-able: rollback must never hard-delete a+ // live generation's evidence out from under it. The soft-claim hides the row and+ // the ordinary erase-on-settle/sweep teardown erases it once the run settles.+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: undefined }));+ await ScheduleRun.create(runData(schedule, { status: 'started' }));++ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('draining');+ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.deleting).toBe(true);+ expect(await ScheduleRun.findOne({ scheduleId: schedule.id }).lean()).not.toBeNull();+ });++ it('refuses to delete a row that has since been armed', async () => {+ const schedule = await methods.createSchedule(scheduleData());++ // An ambiguously-committed arm (or a concurrent replay's arm) landed: the row is a+ // live schedule another response may have confirmed, so the rollback must not+ // erase it.+ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('kept');+ expect(await Schedule.findOne({ id: schedule.id }).lean()).not.toBeNull();+ });++ it('refuses to delete an unarmed row a concurrent edit has moved past', async () => {+ const schedule = await methods.createSchedule(+ scheduleData({ nextRunAt: undefined, configRevision: 0 }),+ );+ // A PATCH that disables the schedule leaves it UNARMED but bumps the revision —+ // the row now embodies the owner's edit, and the POST's rollback must not erase it.+ await methods.updateScheduleById(schedule.id, schedule.user, { enabled: false });++ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user, 0)).toBe('kept');+ expect(await Schedule.findOne({ id: schedule.id }).lean()).not.toBeNull();+ });++ it('arms through the shared CAS without rotating the claim token or revision', async () => {+ const schedule = await methods.createSchedule(+ scheduleData({ nextRunAt: undefined, claimToken: 'ct-live', configRevision: 0 }),+ );+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 60_000), 0)).toBe(true);+ // A second armer (the original POST racing a replay) must collapse to a no-op.+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 120_000), 0)).toBe(false);+ const row = await Schedule.findOne({ id: schedule.id }).lean();+ expect(row?.claimToken).toBe('ct-live');+ expect(row?.configRevision).toBe(0);+ });++ it('refuses the revision-fenced arm after a concurrent edit', async () => {+ const schedule = await methods.createSchedule(+ scheduleData({ nextRunAt: undefined, configRevision: 0 }),+ );+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 60_000), 0)).toBe(false);+ });++ it('reports a missing row distinctly', async () => {+ expect(await methods.deleteUnarmedSchedule('sched_gone', new mongoose.Types.ObjectId())).toBe(+ 'missing',+ );+ });+});++describe('erasure sweep rotation and idempotency-key lookup', () => {+ it('rotates the deleting window so undrainable rows cannot starve later ones', async () => {+ const user = new mongoose.Types.ObjectId();+ const first = scheduleData({ user, deleting: true, nextRunAt: undefined });+ const second = scheduleData({ user, deleting: true, nextRunAt: undefined });+ await Schedule.create(first);+ await Schedule.create(second);++ const window = await methods.getDeletingSchedules(1);+ expect(window).toHaveLength(1);+ await methods.markEraseAttempted(window.map((row) => row.id));++ const rotated = await methods.getDeletingSchedules(1);+ expect(rotated[0].id).not.toBe(window[0].id);+ });++ it('rotates the per-owner deletion-retry window the same way', async () => {+ const user = new mongoose.Types.ObjectId();+ await Schedule.create(scheduleData({ user, deleting: true, nextRunAt: undefined }));+ await Schedule.create(scheduleData({ user, deleting: true, nextRunAt: undefined }));++ const window = await methods.getDeletingScheduleIds(user, 1);+ expect(window).toHaveLength(1);+ await methods.markEraseAttempted(window);++ // The list-path retry is bounded too; without least-recently-attempted+ // ordering an owner with more stuck rows than the limit re-drives the same+ // unconfirmable few on every list and never reaches the rest.+ const rotated = await methods.getDeletingScheduleIds(user, 1);+ expect(rotated[0]).not.toBe(window[0]);+ });++ it('resolves an idempotency key even while its row is draining', async () => {+ const user = new mongoose.Types.ObjectId();+ const schedule = scheduleData({ user, deleting: true, clientRequestId: 'intent-9' });+ await Schedule.create(schedule);++ const found = await methods.getScheduleByClientRequestId(user, 'intent-9');+ expect(found?.id).toBe(schedule.id);+ });+});diff --git a/packages/data-schemas/src/methods/schedule.ts b/packages/data-schemas/src/methods/schedule.tsnew file mode 100644index 0000000000..222a8c5275--- /dev/null+++ b/packages/data-schemas/src/methods/schedule.ts@@ -0,0 +1,2058 @@+import { randomUUID } from 'node:crypto';+import type { ScheduleRunStatus, ScheduleDisabledReason } from 'librechat-data-provider';+import type { Model, Types, AnyBulkWriteOperation } from 'mongoose';+import type {+ ISchedule,+ IScheduleDocument,+ IScheduleRun,+ IScheduleRunDocument,+} from '~/types/schedule';+import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';+import { createIndexesWithRetry } from '~/utils/retry';++const DUPLICATE_KEY = 11000;++/**+ * Tolerance for inter-replica wall-clock skew on the worker-clock lease+ * comparisons (DocumentDB rules out the server-clock `$$NOW` CAS). Applied only+ * where one worker judges ANOTHER worker's lease expired — taking over a due+ * claim, or declaring a deleting schedule drained — so a clock-ahead worker+ * cannot steal a lease its holder still considers live. Never applied where a+ * worker checks its OWN lease (revalidateClaim), which is self-consistent.+ * NTP-managed fleets sit orders of magnitude under this.+ */+const LEASE_SKEW_MARGIN_MS = 30_000;++/**+ * Age past which an unresolved interactive-stop attempt is presumed dead and its+ * per-run serialization claim becomes takeable again. Mirrors the api layer's+ * ABORT_OWNER_PRESUMED_ALIVE_MS — one presumption, two enforcement points.+ */+const ABORT_STAMP_STALE_MS = 30 * 60_000;++/**+ * Upper bound on the per-schedule `countedFor` idempotency set. Far larger than+ * the number of a single schedule's occurrences that can be terminal (or awaiting+ * reconciliation) at once, so an occurrence's marker is never evicted before any+ * possible crash-replay, while the array stays bounded.+ */+const COUNTED_FOR_WINDOW = 64;++/** Statuses that occupy a schedule's live capacity / block a concurrent occurrence. */+const ACTIVE_RUN_STATUSES: ScheduleRunStatus[] = ['started', 'requires_action'];++/** Card statuses that represent a SETTLED occurrence — everything except the pause. */+const TERMINAL_CARD_STATUSES: ScheduleRunStatus[] = [+ 'success',+ 'error',+ 'interrupted',+ 'skipped_overlap',+ 'skipped_balance',+];++type DuplicateKeyError = { code?: number; keyPattern?: Record<string, unknown> };++/** A duplicate-key error whose conflict is the {scheduleId, scheduledFor} occurrence index. */+function isOccurrenceDuplicate(error: unknown): boolean {+ const err = error as DuplicateKeyError;+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'scheduledFor' in err.keyPattern;+}++/** A duplicate-key error whose conflict is the single-active-run partial index+ * ({scheduleId} where status:'started'). Matched EXACTLY on scheduleId so the+ * global {capacitySlot} index below is never misread as a per-schedule overlap. */+function isActiveRunConflict(error: unknown): boolean {+ const err = error as DuplicateKeyError;+ return (+ err?.code === DUPLICATE_KEY &&+ err.keyPattern != null &&+ 'scheduleId' in err.keyPattern &&+ !('scheduledFor' in err.keyPattern)+ );+}++/** A duplicate-key error whose conflict is the GLOBAL {capacitySlot} cap index. */+function isCapacitySlotConflict(error: unknown): boolean {+ const err = error as DuplicateKeyError;+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'capacitySlot' in err.keyPattern;+}++/** A duplicate-key error whose conflict is the per-user {user, slot} cap index. */+function isSlotConflict(error: unknown): boolean {+ const err = error as DuplicateKeyError;+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'slot' in err.keyPattern;+}++/** A duplicate-key error whose conflict is the per-user create-idempotency index. */+function isClientRequestConflict(error: unknown): boolean {+ const err = error as DuplicateKeyError;+ return (+ err?.code === DUPLICATE_KEY && err.keyPattern != null && 'clientRequestId' in err.keyPattern+ );+}++export interface ClaimDueScheduleParams {+ instanceId: string;+ leaseMs: number;+}++export interface RecordRunOutcomeParams {+ scheduleId: string;+ scheduledFor: Date;+ status: Extract<+ ScheduleRunStatus,+ 'success' | 'error' | 'requires_action' | 'interrupted' | 'skipped_balance' | 'skipped_overlap'+ >;+ conversationId?: string;+ /** Erase the run row's RESERVED conversationId in the same terminal write: a+ * pre-start abort reserved an id but never created the conversation, and any+ * recovery replay that reads the row would otherwise project a dead link. */+ clearConversationId?: boolean;+ error?: string;+ durationMs?: number;+ autoDisableAfterFailures: number;+ /** Consecutive-balance-skip auto-disable threshold; required to settle a run as+ * `skipped_balance` (a mid-generation balance refusal), ignored otherwise. */+ balanceSkipDisableThreshold?: number;+ /**+ * RECOVERY REPLAY of a pause, from an actor holding a possibly-stale row snapshot:+ * apply the transition only while no FRESH resume claim owns the row — that is, the+ * row carries no `resumeClaimedAt`, or one older than this cutoff. Without the fence,+ * two clustered sweepers observing the same unprojected pause race: the first projects+ * it, the owner's approval then claims a fresh slot (`markRunResumeClaimed` sets+ * `started` + `resumeClaimedAt` in ONE write), and the second still passes its+ * snapshot-based hand-off check and unsets the capacity slot and claim stamp out from+ * under the running continuation.+ *+ * A CUTOFF rather than a mere existence check, because the stamp outlives its meaning:+ * a worker that dies between claiming and resuming leaves it set forever, and rejecting+ * on existence alone would strand that row `started` — holding its capacity slot, with+ * its approval unresumable (`markRunResumeClaimed` only matches `requires_action`) —+ * which is the very state this replay exists to recover. Pass the same staleness bound+ * the caller's own in-flight check uses so the two agree.+ *+ * Never set by the generation owner: its own re-pause legitimately clears the stamp as+ * the hand-off's completion signal.+ */+ resumeClaimStaleBefore?: Date;+}++/** Result of claiming/leasing a schedule: the snapshot plus the fencing token to carry. */+export interface ScheduleClaim {+ schedule: ISchedule;+ claimToken: string;+}++/** Outcome of reserving the single-active-run slot for a fired occurrence. */+export type StartedRunReservation =+ | { run: IScheduleRun }+ | {+ conflict: 'duplicate' | 'overlap' | 'slot-taken';+ /** For a `duplicate`, the status of the row that already holds this occurrence.+ * A TERMINAL status means the occurrence is finished and merely never advanced+ * past; an active one means another worker is still running it. */+ existingStatus?: ScheduleRunStatus;+ };++/** Outcome of promoting a paused occurrence back into the capacity-consuming+ * `started` set. Both conflicts are database-arbitrated by the same partial+ * unique indexes used for a first fire. */+export type ResumedRunReservation =+ | { capacitySlot: number }+ | { conflict: 'not-paused' | 'overlap' | 'slot-taken' };++export type ScheduleMethods = {+ ensureScheduleIndexes: () => Promise<void>;+ createSchedule: (data: Partial<ISchedule>) => Promise<ISchedule>;+ createScheduleWithSlot: (+ data: Partial<ISchedule>,+ maxPerUser: number,+ ) => Promise<ISchedule | 'limit'>;+ updateScheduleById: (+ id: string,+ userId: string | Types.ObjectId,+ update: Partial<ISchedule>,+ unset?: Record<string, 1>,+ options?: { expectedConfigRevision?: number },+ ) => Promise<ISchedule | null>;+ deleteScheduleById: (id: string, userId: string | Types.ObjectId) => Promise<boolean>;+ deleteUnarmedSchedule: (+ id: string,+ userId: string | Types.ObjectId,+ expectedConfigRevision?: number,+ ) => Promise<'deleted' | 'draining' | 'kept' | 'missing'>;+ getScheduleById: (id: string, userId?: string | Types.ObjectId) => Promise<ISchedule | null>;+ getSchedulesByUser: (userId: string | Types.ObjectId) => Promise<ISchedule[]>;+ countSchedulesByUser: (userId: string | Types.ObjectId) => Promise<number>;+ claimDueSchedule: (params: ClaimDueScheduleParams) => Promise<ISchedule | null>;+ acquireManualRunLease: (+ id: string,+ userId: string | Types.ObjectId,+ leaseMs: number,+ ) => Promise<ISchedule | null>;+ acquireResumeLease: (+ id: string,+ expectedConfigRevision: number | undefined,+ requireEnabled: boolean,+ leaseMs: number,+ ) => Promise<ISchedule | null>;+ consumeResumeLease: (+ id: string,+ expectedClaimToken: string,+ expectedLeaseBy: string,+ requireEnabled: boolean,+ expectedConfigRevision?: number,+ ) => Promise<boolean>;+ releaseLease: (id: string, expectedClaimToken?: string) => Promise<boolean>;+ releaseLeaseByHolder: (id: string, leaseBy: string) => Promise<void>;+ revalidateClaim: (id: string, claimToken: string, requireEnabled?: boolean) => Promise<boolean>;+ advanceSchedule: (+ id: string,+ nextRunAt: Date | null,+ expectedNextRunAt?: Date | null,+ expectedClaimToken?: string,+ ) => Promise<boolean>;+ disableSchedule: (+ id: string,+ reason: ScheduleDisabledReason,+ expectedClaimToken?: string,+ expectedConfigRevision?: number,+ /** Extra filter pinning the counter the disable DECISION was made on, so a+ * concurrent outcome that reset the streak invalidates this stale write. */+ counterGuard?: Record<string, unknown>,+ ) => Promise<void>;+ insertScheduleRun: (data: Partial<IScheduleRun>) => Promise<IScheduleRun | null>;+ reserveStartedRun: (data: Partial<IScheduleRun>) => Promise<StartedRunReservation>;+ getCapacityOccupancy: () => Promise<{ takenSlots: number[]; unslotted: number }>;+ requestRunAbort: (+ scheduleId: string,+ scheduledFor: Date,+ source?: IScheduleRun['abortSource'],+ ) => Promise<boolean | 'in_progress'>;+ getScheduleRunAbortState: (+ scheduleId: string,+ scheduledFor: Date,+ ) => Promise<Pick<+ IScheduleRun,+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'+ > | null>;+ markRunResumeClaimed: (+ scheduleId: string,+ scheduledFor: Date,+ capacitySlot: number,+ ) => Promise<ResumedRunReservation>;+ releaseRunResumeClaim: (+ scheduleId: string,+ scheduledFor: Date,+ capacitySlot: number,+ ) => Promise<boolean>;+ markRunAbortPersisted: (scheduleId: string, scheduledFor: Date) => Promise<void>;+ setRunFireDetails: (+ scheduleId: string,+ scheduledFor: Date,+ details: { conversationId: string; droppedFileIds?: string[] },+ ) => Promise<void>;+ countActiveRuns: () => Promise<number>;+ deleteScheduleRun: (+ scheduleId: string,+ scheduledFor: Date,+ expectedStatus?: ScheduleRunStatus,+ expectedConversationId?: string,+ ) => Promise<void>;+ markScheduleDeleting: (id: string, userId: string | Types.ObjectId) => Promise<ISchedule | null>;+ getActiveRunsForSchedule: (scheduleId: string) => Promise<IScheduleRun[]>;+ getActiveRunsForUser: (userId: string | Types.ObjectId) => Promise<IScheduleRun[]>;+ suspendUserSchedulesForDeletion: (+ userId: string | Types.ObjectId,+ token: string,+ ) => Promise<void>;+ restoreUserSchedulesFromDeletion: (+ userId: string | Types.ObjectId,+ token: string,+ ) => Promise<void>;+ getDeletingSchedules: (limit: number) => Promise<ISchedule[]>;+ markEraseAttempted: (ids: string[]) => Promise<void>;+ getScheduleByClientRequestId: (+ userId: string | Types.ObjectId,+ clientRequestId: string,+ ) => Promise<ISchedule | null>;+ getDeletingScheduleIds: (userId: string | Types.ObjectId, limit: number) => Promise<string[]>;+ getUnarmedSchedules: (limit: number) => Promise<ISchedule[]>;+ armSchedule: (id: string, nextRunAt: Date, expectedConfigRevision?: number) => Promise<boolean>;+ eraseScheduleIfDrained: (id: string) => Promise<boolean>;+ deleteSchedulesByUser: (userId: string | Types.ObjectId) => Promise<void>;+ getUnbookkeptRuns: (olderThan: Date, limit: number) => Promise<IScheduleRun[]>;+ finalizeBookkeeping: (params: RecordRunOutcomeParams) => Promise<void>;+ recordRunOutcome: (params: RecordRunOutcomeParams) => Promise<void>;+ recordSkippedRun: (+ data: Partial<IScheduleRun> & {+ scheduleId: string;+ scheduledFor: Date;+ status: Extract<ScheduleRunStatus, 'skipped_overlap' | 'skipped_balance'>;+ },+ balanceSkipDisableThreshold?: number,+ ) => Promise<void>;+ getRunsForReconciliation: (olderThan: Date, limit: number) => Promise<IScheduleRun[]>;+ markRunsReconciled: (runs: Array<Pick<IScheduleRun, '_id'>>) => Promise<void>;+};++export function createScheduleMethods(mongoose: typeof import('mongoose')): ScheduleMethods {+ const Schedule = () => mongoose.models.Schedule as Model<IScheduleDocument>;+ const ScheduleRun = () => mongoose.models.ScheduleRun as Model<IScheduleRunDocument>;++ /**+ * Explicitly builds the Schedule/ScheduleRun indexes. Required because the+ * standard production setting `MONGO_AUTO_INDEX=` (empty) disables Mongoose's+ * automatic index creation — without this the unique idempotency index and the+ * TTL retention index would never exist. Called once before the engine starts.+ */+ async function ensureScheduleIndexes(): Promise<void> {+ await createIndexesWithRetry(Schedule());+ await createIndexesWithRetry(ScheduleRun());+ }++ async function createSchedule(data: Partial<ISchedule>): Promise<ISchedule> {+ const doc = await Schedule().create(data);+ return doc.toObject();+ }++ /**+ * Creates a schedule in the lowest free per-user slot, enforcing maxPerUser+ * ATOMICALLY: the {user, slot} partial unique index is the sole arbiter, so+ * concurrent creates that pick the same slot collide (duplicate key) and one+ * retries the next free slot — no read-then-count window, no drift. Returns+ * 'limit' when all slots in [0, maxPerUser) are held by the user's live schedules.+ */+ async function createScheduleWithSlot(+ data: Partial<ISchedule>,+ maxPerUser: number,+ ): Promise<ISchedule | 'limit'> {+ const userId = data.user;+ // Bound the retries by maxPerUser+1: each collision advances to a distinct+ // slot, so a caller can lose at most (slots occupied) races before it either+ // wins a free slot or finds every slot taken.+ for (let attempt = 0; attempt <= maxPerUser; attempt++) {+ const used = await Schedule()+ .find({ user: userId, deleting: { $ne: true } })+ .select('slot')+ .lean<Array<{ slot?: number }>>();+ const taken = new Set(+ used.map((s) => s.slot).filter((s): s is number => typeof s === 'number'),+ );+ if (taken.size >= maxPerUser) {+ return 'limit';+ }+ let slot = 0;+ while (taken.has(slot)) {+ slot++;+ }+ if (slot >= maxPerUser) {+ return 'limit';+ }+ try {+ const doc = await Schedule().create({ ...data, slot, deleting: false });+ return doc.toObject();+ } catch (error) {+ if (isSlotConflict(error)) {+ continue;+ }+ // A RETRY of a create whose first attempt already committed. Hand back the+ // original row so the caller re-arms and reports that one, instead of minting a+ // second recurring schedule for a single user intent.+ if (isClientRequestConflict(error) && data.clientRequestId) {+ const existing = await Schedule()+ .findOne({ user: userId, clientRequestId: data.clientRequestId })+ .lean<ISchedule>();+ if (existing != null) {+ return existing;+ }+ }+ throw error;+ }+ }+ return 'limit';+ }++ /**+ * Owner edit. Rotates `claimToken` on every update so an in-flight engine claim+ * (which captured the prior token) is fenced off: its disable/advance/release+ * writes and its pre-dispatch revalidation no-op, and it cannot fire an edited+ * or re-enabled schedule. Delete removes the row, so those writes no-op too.+ */+ async function updateScheduleById(+ id: string,+ userId: string | Types.ObjectId,+ update: Partial<ISchedule>,+ unset?: Record<string, 1>,+ /** Optional edit-generation fence, so two overlapping owner edits cannot each+ * persist state derived from a row the other has already replaced. */+ options?: { expectedConfigRevision?: number },+ ): Promise<ISchedule | null> {+ const filter = {+ id,+ user: userId,+ deleting: { $ne: true },+ ...(options?.expectedConfigRevision !== undefined+ ? { configRevision: options.expectedConfigRevision }+ : {}),+ };+ const mutation = {+ $set: { ...update, claimToken: randomUUID() },+ // The ONLY writer of configRevision: an owner edit moves the config+ // generation forward atomically with the claim-token rotation, so a run+ // that started under the old config can detect it and skip bookkeeping.+ // Worker/policy writes (claim, lease, advance, disable, bookkeeping) never+ // bump it, and deletion deliberately does not either — a draining run must+ // still be able to record its outcome before erasure.+ $inc: { configRevision: 1 },+ };+ // FIRST branch: if the card still shows a pause, drop that projection in the SAME+ // atomic write as the revision bump. A `requires_action` card present at edit time+ // was projected by a run that captured the pre-edit revision; its terminal outcome+ // is revision-fenced (projectLastRun), so once this bump lands it can never replace+ // the card — and a disabling edit means no future occurrence will either, leaving it+ // stuck on "Needs approval" for a dead action. The clear is fenced on the projection+ // STILL being that pause (not a stale handler pre-read), so a terminal outcome or a+ // newer occurrence that raced in fails this branch and is preserved by the second.+ // DocumentDB rules out a conditional pipeline `$unset`, so this is a classic-operator+ // CAS branch rather than one update.+ const cleared = await Schedule()+ .findOneAndUpdate(+ { ...filter, 'lastRun.status': 'requires_action' },+ { ...mutation, $unset: { lastRun: 1, ...(unset ?? {}) } },+ { new: true },+ )+ .lean<ISchedule>();+ if (cleared != null) {+ return cleared;+ }+ // SECOND branch: no dead pause to clear (terminal history survives untouched), or the+ // row is gone / a concurrent edit moved the revision — a null here is the true+ // gone/conflict signal the caller distinguishes.+ return Schedule()+ .findOneAndUpdate(filter, { ...mutation, ...(unset ? { $unset: unset } : {}) }, { new: true })+ .lean<ISchedule>();+ }++ async function deleteScheduleById(id: string, userId: string | Types.ObjectId): Promise<boolean> {+ const result = await Schedule().deleteOne({ id, user: userId });+ if (result.deletedCount > 0) {+ await ScheduleRun().deleteMany({ scheduleId: id });+ return true;+ }+ return false;+ }++ /**+ * Rolls back a fresh create whose arming write failed — but ONLY while the row is+ * still the unarmed, unedited revision this attempt inserted. An+ * ambiguously-committed arm, or a concurrent create-retry that armed the row and+ * already answered 201, leaves an ARMED row this rollback must not erase; a+ * concurrent PATCH that edited the row while leaving it unarmed (e.g. disabling+ * it) moved the revision, and deleting it would erase the owner's edit. 'kept'+ * tells the caller the row survived in some later state.+ *+ * The claim is a SOFT delete, never a hard one: an unarmed row is visible in the+ * owner's list and Run Now-able, so a live manual generation can already hold its+ * lease and an active run row — hard-deleting would erase a billed generation's+ * evidence out from under it. The erase runs only once drained (no active run, no+ * live lease); otherwise 'draining' hands the hidden row to the ordinary+ * erase-on-settle / sweep teardown.+ */+ async function deleteUnarmedSchedule(+ id: string,+ userId: string | Types.ObjectId,+ expectedConfigRevision?: number,+ ): Promise<'deleted' | 'draining' | 'kept' | 'missing'> {+ const claimed = await Schedule()+ .findOneAndUpdate(+ {+ id,+ user: userId,+ deleting: { $ne: true },+ nextRunAt: { $exists: false },+ ...(expectedConfigRevision != null ? { configRevision: expectedConfigRevision } : {}),+ },+ { $set: { enabled: false, deleting: true, claimToken: randomUUID() } },+ { new: true },+ )+ .select('_id')+ .lean();+ if (claimed == null) {+ const still = await Schedule()+ .findOne({ id, user: userId })+ .select('deleting')+ .lean<Pick<ISchedule, 'deleting'>>();+ if (still == null) {+ return 'missing';+ }+ return still.deleting === true ? 'draining' : 'kept';+ }+ const erased = await eraseScheduleIfDrained(id);+ return erased ? 'deleted' : 'draining';+ }++ async function getScheduleById(+ id: string,+ userId?: string | Types.ObjectId,+ ): Promise<ISchedule | null> {+ const filter: Record<string, unknown> = { id, deleting: { $ne: true } };+ if (userId != null) {+ filter.user = userId;+ }+ return Schedule().findOne(filter).lean<ISchedule>();+ }++ async function getSchedulesByUser(userId: string | Types.ObjectId): Promise<ISchedule[]> {+ // Hide schedules pending erasure (soft-deleted, draining their active runs)+ // so a deleted schedule disappears immediately for the owner.+ return Schedule()+ .find({ user: userId, deleting: { $ne: true } })+ .sort({ updatedAt: -1 })+ .select('-leaseUntil -leaseBy -claimToken')+ .lean<ISchedule[]>();+ }++ async function countSchedulesByUser(userId: string | Types.ObjectId): Promise<number> {+ return Schedule().countDocuments({ user: userId, deleting: { $ne: true } });+ }++ /**+ * Atomically claims one due schedule by taking a lease. The per-document CAS+ * is the sole multi-instance dispatch arbiter: exactly one caller wins each+ * due schedule regardless of replica count, with or without Redis. Stamps a+ * fresh `claimToken` the winner carries through every subsequent write.+ *+ * Due-ness and lease expiry compare against THIS worker's clock. DocumentDB+ * supports neither pipeline-form updates nor `$$NOW` (see+ * misc/documentdb/documentdb-compat.md), so the server-clock CAS is not+ * expressible portably; the CAS itself still arbitrates every race, and the+ * exposure is bounded by inter-replica clock skew (NTP-order) against a+ * 5-minute lease and per-minute cadence granularity. `{ $lte }` matches+ * neither missing nor null `nextRunAt`, so unarmed rows stay unclaimable;+ * a missing `leaseUntil` is claimable via the `$or`.+ */+ async function claimDueSchedule(params: ClaimDueScheduleParams): Promise<ISchedule | null> {+ const claimToken = randomUUID();+ const now = new Date();+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);+ return Schedule()+ .findOneAndUpdate(+ {+ enabled: true,+ deleting: { $ne: true },+ nextRunAt: { $lte: now },+ $or: [+ { leaseUntil: { $exists: false } },+ { leaseUntil: null },+ { leaseUntil: { $lt: takeoverCutoff } },+ ],+ },+ {+ $set: {+ leaseUntil: new Date(now.getTime() + params.leaseMs),+ leaseBy: params.instanceId,+ claimToken,+ },+ },+ { new: true, sort: { nextRunAt: 1 } },+ )+ .lean<ISchedule>();+ }++ /**+ * Takes the schedule's lease for a manual run-now, serializing concurrent+ * `POST /:id/run` requests (and blocking against an engine claim) so a+ * double-click can't start two runs. Owner-scoped. Returns the FRESH schedule row+ * (post-image, with the new claim token) so the caller fires the current snapshot+ * — an edit that committed after the route read the schedule but before this lease+ * is reflected here, not the stale pre-edit prompt/agent. Null if already leased.+ */+ async function acquireManualRunLease(+ id: string,+ userId: string | Types.ObjectId,+ leaseMs: number,+ ): Promise<ISchedule | null> {+ // Same worker-clock CAS shape as claimDueSchedule (DocumentDB rules out `$$NOW`+ // and pipeline updates); the CAS still serializes concurrent run-now clicks, and+ // clock skew only shifts WHEN an expired lease becomes re-acquirable.+ const claimToken = randomUUID();+ // A UNIQUE per-lease holder (not the constant 'manual'): the superseded-fire+ // cleanup releases by holder (leaseBy), so a stale run-now that stalled past its+ // lease must not match — and strip — the fresh lease a newer run-now acquired.+ // The claimToken already fences the lease, so reuse it as the holder discriminator.+ const leaseBy = `manual:${claimToken}`;+ const now = new Date();+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);+ return Schedule()+ .findOneAndUpdate(+ {+ id,+ user: userId,+ deleting: { $ne: true },+ $or: [+ { leaseUntil: { $exists: false } },+ { leaseUntil: null },+ { leaseUntil: { $lt: takeoverCutoff } },+ ],+ },+ { $set: { leaseUntil: new Date(now.getTime() + leaseMs), leaseBy, claimToken } },+ { new: true },+ )+ .lean<ISchedule>();+ }++ /**+ * Takes a short-lived schedule-document fence for a paused run's resume. The+ * approval and its ScheduleRun row live in different stores, so this claim is+ * carried through the approval CAS and atomically consumed immediately after it.+ * An owner edit rotates claimToken; a policy disable clears the lease/enabled bit.+ * Either therefore makes consumeResumeLease fail before provider execution starts.+ */+ async function acquireResumeLease(+ id: string,+ expectedConfigRevision: number | undefined,+ requireEnabled: boolean,+ leaseMs: number,+ ): Promise<ISchedule | null> {+ const claimToken = randomUUID();+ const leaseBy = `resume:${claimToken}`;+ const now = new Date();+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);+ return Schedule()+ .findOneAndUpdate(+ {+ id,+ deleting: { $ne: true },+ ...(requireEnabled ? { enabled: true } : {}),+ ...(expectedConfigRevision !== undefined+ ? { configRevision: expectedConfigRevision }+ : {}),+ $or: [+ { leaseUntil: { $exists: false } },+ { leaseUntil: null },+ { leaseUntil: { $lt: takeoverCutoff } },+ ],+ },+ {+ $set: {+ leaseUntil: new Date(now.getTime() + leaseMs),+ leaseBy,+ claimToken,+ },+ },+ { new: true },+ )+ .lean<ISchedule>();+ }++ /**+ * Atomically linearizes a scheduled resume against owner edits, deletion, policy+ * disable, lease expiry, and takeover, then releases the short-lived fence. A+ * successful match means the approval was consumed under the same live schedule+ * generation that acquired the fence; a miss must stop before provider execution.+ */+ async function consumeResumeLease(+ id: string,+ expectedClaimToken: string,+ expectedLeaseBy: string,+ requireEnabled: boolean,+ expectedConfigRevision?: number,+ ): Promise<boolean> {+ const result = await Schedule().updateOne(+ {+ id,+ claimToken: expectedClaimToken,+ leaseBy: expectedLeaseBy,+ deleting: { $ne: true },+ leaseUntil: { $gt: new Date() },+ ...(requireEnabled ? { enabled: true } : {}),+ ...(expectedConfigRevision !== undefined ? { configRevision: expectedConfigRevision } : {}),+ },+ { $unset: { leaseUntil: 1, leaseBy: 1 } },+ );+ return (result.matchedCount ?? 0) > 0;+ }++ /**+ * Releases a lease WITHOUT advancing nextRunAt (manual runs never reschedule).+ * Fenced on the claim token when provided so a stale worker cannot strip a lease+ * a different claimer now holds.+ */+ async function releaseLease(id: string, expectedClaimToken?: string): Promise<boolean> {+ const filter: Record<string, unknown> = { id };+ if (expectedClaimToken !== undefined) {+ filter.claimToken = expectedClaimToken;+ }+ const result = await Schedule().updateOne(filter, { $unset: { leaseUntil: 1, leaseBy: 1 } });+ return (result.matchedCount ?? 0) > 0;+ }++ /**+ * Releases a lease fenced on the lease HOLDER (`leaseBy`) rather than the claim+ * token. Used when a fire is superseded by an owner edit that rotated the token+ * (so a token-fenced release would no-op): the worker still owns the lease, so it+ * must clear it — otherwise the edited schedule (and Run now) is reported "already+ * in progress" until the lease TTL, even though no run was dispatched. A takeover+ * changed `leaseBy`, so this correctly no-ops and never strips the new holder's lease.+ */+ async function releaseLeaseByHolder(id: string, leaseBy: string): Promise<void> {+ await Schedule().updateOne({ id, leaseBy }, { $unset: { leaseUntil: 1, leaseBy: 1 } });+ }++ /**+ * Whether the caller still holds an authoritative claim on the schedule: it is+ * not being deleted, its claim token is unchanged, and its lease has not expired+ * (Mongo `$$NOW`). Called as the last check before the loopback POST so an owner+ * delete/edit or a lease-expiry re-claim between claim and fire aborts the fire+ * instead of dispatching a stale occurrence. `requireEnabled` additionally checks+ * `enabled` (automatic fires must stop once disabled); a manual run-now passes+ * false since the user explicitly triggered it, but delete/edit still fence it.+ */+ async function revalidateClaim(+ id: string,+ claimToken: string,+ requireEnabled = true,+ ): Promise<boolean> {+ // Worker clock (DocumentDB-portable): `$gt` matches neither a missing nor a null+ // leaseUntil, so a released lease reads as invalid — the fail-closed direction.+ const row = await Schedule()+ .findOne({+ id,+ claimToken,+ deleting: { $ne: true },+ ...(requireEnabled ? { enabled: true } : {}),+ leaseUntil: { $gt: new Date() },+ })+ .select('_id')+ .lean();+ return row != null;+ }++ /**+ * Advances past a fired (or skipped) occurrence and releases the lease. When+ * `expectedNextRunAt` is given, the update is predicated on the schedule still+ * sitting on the claimed occurrence; when `expectedClaimToken` is given it is+ * additionally fenced on the claim, so a stale worker (whose lease expired and+ * was re-claimed, or whose schedule the owner edited) cannot clobber a newer+ * claimer's nextRunAt/lease. A predicate miss simply no-ops the stale write.+ */+ async function advanceSchedule(+ id: string,+ nextRunAt: Date | null,+ expectedNextRunAt?: Date | null,+ expectedClaimToken?: string,+ ): Promise<boolean> {+ const filter: Record<string, unknown> = { id };+ if (expectedNextRunAt !== undefined) {+ filter.nextRunAt = expectedNextRunAt;+ }+ if (expectedClaimToken !== undefined) {+ filter.claimToken = expectedClaimToken;+ }+ const result = await Schedule().updateOne(filter, {+ $set: { ...(nextRunAt ? { nextRunAt } : {}) },+ $unset: { leaseUntil: 1, leaseBy: 1, ...(nextRunAt ? {} : { nextRunAt: 1 }) },+ });+ return (result.matchedCount ?? 0) > 0;+ }++ /**+ * Disables a schedule. Preflight disables from the leased worker pass their+ * `expectedClaimToken` so a stale worker cannot flip a schedule the owner just+ * re-enabled/edited (rotating the token) back to disabled or clear a newer+ * claimer's lease. POLICY disables (auto-disable) instead pass the config+ * generation of the RUN that tripped the threshold: the counter update they follow+ * is revision-fenced, so leaving the disable itself unfenced let a stale run (or a+ * reconciler replay of one) disable a schedule the owner had since edited and+ * re-enabled. Absent on either side disables the fence.+ */+ async function disableSchedule(+ id: string,+ reason: ScheduleDisabledReason,+ expectedClaimToken?: string,+ expectedConfigRevision?: number,+ counterGuard?: Record<string, unknown>,+ ): Promise<void> {+ const filter: Record<string, unknown> = { id, ...counterGuard };+ if (expectedClaimToken !== undefined) {+ filter.claimToken = expectedClaimToken;+ }+ if (expectedConfigRevision !== undefined) {+ filter.configRevision = expectedConfigRevision;+ }+ await Schedule().updateOne(filter, {+ $set: { enabled: false, disabledReason: reason },+ $unset: { leaseUntil: 1, leaseBy: 1 },+ });+ }++ /**+ * Inserts the run row BEFORE firing. The unique {scheduleId, scheduledFor}+ * index makes this the durable idempotency claim: null means this occurrence+ * was already fired (or is in flight) by another claimer or a prior life.+ * Used for non-`started` rows (skips); started rows go through reserveStartedRun.+ */+ async function insertScheduleRun(data: Partial<IScheduleRun>): Promise<IScheduleRun | null> {+ try {+ const doc = await ScheduleRun().create(data);+ return doc.toObject();+ } catch (error) {+ if ((error as { code?: number }).code === DUPLICATE_KEY) {+ return null;+ }+ throw error;+ }+ }++ /**+ * Reserves the single active-run slot for a fired occurrence by inserting a+ * `started` run. Two distinct duplicate-key outcomes: the {scheduleId,+ * scheduledFor} index means this occurrence already fired ('duplicate'); the+ * single-active partial index means another occurrence of the same schedule is+ * already active ('overlap'). The DB enforces both atomically — no read-then-insert.+ */+ async function reserveStartedRun(data: Partial<IScheduleRun>): Promise<StartedRunReservation> {+ try {+ const doc = await ScheduleRun().create({ ...data, status: 'started' });+ return { run: doc.toObject() };+ } catch (error) {+ if (isOccurrenceDuplicate(error)) {+ // Report the EXISTING row's status. A duplicate means another fire owns this+ // occurrence — but "owns" and "already finished" need opposite handling, and the+ // caller cannot distinguish them without this.+ const existing = await ScheduleRun()+ .findOne({ scheduleId: data.scheduleId, scheduledFor: data.scheduledFor })+ .select('status')+ .lean<Pick<IScheduleRun, 'status'>>();+ return { conflict: 'duplicate', existingStatus: existing?.status };+ }+ // Checked BEFORE overlap: the global cap index and the per-schedule active+ // index are different failures and drive different caller behavior (retry the+ // next slot vs skip the occurrence).+ if (isCapacitySlotConflict(error)) {+ return { conflict: 'slot-taken' };+ }+ if (isActiveRunConflict(error)) {+ return { conflict: 'overlap' };+ }+ throw error;+ }+ }++ /** Capacity-slot occupancy for the allocator: which slots are held by `started`+ * runs, plus how many legacy rows hold no slot (they shrink the effective cap so+ * the bound stays conservative during rollout rather than transiently overshooting). */+ async function getCapacityOccupancy(): Promise<{ takenSlots: number[]; unslotted: number }> {+ const rows = await ScheduleRun()+ .find({ status: 'started' })+ .select('capacitySlot')+ .lean<Array<{ capacitySlot?: number }>>();+ const takenSlots: number[] = [];+ let unslotted = 0;+ for (const row of rows) {+ if (typeof row.capacitySlot === 'number') {+ takenSlots.push(row.capacitySlot);+ } else {+ unslotted += 1;+ }+ }+ return { takenSlots, unslotted };+ }++ /**+ * Records that an abort was requested WITHOUT freeing the capacity slot: the run+ * keeps counting against fireConcurrency until its generation owner confirms+ * settlement by writing a terminal outcome.+ *+ * The stamp is RENEWED per attempt, not first-wins: the fences read+ * `abortRequestedAt` for freshness, and a retry abort signalled against a live+ * generation must re-arm them — a 31-minute-old stamp from a failed first attempt+ * must not let a drain read the retry's post-abort job state as settled. Callers+ * therefore only invoke this against a generation they observed LIVE (or unknown);+ * stamping cleanup aborts of already-terminal jobs would re-fence a dead owner+ * forever and block the reconciler's recovery.+ *+ * Source semantics: a 'stop' attempt takes the source and re-opens the persistence+ * marker (the Stop route persists writes the owner's barrier must wait on, and each+ * attempt resolves it anew); a 'deletion' request renews freshness but never steals+ * the source from an UNRESOLVED stop attempt — the route may still be persisting,+ * and the owner's barrier keys off `abortSource === 'stop'`.+ * Classic operators only (DocumentDB).+ */+ async function requestRunAbort(+ scheduleId: string,+ scheduledFor: Date,+ source?: IScheduleRun['abortSource'],+ ): Promise<boolean | 'in_progress'> {+ const now = new Date();+ if (source === 'stop') {+ // SERIALIZED per run: only one unresolved fresh stop attempt may exist. A second+ // Stop stamping over a live one would re-open the persistence marker, and its+ // own lost-CAS resolution would then release the settlement barrier while the+ // FIRST attempt (the terminal-CAS winner) was still pruning and saving. The+ // guarded update is the arbiter; a loser answers 'in_progress' and must neither+ // signal the abort nor resolve anything. A stale unresolved attempt (its route+ // presumed dead) is claimable again.+ const staleCutoff = new Date(now.getTime() - ABORT_STAMP_STALE_MS);+ const stamped = await ScheduleRun().updateOne(+ {+ scheduleId,+ scheduledFor,+ status: { $in: ACTIVE_RUN_STATUSES },+ $nor: [+ {+ abortSource: 'stop',+ abortPersistedAt: { $exists: false },+ abortRequestedAt: { $gt: staleCutoff },+ },+ ],+ },+ { $set: { abortRequestedAt: now, abortSource: 'stop' }, $unset: { abortPersistedAt: 1 } },+ );+ if ((stamped.matchedCount ?? 0) > 0) {+ return true;+ }+ const holder = await ScheduleRun()+ .findOne({ scheduleId, scheduledFor, status: { $in: ACTIVE_RUN_STATUSES } })+ .select('_id')+ .lean();+ return holder != null ? 'in_progress' : false;+ }+ const claimed = await ScheduleRun().updateOne(+ {+ scheduleId,+ scheduledFor,+ status: { $in: ACTIVE_RUN_STATUSES },+ $or: [{ abortSource: { $ne: 'stop' } }, { abortPersistedAt: { $exists: true } }],+ },+ { $set: { abortRequestedAt: now, ...(source ? { abortSource: source } : {}) } },+ );+ if ((claimed.matchedCount ?? 0) > 0) {+ return true;+ }+ // An unresolved stop attempt holds the source; renew freshness only.+ const renewed = await ScheduleRun().updateOne(+ { scheduleId, scheduledFor, status: { $in: ACTIVE_RUN_STATUSES } },+ { $set: { abortRequestedAt: now } },+ );+ return (renewed.matchedCount ?? 0) > 0;+ }++ /** The abort-coordination view of a run row, for the generation owner's settlement+ * barrier (see abortPersistedAt). */+ async function getScheduleRunAbortState(+ scheduleId: string,+ scheduledFor: Date,+ ): Promise<Pick<+ IScheduleRun,+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'+ > | null> {+ return ScheduleRun()+ .findOne({ scheduleId, scheduledFor })+ .select('status abortRequestedAt abortSource abortPersistedAt')+ .lean<Pick<+ IScheduleRun,+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'+ > | null>();+ }++ /**+ * Stamps a paused run as RESUME-CLAIMED: its approval was consumed and a+ * continuation is running. While fresh, a re-paused job's state is a hand-off in+ * flight (segment writes still landing), not settleable evidence. The pause+ * record ($unset below) is the completion signal; a crash leaves the stamp to+ * age out on the caller's staleness bound.+ */+ async function markRunResumeClaimed(+ scheduleId: string,+ scheduledFor: Date,+ capacitySlot: number,+ ): Promise<ResumedRunReservation> {+ try {+ const claimed = await ScheduleRun().findOneAndUpdate(+ { scheduleId, scheduledFor, status: 'requires_action' },+ {+ $set: {+ status: 'started',+ capacitySlot,+ resumeClaimedAt: new Date(),+ },+ },+ { new: true, timestamps: false },+ );+ return claimed == null ? { conflict: 'not-paused' } : { capacitySlot };+ } catch (error) {+ if (isCapacitySlotConflict(error)) {+ return { conflict: 'slot-taken' };+ }+ if (isActiveRunConflict(error)) {+ return { conflict: 'overlap' };+ }+ throw error;+ }+ }++ /** Roll back a resume reservation only while the exact slot this caller claimed+ * still belongs to this occurrence. The approval job is checked separately by the+ * controller before crossing this seam, so a committed/racing resume never has its+ * capacity released by a losing request. */+ async function releaseRunResumeClaim(+ scheduleId: string,+ scheduledFor: Date,+ capacitySlot: number,+ ): Promise<boolean> {+ const released = await ScheduleRun().updateOne(+ { scheduleId, scheduledFor, status: 'started', capacitySlot },+ {+ $set: { status: 'requires_action' },+ $unset: { capacitySlot: 1, resumeClaimedAt: 1 },+ },+ { timestamps: false },+ );+ return (released.modifiedCount ?? 0) > 0;+ }++ /** Stamped by the interactive Stop route once ALL its writes (checkpoint prune,+ * partial-response save) have landed; releases the generation owner's settlement+ * barrier. Bookkeeping only: never touches `updatedAt`. */+ async function markRunAbortPersisted(scheduleId: string, scheduledFor: Date): Promise<void> {+ await ScheduleRun().updateOne(+ { scheduleId, scheduledFor },+ { $set: { abortPersistedAt: new Date() } },+ { timestamps: false },+ );+ }++ /** Count of in-flight scheduled runs (across all schedules) for the fire cap. */+ async function countActiveRuns(): Promise<number> {+ return ScheduleRun().countDocuments({ status: 'started' });+ }++ /**+ * Writes an occurrence's result onto the schedule card, fenced on occurrence order+ * so a delayed outcome can never overwrite a NEWER occurrence's projection. The+ * fence is absent-tolerant: rows written before `lastRun.scheduledFor` existed+ * project once and then start ordering normally.+ */+ async function projectLastRun(+ scheduleId: string,+ lastRun: NonNullable<ISchedule['lastRun']>,+ scheduledFor: Date,+ revisionFilter: Record<string, unknown> = {},+ ): Promise<void> {+ // A PAUSE is the only non-terminal projection, and it is the one that can still+ // arrive late for its own occurrence: the row transition is won first, and a resume+ // can terminalize and project between that CAS and this write. Occurrence ordering+ // alone permits it (same `scheduledFor`), so refuse to walk this occurrence's card+ // backwards from a settled result to "Needs approval" — which nothing would correct+ // until the next occurrence ran.+ const notAlreadySettled =+ lastRun.status === 'requires_action'+ ? {+ $nor: [+ {+ $and: [+ { 'lastRun.scheduledFor': scheduledFor },+ { 'lastRun.status': { $in: TERMINAL_CARD_STATUSES } },+ ],+ },+ ],+ }+ : {};+ await Schedule().updateOne(+ {+ id: scheduleId,+ $or: [+ { 'lastRun.scheduledFor': { $exists: false } },+ { 'lastRun.scheduledFor': { $lte: scheduledFor } },+ ],+ ...notAlreadySettled,+ ...revisionFilter,+ },+ { $set: { lastRun: { ...lastRun, scheduledFor } } },+ // Run-state bookkeeping, not a config edit: re-affirmed pauses would+ // otherwise bump updatedAt (and reorder updated-time listings) every+ // reconciliation pass for as long as an approval sits waiting.+ { timestamps: false },+ );+ }++ /**+ * Applies the schedule-side bookkeeping (lastRun + counters + auto-disable) for+ * a terminal occurrence. Idempotent via the per-occurrence `countedFor` guard:+ * the $inc lands at most once per occurrence no matter how many times it is+ * retried (inline finish, reconciler catch of an un-`bookkept` run, crash-replay),+ * even when a later occurrence's counting interleaves with an earlier paused one.+ */+ async function applyTerminalBookkeeping(+ params: RecordRunOutcomeParams & { firedAt: Date; expectConfigRevision?: number },+ ): Promise<void> {+ const lastRun = {+ conversationId: params.conversationId,+ status: params.status,+ error: params.error,+ firedAt: params.firedAt,+ };+ const isFailure = params.status === 'error';+ const isSuccess = params.status === 'success';+ // The COUNT is idempotent per occurrence: `countedFor` is a bounded set of+ // recently-counted occurrence timestamps, so an interleaved earlier occurrence+ // can't clear this one's marker (a single scalar could). Everything that must+ // land atomically WITH the count goes in this one update so a crash can't leave+ // it half-applied: the balance-skip streak resets on ANY non-balance outcome,+ // and a success clears the failure streak inline (never a lost follow-up).+ // CONFIG-REVISION FENCE: a run that started under an older owner config must not+ // apply counters (or walk toward auto-disable) against a schedule the owner has+ // since edited or re-enabled. Absent on either side disables the fence, so+ // pre-existing rows/schedules keep today's behavior instead of wedging.+ const revisionFilter =+ params.expectConfigRevision != null ? { configRevision: params.expectConfigRevision } : {};+ // ORDER FENCE, the same one the card projection has always had. `failureCount` and+ // `balanceSkipCount` are CONSECUTIVE streaks, so they are order-sensitive in a way+ // per-occurrence idempotency (`countedFor`) cannot express: a `requires_action` run+ // does not block later occurrences, so an older occurrence — a resumed pause, a+ // reconciler replay — routinely settles AFTER a newer one. Unfenced, its late `error`+ // rebuilds a streak a newer success had already cleared, and since+ // `autoDisableAfterFailures` may be configured as low as 1, a single stale failure+ // can disable a healthy schedule.+ //+ // The fence covers the WHOLE counter update rather than the streak fields alone:+ // splitting it would break the atomicity that keeps a crash from half-applying the+ // count, and DocumentDB rules out expressing it as a pipeline update. The cost is+ // that a late occurrence's success does not add to `runCount` — a display total+ // under-counting a rare out-of-order run, which is the cheap side of this trade.+ // Absent-tolerant, so schedules written before the watermark existed count once and+ // then order normally.+ const counterOrderFilter = {+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],+ };+ await Schedule().updateOne(+ {+ id: params.scheduleId,+ countedFor: { $ne: params.scheduledFor },+ ...revisionFilter,+ ...counterOrderFilter,+ },+ {+ $set: {+ balanceSkipCount: 0,+ countersAsOf: params.scheduledFor,+ ...(isSuccess ? { failureCount: 0 } : {}),+ },+ $push: { countedFor: { $each: [params.scheduledFor], $slice: -COUNTED_FOR_WINDOW } },+ ...(isSuccess ? { $inc: { runCount: 1 } } : {}),+ ...(isFailure ? { $inc: { failureCount: 1 } } : {}),+ },+ );+ // The card projection is ORDER-fenced, unlike the counters above (idempotent and+ // order-insensitive): a `requires_action` run doesn't block later occurrences, so a+ // resumed pause — or a reconciler replay — routinely settles AFTER a newer run+ // already finished, and must not roll the card back to its older result.+ await projectLastRun(params.scheduleId, lastRun, params.scheduledFor, revisionFilter);+ // A success clears a transient disable reason ONLY while the schedule is still+ // enabled. An older run (e.g. a resumed pause) can succeed AFTER newer outcomes+ // already auto-disabled the schedule — since `requires_action` runs don't block+ // later occurrences — and must not wipe the reason that explains why it's off.+ // Predicated on `enabled` separately so it can't leak into the count-guarded+ // update above (which must run regardless of enabled state).+ if (isSuccess) {+ await Schedule().updateOne(+ { id: params.scheduleId, enabled: true },+ { $unset: { disabledReason: 1 } },+ );+ }+ // Auto-disable is a POLICY re-evaluated on EVERY call (idempotent), NOT gated+ // on the count guard — so if a crash landed the $inc but not the disable, the+ // reconciler's replay still disables. Reads current state after the count.+ if (isFailure) {+ const schedule = await Schedule().findOne({ id: params.scheduleId }).lean<ISchedule>();+ if (schedule?.enabled && schedule.failureCount >= params.autoDisableAfterFailures) {+ // Carry the COUNT this decision was made on, not just the revision. The read+ // above and this write are separate statements, and a concurrent success resets+ // the failure streak to zero — a revision-only fence would still let this stale+ // decision disable a schedule whose streak had just been cleared.+ await disableSchedule(+ params.scheduleId,+ 'too_many_failures',+ undefined,+ params.expectConfigRevision,+ { failureCount: { $gte: params.autoDisableAfterFailures } },+ );+ }+ }+ }++ /**+ * Balance-skip streak bookkeeping, shared by the pre-fire skip (recordSkippedRun)+ * and a mid-generation balance refusal settling a `started` row (recordRunOutcome+ * with `skipped_balance`): projects the card, walks the consecutive-balance-skip+ * streak (per-occurrence idempotent via `countedFor`, order-fenced via+ * `countersAsOf`), and applies the insufficient_balance auto-disable policy.+ */+ async function applyBalanceSkipBookkeeping(params: {+ scheduleId: string;+ scheduledFor: Date;+ firedAt: Date;+ conversationId?: string;+ rowRevision?: number;+ balanceSkipDisableThreshold?: number;+ }): Promise<void> {+ const revisionFilter = params.rowRevision != null ? { configRevision: params.rowRevision } : {};+ await projectLastRun(+ params.scheduleId,+ {+ ...(params.conversationId ? { conversationId: params.conversationId } : {}),+ status: 'skipped_balance',+ firedAt: params.firedAt,+ },+ params.scheduledFor,+ revisionFilter,+ );+ if (params.balanceSkipDisableThreshold == null) {+ return;+ }+ // Same order fence as the terminal counters: these are consecutive streaks, so an+ // older occurrence settling late must not reset or extend one a newer outcome owns.+ const orderFilter = {+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],+ };+ // Per-occurrence guard: increment the streak at most once for this occurrence even+ // across crash retries (same `countedFor` set the terminal counters use).+ const schedule = await Schedule()+ .findOneAndUpdate(+ {+ id: params.scheduleId,+ countedFor: { $ne: params.scheduledFor },+ ...revisionFilter,+ ...orderFilter,+ },+ {+ $inc: { balanceSkipCount: 1 },+ $set: { countersAsOf: params.scheduledFor },+ $push: { countedFor: { $each: [params.scheduledFor], $slice: -COUNTED_FOR_WINDOW } },+ },+ { new: true },+ )+ .lean<ISchedule>();+ // Auto-disable is a POLICY re-evaluated on EVERY call (idempotent), NOT gated on+ // the count guard — if a crash landed the $inc but not the disable, the replay's+ // guarded update no-ops to null, so re-read and still disable at/over threshold.+ const current =+ schedule ?? (await Schedule().findOne({ id: params.scheduleId }).lean<ISchedule>());+ if (current?.enabled && current.balanceSkipCount >= params.balanceSkipDisableThreshold) {+ // Same read-then-write window as the failure policy: any non-balance outcome+ // resets this streak, so the write must carry the count it decided on.+ await disableSchedule(+ params.scheduleId,+ 'insufficient_balance',+ undefined,+ params.rowRevision,+ { balanceSkipCount: { $gte: params.balanceSkipDisableThreshold } },+ );+ }+ }++ /**+ * Overlap-skip bookkeeping, shared by the pre-fire skip (recordSkippedRun) and its+ * crash-retry replay (finalizeBookkeeping): projects the skip onto the card and+ * BREAKS the consecutive-balance-skip streak (an overlap is an intervening+ * non-balance outcome). Both writes are idempotent and order-fenced, so replays+ * are safe.+ */+ async function applyOverlapSkipBookkeeping(params: {+ scheduleId: string;+ scheduledFor: Date;+ firedAt: Date;+ rowRevision?: number;+ }): Promise<void> {+ const revisionFilter = params.rowRevision != null ? { configRevision: params.rowRevision } : {};+ // Through the ORDERED projection: writing `lastRun` directly dropped the+ // `scheduledFor` marker, after which the next projection read the marker as absent+ // and let an older occurrence's outcome overwrite this newer skip.+ await projectLastRun(+ params.scheduleId,+ { status: 'skipped_overlap', firedAt: params.firedAt },+ params.scheduledFor,+ revisionFilter,+ );+ // Same order fence as the terminal counters: these are consecutive streaks, so an+ // older occurrence settling late must not reset or extend one a newer outcome owns.+ const orderFilter = {+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],+ };+ await Schedule().updateOne(+ { id: params.scheduleId, ...revisionFilter, ...orderFilter },+ { $set: { balanceSkipCount: 0, countersAsOf: params.scheduledFor } },+ );+ }++ /**+ * Terminal (or pause) transition for a run + lastRun/failure bookkeeping.+ * Matches a run row still in `started` OR `requires_action`. Crash-retryable:+ * the run row is marked `bookkept:false` at terminalization and only flipped+ * to `true` after bookkeeping lands, so a crash in between is re-applied by the+ * reconciler (`getUnbookkeptRuns`), while `countedFor` keeps it idempotent.+ * A `skipped_balance` settlement (a mid-generation balance refusal) takes the same+ * guarded row transition but walks the balance-skip streak instead of the failure+ * streak — the owner's credits, not the schedule, are at fault.+ */+ async function recordRunOutcome(params: RecordRunOutcomeParams): Promise<void> {+ const firedAt = new Date();+ if (params.status === 'requires_action') {+ // PAUSE (HITL): win the ROW transition first, then project the card. A read-then-+ // write guard let a concurrent resume terminalize the run between the two, after+ // which the card was pinned to "Needs approval" forever — the row update no-op'd+ // (already terminal) while the card write had landed. Matching `requires_action`+ // too keeps a retried pause re-affirming the card, and keeps the transition+ // crash-recoverable: the reconciler re-invokes this for a row in either state.+ // The match is also the anti-spoof guard a bare scheduleId would bypass.+ const paused = await ScheduleRun().findOneAndUpdate(+ {+ scheduleId: params.scheduleId,+ scheduledFor: params.scheduledFor,+ status: { $in: ['started', 'requires_action'] },+ // See resumeClaimStaleBefore: fences a stale-snapshot recovery replay against a+ // resume that claimed the row after the snapshot was taken, while still letting+ // an ABANDONED claim (its worker died mid-hand-off) be recovered.+ ...(params.resumeClaimStaleBefore != null+ ? {+ $or: [+ { resumeClaimedAt: { $exists: false } },+ { resumeClaimedAt: { $lt: params.resumeClaimStaleBefore } },+ ],+ }+ : {}),+ },+ {+ $set: {+ status: 'requires_action',+ ...(params.conversationId ? { conversationId: params.conversationId } : {}),+ },+ // Leaving `started` frees the global capacity slot; the resume claims a+ // fresh one from the allocator rather than re-adopting a possibly-taken slot.+ // The pause record is also the resume hand-off's COMPLETION signal: the+ // re-pause branch persists its segment before recording, so clearing the+ // claim stamp here re-opens the run to quiesce settling.+ $unset: { capacitySlot: 1, resumeClaimedAt: 1 },+ },+ { new: false },+ );+ if (paused == null) {+ return;+ }+ // Revision-fenced like the terminal path: an owner edit landing between the fire+ // and the approval must not have the OLD config's pause stamped on it. Without+ // this the later terminal write (which IS fenced) could never replace the stale+ // status, so the card stuck on "Needs approval".+ // The ROW's fire time, not the projection time: reconciliation re-affirms a+ // long-lived pause every pass, and a fresh stamp per pass walked the card's+ // timestamp forward for as long as the approval sat waiting.+ await projectLastRun(+ params.scheduleId,+ {+ conversationId: params.conversationId,+ status: params.status,+ firedAt: paused.firedAt ?? firedAt,+ },+ params.scheduledFor,+ paused.configRevision != null ? { configRevision: paused.configRevision } : {},+ );+ return;+ }+ // TERMINAL: flip the run row (match-guarded), then apply bookkeeping. `bookkept` is+ // set false at the flip and true only after bookkeeping lands, so a crash between is+ // re-applied by the reconciler (getUnbookkeptRuns) while countedFor keeps counters idempotent.+ const settled = await ScheduleRun()+ .findOneAndUpdate(+ {+ scheduleId: params.scheduleId,+ scheduledFor: params.scheduledFor,+ status: { $in: ['started', 'requires_action'] },+ },+ {+ $set: {+ status: params.status,+ bookkept: false,+ settledAt: firedAt,+ ...(params.conversationId && !params.clearConversationId+ ? { conversationId: params.conversationId }+ : {}),+ ...(params.error ? { error: params.error } : {}),+ ...(params.durationMs != null ? { durationMs: params.durationMs } : {}),+ },+ // SETTLEMENT: a terminal outcome is the generation owner confirming the run+ // actually stopped, so this is the ONLY place the global capacity slot is+ // released. An abort request alone does not free it (see requestRunAbort).+ $unset: { capacitySlot: 1, ...(params.clearConversationId ? { conversationId: 1 } : {}) },+ },+ { new: false },+ )+ .lean<IScheduleRun>();+ // No-match guard: never touch schedule bookkeeping without a matching run+ // (protects against a spoofed scheduleId on a normal chat).+ if (settled == null) {+ return;+ }+ // SINGLE SEAM: the config fence is DERIVED here from the row being settled, not+ // passed in by each caller. Callers only say "this occurrence reached status X" and+ // structurally cannot forget a token — which is exactly how the reconcile and+ // balance-skip paths previously shipped unfenced.+ if (params.status === 'skipped_balance') {+ await applyBalanceSkipBookkeeping({+ scheduleId: params.scheduleId,+ scheduledFor: params.scheduledFor,+ firedAt: settled.firedAt ?? firedAt,+ conversationId: params.conversationId,+ rowRevision: settled.configRevision,+ balanceSkipDisableThreshold: params.balanceSkipDisableThreshold,+ });+ } else {+ await applyTerminalBookkeeping({+ ...params,+ firedAt: settled.firedAt ?? firedAt,+ expectConfigRevision: settled.configRevision,+ });+ }+ await ScheduleRun().updateOne(+ { scheduleId: params.scheduleId, scheduledFor: params.scheduledFor },+ { $set: { bookkept: true } },+ );+ }++ async function recordSkippedRun(+ data: Partial<IScheduleRun> & {+ scheduleId: string;+ scheduledFor: Date;+ status: Extract<ScheduleRunStatus, 'skipped_overlap' | 'skipped_balance'>;+ },+ balanceSkipDisableThreshold?: number,+ ): Promise<void> {+ const firedAt = new Date();+ // A duplicate {scheduleId, scheduledFor} row means a prior attempt inserted it.+ // Proceed as a retry ONLY when that row is the SAME skip — otherwise this claim+ // is a retry of an occurrence that actually started/terminalized (e.g. the POST+ // was accepted but advanceSchedule failed before releasing the lease), and+ // rewriting lastRun/counters would mislabel a real run as a skip (and could+ // walk it toward auto-disable). The streak $inc is separately guarded per+ // occurrence by `countedFor`, so a genuine same-skip retry can't double-count.+ // `bookkept: false` until the schedule-side writes land: a crash in between is+ // replayed by the reconciler (getUnbookkeptRuns), exactly like a terminal run —+ // a duplicate claim of this occurrence advances past the terminal row without+ // re-running bookkeeping, so nothing else would ever repair a half-applied skip.+ const inserted = await insertScheduleRun({+ ...data,+ firedAt,+ settledAt: firedAt,+ bookkept: false,+ });+ let rowRevision = inserted?.configRevision;+ let rowFiredAt = firedAt;+ if (inserted == null) {+ const existing = await ScheduleRun()+ .findOne({ scheduleId: data.scheduleId, scheduledFor: data.scheduledFor })+ .select('status configRevision firedAt')+ .lean<Pick<IScheduleRun, 'status' | 'configRevision' | 'firedAt'>>();+ if (existing == null || existing.status !== data.status) {+ return;+ }+ rowRevision = existing.configRevision;+ // A retry re-projects the ORIGINAL occurrence, not the retry moment.+ rowFiredAt = existing.firedAt ?? firedAt;+ }+ // Same single seam as recordRunOutcome: the config fence is DERIVED from the row,+ // never passed by the caller. A skip decided under an older owner config must not+ // stamp the card (or walk the balance streak toward auto-disable) on a schedule the+ // owner has since edited. Absent on either side disables the fence.+ if (data.status === 'skipped_balance') {+ await applyBalanceSkipBookkeeping({+ scheduleId: data.scheduleId,+ scheduledFor: data.scheduledFor,+ firedAt: rowFiredAt,+ rowRevision,+ balanceSkipDisableThreshold,+ });+ } else {+ await applyOverlapSkipBookkeeping({+ scheduleId: data.scheduleId,+ scheduledFor: data.scheduledFor,+ firedAt: rowFiredAt,+ rowRevision,+ });+ }+ await ScheduleRun().updateOne(+ { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor },+ { $set: { bookkept: true } },+ );+ }++ async function setRunFireDetails(+ scheduleId: string,+ scheduledFor: Date,+ details: { conversationId: string; droppedFileIds?: string[] },+ ): Promise<void> {+ await ScheduleRun().updateOne(+ { scheduleId, scheduledFor },+ {+ $set: {+ conversationId: details.conversationId,+ ...(details.droppedFileIds?.length ? { droppedFileIds: details.droppedFileIds } : {}),+ },+ },+ );+ }++ /**+ * Non-terminal runs old enough to need a job-store status check. Fetches+ * `started` (capacity-consuming) and `requires_action` (paused) in separate+ * budgeted, firedAt-ordered buckets so a backlog of long-lived paused rows+ * can't starve orphaned `started` runs out of every sweep.+ */+ async function getRunsForReconciliation(olderThan: Date, limit: number): Promise<IScheduleRun[]> {+ const [started, paused] = await Promise.all([+ // `started` runs are bounded by the global fireConcurrency cap, so this window+ // can never fill with rows that have nothing to do — oldest-first is right here.+ ScheduleRun()+ .find({ status: 'started', firedAt: { $lt: olderThan } })+ .sort({ firedAt: 1 })+ .limit(limit)+ .lean<IScheduleRun[]>(),+ // ROUND-ROBIN, not oldest-first. A paused run holds no capacity slot and does not+ // block its schedule's later occurrences, so pauses accumulate without bound — a+ // handful of hourly schedules can leave hundreds live at once, and the approval+ // TTL that ends them is operator-configured with no ceiling. Under oldest-first+ // the same full window of still-live pauses is re-fetched every tick while an+ // ABANDONED row behind them is never reached, so its run stays active forever.+ // Ordering by when each row was last examined (never-examined first) gives every+ // row a turn, which makes that starvation impossible for any number of pauses.+ ScheduleRun()+ .find({ status: 'requires_action', firedAt: { $lt: olderThan } })+ .sort({ reconciledAt: 1, firedAt: 1 })+ .limit(limit)+ .lean<IScheduleRun[]>(),+ ]);+ return [...started, ...paused];+ }++ /** Stamps rows as examined, so the paused window rotates instead of re-serving the+ * same rows forever. Bookkeeping only: never touches `updatedAt`. */+ async function markRunsReconciled(runs: Array<Pick<IScheduleRun, '_id'>>): Promise<void> {+ const ids = runs.map((run) => run._id).filter((id) => id != null);+ if (ids.length === 0) {+ return;+ }+ await ScheduleRun().updateMany(+ { _id: { $in: ids } },+ { $set: { reconciledAt: new Date() } },+ { timestamps: false },+ );+ }++ /** Settled runs (terminal outcomes AND skips) whose schedule bookkeeping never+ * landed — a crash between the row write and the card/counter writes.+ * Least-recently-attempted first (the paused window's rotation): a row whose+ * replay keeps failing must rotate to the back, or a batch of them re-fills+ * this bounded window forever and every later row's counters never land. */+ async function getUnbookkeptRuns(olderThan: Date, limit: number): Promise<IScheduleRun[]> {+ return ScheduleRun()+ .find({+ status: {+ $in: ['success', 'error', 'interrupted', 'skipped_balance', 'skipped_overlap'],+ },+ bookkept: false,+ firedAt: { $lt: olderThan },+ })+ .sort({ reconciledAt: 1, firedAt: 1 })+ .limit(limit)+ .lean<IScheduleRun[]>();+ }++ /** Re-applies (idempotent) bookkeeping for a terminal run and marks it bookkept. */+ async function finalizeBookkeeping(params: RecordRunOutcomeParams): Promise<void> {+ // Same single seam as recordRunOutcome: derive the config fence from the row, so the+ // crash-retry path cannot apply bookkeeping the inline path would have refused.+ // The row's ORIGINAL firedAt is reused too: this replay runs minutes (or many+ // retries) after the fact, and stamping the replay time onto lastRun would keep+ // walking the card's timestamp forward every time the recovery path fires.+ const run = await ScheduleRun()+ .findOne({ scheduleId: params.scheduleId, scheduledFor: params.scheduledFor })+ .select('configRevision firedAt')+ .lean<Pick<IScheduleRun, 'configRevision' | 'firedAt'>>();+ const firedAt = run?.firedAt ?? new Date();+ if (params.status === 'skipped_balance') {+ await applyBalanceSkipBookkeeping({+ scheduleId: params.scheduleId,+ scheduledFor: params.scheduledFor,+ firedAt,+ conversationId: params.conversationId,+ rowRevision: run?.configRevision,+ balanceSkipDisableThreshold: params.balanceSkipDisableThreshold,+ });+ } else if (params.status === 'skipped_overlap') {+ await applyOverlapSkipBookkeeping({+ scheduleId: params.scheduleId,+ scheduledFor: params.scheduledFor,+ firedAt,+ rowRevision: run?.configRevision,+ });+ } else {+ await applyTerminalBookkeeping({+ ...params,+ firedAt,+ expectConfigRevision: run?.configRevision,+ });+ }+ await ScheduleRun().updateOne(+ { scheduleId: params.scheduleId, scheduledFor: params.scheduledFor },+ { $set: { bookkept: true } },+ );+ }++ /**+ * Deletes a run row (used to roll back a capacity reservation). Status-fenced+ * when `expectedStatus` is provided so a rollback cannot delete a row a+ * concurrent process already advanced (e.g. to a terminal outcome). Fenced+ * additionally on `expectedConversationId` so a fire only ever deletes the+ * reservation IT inserted.+ */+ async function deleteScheduleRun(+ scheduleId: string,+ scheduledFor: Date,+ expectedStatus?: ScheduleRunStatus,+ expectedConversationId?: string,+ ): Promise<void> {+ const filter: Record<string, unknown> = { scheduleId, scheduledFor };+ if (expectedStatus !== undefined) {+ filter.status = expectedStatus;+ }+ if (expectedConversationId !== undefined) {+ filter.conversationId = expectedConversationId;+ }+ await ScheduleRun().deleteOne(filter);+ }++ /**+ * Soft-deletes a schedule for the owner: disables it (so the engine can no+ * longer claim it), rotates the claim token (fencing any in-flight worker), and+ * marks it `deleting` so it is hidden and awaits erasure once its active runs+ * drain. Returns the updated row (for aborting its in-flight jobs) or null.+ */+ async function markScheduleDeleting(+ id: string,+ userId: string | Types.ObjectId,+ ): Promise<ISchedule | null> {+ // Keep leaseUntil/leaseBy so a fire that already leased this occurrence still+ // serializes against a re-claim while it unwinds. Only clear nextRunAt+ // (belt-and-suspenders atop enabled:false to stop new claims); the lease+ // releases itself when the fire finishes, or via TTL.+ //+ // IDEMPOTENT: deliberately NOT gated on `deleting: { $ne: true }`. The mark is the+ // first of several steps (read active runs, abort their jobs, prune checkpoints,+ // erase when drained), and if any of those threw — or the process exited — a+ // one-shot mark made every retry answer 404, stranding the schedule with its job+ // and checkpoint alive. Re-marking an already-deleting schedule is harmless (the+ // fields are already at these values; the token rotates again, which only re-fences+ // stale workers), and it lets the caller re-drive the rest of the teardown.+ return Schedule()+ .findOneAndUpdate(+ { id, user: userId },+ {+ $set: { enabled: false, deleting: true, claimToken: randomUUID() },+ $unset: { nextRunAt: 1 },+ },+ { new: true },+ )+ .lean<ISchedule>();+ }++ /** In-flight (non-terminal) runs of a schedule — the jobs a delete must abort. */+ async function getActiveRunsForSchedule(scheduleId: string): Promise<IScheduleRun[]> {+ return ScheduleRun()+ .find({ scheduleId, status: { $in: ACTIVE_RUN_STATUSES } })+ .lean<IScheduleRun[]>();+ }++ /** In-flight runs across all of a user's schedules — for account-deletion quiescing. */+ async function getActiveRunsForUser(userId: string | Types.ObjectId): Promise<IScheduleRun[]> {+ return ScheduleRun()+ .find({ user: userId, status: { $in: ACTIVE_RUN_STATUSES } })+ .lean<IScheduleRun[]>();+ }++ /**+ * Marks all of a user's schedules non-claimable ahead of account deletion, so+ * the engine cannot fire a new occurrence while the cascade runs. Rotates each+ * claim token to fence any in-flight worker.+ */+ /**+ * REVERSIBLE quiesce ahead of account deletion. Snapshots each schedule's pre-suspension+ * enabled/next-run state under this attempt's `token`, then fences firing (disable, clear+ * `nextRunAt`, rotate `claimToken`). It deliberately does NOT set `deleting`: a suspended+ * row must not be erased, because a deletion the controller later cancels has to restore+ * it. On a successful deletion the hard-delete cascade removes the row (and this snapshot)+ * anyway. Snapshotting needs each row's current values, which a classic (DocumentDB-safe)+ * update cannot copy, so it reads then `bulkWrite`s — fenced per row so a row already+ * suspended by this token, soft-deleted, or edited out from under the read is left alone.+ */+ async function suspendUserSchedulesForDeletion(+ userId: string | Types.ObjectId,+ token: string,+ ): Promise<void> {+ const schedules = await Schedule()+ .find({+ user: userId,+ deleting: { $ne: true },+ 'deletionSuspension.token': { $ne: token },+ })+ .select('id enabled nextRunAt deletionSuspension')+ .lean<Array<Pick<ISchedule, 'id' | 'enabled' | 'nextRunAt' | 'deletionSuspension'>>>();+ if (schedules.length === 0) {+ return;+ }+ const ops = schedules.map((schedule) => {+ // A row already suspended by an ABANDONED earlier attempt (its process died before+ // restoring) still holds the only record of the true pre-suspension state. Carry that+ // snapshot forward and merely take ownership with this attempt's token — re-reading+ // the row's CURRENT values would snapshot the suspended state itself (disabled, no+ // next run) and permanently strand the schedule when this attempt later restores.+ const existing = schedule.deletionSuspension;+ const snapshotEnabled =+ existing != null ? existing.enabled === true : schedule.enabled === true;+ const snapshotNextRunAt = existing != null ? existing.nextRunAt : schedule.nextRunAt;+ return {+ updateOne: {+ filter: {+ id: schedule.id,+ user: userId,+ deleting: { $ne: true },+ 'deletionSuspension.token': { $ne: token },+ },+ update: {+ $set: {+ enabled: false,+ claimToken: randomUUID(),+ deletionSuspension: {+ token,+ enabled: snapshotEnabled,+ ...(snapshotNextRunAt != null ? { nextRunAt: snapshotNextRunAt } : {}),+ },+ },+ $unset: { nextRunAt: 1 as const },+ },+ },+ };+ });+ await tenantSafeBulkWrite(Schedule(), ops as AnyBulkWriteOperation[], { ordered: false });+ }++ const RESTORE_ATTEMPTS = 3;++ /**+ * Reverses {@link suspendUserSchedulesForDeletion} when a deletion attempt is cancelled.+ * Restores enabled/next-run from the snapshot and clears the suspension — but ONLY for+ * rows still carrying this exact attempt's token and not independently soft-deleted, so a+ * schedule the owner deleted (or a newer attempt re-suspended) is never resurrected.+ *+ * RETRIED, because its callers cannot retry it later. A cancelled deletion restores while+ * the user-deletion fence is still armed and then releases that fence; once released,+ * nothing re-drives this, and a single transient failure would strand the owner's live+ * account with silently disabled, next-run-less schedules. Retrying is safe: every attempt+ * re-reads only the rows STILL carrying this token, so a partially-applied unordered write+ * converges on exactly the stragglers, and a fully-applied one finds nothing and returns.+ * Throws the last failure so callers still surface it.+ */+ async function restoreUserSchedulesFromDeletion(+ userId: string | Types.ObjectId,+ token: string,+ ): Promise<void> {+ let lastError: unknown;+ for (let attempt = 1; attempt <= RESTORE_ATTEMPTS; attempt++) {+ try {+ await restoreSuspendedBatch(userId, token);+ return;+ } catch (error) {+ lastError = error;+ if (attempt < RESTORE_ATTEMPTS) {+ await new Promise((resolve) => setTimeout(resolve, 100 * attempt));+ }+ }+ }+ throw lastError;+ }++ async function restoreSuspendedBatch(+ userId: string | Types.ObjectId,+ token: string,+ ): Promise<void> {+ const suspended = await Schedule()+ .find({ user: userId, 'deletionSuspension.token': token, deleting: { $ne: true } })+ .select('id deletionSuspension')+ .lean<Array<Pick<ISchedule, 'id' | 'deletionSuspension'>>>();+ if (suspended.length === 0) {+ return;+ }+ const ops = suspended.map((schedule) => {+ const snapshot = schedule.deletionSuspension;+ const restoreNextRunAt = snapshot?.nextRunAt;+ return {+ updateOne: {+ filter: {+ id: schedule.id,+ user: userId,+ 'deletionSuspension.token': token,+ deleting: { $ne: true },+ },+ update: {+ $set: {+ enabled: snapshot?.enabled === true,+ claimToken: randomUUID(),+ ...(restoreNextRunAt != null ? { nextRunAt: restoreNextRunAt } : {}),+ },+ $unset: {+ deletionSuspension: 1 as const,+ ...(restoreNextRunAt == null ? { nextRunAt: 1 as const } : {}),+ },+ },+ },+ };+ });+ await tenantSafeBulkWrite(Schedule(), ops as AnyBulkWriteOperation[], { ordered: false });+ }++ /**+ * Enabled schedules with no `nextRunAt`. Creation arms in a second write, so a crash+ * or a failed arm leaves this state — and `claimDueSchedule` sorts on `nextRunAt`, so+ * the row is permanently inert while still occupying the owner's slot.+ */+ async function getUnarmedSchedules(limit: number): Promise<ISchedule[]> {+ return Schedule()+ .find({ enabled: true, deleting: { $ne: true }, nextRunAt: { $exists: false } })+ .limit(limit)+ .lean<ISchedule[]>();+ }++ /**+ * Arms an unarmed schedule — the ONE arming CAS every creation path shares (the+ * original POST, its idempotent replay, and the reconciler's unarmed sweep).+ * Conditional on still being unarmed, so concurrent armers collapse to a single+ * write: it never rotates the claim token (which would fence an occurrence the+ * engine already claimed off the freshly-armed row) and never bumps the config+ * revision (which would break a concurrent PATCH's CAS). `expectedConfigRevision`+ * additionally fences the original POST's arm against a PATCH that landed after+ * the insert — the edit's own arming governs then. Returns whether THIS call armed.+ */+ async function armSchedule(+ id: string,+ nextRunAt: Date,+ expectedConfigRevision?: number,+ ): Promise<boolean> {+ const result = await Schedule().updateOne(+ {+ id,+ enabled: true,+ deleting: { $ne: true },+ nextRunAt: { $exists: false },+ ...(expectedConfigRevision != null ? { configRevision: expectedConfigRevision } : {}),+ },+ { $set: { nextRunAt } },+ );+ return (result.modifiedCount ?? 0) > 0;+ }++ /** Soft-deleted schedules awaiting erasure (drained of active runs). ROUND-ROBIN+ * on the last sweep attempt (never-attempted first): an unsorted window re-served+ * the same first rows every pass, so a batch of undrainable rows (live runs or+ * leases) starved every row behind it out of the sweep indefinitely. */+ async function getDeletingSchedules(limit: number): Promise<ISchedule[]> {+ return Schedule()+ .find({ deleting: true, erased: { $ne: true } })+ .sort({ eraseAttemptedAt: 1 })+ .limit(limit)+ .lean<ISchedule[]>();+ }++ /** Stamps deleting rows as attempted so the erasure window rotates. Bookkeeping+ * only: never touches `updatedAt`. */+ async function markEraseAttempted(ids: string[]): Promise<void> {+ if (ids.length === 0) {+ return;+ }+ await Schedule().updateMany(+ { id: { $in: ids }, deleting: true },+ { $set: { eraseAttemptedAt: new Date() } },+ { timestamps: false },+ );+ }++ /** Resolves the schedule a prior create attempt with this idempotency key committed,+ * including one already soft-deleted (a retry must see its key as claimed while the+ * row drains, mirroring the unique index, which deliberately spans deleting rows). */+ async function getScheduleByClientRequestId(+ userId: string | Types.ObjectId,+ clientRequestId: string,+ ): Promise<ISchedule | null> {+ return Schedule().findOne({ user: userId, clientRequestId }).lean<ISchedule>();+ }++ /** Ids of an owner's soft-deleted schedules, for a lazy erase retry on a read path+ * in topologies that run no reconciler. Projected to ids: the caller only re-drives+ * eraseScheduleIfDrained, and the rows carry prompt text this need not load. */+ async function getDeletingScheduleIds(+ userId: string | Types.ObjectId,+ limit: number,+ ): Promise<string[]> {+ // Least-recently-attempted first (missing sorts before any date), for the same+ // reason the erasure sweep rotates: an unsorted `.limit()` window pins the same+ // stuck rows forever once the owner has more deleting rows than the limit, and+ // the ones beyond it never get their deletion re-driven. Callers stamp+ // markEraseAttempted after each attempt to rotate the window.+ const rows = await Schedule()+ .find({ user: userId, deleting: true, erased: { $ne: true } })+ .sort({ eraseAttemptedAt: 1 })+ .select('id')+ .limit(limit)+ .lean<Array<Pick<ISchedule, 'id'>>>();+ return rows.map((row) => row.id);+ }++ /**+ * Erases a soft-deleted schedule and its runs ONLY once it has fully drained, so a+ * live loopback generation's evidence is never destroyed out from under it. Drained+ * means BOTH: (a) no run is active, and (b) no LIVE lease is held. The lease check+ * is essential — a worker can have CLAIMED the schedule but not yet inserted its+ * `started` reservation (or be mid-rollback of one); erasing in that window would+ * let the worker then insert a ghost row against a gone schedule that it can no+ * longer prove it owns. Returns whether it erased.+ */+ /** The ONLY fields an erasure tombstone keeps: replay-detection identity plus the+ * flags that keep it inert to sweeps and resolvable by the create replay path. */+ const TOMBSTONE_IDENTITY_FIELDS = new Set([+ '_id',+ '__v',+ 'id',+ 'user',+ 'tenantId',+ 'clientRequestId',+ 'clientRequestDigest',+ 'deleting',+ 'erased',+ 'erasedAt',+ 'enabled',+ 'createdAt',+ 'updatedAt',+ ]);++ async function eraseScheduleIfDrained(id: string): Promise<boolean> {+ // A live lease (leaseUntil in the future) means a worker still holds the claim.+ // Worker clock, DocumentDB-portable: `$gt` matches neither missing nor null, so a+ // released lease reads as drained. Skew margin: a lease reads as live until+ // MARGIN past its expiry, so a clock-ahead erasure worker cannot destroy a row a+ // skew-behind holder still legitimately claims.+ const leased = await Schedule()+ .findOne({+ id,+ deleting: true,+ leaseUntil: { $gt: new Date(Date.now() - LEASE_SKEW_MARGIN_MS) },+ })+ .select('_id')+ .lean();+ if (leased != null) {+ return false;+ }+ const active = await ScheduleRun()+ .findOne({ scheduleId: id, status: { $in: ACTIVE_RUN_STATUSES } })+ .select('_id')+ .lean();+ if (active != null) {+ return false;+ }+ await ScheduleRun().deleteMany({ scheduleId: id });+ // Rows carrying a create-idempotency key leave a content-free TOMBSTONE instead+ // of vanishing: a create whose response was lost retries with the same key, and+ // if the owner deleted the schedule before that retry arrived, a hard delete+ // would let the retry recreate the recurring work they just removed. The+ // tombstone keeps only the identity fields (key, digest, owner) for a bounded+ // window (TTL on erasedAt); the replay path answers "deleted" against it.+ // ALLOWLIST, not a blocklist: everything except the replay-detection identity+ // is unset, derived from the live schema paths so a field added later (tools,+ // cron, anything) cannot silently survive into the tombstone. The deletion path+ // promises that ONLY the idempotency identity remains.+ const contentFields = [+ ...new Set(Object.keys(Schedule().schema.paths).map((path) => path.split('.')[0])),+ ].filter((field) => !TOMBSTONE_IDENTITY_FIELDS.has(field));+ const tombstoned = await Schedule().updateOne(+ { id, deleting: true, clientRequestId: { $exists: true } },+ {+ $set: { erased: true, erasedAt: new Date(), enabled: false },+ $unset: Object.fromEntries(contentFields.map((field) => [field, 1])),+ // Not a config edit; the tombstone must not surface in updated-time listings.+ },+ { timestamps: false },+ );+ if (tombstoned.matchedCount === 0) {+ await Schedule().deleteOne({ id, deleting: true });+ }+ return true;+ }++ /** Cascade for account deletion: removes a user's schedules and their runs. */+ async function deleteSchedulesByUser(userId: string | Types.ObjectId): Promise<void> {+ // Delete RUNS before SCHEDULES so a partial failure is retryable: both are+ // user-scoped and idempotent, so a crash after the runs delete leaves the+ // schedules for a retry to re-delete (deleting schedules first would orphan the+ // runs — a re-run finds no schedules and never removes the leftover run rows).+ await ScheduleRun().deleteMany({ user: userId });+ await Schedule().deleteMany({ user: userId });+ }++ return {+ ensureScheduleIndexes,+ createSchedule,+ createScheduleWithSlot,+ updateScheduleById,+ deleteScheduleById,+ deleteUnarmedSchedule,+ getScheduleById,+ getSchedulesByUser,+ countSchedulesByUser,+ claimDueSchedule,+ acquireManualRunLease,+ acquireResumeLease,+ consumeResumeLease,+ releaseLease,+ releaseLeaseByHolder,+ revalidateClaim,+ advanceSchedule,+ disableSchedule,+ insertScheduleRun,+ reserveStartedRun,+ getCapacityOccupancy,+ requestRunAbort,+ getScheduleRunAbortState,+ markRunResumeClaimed,+ releaseRunResumeClaim,+ markRunAbortPersisted,+ setRunFireDetails,+ countActiveRuns,+ deleteScheduleRun,+ markScheduleDeleting,+ getActiveRunsForSchedule,+ getActiveRunsForUser,+ suspendUserSchedulesForDeletion,+ restoreUserSchedulesFromDeletion,+ getDeletingSchedules,+ markEraseAttempted,+ getScheduleByClientRequestId,+ getDeletingScheduleIds,+ getUnarmedSchedules,+ armSchedule,+ eraseScheduleIfDrained,+ deleteSchedulesByUser,+ getUnbookkeptRuns,+ finalizeBookkeeping,+ recordRunOutcome,+ recordSkippedRun,+ getRunsForReconciliation,+ markRunsReconciled,+ };+}diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.tsindex 06d6a818f8..1da5a3eb3a 100644--- a/packages/data-schemas/src/models/index.ts+++ b/packages/data-schemas/src/models/index.ts@@ -1,4 +1,5 @@import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';+import { createScheduleModel, createScheduleRunModel } from './schedule';import { createSkillSyncCredentialModel } from './skillSyncCredential';import { createAgentTriggerUserPurgeModel } from './triggerUserPurge';import { createAgentTriggerDeliveryModel } from './triggerDelivery';@@ -84,6 +85,8 @@ export function createModels(mongoose: typeof import('mongoose')): {AgentTriggerDelivery: ReturnType<typeof createAgentTriggerDeliveryModel>;AgentTriggerLaneSequence: ReturnType<typeof createAgentTriggerLaneSequenceModel>;AgentTriggerUserPurge: ReturnType<typeof createAgentTriggerUserPurgeModel>;+ Schedule: ReturnType<typeof createScheduleModel>;+ ScheduleRun: ReturnType<typeof createScheduleRunModel>;} {const models = {User: createUserModel(mongoose),@@ -126,6 +129,8 @@ export function createModels(mongoose: typeof import('mongoose')): {AgentTriggerDelivery: createAgentTriggerDeliveryModel(mongoose),AgentTriggerLaneSequence: createAgentTriggerLaneSequenceModel(mongoose),AgentTriggerUserPurge: createAgentTriggerUserPurgeModel(mongoose),+ Schedule: createScheduleModel(mongoose),+ ScheduleRun: createScheduleRunModel(mongoose),};/*** Background index builds fail silently unless an 'index' listener isdiff --git a/packages/data-schemas/src/models/schedule.ts b/packages/data-schemas/src/models/schedule.tsnew file mode 100644index 0000000000..67672aa251--- /dev/null+++ b/packages/data-schemas/src/models/schedule.ts@@ -0,0 +1,20 @@+import { Model } from 'mongoose';+import type { IScheduleDocument, IScheduleRunDocument } from '~/types/schedule';+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';+import scheduleRunSchema from '~/schema/scheduleRun';+import scheduleSchema from '~/schema/schedule';++export function createScheduleModel(mongoose: typeof import('mongoose')): Model<IScheduleDocument> {+ applyTenantIsolation(scheduleSchema);+ return mongoose.models.Schedule || mongoose.model<IScheduleDocument>('Schedule', scheduleSchema);+}++export function createScheduleRunModel(+ mongoose: typeof import('mongoose'),+): Model<IScheduleRunDocument> {+ applyTenantIsolation(scheduleRunSchema);+ return (+ mongoose.models.ScheduleRun ||+ mongoose.model<IScheduleRunDocument>('ScheduleRun', scheduleRunSchema)+ );+}diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.tsindex d9171d0099..84aa7d4ec0 100644--- a/packages/data-schemas/src/schema/index.ts+++ b/packages/data-schemas/src/schema/index.ts@@ -35,3 +35,5 @@ export { default as configSchema } from './config';export { default as triggerDeliverySchema } from './triggerDelivery';export { default as triggerLaneSequenceSchema } from './triggerLaneSequence';export { default as triggerUserPurgeSchema } from './triggerUserPurge';+export { default as scheduleSchema } from './schedule';+export { default as scheduleRunSchema } from './scheduleRun';diff --git a/packages/data-schemas/src/schema/role.ts b/packages/data-schemas/src/schema/role.tsindex c931179631..9b55d7112c 100644--- a/packages/data-schemas/src/schema/role.ts+++ b/packages/data-schemas/src/schema/role.ts@@ -79,6 +79,10 @@ const rolePermissionsSchema = new Schema([Permissions.SHARE]: { type: Boolean },[Permissions.SHARE_PUBLIC]: { type: Boolean },},+ [PermissionTypes.SCHEDULES]: {+ [Permissions.USE]: { type: Boolean },+ [Permissions.CREATE]: { type: Boolean },+ },},{ _id: false },);diff --git a/packages/data-schemas/src/schema/schedule.ts b/packages/data-schemas/src/schema/schedule.tsnew file mode 100644index 0000000000..4fabb22065--- /dev/null+++ b/packages/data-schemas/src/schema/schedule.ts@@ -0,0 +1,252 @@+import { Schema } from 'mongoose';+import type { IScheduleDocument } from '~/types/schedule';++const scheduleSchema: Schema<IScheduleDocument> = new Schema(+ {+ id: {+ type: String,+ required: true,+ index: true,+ },+ user: {+ type: Schema.Types.ObjectId,+ ref: 'User',+ required: true,+ index: true,+ },+ tenantId: {+ type: String,+ index: true,+ },+ name: {+ type: String,+ required: true,+ maxlength: 256,+ },+ prompt: {+ type: String,+ required: true,+ maxlength: 32000,+ },+ agent_id: {+ type: String,+ required: true,+ },+ cadence: {+ frequency: {+ type: String,+ enum: ['hourly', 'daily', 'weekdays', 'weekly'],+ required: true,+ },+ hour: { type: Number, min: 0, max: 23, required: true },+ minute: { type: Number, min: 0, max: 59, required: true },+ daysOfWeek: { type: [Number], default: undefined },+ },+ timezone: {+ type: String,+ required: true,+ },+ target: {+ type: String,+ enum: ['new'],+ default: 'new',+ required: true,+ },+ file_ids: {+ type: [String],+ default: undefined,+ },+ tools: {+ type: [String],+ default: undefined,+ },+ cron: {+ type: String,+ },+ enabled: {+ type: Boolean,+ default: true,+ required: true,+ },+ disabledReason: {+ type: String,+ enum: [+ 'too_many_failures',+ 'agent_deleted',+ 'invalid_schedule',+ 'permission_revoked',+ 'insufficient_balance',+ ],+ },+ nextRunAt: {+ type: Date,+ },+ leaseUntil: {+ type: Date,+ },+ leaseBy: {+ type: String,+ },+ /**+ * Per-claim fencing token. Set fresh on every lease acquisition (engine or+ * manual) and rotated on every owner edit, so a stale/expired-lease worker's+ * writes (disable/advance/release) and its pre-dispatch revalidation no-op+ * once the schedule was re-claimed, edited, re-enabled, or deleted.+ */+ claimToken: {+ type: String,+ },+ /**+ * Owner-config generation. Bumped ONLY by an owner edit (updateScheduleById),+ * atomically with the claimToken rotation. Distinct from claimToken, which also+ * rotates on every lease acquisition. A run captures this at claim time so its+ * terminal bookkeeping / auto-disable cannot act on config it never ran under.+ */+ configRevision: {+ type: Number,+ default: 0,+ min: 0,+ },+ /**+ * Soft-delete marker. A delete disables + marks the schedule `deleting` (so+ * it is hidden from the owner and never re-claimed) and aborts in-flight+ * runs; the reconciler erases it only once no run is still active, so a live+ * loopback generation's evidence is never destroyed out from under it.+ * Defaults false so the per-user `slot` partial unique index covers it.+ */+ deleting: {+ type: Boolean,+ default: false,+ },+ /**+ * Reversible account-deletion suspension. Set at quiesce under a per-attempt token+ * that snapshots the pre-suspension enabled/next-run state. A deletion cancelled by a+ * controller failure restores exactly the rows carrying its token; a successful+ * deletion hard-deletes the row (and this snapshot with it). Distinct from `deleting`:+ * a suspended row is NOT eligible for erasure, so a failed attempt cannot lose it.+ */+ deletionSuspension: {+ type: {+ token: { type: String, required: true },+ enabled: { type: Boolean, required: true },+ nextRunAt: { type: Date },+ },+ default: undefined,+ _id: false,+ },+ /** Erased tombstone: content is gone, only the create-idempotency identity+ * remains for a bounded retry window (TTL below). A delayed create retry that+ * matches this key must answer "deleted", not resurrect the recurring work. */+ erased: {+ type: Boolean,+ },+ erasedAt: {+ type: Date,+ },+ /**+ * Per-user occupancy slot in [0, maxPerUser). Assigned atomically via the+ * partial unique index below so concurrent creates cannot exceed the cap: two+ * racers claiming the same slot collide on the index and one retries the next+ * free slot. Freed (excluded from the index) when the schedule is `deleting`.+ */+ slot: {+ type: Number,+ min: 0,+ },+ /** Client-supplied idempotency key for the CREATE that produced this row, unique+ * per user via the partial index below. Creation commits the row and arms it in+ * two writes, so a retry after a failure between them would otherwise mint a+ * second recurring schedule; the index makes that retry collide and resolve to+ * this row instead. */+ clientRequestId: {+ type: String,+ },+ /** Digest of the original create payload, stamped at insert and never edited, so a+ * retry is matched against what was actually requested rather than a row a+ * concurrent PATCH may have already reshaped. */+ clientRequestDigest: {+ type: String,+ },+ /** Last erasure-sweep attempt; rotates the `deleting` window (see eraseAttemptedAt+ * ordering in getDeletingSchedules) so undrainable rows can't starve later ones. */+ eraseAttemptedAt: {+ type: Date,+ },+ lastRun: {+ type: {+ conversationId: { type: String },+ status: { type: String, required: true },+ error: { type: String },+ firedAt: { type: Date, required: true },+ scheduledFor: { type: Date },+ },+ default: undefined,+ _id: false,+ },+ runCount: {+ type: Number,+ default: 0,+ min: 0,+ },+ /**+ * Bounded set of recently-counted occurrence timestamps. Per-occurrence+ * idempotency guard for `recordRunOutcome` counter increments: a single+ * scalar could be moved by an interleaved earlier occurrence, letting the+ * reconciler double-count a crashed later one. Kept bounded via `$slice`.+ */+ countedFor: {+ type: [Date],+ default: undefined,+ },+ /** Newest occurrence whose outcome has been applied to the streak counters.+ * `countedFor` makes counting idempotent per occurrence; this makes it ORDERED,+ * so an older occurrence settling late cannot rebuild a streak a newer one+ * cleared. Absent on rows written before it existed (fence disabled). */+ countersAsOf: {+ type: Date,+ },+ failureCount: {+ type: Number,+ default: 0,+ min: 0,+ },+ balanceSkipCount: {+ type: Number,+ default: 0,+ min: 0,+ },+ },+ {+ timestamps: true,+ },+);++// Idempotency tombstones expire after the bounded retry window; live rows never match.+scheduleSchema.index(+ { erasedAt: 1 },+ { expireAfterSeconds: 24 * 60 * 60, partialFilterExpression: { erased: true } },+);+scheduleSchema.index({ id: 1, tenantId: 1 }, { unique: true });+scheduleSchema.index({ enabled: 1, nextRunAt: 1 });+// Atomic per-user create cap: a live (non-deleting) schedule occupies a unique+// slot, so concurrent creates cannot collectively exceed maxPerUser — a second+// claimant of the same slot collides and retries the next free one. Scoped to+// slot-bearing docs so the (non-user-facing) slotless create path is unaffected.+scheduleSchema.index(+ { user: 1, slot: 1 },+ { unique: true, partialFilterExpression: { deleting: false, slot: { $exists: true } } },+);++// Makes a client's create RETRY idempotent: the second attempt collides here instead of+// committing a second recurring schedule. Deliberately NOT filtered on `deleting` —+// a key must stay claimed while its row is being erased, or a retry arriving mid-erase+// would create a duplicate that outlives it.+scheduleSchema.index(+ { user: 1, clientRequestId: 1 },+ { unique: true, partialFilterExpression: { clientRequestId: { $exists: true } } },+);++// Erasure sweeps read `deleting: true` ordered by last attempt (rotation).+scheduleSchema.index({ deleting: 1, eraseAttemptedAt: 1 });++export default scheduleSchema;diff --git a/packages/data-schemas/src/schema/scheduleRun.ts b/packages/data-schemas/src/schema/scheduleRun.tsnew file mode 100644index 0000000000..cf5e51a9a0--- /dev/null+++ b/packages/data-schemas/src/schema/scheduleRun.ts@@ -0,0 +1,154 @@+import { Schema } from 'mongoose';+import type { IScheduleRunDocument } from '~/types/schedule';++const SCHEDULE_RUN_TTL_SECONDS = 90 * 24 * 60 * 60;++const scheduleRunSchema: Schema<IScheduleRunDocument> = new Schema(+ {+ scheduleId: {+ type: String,+ required: true,+ },+ user: {+ type: Schema.Types.ObjectId,+ ref: 'User',+ required: true,+ index: true,+ },+ tenantId: {+ type: String,+ index: true,+ },+ scheduledFor: {+ type: Date,+ required: true,+ },+ firedAt: {+ type: Date,+ },+ conversationId: {+ type: String,+ },+ /** Deterministic durable-trigger delivery key for this occurrence, stamped at+ * reservation before enqueue so reconciliation can read the delivery's live/dead+ * state rather than orphan-settling a deferred or dead-lettered run. */+ deliveryKey: {+ type: String,+ },+ /** Fresh while a RESUME of this paused run is mid-flight; a re-pause hand-off's+ * writes are still landing, so quiesce must not settle on the paused job state.+ * Cleared by the pause record (the hand-off completion) or aged out. */+ resumeClaimedAt: {+ type: Date,+ },+ status: {+ type: String,+ enum: [+ 'started',+ 'requires_action',+ 'success',+ 'error',+ 'interrupted',+ 'skipped_overlap',+ 'skipped_balance',+ ],+ required: true,+ },+ error: {+ type: String,+ maxlength: 2048,+ },+ droppedFileIds: {+ type: [String],+ default: undefined,+ },+ durationMs: {+ type: Number,+ min: 0,+ },+ /** False on a terminal run whose schedule bookkeeping hasn't landed yet (crash-retry marker). */+ bookkept: {+ type: Boolean,+ },+ /** Set ONLY when the row reaches a terminal status; the TTL index below expires on+ * this field, so live rows (which never carry it) never expire. */+ settledAt: {+ type: Date,+ },+ /** Global concurrency slot held while `started`. The unique partial index below+ * turns fireConcurrency into a DB-enforced bound instead of a racy count. */+ capacitySlot: {+ type: Number,+ min: 0,+ },+ /** When an abort was requested. The run keeps holding its capacity slot until the+ * generation owner confirms settlement, so capacity is never freed early. */+ abortRequestedAt: {+ type: Date,+ },+ /** Who requested the abort ('stop' = interactive route, 'deletion' = delete/quiesce). */+ abortSource: {+ type: String,+ enum: ['stop', 'deletion'],+ },+ /** Stamped once the interactive Stop route has persisted everything it writes;+ * the generation owner defers settlement until this appears (bounded). */+ abortPersistedAt: {+ type: Date,+ },+ /** When reconciliation last examined this row. Orders the paused window so a full+ * batch of still-live pauses cannot starve an abandoned row behind them. */+ reconciledAt: {+ type: Date,+ },+ /** The schedule's configRevision at claim time. Fences terminal bookkeeping and+ * auto-disable from owner edits/re-enables that landed after this run started. */+ configRevision: {+ type: Number,+ min: 0,+ },+ },+ {+ timestamps: true,+ },+);++scheduleRunSchema.index({ scheduleId: 1, scheduledFor: 1 }, { unique: true });+// TTL retention applies only to SETTLED rows: an approval window configured longer+// than the retention period would otherwise let Mongo expire a still-live+// `requires_action` run — its retained job could then resume with no run row (no+// bookkeeping), and deletion quiescing could no longer discover or abort it.+// Expressed as a PLAIN TTL index on `settledAt` (stamped only by terminal writes):+// TTL expiry skips documents that lack the indexed date, so live rows never expire+// without needing a partialFilterExpression — whose `$in` predicate Amazon+// DocumentDB 5.0 cannot build (misc/documentdb/documentdb-compat.md), which would+// leave initializeScheduleEngine refusing to arm on that target.+scheduleRunSchema.index({ settledAt: 1 }, { expireAfterSeconds: SCHEDULE_RUN_TTL_SECONDS });+// At most ONE active (`started`) run per schedule, enforced by the DB rather than a+// read-then-write check: a second occurrence inserting while one is already active+// fails with a duplicate-key error instead of racing.+scheduleRunSchema.index(+ { scheduleId: 1 },+ { unique: true, partialFilterExpression: { status: 'started' } },+);+// GLOBAL fireConcurrency, enforced by the DB rather than a read-then-compare count.+// A fire claims a slot in [0, fireConcurrency) in the SAME write that inserts the run;+// a duplicate slot is rejected atomically, so two concurrent admissions of DIFFERENT+// schedules can never both pass a cap-1 check.+// Partial + $exists so legacy slotless rows (written before this field) never collide.+scheduleRunSchema.index(+ { capacitySlot: 1 },+ {+ unique: true,+ partialFilterExpression: { status: 'started', capacitySlot: { $exists: true } },+ },+);+scheduleRunSchema.index({ scheduleId: 1, firedAt: -1 });+// Reconciliation sweeps by status; keeps `started` (capacity) fetch cheap and+// prevents long-lived `requires_action` rows from starving the scan.+scheduleRunSchema.index({ status: 1, firedAt: 1 });+// The paused reconciliation window sorts on {reconciledAt, firedAt} within a status;+// without this the round-robin rotation re-sorts the whole paused set every tick.+scheduleRunSchema.index({ status: 1, reconciledAt: 1, firedAt: 1 });++export default scheduleRunSchema;diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.tsindex cecdab0a50..c3c9aecd61 100644--- a/packages/data-schemas/src/types/index.ts+++ b/packages/data-schemas/src/types/index.ts@@ -30,6 +30,7 @@ export * from './prompts';export * from './skill';export * from './skillSync';export * from './triggerDelivery';+export * from './schedule';/* Access Control */export * from './accessRole';export * from './aclEntry';diff --git a/packages/data-schemas/src/types/role.ts b/packages/data-schemas/src/types/role.tsindex 2f6b6c813c..9a5807013d 100644--- a/packages/data-schemas/src/types/role.ts+++ b/packages/data-schemas/src/types/role.ts@@ -77,6 +77,10 @@ export interface IRole extends Document {[Permissions.SHARE]?: boolean;[Permissions.SHARE_PUBLIC]?: boolean;};+ [PermissionTypes.SCHEDULES]?: {+ [Permissions.USE]?: boolean;+ [Permissions.CREATE]?: boolean;+ };};tenantId?: string;}diff --git a/packages/data-schemas/src/types/schedule.ts b/packages/data-schemas/src/types/schedule.tsnew file mode 100644index 0000000000..e806a82a69--- /dev/null+++ b/packages/data-schemas/src/types/schedule.ts@@ -0,0 +1,118 @@+import type {+ ScheduleRunStatus,+ ScheduleDisabledReason,+ TScheduleCadence,+} from 'librechat-data-provider';+import type { Document, Types } from 'mongoose';++export interface ISchedule {+ _id?: Types.ObjectId;+ id: string;+ user: Types.ObjectId;+ tenantId?: string;+ name: string;+ prompt: string;+ agent_id: string;+ cadence: TScheduleCadence;+ timezone: string;+ target: 'new';+ file_ids?: string[];+ tools?: string[];+ cron?: string;+ enabled: boolean;+ disabledReason?: ScheduleDisabledReason;+ nextRunAt?: Date;+ leaseUntil?: Date;+ leaseBy?: string;+ claimToken?: string;+ /** Owner-config generation; bumped only by an owner edit. */+ configRevision?: number;+ deleting?: boolean;+ /** Reversible account-deletion suspension. Set at quiesce under a per-attempt token+ * that snapshots the pre-suspension enabled/next-run state, so a deletion that is+ * later cancelled restores exactly this row (fenced to its token) instead of leaving+ * a live user with silently disabled schedules. Distinct from `deleting`, which marks+ * a row for erasure. */+ deletionSuspension?: {+ token: string;+ enabled: boolean;+ nextRunAt?: Date;+ };+ erased?: boolean;+ erasedAt?: Date;+ slot?: number;+ /** Client-supplied idempotency key of the create that produced this row. */+ clientRequestId?: string;+ /** Digest of the ORIGINAL create payload, stamped at insert and never edited.+ * Replay matching compares against this rather than mutable schedule state, so a+ * PATCH landing between the first attempt and its retry cannot fail the retry. */+ clientRequestDigest?: string;+ /** When an erasure sweep last attempted this soft-deleted row; orders the sweep+ * window so undrainable rows cannot starve the ones behind them. */+ eraseAttemptedAt?: Date;+ lastRun?: {+ conversationId?: string;+ status: ScheduleRunStatus;+ error?: string;+ firedAt: Date;+ /** The OCCURRENCE this projection came from; orders the card against delayed+ * outcomes (a resumed pause, a reconciler replay) arriving after a newer run. */+ scheduledFor?: Date;+ };+ runCount: number;+ failureCount: number;+ balanceSkipCount: number;+ countedFor?: Date[];+ /** Newest occurrence applied to the streak counters; orders them like the card. */+ countersAsOf?: Date;+ createdAt?: Date;+ updatedAt?: Date;+}++export interface IScheduleDocument extends Omit<ISchedule, 'id' | '_id'>, Document {+ id: string;+}++export interface IScheduleRun {+ _id?: Types.ObjectId;+ scheduleId: string;+ user: Types.ObjectId;+ tenantId?: string;+ scheduledFor: Date;+ firedAt?: Date;+ conversationId?: string;+ status: ScheduleRunStatus;+ error?: string;+ /** Deterministic durable-trigger delivery key for this occurrence, stamped at+ * reservation (before enqueue). Lets reconciliation read the delivery's live/dead+ * state instead of orphan-settling a jobless run that is merely deferred (Retry-After)+ * or that dead-lettered before a generation ever started. */+ deliveryKey?: string;+ droppedFileIds?: string[];+ durationMs?: number;+ bookkept?: boolean;+ /** Set only by terminal writes; the retention TTL index expires on this field. */+ settledAt?: Date;+ /** Global concurrency slot held while `started`. */+ capacitySlot?: number;+ /** When an abort was requested; capacity is held until settlement is confirmed. */+ abortRequestedAt?: Date;+ /** Who requested the abort: the interactive Stop route ('stop', which persists a+ * partial response before the run may settle) or a deletion path ('deletion'). */+ abortSource?: 'stop' | 'deletion';+ /** Stamped by the interactive Stop route once every write it makes (checkpoint+ * prune, partial-response save) has landed; the generation owner defers its+ * terminal settlement until this appears so the run can never leave the active+ * set while the route is still persisting. */+ abortPersistedAt?: Date;+ /** The schedule's configRevision at claim time. */+ configRevision?: number;+ /** When reconciliation last examined this row; orders the paused window so no row+ * can be starved by a full batch of still-live pauses ahead of it. */+ reconciledAt?: Date;+ resumeClaimedAt?: Date;+ createdAt?: Date;+ updatedAt?: Date;+}++export interface IScheduleRunDocument extends Omit<IScheduleRun, '_id'>, Document {}