Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

⏱️ feat: Run Scheduled Chats Through Durable Agent Triggers (#14939)

Danny Avilac5276fc63d26
diff --git a/.env.example b/.env.example
index 7091d3f9ad..22b4265882 100644
--- a/.env.example
+++ b/.env.example
@@ -1059,6 +1059,11 @@ HELP_AND_FAQ_URL=https://librechat.ai
# Enable Redis for resumable LLM streams (defaults to USE_REDIS value if not set)
# Set to false to use in-memory storage for streams while keeping Redis for other caches
# USE_REDIS_STREAMS=true
+# Scheduled chats require shared Redis streams in multi-replica deployments.
+# Set this only when the deployment truly runs one LibreChat process without Redis.
+# SCHEDULES_SINGLE_PROCESS=true
+# Emergency global stop for both automatic and manual scheduled runs.
+# SCHEDULES_DISABLED=true
# Generation stream wire/state protocol. Redis-backed deployments default to the
# rolling-upgrade-safe v1 protocol when this is unset; in-memory deployments use v2.
diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js
index 9821f47db7..040b2ae21b 100644
--- a/api/server/controllers/UserController.js
+++ b/api/server/controllers/UserController.js
@@ -33,6 +33,11 @@ const {
purgeAgentTriggerDeliveriesForUser,
} = require('~/server/services/Agents/triggers');
const { getAppConfig } = require('~/server/services/Config');
+const { randomUUID } = require('node:crypto');
+const {
+ quiesceUserSchedules,
+ restoreUserSchedulesFromDeletion,
+} = require('~/server/services/Schedules');
const { getLogStores } = require('~/cache');
const db = require('~/models');
@@ -360,6 +365,7 @@ const updateUserPluginsController = async (req, res) => {
const deleteUserController = async (req, res) => {
const { user } = req;
let triggerDeletionFence;
+ let scheduleSuspensionToken;
let userDeleted = false;
try {
@@ -395,6 +401,14 @@ const deleteUserController = async (req, res) => {
}
await drainAgentTriggerDeliveriesForUser(user.id);
await subagentThreadTaskStore.cancelAndDrainForOwner(user.id, user.tenantId);
+ // Reversibly suspend the user's schedules under a per-attempt token BEFORE draining.
+ // A later cascade step (or this drain) can still fail and cancel the deletion, and the
+ // catch below restores exactly this attempt's rows — so a failed deletion never leaves
+ // a live user with silently disabled, erasure-eligible schedules.
+ scheduleSuspensionToken = randomUUID();
+ if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) {
+ throw new Error('Scheduled executions could not be confirmed stopped');
+ }
const activeAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(
user.id,
user.tenantId,
@@ -446,6 +460,7 @@ const deleteUserController = async (req, res) => {
await db.deleteTokens({ userId: user.id });
await db.removeUserFromAllGroups(user.id);
await db.deleteAclEntries({ principalId: user._id });
+ await db.deleteSchedulesByUser(user.id);
const deleteResult = await db.deleteUserById(user.id);
if (deleteResult.deletedCount !== 1) {
throw new Error('User disappeared before account deletion could commit');
@@ -455,6 +470,33 @@ const deleteUserController = async (req, res) => {
logger.info(`User deleted account. Email: ${user.email} ID: ${user.id}`);
res.status(200).send({ message: 'User deleted' });
} catch (err) {
+ // The account survives this failed attempt, so its schedules must too: restore the
+ // exact rows this attempt suspended (re-enabled/re-armed from their snapshot). Fenced
+ // to the token, so a schedule the owner deleted meanwhile is not resurrected. A
+ // successful deletion never reaches here (userDeleted short-circuits it).
+ //
+ // RESTORE BEFORE RELEASING THE DELETION FENCE. That fence is what refuses new schedule
+ // writes/claims for this user; releasing it first opens a window where an owner PATCH
+ // could edit a still-suspended row and then have its enabled/next-run state overwritten
+ // by this older snapshot, and where a second deletion attempt could re-suspend these
+ // rows under a new token — making this restore a no-op and stranding the disabled
+ // snapshot permanently.
+ if (scheduleSuspensionToken != null && !userDeleted) {
+ try {
+ await restoreUserSchedulesFromDeletion(user.id, scheduleSuspensionToken);
+ } catch (restoreError) {
+ // Every retry is exhausted at this point. The fence is still released below on
+ // purpose: retaining it would refuse this live account's schedule writes AND make
+ // `beginAgentTriggerUserDeletion` report `in_progress` forever, blocking the retry
+ // that is the convergence path — a later attempt re-suspends by ADOPTING this
+ // snapshot, so its cancel restores these exact rows. Log the token so the state is
+ // recoverable directly if that never happens.
+ logger.error(
+ `[deleteUserController] Failed to restore suspended schedules after a cancelled deletion; they remain disabled for user ${user.id} under suspension token ${scheduleSuspensionToken}`,
+ restoreError,
+ );
+ }
+ }
if (triggerDeletionFence != null && !userDeleted) {
try {
await cancelAgentTriggerUserPurge(user.id, triggerDeletionFence);
diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.js
index c9a3f6eb44..89af705a56 100644
--- a/api/server/controllers/UserController.spec.js
+++ b/api/server/controllers/UserController.spec.js
@@ -8,6 +8,8 @@ const mockPrepareAgentTriggerUserPurge = jest.fn().mockResolvedValue(undefined);
const mockCancelAgentTriggerUserPurge = jest.fn().mockResolvedValue(true);
const mockPurgeAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);
const mockCancelAndDrainSubagentThreads = jest.fn().mockResolvedValue(undefined);
+const mockQuiesceUserSchedules = jest.fn().mockResolvedValue(true);
+const mockRestoreUserSchedules = jest.fn().mockResolvedValue(undefined);
jest.mock('@librechat/data-schemas', () => {
const actual = jest.requireActual('@librechat/data-schemas');
@@ -30,6 +32,7 @@ jest.mock('~/models', () => {
deleteAllAgentApiKeys: jest.fn().mockResolvedValue(undefined),
deleteConversationTags: jest.fn().mockResolvedValue(undefined),
deleteAllUserMemories: jest.fn().mockResolvedValue(undefined),
+ deleteSchedulesByUser: jest.fn().mockResolvedValue(undefined),
deleteTransactions: jest.fn().mockResolvedValue(undefined),
deleteAclEntries: jest.fn().mockResolvedValue(undefined),
updateUserPlugins: jest.fn(),
@@ -100,6 +103,11 @@ jest.mock('~/server/services/Endpoints/agents/subagentThreadStore', () => ({
cancelAndDrainForOwner: (...args) => mockCancelAndDrainSubagentThreads(...args),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ quiesceUserSchedules: (...args) => mockQuiesceUserSchedules(...args),
+ restoreUserSchedulesFromDeletion: (...args) => mockRestoreUserSchedules(...args),
+}));
+
jest.mock('~/server/services/Files/process', () => ({
processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),
}));
@@ -160,6 +168,7 @@ describe('verifyEmailController', () => {
beforeEach(() => {
jest.clearAllMocks();
+ mockQuiesceUserSchedules.mockResolvedValue(true);
});
it('returns the generic verification error message from service failures', async () => {
@@ -343,6 +352,7 @@ describe('deleteUserController', () => {
beforeEach(() => {
jest.clearAllMocks();
+ mockQuiesceUserSchedules.mockResolvedValue(true);
});
it('should return 200 on successful deletion', async () => {
@@ -361,6 +371,7 @@ describe('deleteUserController', () => {
);
expect(mockDrainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());
expect(mockCancelAndDrainSubagentThreads).toHaveBeenCalledWith(userId.toString(), undefined);
+ expect(mockQuiesceUserSchedules).toHaveBeenCalledWith(userId.toString(), expect.any(String));
expect(beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(
mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],
);
@@ -371,6 +382,9 @@ describe('deleteUserController', () => {
mockCancelAndDrainSubagentThreads.mock.invocationCallOrder[0],
);
expect(mockCancelAndDrainSubagentThreads.mock.invocationCallOrder[0]).toBeLessThan(
+ mockQuiesceUserSchedules.mock.invocationCallOrder[0],
+ );
+ expect(mockQuiesceUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(
deleteMessages.mock.invocationCallOrder[0],
);
expect(deleteMessages.mock.invocationCallOrder[0]).toBeLessThan(
@@ -382,6 +396,8 @@ describe('deleteUserController', () => {
expect(mockPurgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());
expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();
+ // A successful deletion hard-deletes the schedules; it must never restore them.
+ expect(mockRestoreUserSchedules).not.toHaveBeenCalled();
});
it('aborts generations admitted before the deletion fence before erasing messages', async () => {
@@ -428,6 +444,17 @@ describe('deleteUserController', () => {
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
expect(deleteUserById).not.toHaveBeenCalled();
+ // Account survives -> its suspended schedules are restored under the quiesce token.
+ expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
+ userIdString,
+ mockQuiesceUserSchedules.mock.calls[0][1],
+ );
+ // BEFORE the deletion fence is released: that fence is what refuses new schedule writes,
+ // so restoring after it would let an owner PATCH — or a second deletion attempt
+ // re-suspending under a new token — race the restore and strand the disabled snapshot.
+ expect(mockRestoreUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(
+ cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],
+ );
});
it('fails closed before data cleanup when detached subagents do not drain', async () => {
@@ -453,6 +480,41 @@ describe('deleteUserController', () => {
expect(deleteUserById).not.toHaveBeenCalled();
});
+ it('fails closed and releases deletion fences when schedules cannot be quiesced', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const userIdString = userId.toString();
+ mockQuiesceUserSchedules.mockResolvedValueOnce(false);
+ const req = {
+ user: {
+ id: userIdString,
+ _id: userId,
+ email: '[email protected]',
+ tenantId: 'tenant-1',
+ },
+ };
+
+ await deleteUserController(req, mockRes);
+
+ expect(mockRes.status).toHaveBeenCalledWith(500);
+ expect(deleteMessages).not.toHaveBeenCalled();
+ expect(mockGetActiveJobIdsForUser).not.toHaveBeenCalled();
+ const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
+ expect(deleteUserById).not.toHaveBeenCalled();
+ // Account survives -> its suspended schedules are restored under the quiesce token.
+ expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
+ userIdString,
+ mockQuiesceUserSchedules.mock.calls[0][1],
+ );
+ // BEFORE the deletion fence is released: that fence is what refuses new schedule writes,
+ // so restoring after it would let an owner PATCH — or a second deletion attempt
+ // re-suspending under a new token — race the restore and strand the disabled snapshot.
+ expect(mockRestoreUserSchedules.mock.invocationCallOrder[0]).toBeLessThan(
+ cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],
+ );
+ });
+
it('should remove the user from all groups via $pullAll', async () => {
const userId = new mongoose.Types.ObjectId();
const userIdStr = userId.toString();
diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js
index f6777172a6..c64a40de8d 100644
--- a/api/server/controllers/__tests__/deleteUser.spec.js
+++ b/api/server/controllers/__tests__/deleteUser.spec.js
@@ -28,6 +28,8 @@ const mockPurgeAgentTriggerDeliveriesForUser = jest.fn();
const mockBeginAgentTriggerUserDeletion = jest.fn();
const mockCancelAgentTriggerUserDeletion = jest.fn();
const mockCancelAndDrainSubagentThreads = jest.fn();
+const mockQuiesceUserSchedules = jest.fn();
+const mockDeleteSchedulesByUser = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
logger: { error: jest.fn(), info: jest.fn() },
@@ -85,6 +87,7 @@ jest.mock('~/models', () => ({
deleteTokens: jest.fn(),
removeUserFromAllGroups: jest.fn(),
deleteAclEntries: jest.fn(),
+ deleteSchedulesByUser: (...args) => mockDeleteSchedulesByUser(...args),
getSoleOwnedResourceIds: jest.fn().mockResolvedValue([]),
}));
@@ -127,6 +130,10 @@ jest.mock('~/server/services/Endpoints/agents/subagentThreadStore', () => ({
cancelAndDrainForOwner: (...args) => mockCancelAndDrainSubagentThreads(...args),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ quiesceUserSchedules: (...args) => mockQuiesceUserSchedules(...args),
+}));
+
jest.mock('~/server/services/Config', () => ({
getAppConfig: jest.fn(),
}));
@@ -171,6 +178,8 @@ function stubDeletionMocks() {
mockBeginAgentTriggerUserDeletion.mockResolvedValue('acquired');
mockCancelAgentTriggerUserDeletion.mockResolvedValue(true);
mockCancelAndDrainSubagentThreads.mockResolvedValue();
+ mockQuiesceUserSchedules.mockResolvedValue(true);
+ mockDeleteSchedulesByUser.mockResolvedValue();
}
beforeEach(() => {
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
index c2f1173257..7428c2287f 100644
--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
@@ -47,6 +47,8 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
sendEvent: jest.fn(),
+ isScheduleFireRequest: jest.fn(() => false),
+ exemptFromConcurrencyLimiter: jest.fn(() => false),
toPendingSteer: jest.fn((item) => item),
isSteerPreemptSupported: jest.fn(() => true),
buildRecoveredSteerPayload: jest.fn(() => null),
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index e659242977..5b6fb7dab6 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -64,6 +64,10 @@ const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockSaveMessage = jest.fn();
const mockIsAgentTriggerPrincipalActive = jest.fn();
+const mockIsScheduleFireRequest = jest.fn();
+const mockExemptFromConcurrencyLimiter = jest.fn();
+const mockRecordScheduleOutcome = jest.fn();
+const mockIsScheduleLive = jest.fn();
const mockDeleteAgentCheckpoint = jest.fn();
const mockStartupTelemetry = {
mark: jest.fn(),
@@ -132,6 +136,8 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
sendEvent: jest.fn(),
+ isScheduleFireRequest: (...args) => mockIsScheduleFireRequest(...args),
+ exemptFromConcurrencyLimiter: (...args) => mockExemptFromConcurrencyLimiter(...args),
toPendingSteer: jest.fn((item) => item),
/** Recorded onto the job so the steer route can honour the OWNING replica's
* seal capability rather than its own probe. */
@@ -214,6 +220,11 @@ jest.mock('~/models', () => ({
isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),
+ isScheduleLive: (...args) => mockIsScheduleLive(...args),
+}));
+
const AgentController = require('../request');
const { ErrorTypes } = require('librechat-data-provider');
const { disposeClient: mockDisposeClient } = require('~/server/cleanup');
@@ -250,6 +261,12 @@ describe('ResumableAgentController resume metadata', () => {
mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });
mockGetMessages.mockResolvedValue([]);
mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);
+ mockIsScheduleFireRequest.mockImplementation((req) => req?._isScheduledFire === true);
+ mockExemptFromConcurrencyLimiter.mockImplementation(
+ (req) => req?._isScheduledFire === true && req?._isManualScheduledFire !== true,
+ );
+ mockRecordScheduleOutcome.mockResolvedValue(true);
+ mockIsScheduleLive.mockResolvedValue(true);
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
metadata: {
@@ -286,7 +303,7 @@ describe('ResumableAgentController resume metadata', () => {
}),
);
mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
- mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.completeJob.mockResolvedValue(true);
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
@@ -797,6 +814,73 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
});
+ it('rejects a superseded automatic occurrence after durable job creation and preserves its outcome', async () => {
+ const conversationId = 'scheduled-conversation-123';
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(
+ wonGenerationClaim({ streamId: conversationId, conversationId }),
+ );
+ mockIsScheduleLive.mockResolvedValue(false);
+ const req = {
+ _isScheduledFire: true,
+ _isManualScheduledFire: false,
+ user: { id: 'user-123' },
+ body: {
+ text: 'Run the scheduled digest.',
+ messageId: 'scheduled-user-message',
+ clientRequestId: 'sched:schedule-1:2026-08-17T12:00:00-000Z',
+ conversationId: 'new',
+ newConversationId: conversationId,
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleConfigRevision: 7,
+ endpointOption: { endpoint: 'agents', agent_id: 'agent-1' },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+ const initializeClient = jest.fn();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ conversationId,
+ 'user-123',
+ conversationId,
+ expect.objectContaining({
+ initialMetadata: expect.objectContaining({
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleConfigRevision: 7,
+ preserveForScheduleReconcile: true,
+ }),
+ }),
+ );
+ expect(mockIsScheduleLive).toHaveBeenCalledWith('schedule-1', 7, {
+ automatic: true,
+ policy: true,
+ });
+ expect(initializeClient).not.toHaveBeenCalled();
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ status: 409,
+ code: 'SCHEDULE_NO_LONGER_ACTIVE',
+ error: 'This scheduled occurrence is no longer active',
+ generationProtocolVersion: 1,
+ });
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ streamId: conversationId,
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId,
+ clearConversationId: false,
+ error: 'This scheduled occurrence is no longer active',
+ });
+ expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
+ });
+
it('does not start a provider when account deletion or replacement wins the startup CAS', async () => {
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(false);
const req = {
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index eb30808054..034c53189e 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -60,6 +60,7 @@ const mockGenerationJobManager = {
publishTerminalClaim: jest.fn(),
finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
+ abortJob: jest.fn(),
beginProviderExecution: jest.fn(),
markProviderExecutionDrained: jest.fn(),
failPausePersistence: jest.fn(),
@@ -82,6 +83,12 @@ const mockGetMessages = jest.fn();
const mockDisposeClient = jest.fn();
const mockGetMCPRequestContext = jest.fn();
const mockCleanupMCPRequestContextForReq = jest.fn();
+const mockRecordScheduleOutcome = jest.fn();
+const mockIsScheduleLive = jest.fn();
+const mockClaimScheduleResume = jest.fn();
+const mockReleaseScheduleResumeClaim = jest.fn();
+const mockFinalizeScheduleResumeClaim = jest.fn();
+const mockReleaseScheduleResumeFence = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
@@ -104,6 +111,15 @@ jest.mock('~/models', () => ({
getMessages: (...args) => mockGetMessages(...args),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),
+ claimScheduleResume: (...args) => mockClaimScheduleResume(...args),
+ releaseScheduleResumeClaim: (...args) => mockReleaseScheduleResumeClaim(...args),
+ finalizeScheduleResumeClaim: (...args) => mockFinalizeScheduleResumeClaim(...args),
+ releaseScheduleResumeFence: (...args) => mockReleaseScheduleResumeFence(...args),
+ isScheduleLive: (...args) => mockIsScheduleLive(...args),
+}));
+
jest.mock('~/server/cleanup', () => ({
disposeClient: (...args) => mockDisposeClient(...args),
}));
@@ -248,12 +264,23 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
);
mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(true);
+ mockGenerationJobManager.abortJob.mockResolvedValue({ success: true });
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);
mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);
+ mockRecordScheduleOutcome.mockResolvedValue(true);
+ mockIsScheduleLive.mockResolvedValue(true);
+ mockClaimScheduleResume.mockResolvedValue({
+ capacitySlot: 0,
+ claimToken: 'resume-token',
+ leaseBy: 'resume:resume-token',
+ });
+ mockReleaseScheduleResumeClaim.mockResolvedValue(true);
+ mockFinalizeScheduleResumeClaim.mockResolvedValue(true);
+ mockReleaseScheduleResumeFence.mockResolvedValue(undefined);
// `decrementPendingRequest` runs in the controller's `finally` on every
// post-ACK path, so resolving on it signals the async continuation is done.
@@ -304,6 +331,222 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
...extra,
});
+ describe('scheduled occurrence lifecycle', () => {
+ const scheduledFor = '2026-08-17T12:00:00.000Z';
+ const makeScheduledJob = () =>
+ makeToolApprovalJob({
+ metadata: {
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ scheduleConfigRevision: 4,
+ checkpointNamespace: '1000',
+ },
+ });
+
+ it('stops and settles an occurrence that became inactive while awaiting approval', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockIsScheduleLive.mockResolvedValue(false);
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(409);
+ expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });
+ expect(mockIsScheduleLive).toHaveBeenCalledWith('schedule-1', 4, {
+ automatic: true,
+ policy: true,
+ });
+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {
+ expectedCreatedAt: 1000,
+ awaitProviderDrain: true,
+ });
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId: CONVO_ID,
+ error: 'Schedule was disabled, changed, or deleted before approval',
+ });
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ CONVO_ID,
+ { type: 'mongo' },
+ undefined,
+ { checkpointNamespace: '1000' },
+ );
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ });
+
+ it('fails closed without settling or pruning when provider drain cannot be confirmed', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockIsScheduleLive.mockResolvedValue(false);
+ mockGenerationJobManager.abortJob.mockRejectedValue(new Error('drain timed out'));
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(503);
+ expect(res.headers['retry-after']).toBe('1');
+ expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });
+ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ });
+
+ it('records success after resumed persistence and before terminal publication', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+
+ expect(mockClaimScheduleResume).toHaveBeenCalledWith('schedule-1', scheduledFor, {
+ expectedConfigRevision: 4,
+ automatic: true,
+ });
+ expect(mockFinalizeScheduleResumeClaim).toHaveBeenCalledWith(
+ 'schedule-1',
+ 'resume-token',
+ 'resume:resume-token',
+ { expectedConfigRevision: 4, automatic: true },
+ );
+
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'success',
+ conversationId: CONVO_ID,
+ });
+ expect(mockRecordScheduleOutcome.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('keeps the approval paused when global scheduled-run capacity is full', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockClaimScheduleResume.mockResolvedValue({ conflict: 'capacity' });
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(429);
+ expect(res.headers['retry-after']).toBe('1');
+ expect(res.body).toMatchObject({ code: 'SCHEDULE_CAPACITY' });
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
+ });
+
+ it('rolls back scheduled capacity when the approval CAS does not consume the action', async () => {
+ const job = makeScheduledJob();
+ mockGenerationJobManager.getJob.mockResolvedValue(job);
+ mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(409);
+ expect(mockReleaseScheduleResumeClaim).toHaveBeenCalledWith('schedule-1', scheduledFor, 0);
+ expect(mockReleaseScheduleResumeFence).toHaveBeenCalledWith(
+ 'schedule-1',
+ 'resume:resume-token',
+ );
+ });
+
+ it('stops before provider execution when an edit wins the final resume handoff', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockFinalizeScheduleResumeClaim.mockResolvedValue(false);
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(409);
+ expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalled();
+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {
+ expectedCreatedAt: 1000,
+ awaitProviderDrain: true,
+ });
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId: CONVO_ID,
+ error: 'Schedule was disabled, changed, or deleted before approval',
+ });
+ expect(mockInitializeClient).not.toHaveBeenCalled();
+ });
+
+ it('settles a scheduled continuation stopped during its resumed segment', async () => {
+ const job = makeScheduledJob();
+ mockGenerationJobManager.getJob.mockResolvedValue(job);
+ mockInitializeClient.mockImplementation(async () => {
+ job.abortController.abort();
+ return { client: makeClient(), userMCPAuthMap: {} };
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId: CONVO_ID,
+ error: 'Scheduled run was stopped',
+ });
+ });
+
+ it('records an empty-preempt resumed segment as interrupted, not successful', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ run: {
+ getPreemptStats: () => ({ emptyBoundaries: 1 }),
+ getHaltReason: () => 'preempt_incomplete',
+ },
+ }),
+ userMCPAuthMap: {},
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ unfinished: true }),
+ expect.anything(),
+ );
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId: CONVO_ID,
+ error: 'Scheduled run was interrupted before completion',
+ });
+ });
+
+ it('does not overwrite the terminal winner when failed-resume finalization loses its CAS', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
+ mockGenerationJobManager.completeJob.mockResolvedValue(false);
+ mockInitializeClient.mockRejectedValue(new Error('resume reconstruction failed'));
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();
+ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
+ });
+ });
+
describe('temporal context restore', () => {
it('restores req.conversationCreatedAt from the convo before initializeClient', async () => {
// Temporal prompt vars must resolve against the paused anchor, not resume wall-clock.
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index 3da58904e8..bb9201ab89 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -2884,7 +2884,9 @@ class AgentClient extends BaseClient {
// the flag; if it fails here, the teardown still releases (it checks the flag).
if (!this.pendingRequestReleased) {
try {
- await decrementPendingRequest(this.options.req?.user?.id);
+ if (this.options.req?._scheduleConcurrencyExempt !== true) {
+ await decrementPendingRequest(this.options.req?.user?.id);
+ }
this.pendingRequestReleased = true;
} catch (err) {
logger.error(`[AgentClient] Failed to release request slot on pause ${streamId}`, err);
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index 580b545f4c..5c99383180 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -18,6 +18,8 @@ const {
decrementPendingRequest,
sanitizeMessageForTransmit,
checkAndIncrementPendingRequest,
+ exemptFromConcurrencyLimiter,
+ isScheduleFireRequest,
isUnpersistedPreliminaryParent,
resolveConversationAnchor,
getAgentStartupTelemetry,
@@ -33,6 +35,7 @@ const {
cleanupMCPRequestContextForReq,
} = require('~/server/services/MCPRequestContext');
const { logViolation } = require('~/cache');
+const { recordScheduleOutcome, isScheduleLive } = require('~/server/services/Schedules');
const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models');
const {
GENERATION_PROTOCOL_HEADER,
@@ -182,8 +185,20 @@ async function finishResumableRequest(req, userId) {
try {
await cleanupMCPRequestContextForReq(req);
} finally {
- await decrementPendingRequest(userId);
+ if (req._scheduleConcurrencyExempt !== true) {
+ await decrementPendingRequest(userId);
+ }
+ }
+}
+
+function classifyScheduledFailure(error, aborted = false) {
+ if (aborted || error?.code === 'SCHEDULE_NO_LONGER_ACTIVE') {
+ return { status: 'interrupted', error: error?.message };
}
+ if (error?.message?.includes(ViolationTypes.TOKEN_BALANCE)) {
+ return { status: 'skipped_balance' };
+ }
+ return { status: 'error', error: error?.message || 'Generation failed' };
}
const JOB_RECORD_WAIT_ATTEMPTS = 5;
@@ -338,8 +353,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
parentMessageId = null,
overrideParentMessageId = null,
responseMessageId: editedResponseMessageId = null,
+ scheduleId: bodyScheduleId = null,
+ scheduledFor: bodyScheduledFor = null,
+ scheduleConfigRevision: bodyScheduleConfigRevision = null,
} = req.body;
+ const isScheduledFire = isScheduleFireRequest(req);
+ const scheduleId = isScheduledFire ? bodyScheduleId : null;
+ const scheduledFor = isScheduledFire ? bodyScheduledFor : null;
+ const scheduleConfigRevision = isScheduledFire ? bodyScheduleConfigRevision : undefined;
+
const userId = req.user.id;
const tenantId = req.user.tenantId;
const rawClientRequestId = req.body?.clientRequestId;
@@ -446,12 +469,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
req.body.overrideUserMessageId = `${recoveredSteerId}${Constants.COMMON_DIVIDER}0`;
}
const isNewConvo = !reqConversationId || reqConversationId === 'new';
+ const scheduledNewConversationId =
+ isScheduledFire && typeof req.body?.newConversationId === 'string'
+ ? req.body.newConversationId
+ : null;
let conversationId = reqConversationId;
if (isNewConvo) {
conversationId =
- typeof clientRequestId === 'string' && clientRequestId.length > 0
+ scheduledNewConversationId ??
+ (typeof clientRequestId === 'string' && clientRequestId.length > 0
? uuidv5(`${userId}:${clientRequestId}`, NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE)
- : crypto.randomUUID();
+ : crypto.randomUUID());
}
const conversationAnchorPromise = resolveConversationCreatedAt({
userId,
@@ -936,26 +964,53 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
}
}
- const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);
- if (!allowed) {
- if (ownedIdempotencyClaim) {
- await GenerationJobManager.releaseGeneration(
- userId,
- clientRequestId,
- streamId,
- ownedIdempotencyClaim,
- ).catch(() => {});
+ const scheduleConcurrencyExempt = exemptFromConcurrencyLimiter(req);
+ req._scheduleConcurrencyExempt = scheduleConcurrencyExempt;
+ if (!scheduleConcurrencyExempt) {
+ const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);
+ if (!allowed) {
+ if (ownedIdempotencyClaim) {
+ await GenerationJobManager.releaseGeneration(
+ userId,
+ clientRequestId,
+ streamId,
+ ownedIdempotencyClaim,
+ ).catch(() => {});
+ }
+ const violationInfo = getViolationInfo(pendingRequests, limit);
+ await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);
+ startupTelemetry?.end('rejected');
+ return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);
}
- const violationInfo = getViolationInfo(pendingRequests, limit);
- await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);
- startupTelemetry?.end('rejected');
- return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);
}
startupTelemetry?.mark('request_admitted');
let client = null;
let jobCreatedAt;
let providerExecutionId;
+ let scheduleTerminalOutcomeRecorded = false;
+ const settleScheduledRun = async ({ status, error, clearConversationId = false }) => {
+ if (!scheduleId) {
+ return true;
+ }
+ if (status !== 'requires_action' && scheduleTerminalOutcomeRecorded) {
+ return true;
+ }
+ const recorded = await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt,
+ status,
+ conversationId,
+ clearConversationId,
+ error,
+ });
+ if (recorded && status !== 'requires_action') {
+ scheduleTerminalOutcomeRecorded = true;
+ }
+ return recorded;
+ };
try {
logger.debug(`[ResumableAgentController] Creating job`, {
@@ -995,6 +1050,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// Persist temporary-chat state so a HITL resume keeps the resumed response
// non-persisted instead of trusting the resume request to re-send the flag.
isTemporary: req.body?.isTemporary,
+ ...(scheduleId
+ ? {
+ scheduleId,
+ scheduledFor,
+ preserveForScheduleReconcile: true,
+ ...(Number.isSafeInteger(scheduleConfigRevision) && {
+ scheduleConfigRevision,
+ }),
+ ...(req._isManualScheduledFire === true && { scheduleManual: true }),
+ }
+ : {}),
responseMessageId: preliminaryResponseMessageId,
userMessage: preliminaryUserMessage,
},
@@ -1016,6 +1082,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
status: 409,
});
}
+ if (
+ scheduleId &&
+ !(await isScheduleLive(scheduleId, scheduleConfigRevision, {
+ automatic: req._isManualScheduledFire !== true,
+ policy: true,
+ }))
+ ) {
+ throw Object.assign(new Error('This scheduled occurrence is no longer active'), {
+ code: 'SCHEDULE_NO_LONGER_ACTIVE',
+ status: 409,
+ });
+ }
if (
providerExecutionId &&
!(await GenerationJobManager.beginProviderExecution(
@@ -1187,6 +1265,11 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
completeErr,
);
});
+ await settleScheduledRun({
+ status: 'interrupted',
+ error: 'Request aborted during initialization',
+ clearConversationId: job.createdEventEmitted !== true,
+ });
startupTelemetry?.end('aborted');
try {
await finishResumableRequest(req, userId);
@@ -1260,6 +1343,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
* may call completeJob: the pause may already have been replaced by a newer
* action or generation by the time the persistence failure is observed. */
let pausePersistenceFailed = false;
+ let pausePersistenceFailureFinalized = false;
const finishOwnedTerminalClaim = async () => {
if (!terminalClaim || terminalClaimFinished) {
return;
@@ -1586,21 +1670,21 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
await commitRecoveredSteer();
} catch (pausePersistenceError) {
pausePersistenceFailed = true;
- let failed;
try {
- failed = await GenerationJobManager.failPausePersistence(
- streamId,
- pauseActionId,
- pausePersistenceError?.message ?? 'Pause persistence failed',
- pauseCreatedAt,
- );
+ pausePersistenceFailureFinalized =
+ (await GenerationJobManager.failPausePersistence(
+ streamId,
+ pauseActionId,
+ pausePersistenceError?.message ?? 'Pause persistence failed',
+ pauseCreatedAt,
+ )) === true;
} catch (failError) {
logger.error(
`[ResumableAgentController] Failed to terminalize pause persistence error for ${streamId}`,
failError,
);
}
- if (failed === true) {
+ if (pausePersistenceFailureFinalized) {
/** Namespaced checkpoints belong exclusively to this epoch,
* so the exact pause-failure CAS winner can safely remove the
* now-unresumable graph state. Legacy shared namespaces are
@@ -1621,7 +1705,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
}
}
- } else if (failed === false) {
+ } else if (pausePersistenceFailureFinalized === false) {
logger.warn(
`[ResumableAgentController] Skipping stale pause persistence failure — ${streamId} no longer owns its barrier`,
);
@@ -1638,6 +1722,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
`[ResumableAgentController] Pause persistence barrier changed before release: ${streamId}`,
);
}
+ // The pause projection is what moves the run row off `started` and frees its
+ // GLOBAL capacity slot. recordScheduleOutcome already retried it; a `false`
+ // here means every attempt failed, leaving the row `started` while the job
+ // sits `requires_action`. Surface it — the armed engine's reconciler replays
+ // this state, and the clustered sweep now converges it too, but a silent drop
+ // gave neither a reason to look.
+ if (!(await settleScheduledRun({ status: 'requires_action' }))) {
+ logger.error(
+ `[ResumableAgentController] Failed to project the scheduled pause for ${streamId}; run stays active until reconciliation replays it`,
+ );
+ }
} else {
logger.debug(
`[ResumableAgentController] Skipping stale pause persistence — ${streamId} no longer owns its barrier`,
@@ -1650,7 +1745,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// (so a fast /resume isn't 429'd); only release here if that didn't happen.
// Always run the MCP request-context cleanup.
await cleanupMCPRequestContextForReq(req);
- if (!client?.pendingRequestReleased) {
+ if (!client?.pendingRequestReleased && req._scheduleConcurrencyExempt !== true) {
await decrementPendingRequest(userId);
}
if (client) {
@@ -1781,6 +1876,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
await titleEventPromise;
}
+ let scheduleCompletionError;
+ if (terminalWasAborted) {
+ scheduleCompletionError = 'Scheduled run was stopped';
+ } else if (preemptIncomplete) {
+ scheduleCompletionError = 'Scheduled run was interrupted before completion';
+ }
+ await settleScheduledRun({
+ status: terminalWasAborted || preemptIncomplete ? 'interrupted' : 'success',
+ ...(scheduleCompletionError != null && { error: scheduleCompletionError }),
+ });
+
let terminalPublicationStarted = false;
try {
const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);
@@ -1893,7 +1999,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// transition can win. Settle its pending marker with conservative
// reconciliation on any required-write/final-construction failure,
// then release exactly that claim.
+ let ownsScheduledFailure = false;
if (terminalClaim && !terminalClaimFinished) {
+ ownsScheduledFailure = true;
try {
await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
} catch (publishError) {
@@ -1915,6 +2023,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
startupTelemetry?.end('error', error);
} else if (pausePersistenceFailed) {
+ ownsScheduledFailure = pausePersistenceFailureFinalized;
// failPausePersistence owns the only legal requires_action -> error
// transition for this exact action/epoch. Never fall through to
// completeJob, which could race a newer action or replacement job.
@@ -1924,6 +2033,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
startupTelemetry?.end('error', error);
} else if (job.abortController.signal.aborted || error.message?.includes('abort')) {
+ ownsScheduledFailure = true;
logger.debug(`[ResumableAgentController] Generation aborted for ${streamId}`);
startupTelemetry?.end('aborted');
// abortJob already handled emitDone and completeJob
@@ -1933,7 +2043,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
try {
// completeJob first wins running -> error and atomically parks
// steers, then publishes. A competing abort/pause emits nothing.
- await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt);
+ ownsScheduledFailure =
+ (await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt)) ===
+ true;
} catch (completeErr) {
logger.warn(
'[ResumableAgentController] completeJob failed during generation-error cleanup',
@@ -1944,6 +2056,14 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
}
}
+ if (ownsScheduledFailure && !scheduleTerminalOutcomeRecorded) {
+ const scheduledFailure = classifyScheduledFailure(
+ error,
+ job.abortController.signal.aborted,
+ );
+ await settleScheduledRun(scheduledFailure);
+ }
+
try {
await finishResumableRequest(req, userId);
} finally {
@@ -1962,15 +2082,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
`[ResumableAgentController] Unhandled error in background generation: ${err.message}`,
);
startupTelemetry?.end('error', err);
+ let errorFinalized = false;
if (!pausePersistenceFailed) {
- await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
- (completeErr) => {
- logger.warn(
- '[ResumableAgentController] completeJob failed during background-error cleanup',
- completeErr,
- );
- },
- );
+ errorFinalized =
+ (await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
+ (completeErr) => {
+ logger.warn(
+ '[ResumableAgentController] completeJob failed during background-error cleanup',
+ completeErr,
+ );
+ return false;
+ },
+ )) === true;
+ }
+ if (
+ (errorFinalized || (pausePersistenceFailed && pausePersistenceFailureFinalized)) &&
+ !scheduleTerminalOutcomeRecorded
+ ) {
+ await settleScheduledRun(classifyScheduledFailure(err));
}
try {
await finishResumableRequest(req, userId);
@@ -2084,18 +2213,24 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// release + pending-request decrement below, or the retry stays wedged behind the claim
// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not
// finalize anything, so releasing afterward can't let it abort a later replacement.)
+ let initializationFinalized = jobCreatedAt == null;
if (jobCreatedAt != null) {
const initializationError = initializationFailure
? JSON.stringify(initializationFailure)
: error.message || 'Failed to start generation';
- await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(
- (completeErr) => {
- logger.warn(
- '[ResumableAgentController] completeJob failed during init-error cleanup',
- completeErr,
- );
- },
- );
+ initializationFinalized =
+ (await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(
+ (completeErr) => {
+ logger.warn(
+ '[ResumableAgentController] completeJob failed during init-error cleanup',
+ completeErr,
+ );
+ return false;
+ },
+ )) === true;
+ }
+ if (initializationFinalized && !scheduleTerminalOutcomeRecorded) {
+ await settleScheduledRun(classifyScheduledFailure(error));
}
if (ownedIdempotencyClaim) {
await GenerationJobManager.releaseGeneration(
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index cb3ea82908..e4eb76e295 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -1,6 +1,6 @@
const { randomUUID } = require('crypto');
const { logger } = require('@librechat/data-schemas');
-const { Constants, EModelEndpoint } = require('librechat-data-provider');
+const { Constants, EModelEndpoint, ViolationTypes } = require('librechat-data-provider');
const {
GenerationJobManager,
isPendingActionStale,
@@ -30,6 +30,14 @@ const {
cleanupMCPRequestContextForReq,
} = require('~/server/services/MCPRequestContext');
const { saveMessage, getConvo, getMessages } = require('~/models');
+const {
+ recordScheduleOutcome,
+ claimScheduleResume,
+ releaseScheduleResumeClaim,
+ finalizeScheduleResumeClaim,
+ releaseScheduleResumeFence,
+ isScheduleLive,
+} = require('~/server/services/Schedules');
const {
GENERATION_PROTOCOL_HEADER,
negotiateNewGenerationProtocol,
@@ -422,6 +430,20 @@ async function finalizeResumedTurn({
}
conversation.title = conversation.title || 'New Chat';
+ if (meta.scheduleId) {
+ await recordScheduleOutcome({
+ scheduleId: meta.scheduleId,
+ scheduledFor: meta.scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: preemptIncomplete ? 'interrupted' : 'success',
+ conversationId,
+ ...(preemptIncomplete && {
+ error: 'Scheduled run was interrupted before completion',
+ }),
+ });
+ }
+
const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);
const finalEvent = {
final: true,
@@ -571,6 +593,65 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
);
}
+ const scheduleId = job.metadata?.scheduleId;
+ const scheduledFor = job.metadata?.scheduledFor;
+ if (
+ scheduleId &&
+ !(await isScheduleLive(scheduleId, job.metadata?.scheduleConfigRevision, {
+ automatic: job.metadata?.scheduleManual !== true,
+ policy: true,
+ }))
+ ) {
+ let stopped = false;
+ try {
+ const abortResult = await GenerationJobManager.abortJob(streamId, {
+ expectedCreatedAt: job.createdAt,
+ awaitProviderDrain: true,
+ });
+ stopped = abortResult != null && abortResult.failureReason == null;
+ } catch (error) {
+ logger.warn('[ResumeAgentController] Failed to stop inactive scheduled run', error);
+ }
+ if (!stopped) {
+ res.set('Retry-After', '1');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SCHEDULE_STOP_UNCONFIRMED',
+ error: 'The inactive scheduled run could not be confirmed stopped. Please retry.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'interrupted',
+ conversationId,
+ error: 'Schedule was disabled, changed, or deleted before approval',
+ });
+ const checkpointNamespace = job.metadata?.checkpointNamespace;
+ if (typeof checkpointNamespace === 'string' && checkpointNamespace !== '') {
+ await deleteAgentCheckpoint(
+ conversationId,
+ req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
+ undefined,
+ { checkpointNamespace },
+ ).catch((error) => {
+ logger.warn('[ResumeAgentController] Failed to prune inactive schedule checkpoint', error);
+ });
+ }
+ return sendGenerationJson(
+ res,
+ 409,
+ { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' },
+ generationProtocolVersion,
+ );
+ }
+
const pendingAction = job.metadata?.pendingAction;
if (job.status !== 'requires_action') {
return sendGenerationJson(
@@ -720,6 +801,109 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
);
}
+ // Finish the legacy checkpoint snapshot before claiming scheduled capacity.
+ // It is independent of the approval claim, and holding a deployment-wide slot
+ // while an indexed saver read stalls would unnecessarily block other schedules
+ // and lengthen the Mongo-claim -> approval-CAS hand-off window below.
+ const checkpointGeneration = await checkpointGenerationPromise;
+
+ // A pause frees its scheduled-run capacity slot. Before consuming the approval,
+ // atomically promote the run row back to `started` and claim a fresh global slot.
+ // The database's partial unique indexes arbitrate both deployment capacity and a
+ // concurrent active occurrence of the same schedule.
+ let scheduleCapacitySlot;
+ let scheduleResumeClaimToken;
+ let scheduleResumeLeaseBy;
+ const scheduleResumeOptions = {
+ expectedConfigRevision: job.metadata?.scheduleConfigRevision,
+ automatic: job.metadata?.scheduleManual !== true,
+ };
+ if (scheduleId) {
+ let scheduleClaim;
+ try {
+ scheduleClaim = await claimScheduleResume(scheduleId, scheduledFor, scheduleResumeOptions);
+ } catch (err) {
+ await decrementPendingRequest(userId);
+ logger.error('[ResumeAgentController] Failed to claim scheduled resume capacity', err);
+ return sendGenerationJson(
+ res,
+ 500,
+ { error: 'Failed to reserve scheduled-run capacity' },
+ generationProtocolVersion,
+ );
+ }
+ if ('conflict' in scheduleClaim) {
+ await decrementPendingRequest(userId);
+ if (scheduleClaim.conflict === 'capacity' || scheduleClaim.conflict === 'overlap') {
+ res.set('Retry-After', '1');
+ return sendGenerationJson(
+ res,
+ 429,
+ {
+ code:
+ scheduleClaim.conflict === 'capacity'
+ ? 'SCHEDULE_CAPACITY'
+ : 'SCHEDULE_OCCURRENCE_ACTIVE',
+ error:
+ scheduleClaim.conflict === 'capacity'
+ ? 'Scheduled-run capacity is currently full. Please retry.'
+ : 'Another occurrence of this schedule is still running. Please retry.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ return sendGenerationJson(
+ res,
+ 409,
+ {
+ code:
+ scheduleClaim.conflict === 'inactive'
+ ? 'SCHEDULE_NO_LONGER_ACTIVE'
+ : 'SCHEDULE_RUN_NOT_PAUSED',
+ error: 'This scheduled run can no longer be resumed',
+ },
+ generationProtocolVersion,
+ );
+ }
+ scheduleCapacitySlot = scheduleClaim.capacitySlot;
+ scheduleResumeClaimToken = scheduleClaim.claimToken;
+ scheduleResumeLeaseBy = scheduleClaim.leaseBy;
+ }
+
+ const releaseScheduleFence = async () => {
+ if (scheduleId == null || scheduleResumeLeaseBy == null) {
+ return;
+ }
+ try {
+ await releaseScheduleResumeFence(scheduleId, scheduleResumeLeaseBy);
+ } catch (releaseError) {
+ logger.warn('[ResumeAgentController] Failed to release scheduled resume fence', releaseError);
+ }
+ };
+
+ /** Release only when the exact generation demonstrably remains paused. If the
+ * approval CAS reply is ambiguous and the job cannot be read, retaining the slot
+ * until reconciliation is the safe direction: releasing it could exceed the cap
+ * while a committed continuation is already running. */
+ const rollbackUnconsumedScheduleClaim = async (currentJob) => {
+ if (
+ scheduleId == null ||
+ scheduleCapacitySlot == null ||
+ currentJob?.createdAt !== job.createdAt ||
+ currentJob?.status !== 'requires_action'
+ ) {
+ return;
+ }
+ try {
+ await releaseScheduleResumeClaim(scheduleId, scheduledFor, scheduleCapacitySlot);
+ } catch (rollbackError) {
+ logger.warn(
+ '[ResumeAgentController] Failed to release unconsumed scheduled resume capacity',
+ rollbackError,
+ );
+ }
+ };
+
// Atomically claim the resume. The single winner drives the run; a racing second
// submit (double-click, two tabs) gets false and must not re-drive — that would
// re-execute tools and double-bill.
@@ -729,10 +913,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// would leak the concurrency slot until the counter TTL expires — spuriously 429'ing
// the user when they retry the still-paused approval. Release the slot on that path too.
let claimed;
- let checkpointGeneration;
const providerExecutionId = randomUUID();
try {
- checkpointGeneration = await checkpointGenerationPromise;
/** The CAS that reopens steering must also publish THIS owner's seal
* capability. A separate write after status=`running` leaves a window in
* which steer/arm requests read the previous replica's capability. */
@@ -748,6 +930,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
job.createdAt,
);
} catch (err) {
+ const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);
+ await rollbackUnconsumedScheduleClaim(currentJob);
+ await releaseScheduleFence();
await decrementPendingRequest(userId);
logger.error('[ResumeAgentController] Failed to claim resume', err);
return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion);
@@ -755,6 +940,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
if (!claimed) {
await decrementPendingRequest(userId);
const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);
+ await rollbackUnconsumedScheduleClaim(currentJob);
+ await releaseScheduleFence();
if (currentJob != null && currentJob.createdAt !== job.createdAt) {
return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion);
}
@@ -766,6 +953,73 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
);
}
+ // Linearize the consumed approval against the schedule's live config. The schedule
+ // document fence was acquired only after all async policy reads, and this atomic
+ // consume checks its token/revision/enabled state immediately after the approval CAS.
+ // An edit/disable that won first makes this fail; one that lands afterward is ordered
+ // after the continuation has started. Never begin provider execution on a stale claim.
+ if (scheduleId) {
+ let scheduleClaimCurrent = false;
+ try {
+ scheduleClaimCurrent = await finalizeScheduleResumeClaim(
+ scheduleId,
+ scheduleResumeClaimToken,
+ scheduleResumeLeaseBy,
+ scheduleResumeOptions,
+ );
+ } catch (error) {
+ logger.error('[ResumeAgentController] Failed to finalize scheduled resume fence', error);
+ await releaseScheduleFence();
+ }
+ if (!scheduleClaimCurrent) {
+ await decrementPendingRequest(userId);
+ let stopped = false;
+ try {
+ const abortResult = await GenerationJobManager.abortJob(streamId, {
+ expectedCreatedAt: job.createdAt,
+ awaitProviderDrain: true,
+ });
+ stopped = abortResult != null && abortResult.failureReason == null;
+ } catch (error) {
+ logger.warn('[ResumeAgentController] Failed to stop stale scheduled resume', error);
+ }
+ if (!stopped) {
+ res.set('Retry-After', '1');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SCHEDULE_STOP_UNCONFIRMED',
+ error: 'The stale scheduled resume could not be confirmed stopped.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'interrupted',
+ conversationId,
+ error: 'Schedule was disabled, changed, or deleted before approval',
+ });
+ if (checkpointNamespace !== '') {
+ await deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, {
+ checkpointNamespace,
+ }).catch((error) => {
+ logger.warn('[ResumeAgentController] Failed to prune stale schedule checkpoint', error);
+ });
+ }
+ return sendGenerationJson(
+ res,
+ 409,
+ { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' },
+ generationProtocolVersion,
+ );
+ }
+ }
+
/**
* An interrupt steer enqueued just before the pause survives durably with
* its `preempt` flag, but the ARM lived only in the previous owner's
@@ -1018,6 +1272,16 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
`[ResumeAgentController] Re-pause persistence barrier changed before release: ${streamId}`,
);
}
+ if (scheduleId) {
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'requires_action',
+ conversationId,
+ });
+ }
} else {
logger.debug(
`[ResumeAgentController] Skipping stale re-pause persistence — ${streamId} no longer owns its barrier`,
@@ -1027,11 +1291,25 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
}
// If the user aborted mid-resume, the abort route already emitted the terminal
- // event and finalized the job — don't double-save / double-finalize here.
+ // event and finalized the job — don't double-save / double-finalize here. This
+ // continuation is nevertheless the scheduled-run owner, so it must settle the
+ // run row after observing its own abort; the generic Stop route deliberately
+ // delegates a running generation's settlement to that generation owner.
if (job.abortController.signal.aborted) {
logger.debug(
`[ResumeAgentController] Aborted during resume; abort route finalizes: ${streamId}`,
);
+ if (scheduleId) {
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'interrupted',
+ conversationId,
+ error: 'Scheduled run was stopped',
+ });
+ }
return;
}
@@ -1061,6 +1339,17 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
're-pause persistence failure',
);
}
+ if (scheduleId && pausePersistenceFailureFinalized) {
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'error',
+ conversationId,
+ error: err?.message ?? 'Re-pause persistence failed',
+ });
+ }
return;
}
// Job-replacement guard (mirrors finalizeResumedTurn's success-path guard): if a
@@ -1104,6 +1393,18 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
'failed resume finalization',
);
}
+ if (scheduleId && errorFinalized) {
+ const balanceRefusal = err?.message?.includes(ViolationTypes.TOKEN_BALANCE);
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: balanceRefusal ? 'skipped_balance' : 'error',
+ conversationId,
+ ...(!balanceRefusal && { error: err?.message ?? 'Resume failed' }),
+ });
+ }
}
} finally {
try {
diff --git a/api/server/experimental.js b/api/server/experimental.js
index 698cb52ff2..8d03f8ae59 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -28,6 +28,7 @@ const {
setupGracefulShutdown,
configureMessageFilterRegexValidator,
configureFileConfigRegexEngine,
+ GenerationJobManager,
waitForKeyvRedisClient,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
@@ -37,6 +38,10 @@ const createValidateImageRequest = require('./middleware/validateImageRequest');
const { startExpiredFileSweep } = require('./services/Files/process');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
+const {
+ recordExpiredScheduleApproval,
+ initializeScheduleErasureSweep,
+} = require('./services/Schedules');
const { configureSubagentTaskRouting } = require('./services/Endpoints/agents/subagentThreadStore');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api');
@@ -276,6 +281,11 @@ if (cluster.isMaster) {
* Each worker runs a full Express server instance
*/
const app = express();
+ // The clustered entrypoint deliberately does not arm the v1 schedule engine,
+ // but an already-fired scheduled generation can still reach HITL here. Settle
+ // its durable run when the generic approval runtime expires it.
+ GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
+ GenerationJobManager.initialize();
/**
* The master may assign the sweep worker before or after this worker has
* loaded app config. These flags join the IPC assignment with config
@@ -284,6 +294,17 @@ if (cluster.isMaster) {
let shouldStartExpiredFileSweep = false;
let expiredFileSweepOptions = null;
let expiredFileSweepStarted = false;
+ const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);
+
+ const rejectScheduleWritesUntilReady = (req, res, next) => {
+ if (SCHEDULE_ENGINE_OPTIONAL_METHODS.has(req.method)) {
+ return next();
+ }
+ return res.status(501).json({
+ code: 'SCHEDULES_NOT_SUPPORTED',
+ error: 'Scheduled chats are not available in clustered mode.',
+ });
+ };
const startExpiredFileSweepOnce = () => {
if (!shouldStartExpiredFileSweep || expiredFileSweepStarted || !expiredFileSweepOptions) {
@@ -322,6 +343,13 @@ if (cluster.isMaster) {
logger.error(`[Worker ${process.pid}][indexSync] Background sync failed:`, err);
});
+ // This entrypoint deliberately does not arm the schedule engine, but DELETE stays
+ // open — so soft-deleted rows still accrue with no reconciler to erase them. Start
+ // the erasure-ONLY sweep (Mongo is up, GenerationJobManager was initialized above):
+ // it never claims, fires, advances, or infers owner death from a process-local
+ // missing job, and its idempotent guard makes this safe once per worker.
+ initializeScheduleErasureSweep();
+
app.disable('x-powered-by');
app.set('trust proxy', trusted_proxy);
@@ -488,6 +516,7 @@ if (cluster.isMaster) {
app.use('/api/agents', routes.agents);
app.use('/api/banner', routes.banner);
app.use('/api/memories', routes.memories);
+ app.use('/api/schedules', rejectScheduleWritesUntilReady, routes.schedules);
app.use('/api/permissions', routes.accessPermissions);
app.use('/api/tags', routes.tags);
app.use('/api/mcp', routes.mcp);
diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.js
index 4ff9d6b924..58349d3091 100644
--- a/api/server/experimental.spec.js
+++ b/api/server/experimental.spec.js
@@ -23,6 +23,34 @@ describe('Experimental server configuration', () => {
expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);
});
+ it('starts approval expiry after installing the scheduled-run callback', () => {
+ const handlerIndex = source.indexOf(
+ 'GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);',
+ );
+ const initializeIndex = source.indexOf('GenerationJobManager.initialize();');
+
+ expect(handlerIndex).toBeGreaterThan(-1);
+ expect(initializeIndex).toBeGreaterThan(handlerIndex);
+ });
+
+ it('starts erasure-only schedule maintenance after connecting to Mongo, once per worker', () => {
+ const connectIndex = source.indexOf('await connectDb();');
+ const sweepIndex = source.indexOf('initializeScheduleErasureSweep();');
+
+ expect(connectIndex).toBeGreaterThan(-1);
+ expect(sweepIndex).toBeGreaterThan(-1);
+ // Mongo must be up before the sweep reads soft-deleted rows.
+ expect(sweepIndex).toBeGreaterThan(connectIndex);
+ // Idempotent guard lives in the service; started exactly once from this entrypoint.
+ expect(source.match(/initializeScheduleErasureSweep\(\);/g)).toHaveLength(1);
+ });
+
+ it('never arms the full schedule engine in a clustered worker', () => {
+ // The clustered entrypoint runs erasure-only maintenance: arming the engine here
+ // would claim/fire/absence-reconcile runs whose peer generations it cannot see.
+ expect(source).not.toContain('initializeScheduleEngine(');
+ });
+
it('runs cross-tenant startup work in the system context', () => {
expect(source).toContain('await runAsSystem(seedDatabase);');
expect(source).toMatch(
diff --git a/api/server/index.js b/api/server/index.js
index 0768a27c50..9203250d70 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -57,6 +57,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilities
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
+const { initializeScheduleEngine, recordExpiredScheduleApproval } = require('./services/Schedules');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { startExpiredFileSweep } = require('./services/Files/process');
const { checkMigrations } = require('./services/start/migration');
@@ -85,9 +86,12 @@ const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default
const app = express();
let serverReady = false;
+let schedulesReady = false;
const SERVER_NOT_READY_CODE = 'SERVER_NOT_READY';
const CHAT_START_RETRY_AFTER_SECONDS = '1';
+const SCHEDULES_NOT_READY_CODE = 'SCHEDULES_NOT_READY';
+const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']);
const rejectChatStartsUntilReady = (req, res, next) => {
if (serverReady || req.method !== 'POST' || req.path === '/abort') {
@@ -101,12 +105,24 @@ const rejectChatStartsUntilReady = (req, res, next) => {
});
};
+const rejectScheduleWritesUntilReady = (req, res, next) => {
+ if (schedulesReady || SCHEDULE_ENGINE_OPTIONAL_METHODS.has(req.method)) {
+ return next();
+ }
+ res.set('Retry-After', CHAT_START_RETRY_AFTER_SECONDS);
+ return res.status(503).json({
+ code: SCHEDULES_NOT_READY_CODE,
+ error: 'Scheduler is still starting. Please retry shortly.',
+ });
+};
+
const configureGenerationStreams = () => {
const streamServices = createStreamServices();
GenerationJobManager.configure({
...streamServices,
cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),
});
+ GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
GenerationJobManager.initialize();
// Stop active generations and close their SSE streams while the HTTP server drains.
registerShutdownTask(
@@ -345,6 +361,7 @@ const startServer = async () => {
app.use('/api/agents', routes.agents);
app.use('/api/banner', routes.banner);
app.use('/api/memories', routes.memories);
+ app.use('/api/schedules', rejectScheduleWritesUntilReady, routes.schedules);
app.use('/api/permissions', routes.accessPermissions);
app.use('/api/tags', routes.tags);
@@ -402,6 +419,10 @@ const startServer = async () => {
memoryDiagnostics.start();
}
await initializeAgentTriggerService({ address: server.address() });
+ schedulesReady = (await initializeScheduleEngine()) != null;
+ if (!schedulesReady) {
+ logger.warn('[schedules] write routes remain unavailable because the engine did not arm.');
+ }
serverReady = true;
logger.info('Server readiness checks passing.');
} catch (initErr) {
diff --git a/api/server/index.metrics.spec.js b/api/server/index.metrics.spec.js
index 6b95104c3d..80a386c607 100644
--- a/api/server/index.metrics.spec.js
+++ b/api/server/index.metrics.spec.js
@@ -38,6 +38,10 @@ jest.mock('~/server/services/Agents/triggers', () => ({
initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ initializeScheduleEngine: jest.fn().mockResolvedValue(undefined),
+}));
+
describe('Server metrics route', () => {
jest.setTimeout(30_000);
diff --git a/api/server/index.spec.js b/api/server/index.spec.js
index 013ece204e..73ad042865 100644
--- a/api/server/index.spec.js
+++ b/api/server/index.spec.js
@@ -39,6 +39,10 @@ jest.mock('~/server/services/Agents/triggers', () => ({
initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ initializeScheduleEngine: jest.fn().mockResolvedValue(undefined),
+}));
+
jest.mock(
'@librechat/api/telemetry',
() => ({
diff --git a/api/server/routes/__tests__/mcp.spec.js b/api/server/routes/__tests__/mcp.spec.js
index 44dcfe2628..99ebff511a 100644
--- a/api/server/routes/__tests__/mcp.spec.js
+++ b/api/server/routes/__tests__/mcp.spec.js
@@ -228,6 +228,12 @@ describe('MCP Routes', () => {
currentUser = undefined;
mockResolveAllMcpConfigs.mockResolvedValue({});
mockResolveMcpConfigNames.mockResolvedValue([]);
+ // `clearAllMocks` preserves queued `mockReturnValueOnce` entries. A callback
+ // test can legitimately leave one unconsumed, which then masks a later test's
+ // default implementation when this file shares a CI shard with other suites.
+ // Reset the cache accessor itself so every test starts from a deterministic
+ // default and can opt into a one-shot value deliberately.
+ require('~/cache').getLogStores.mockReset().mockReturnValue({});
const { MCPOAuthHandler } = require('@librechat/api');
const { getTenantId } = require('@librechat/data-schemas');
getTenantId.mockReturnValue(undefined);
diff --git a/api/server/routes/agents/__tests__/abort.spec.js b/api/server/routes/agents/__tests__/abort.spec.js
index f6a3ca559f..43516fbb16 100644
--- a/api/server/routes/agents/__tests__/abort.spec.js
+++ b/api/server/routes/agents/__tests__/abort.spec.js
@@ -25,6 +25,10 @@ const mockGenerationJobManager = {
const mockSaveMessage = jest.fn();
+const mockRecordScheduleOutcome = jest.fn();
+const mockBeginScheduledStop = jest.fn();
+const mockAcknowledgeScheduledStopPersistence = jest.fn();
+
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
logger: mockLogger,
@@ -33,6 +37,11 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
isEnabled: jest.fn().mockReturnValue(false),
+ isAgentTriggerRequest: jest.fn(() => false),
+ captureScheduleFireContext: jest.fn((req) => {
+ req._isScheduledFire = false;
+ req._isManualScheduledFire = false;
+ }),
GenerationJobManager: mockGenerationJobManager,
}));
@@ -40,6 +49,13 @@ jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
}));
+jest.mock('~/server/services/Schedules', () => ({
+ recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),
+ beginScheduledStop: (...args) => mockBeginScheduledStop(...args),
+ acknowledgeScheduledStopPersistence: (...args) =>
+ mockAcknowledgeScheduledStopPersistence(...args),
+}));
+
jest.mock('~/server/middleware', () => ({
uaParser: (req, res, next) => next(),
checkBan: (req, res, next) => next(),
@@ -81,6 +97,12 @@ describe('Agent Abort Endpoint', () => {
mockGenerationJobManager.getActiveJobIdsForUser.mockReset();
mockSaveMessage.mockReset();
mockSaveMessage.mockImplementation(async (_context, message) => message);
+ mockRecordScheduleOutcome.mockReset();
+ mockRecordScheduleOutcome.mockResolvedValue(true);
+ mockBeginScheduledStop.mockReset();
+ mockBeginScheduledStop.mockResolvedValue(true);
+ mockAcknowledgeScheduledStopPersistence.mockReset();
+ mockAcknowledgeScheduledStopPersistence.mockResolvedValue(undefined);
});
describe('POST /chat/abort', () => {
@@ -841,5 +863,140 @@ describe('Agent Abort Endpoint', () => {
});
});
});
+
+ describe('Scheduled Stop persistence protocol', () => {
+ const scheduledJob = {
+ status: 'running',
+ createdAt: 111,
+ metadata: {
+ userId: 'test-user-123',
+ generationProtocolVersion: 2,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ conversationId: 'conv-1',
+ },
+ };
+
+ it('stamps the Stop before signalling abort, then acknowledges after persistence', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ jobData: { conversationId: 'conv-1' },
+ content: [],
+ text: '',
+ });
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(200);
+ expect(mockBeginScheduledStop).toHaveBeenCalledWith({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ });
+ // Stamp BEFORE the abort signal; acknowledgement AFTER it (persistence done).
+ expect(mockBeginScheduledStop.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.abortJob.mock.invocationCallOrder[0],
+ );
+ // The ack also carries a terminal outcome to re-drive: the owner calls
+ // recordScheduleOutcome once, and if that call's Stop barrier deferred (slow
+ // beforePublish), nothing would settle the run where no reconciler is armed.
+ expect(mockAcknowledgeScheduledStopPersistence).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ settle: expect.objectContaining({ status: 'interrupted' }),
+ }),
+ );
+ expect(mockGenerationJobManager.abortJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockAcknowledgeScheduledStopPersistence.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('returns STOP_IN_PROGRESS and never signals a second abort when a Stop is already live', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);
+ mockBeginScheduledStop.mockResolvedValue('in_progress');
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(409);
+ expect(response.body.code).toBe('STOP_IN_PROGRESS');
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();
+ });
+
+ it('does NOT acknowledge when the abort persistence failed (run stays preserved)', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(scheduledJob);
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ persistenceFailed: true,
+ jobData: { conversationId: 'conv-1' },
+ content: [],
+ text: '',
+ });
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(200);
+ expect(response.body.persistenceFailed).toBe(true);
+ expect(mockBeginScheduledStop).toHaveBeenCalled();
+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();
+ });
+
+ it('does not re-drive settlement for a PAUSED job, which settles explicitly', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ ...scheduledJob,
+ status: 'requires_action',
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ jobData: { conversationId: 'conv-1' },
+ content: [],
+ text: '',
+ });
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'conv-1', generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(200);
+ const ack = mockAcknowledgeScheduledStopPersistence.mock.calls[0][0];
+ expect(ack.settle).toBeUndefined();
+ // The paused path records its own outcome explicitly.
+ expect(mockRecordScheduleOutcome).toHaveBeenCalled();
+ });
+
+ it('leaves non-scheduled aborts untouched by the Stop protocol', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ status: 'running',
+ createdAt: 1,
+ metadata: { userId: 'test-user-123', generationProtocolVersion: 2 },
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ jobData: { conversationId: 'conv-x' },
+ content: [],
+ text: '',
+ });
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'conv-x', generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(200);
+ expect(mockBeginScheduledStop).not.toHaveBeenCalled();
+ expect(mockAcknowledgeScheduledStopPersistence).not.toHaveBeenCalled();
+ });
+ });
});
});
diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.js
index ee3ed4e7bb..1d73ea5d98 100644
--- a/api/server/routes/agents/index.js
+++ b/api/server/routes/agents/index.js
@@ -15,6 +15,8 @@ const {
createMessageFilterPii,
isAgentTriggerRequest,
exemptAgentTriggerFromIpLimiter,
+ captureScheduleFireContext,
+ exemptFromUserLimiter: exemptScheduleFromUserLimiter,
} = require('@librechat/api');
const { createSseStreamTelemetry } = require('@librechat/api/telemetry');
const { logger } = require('@librechat/data-schemas');
@@ -36,6 +38,11 @@ const {
negotiateExistingGenerationProtocol,
} = require('~/server/controllers/agents/protocol');
const { saveMessage } = require('~/models');
+const {
+ recordScheduleOutcome,
+ beginScheduledStop,
+ acknowledgeScheduledStopPersistence,
+} = require('~/server/services/Schedules');
const responses = require('./responses');
const openai = require('./openai');
const { v1 } = require('./v1');
@@ -119,6 +126,7 @@ router.use(requireJwtAuth);
// middleware reads this stable decision instead of re-verifying an expired token.
router.use((req, _res, next) => {
req._isAgentTrigger = isAgentTriggerRequest(req);
+ captureScheduleFireContext(req);
next();
});
router.use(checkBan);
@@ -669,6 +677,27 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {
throwOnError: true,
})
: undefined;
+ // Stamp a scheduled run's Stop BEFORE signalling the abort. `abortJob` flips the job
+ // to `aborted` immediately, then runs the partial-message/checkpoint persistence in
+ // `beforePublish`. Without this stamp the owner settlement, reconciliation, and
+ // schedule/account deletion could observe `aborted` and terminalize/erase the run
+ // mid-write. The stamp is serialized: a fresh Stop already owning it means another
+ // request is persisting, so we must not signal a second abort.
+ const stopScheduleId = job.metadata?.scheduleId;
+ const stopScheduledFor = job.metadata?.scheduledFor;
+ const isScheduledStop = stopScheduleId != null && stopScheduledFor != null;
+ let scheduledStopStamped = false;
+ if (isScheduledStop) {
+ const stopStamp = await beginScheduledStop({
+ scheduleId: stopScheduleId,
+ scheduledFor: stopScheduledFor,
+ });
+ if (stopStamp === 'in_progress') {
+ res.set('Retry-After', '1');
+ return res.status(409).json({ code: 'STOP_IN_PROGRESS', generationProtocolVersion });
+ }
+ scheduledStopStamped = stopStamp === true;
+ }
const abortResult = await GenerationJobManager.abortJob(jobStreamId, {
expectedCreatedAt: job.createdAt,
transformAbortContent: (content, abortJobData) => {
@@ -807,6 +836,18 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {
}
},
});
+ // The abort did not land (replaced/still-active/already-settled), so no persistence
+ // is in flight: release the Stop barrier we armed rather than deferring this
+ // occurrence's settlement for the full stale-owner window. A retry or a replacement
+ // generation re-stamps its own; the acknowledgement is fenced to the occurrence, not
+ // a generation, so it cannot settle a successor through its predecessor.
+ if (scheduledStopStamped && !abortResult.success) {
+ await acknowledgeScheduledStopPersistence({
+ scheduleId: stopScheduleId,
+ scheduledFor: stopScheduledFor,
+ });
+ scheduledStopStamped = false;
+ }
if (abortResult.failureReason === 'generation_replaced') {
return res.status(409).json({ code: 'RUN_REPLACED', generationProtocolVersion });
}
@@ -858,6 +899,53 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {
abortResultResponseMessageId: abortResult.jobData?.responseMessageId,
});
+ // `beforePublish` has run: its partial-message and checkpoint writes have either
+ // landed or failed. Acknowledge the Stop ONLY on success — that releases the owner's
+ // settlement barrier so the run can terminalize. On a persistence failure we leave
+ // the barrier unresolved: the run stays preserved (client retries; the stale-owner
+ // timeout is the bounded recovery) rather than settling over an incomplete write.
+ if (scheduledStopStamped && !abortResult.persistenceFailed) {
+ await acknowledgeScheduledStopPersistence({
+ scheduleId: stopScheduleId,
+ scheduledFor: stopScheduledFor,
+ // Re-drive the terminal outcome from here for a RUNNING generation: its owner
+ // calls recordScheduleOutcome once, and if that call's Stop barrier deferred
+ // (slow beforePublish), nothing would settle the run where no schedule
+ // reconciler is armed. recordRunOutcome is match-guarded and idempotent, so an
+ // owner that already settled makes this a no-op. A paused job is settled
+ // explicitly below and needs no re-drive here.
+ ...(job.status !== 'requires_action' && {
+ settle: {
+ status: 'interrupted',
+ conversationId: job.metadata?.conversationId ?? jobStreamId,
+ error: 'Scheduled run was stopped',
+ },
+ }),
+ });
+ }
+
+ // A paused generation has no live provider owner left to report the stop.
+ // Persist its scheduled occurrence here, after abortJob's required partial
+ // response/checkpoint work AND its acknowledgement above, while running generations
+ // continue to settle from their owning request/resume controller. A failed
+ // persistence skips settlement so the incomplete run is not terminalized.
+ if (
+ job.status === 'requires_action' &&
+ job.metadata?.scheduleId &&
+ !abortResult.persistenceFailed
+ ) {
+ await recordScheduleOutcome({
+ scheduleId: job.metadata.scheduleId,
+ scheduledFor: job.metadata.scheduledFor,
+ streamId: jobStreamId,
+ jobCreatedAt: job.createdAt,
+ status: 'interrupted',
+ conversationId: job.metadata.conversationId ?? jobStreamId,
+ clearConversationId: abortResult.jobData?.createdEventEmitted !== true,
+ error: 'Scheduled run was stopped while awaiting approval',
+ });
+ }
+
if (abortResult.persistenceFailed && generationProtocolVersion < GENERATION_PROTOCOL_V2) {
res.set('Retry-After', '1');
return res.status(409).json({
@@ -977,7 +1065,7 @@ if (isEnabled(LIMIT_MESSAGE_IP)) {
}
if (isEnabled(LIMIT_MESSAGE_USER)) {
- chatRouter.use(messageUserLimiter);
+ chatRouter.use(unless(exemptScheduleFromUserLimiter, messageUserLimiter));
}
chatRouter.use('/', chat);
diff --git a/api/server/routes/index.js b/api/server/routes/index.js
index 96baaf92b6..6b305686b6 100644
--- a/api/server/routes/index.js
+++ b/api/server/routes/index.js
@@ -17,6 +17,7 @@ const memories = require('./memories');
const presets = require('./presets');
const projects = require('./projects');
const prompts = require('./prompts');
+const schedules = require('./schedules');
const skills = require('./skills');
const balance = require('./balance');
const actions = require('./actions');
@@ -69,6 +70,7 @@ module.exports = {
models,
prompts,
projects,
+ schedules,
skills,
actions,
presets,
diff --git a/api/server/routes/schedules.js b/api/server/routes/schedules.js
new file mode 100644
index 0000000000..b426710990
--- /dev/null
+++ b/api/server/routes/schedules.js
@@ -0,0 +1,102 @@
+const express = require('express');
+const { Permissions, PermissionTypes } = require('librechat-data-provider');
+const {
+ isEnabled,
+ SCHEDULE_FILE_HOLD,
+ generateCheckAccess,
+ createSchedulesHandlers,
+} = require('@librechat/api');
+const { requireJwtAuth, configMiddleware, messageIpLimiter } = require('~/server/middleware');
+const {
+ getLimits,
+ fireScheduleNow,
+ deleteScheduleForOwner,
+ isUserDeleting,
+} = require('~/server/services/Schedules');
+const { resolveAgentFireAccess } = require('~/server/services/Schedules/access');
+const methods = require('~/models');
+
+const { getRoleByName } = methods;
+
+const router = express.Router();
+router.use(requireJwtAuth);
+router.use(configMiddleware);
+
+const checkSchedulesAccess = generateCheckAccess({
+ permissionType: PermissionTypes.SCHEDULES,
+ permissions: [Permissions.USE],
+ getRoleByName,
+});
+const checkSchedulesCreate = generateCheckAccess({
+ permissionType: PermissionTypes.SCHEDULES,
+ permissions: [Permissions.USE, Permissions.CREATE],
+ getRoleByName,
+});
+
+const handlers = createSchedulesHandlers({
+ methods,
+ getLimits,
+ // Full fire-equivalent access check (not mere existence): the body-based agent
+ // middleware is a no-op for schedule payloads (no `endpoint: 'agents'`), so
+ // enforce the same role AGENTS:USE + resource VIEW (with manage:agents bypass)
+ // the actual fire requires — otherwise a role without AGENTS:USE could schedule
+ // runs the chat route rejects and walks toward auto-disable.
+ canViewAgent: async (agentId, req) => (await resolveAgentFireAccess(agentId, req.user)) === 'ok',
+ filterOwnedFileIds: async (fileIds, userId) => {
+ const files = await methods.getFiles({ file_id: { $in: fileIds }, user: userId }, null, {
+ file_id: 1,
+ });
+ return (files ?? []).map((file) => file.file_id);
+ },
+ markFilesUsed: async (fileIds, userId) => {
+ // BOUNDED renewable hold (extendFilesTTL), not a permanent `$unset` of the upload
+ // TTL: permanence made a schedule deleted before its first run, an edit that
+ // replaced file_ids, or a failed creation leak the upload forever, since nothing
+ // ever restored an expiry. The first fire that actually SENDS the file clears its
+ // TTL through the ordinary consumption path; until then the hold is renewed at
+ // create/edit and each fire preflight, and lapses when the schedule stops touching
+ // it. Files already made permanent by a real send are skipped by construction.
+ // Idempotent, and never touches the usage counter (a retention is not a consumption).
+ // Then VERIFY every requested file still exists: one can be deleted between the
+ // ownership check and here, and a silent success would persist a schedule whose
+ // attachments the first fire drops. Existence is the check — not the hold's
+ // modified-count, which reads 0 for an already-permanent or already-held file.
+ const unique = [...new Set(fileIds)];
+ await methods.extendFilesTTL(unique, SCHEDULE_FILE_HOLD, { user: userId });
+ const files = await methods.getFiles({ file_id: { $in: unique }, user: userId }, null, {
+ file_id: 1,
+ });
+ const present = (files ?? []).length;
+ if (present !== unique.length) {
+ throw new Error(`attachment retention incomplete: ${present}/${unique.length} files exist`);
+ }
+ },
+ fireNow: fireScheduleNow,
+ // Quiesce-then-erase delete: stops new claims, settles provably job-less runs
+ // synchronously, aborts live ones, and reports honestly (see ScheduleDeleteResult);
+ // a delivered abort erases on the generation's own outcome write, in any topology.
+ deleteSchedule: deleteScheduleForOwner,
+ // Durable account-deletion barrier. A one-shot disable scan cannot close the
+ // create race, so every scheduling WRITE consults the user-level flag instead.
+ isUserDeleting,
+});
+
+router.get('/', checkSchedulesAccess, handlers.listSchedules);
+router.get('/:id', checkSchedulesAccess, handlers.getSchedule);
+router.post('/', checkSchedulesCreate, handlers.createSchedule);
+router.patch('/:id', checkSchedulesCreate, handlers.updateSchedule);
+router.delete('/:id', checkSchedulesCreate, handlers.deleteSchedule);
+// Run-now mutates runtime state; gate it on CREATE like the UI does (not USE).
+// This is also where LIMIT_MESSAGE_IP has to apply to a manual run: the fire itself is a
+// loopback POST carrying the server's address, so limiting by IP there would pool every
+// user into one bucket. Here the initiating client's address is still on the request.
+// The USER limiter is NOT duplicated here; the fire token carries the authenticated id,
+// so the chat router applies it to the loopback exactly once.
+router.post(
+ '/:id/run',
+ ...(isEnabled(process.env.LIMIT_MESSAGE_IP) ? [messageIpLimiter] : []),
+ checkSchedulesCreate,
+ handlers.runScheduleNow,
+);
+
+module.exports = router;
diff --git a/api/server/services/Schedules/access.js b/api/server/services/Schedules/access.js
new file mode 100644
index 0000000000..a278b4e339
--- /dev/null
+++ b/api/server/services/Schedules/access.js
@@ -0,0 +1,18 @@
+const mongoose = require('mongoose');
+const { createResolveAgentFireAccess } = require('@librechat/api');
+const { checkPermission } = require('~/server/services/PermissionService');
+const { hasCapability } = require('~/server/middleware/roles/capabilities');
+const { getRoleByName } = require('~/models');
+
+// Thin wiring over the TypeScript implementation in packages/api: inject the
+// api-layer lookups (agent id, role, capability, resource ACL); the authorization
+// logic itself lives in @librechat/api so it stays type-checked and in-boundary.
+const resolveAgentFireAccess = createResolveAgentFireAccess({
+ findAgentObjectId: (agentId) =>
+ mongoose.models.Agent.findOne({ id: agentId }).select('_id').lean(),
+ getRoleByName,
+ hasCapability,
+ checkPermission,
+});
+
+module.exports = { resolveAgentFireAccess };
diff --git a/api/server/services/Schedules/index.js b/api/server/services/Schedules/index.js
new file mode 100644
index 0000000000..72f76fa7dc
--- /dev/null
+++ b/api/server/services/Schedules/index.js
@@ -0,0 +1,112 @@
+let service;
+
+/**
+ * Build the schedule service on first use. Controllers import this facade in many
+ * focused test/runtime paths that never use scheduling; eagerly loading Config and
+ * the trigger worker made those paths initialize two unrelated service graphs just
+ * by requiring a controller.
+ */
+function getService() {
+ if (service != null) {
+ return service;
+ }
+ const mongoose = require('mongoose');
+ const { createSchedulesService } = require('@librechat/api');
+ const { getAppConfig } = require('~/server/services/Config/app');
+ const {
+ enqueueAgentTrigger,
+ getAgentTriggerDelivery,
+ } = require('~/server/services/Agents/triggers');
+ const { resolveAgentFireAccess } = require('./access');
+ const methods = require('~/models');
+ const isUserDeleting = async (userId) => !(await methods.isAgentTriggerPrincipalActive(userId));
+
+ service = createSchedulesService({
+ methods,
+ getAppConfig,
+ findUserById: (userId) =>
+ mongoose.models.User.findById(userId).select('_id tenantId role').lean(),
+ findBalance: (userId) => mongoose.models.Balance.findOne({ user: userId }).lean(),
+ upsertBalance: (userId, { set, setOnInsert }) =>
+ mongoose.models.Balance.findOneAndUpdate(
+ { user: userId },
+ {
+ ...(set && Object.keys(set).length > 0 ? { $set: set } : {}),
+ ...(setOnInsert && Object.keys(setOnInsert).length > 0
+ ? { $setOnInsert: setOnInsert }
+ : {}),
+ },
+ { upsert: true, new: true },
+ ).lean(),
+ // Compare-and-set: only initialize an existing record while its credit is still null.
+ // No upsert — a CAS miss must re-read the winner, not insert a fresh balance. `null`
+ // in the filter also matches a legacy record whose `tokenCredits` field is absent.
+ initializeNullBalance: (userId, { tokenCredits, sync }) =>
+ mongoose.models.Balance.findOneAndUpdate(
+ { user: userId, tokenCredits: null },
+ { $set: { tokenCredits, ...(sync && Object.keys(sync).length > 0 ? sync : {}) } },
+ { new: true },
+ ).lean(),
+ enqueueAgentTrigger,
+ // Reconciliation reads the durable delivery to tell a still-live admission (a
+ // deferred Retry-After) or a dead-letter apart from a genuinely orphaned run.
+ getTriggerDelivery: getAgentTriggerDelivery,
+ resolveAgentFireAccess,
+ // Reuse the merged trigger/deletion admission fence. A schedule fire and every
+ // generic trigger now fail closed on the same durable principal state.
+ isUserDeleting,
+ });
+ return service;
+}
+
+const invoke =
+ (method) =>
+ (...args) =>
+ getService()[method](...args);
+
+/** Host adapter for the generic generation runtime's approval-expiry seam. The
+ * callback is intentionally idempotent: a replica relay or schedule reconciliation
+ * may re-drive the same retained terminal evidence after a transient failure. */
+async function recordExpiredScheduleApproval(streamId, job) {
+ if (!job?.scheduleId || !job?.scheduledFor) {
+ return;
+ }
+ const recorded = await getService().recordScheduleOutcome({
+ scheduleId: job.scheduleId,
+ scheduledFor: job.scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'interrupted',
+ conversationId: job.conversationId ?? streamId,
+ error: 'Approval expired before a decision was made',
+ });
+ if (!recorded) {
+ throw new Error(`Failed to settle expired scheduled approval ${job.scheduleId}`);
+ }
+}
+
+module.exports = {
+ getLimits: invoke('getLimits'),
+ fireScheduleNow: invoke('fireScheduleNow'),
+ recordScheduleOutcome: invoke('recordScheduleOutcome'),
+ beginScheduledStop: invoke('beginScheduledStop'),
+ acknowledgeScheduledStopPersistence: invoke('acknowledgeScheduledStopPersistence'),
+ claimScheduleResume: invoke('claimScheduleResume'),
+ releaseScheduleResumeClaim: invoke('releaseScheduleResumeClaim'),
+ finalizeScheduleResumeClaim: invoke('finalizeScheduleResumeClaim'),
+ releaseScheduleResumeFence: invoke('releaseScheduleResumeFence'),
+ isScheduleLive: invoke('isScheduleLive'),
+ deleteScheduleForOwner: invoke('deleteScheduleForOwner'),
+ quiesceUserSchedules: invoke('quiesceUserSchedules'),
+ restoreUserSchedulesFromDeletion: invoke('restoreUserSchedulesFromDeletion'),
+ initializeScheduleEngine: invoke('initializeScheduleEngine'),
+ initializeScheduleErasureSweep: invoke('initializeScheduleErasureSweep'),
+ recordExpiredScheduleApproval,
+ isUserDeleting: async (userId) => {
+ const methods = require('~/models');
+ return !(await methods.isAgentTriggerPrincipalActive(userId));
+ },
+ // Identity-fenced delete for a retained job whose run has since been settled
+ // inline; reconciliation only reaches jobs whose run is still active.
+ clearScheduledJob: (...args) => getService().engineDeps.clearReconciledJob(...args),
+};
diff --git a/api/server/services/Schedules/index.spec.js b/api/server/services/Schedules/index.spec.js
new file mode 100644
index 0000000000..194af1ea68
--- /dev/null
+++ b/api/server/services/Schedules/index.spec.js
@@ -0,0 +1,60 @@
+const mockRecordScheduleOutcome = jest.fn();
+const mockCreateSchedulesService = jest.fn(() => ({
+ recordScheduleOutcome: mockRecordScheduleOutcome,
+}));
+
+jest.mock('@librechat/api', () => ({
+ createSchedulesService: (...args) => mockCreateSchedulesService(...args),
+}));
+jest.mock('mongoose', () => ({ models: {} }));
+jest.mock('~/server/services/Config/app', () => ({ getAppConfig: jest.fn() }));
+jest.mock('~/server/services/Agents/triggers', () => ({ enqueueAgentTrigger: jest.fn() }));
+jest.mock('./access', () => ({ resolveAgentFireAccess: jest.fn() }));
+jest.mock('~/models', () => ({ isAgentTriggerPrincipalActive: jest.fn() }));
+
+const { recordExpiredScheduleApproval } = require('./index');
+
+describe('recordExpiredScheduleApproval', () => {
+ beforeEach(() => {
+ mockRecordScheduleOutcome.mockReset().mockResolvedValue(true);
+ });
+
+ it('ignores ordinary generation approvals', async () => {
+ await expect(
+ recordExpiredScheduleApproval('conversation-1', { createdAt: 1000 }),
+ ).resolves.toBeUndefined();
+
+ expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
+ });
+
+ it('settles the exact scheduled generation as interrupted', async () => {
+ await recordExpiredScheduleApproval('conversation-1', {
+ createdAt: 1000,
+ conversationId: 'conversation-1',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ });
+
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ streamId: 'conversation-1',
+ jobCreatedAt: 1000,
+ status: 'interrupted',
+ conversationId: 'conversation-1',
+ error: 'Approval expired before a decision was made',
+ });
+ });
+
+ it('surfaces a failed durable settlement instead of reporting false success', async () => {
+ mockRecordScheduleOutcome.mockResolvedValue(false);
+
+ await expect(
+ recordExpiredScheduleApproval('conversation-1', {
+ createdAt: 1000,
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ }),
+ ).rejects.toThrow('Failed to settle expired scheduled approval schedule-1');
+ });
+});
diff --git a/client/src/components/SidePanel/Agents/Instructions.tsx b/client/src/components/SidePanel/Agents/Instructions.tsx
index e4b0ae58f6..347ad0a1d8 100644
--- a/client/src/components/SidePanel/Agents/Instructions.tsx
+++ b/client/src/components/SidePanel/Agents/Instructions.tsx
@@ -1,185 +1,42 @@
-import { useState, useId } from 'react';
-import * as Menu from '@ariakit/react/menu';
-import { PlusCircle, Maximize2 } from 'lucide-react';
-import { specialVariables } from 'librechat-data-provider';
import { Controller, useFormContext } from 'react-hook-form';
-import {
- Label,
- Button,
- OGDialog,
- Textarea,
- DropdownPopup,
- OGDialogClose,
- TooltipAnchor,
- OGDialogTitle,
- OGDialogHeader,
- OGDialogContent,
-} from '@librechat/client';
-import type { TSpecialVarLabel } from 'librechat-data-provider';
import type { AgentForm } from '~/common';
+import { VariableEditor } from '~/components/Variables';
import { useLocalize } from '~/hooks';
-interface VariableOption {
- label: TSpecialVarLabel;
- value: string;
-}
-
-const variableOptions: VariableOption[] = Object.keys(specialVariables).map((key) => ({
- label: `com_ui_special_var_${key}` as TSpecialVarLabel,
- value: `{{${key}}}`,
-}));
-
export default function Instructions() {
- const menuId = useId();
- const dialogMenuId = useId();
const localize = useLocalize();
- const methods = useFormContext<AgentForm>();
- const { control, setValue, getValues } = methods;
-
- const [isMenuOpen, setIsMenuOpen] = useState(false);
- const [isDialogMenuOpen, setIsDialogMenuOpen] = useState(false);
- const [isDialogOpen, setIsDialogOpen] = useState(false);
-
- const handleAddVariable = (label: TSpecialVarLabel, value: string) => {
- const currentInstructions = getValues('instructions') || '';
- const spacer = currentInstructions.length > 0 ? '\n' : '';
- const prefix = localize(label);
- setValue('instructions', currentInstructions + spacer + prefix + ': ' + value);
- setIsMenuOpen(false);
- setIsDialogMenuOpen(false);
- };
-
- const variableItems = variableOptions.map((option) => ({
- label: localize(option.label) || option.label,
- onClick: () => handleAddVariable(option.label, option.value),
- }));
+ const { control } = useFormContext<AgentForm>();
return (
- <div className="mb-3 flex flex-col">
- <div className="mb-1 flex items-center justify-between">
- <Label
- className="block text-[11px] font-medium uppercase tracking-wide text-text-secondary"
- htmlFor="instructions"
- >
- {localize('com_ui_instructions')}
- </Label>
- <div className="flex items-center gap-0.5">
- <DropdownPopup
- portal={true}
- mountByState={true}
- unmountOnHide={true}
- preserveTabOrder={true}
- isOpen={isMenuOpen}
- setIsOpen={setIsMenuOpen}
- trigger={
- <Menu.MenuButton
- id="variables-menu-button"
- aria-label={localize('com_ui_variables')}
- title={localize('com_ui_variables')}
- className="inline-flex h-7 w-7 items-center justify-center rounded-lg text-text-secondary transition-colors hover:bg-surface-secondary hover:text-text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring-primary"
- >
- <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
- </Menu.MenuButton>
- }
- items={variableItems}
- menuId={menuId}
- className="pointer-events-auto z-30"
- />
- <TooltipAnchor
- description={localize('com_ui_expand_editor')}
- render={
- <Button
- variant="ghost"
- onClick={() => setIsDialogOpen(true)}
- aria-label={localize('com_ui_expand_editor')}
- className="h-7 w-7 p-0 text-text-secondary hover:bg-surface-secondary hover:text-text-primary"
- >
- <Maximize2 className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
- </Button>
- }
+ <Controller
+ name="instructions"
+ control={control}
+ render={({ field, fieldState: { error } }) => (
+ <div className="mb-3 flex flex-col">
+ <VariableEditor
+ id="instructions"
+ label={localize('com_ui_instructions')}
+ value={field.value ?? ''}
+ onChange={field.onChange}
+ onBlur={field.onBlur}
+ inputRef={field.ref}
+ placeholder={localize('com_agents_instructions_placeholder')}
+ className="min-h-[88px] resize-y"
+ labelClassName="block text-[11px] font-medium uppercase tracking-wide text-text-secondary"
+ rows={3}
+ required={true}
+ invalid={error != null}
/>
+ {error && (
+ <span
+ className="mt-1 text-xs text-text-destructive transition duration-300 ease-in-out"
+ role="alert"
+ >
+ {localize('com_ui_field_required')}
+ </span>
+ )}
</div>
- </div>
- <Controller
- name="instructions"
- control={control}
- render={({ field, fieldState: { error } }) => (
- <>
- <Textarea
- {...field}
- value={field.value ?? ''}
- className="min-h-[88px] resize-y"
- id="instructions"
- placeholder={localize('com_agents_instructions_placeholder')}
- rows={3}
- aria-label={localize('com_ui_instructions')}
- aria-required="true"
- aria-invalid={error ? 'true' : 'false'}
- />
- {error && (
- <span
- className="mt-1 text-xs text-text-destructive transition duration-300 ease-in-out"
- role="alert"
- >
- {localize('com_ui_field_required')}
- </span>
- )}
- </>
- )}
- />
-
- <OGDialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>
- <OGDialogContent
- className="flex h-[85vh] max-h-[85vh] w-11/12 max-w-6xl flex-col gap-4 p-6"
- showCloseButton={false}
- >
- <OGDialogHeader className="mb-2 pr-14">
- <OGDialogTitle className="text-left text-2xl font-semibold">
- {localize('com_ui_instructions')}
- </OGDialogTitle>
- </OGDialogHeader>
- <Controller
- name="instructions"
- control={control}
- render={({ field }) => (
- <Textarea
- {...field}
- value={field.value ?? ''}
- className="min-h-0 flex-1 resize-none text-base leading-relaxed"
- placeholder={localize('com_agents_instructions_placeholder')}
- aria-label={localize('com_ui_instructions')}
- />
- )}
- />
- <div className="flex items-center justify-between">
- <DropdownPopup
- portal={true}
- mountByState={true}
- unmountOnHide={true}
- preserveTabOrder={true}
- isOpen={isDialogMenuOpen}
- setIsOpen={setIsDialogMenuOpen}
- trigger={
- <Menu.MenuButton
- id="variables-menu-button-dialog"
- render={
- <Button variant="outline" className="gap-1.5">
- <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
- {localize('com_ui_variables')}
- </Button>
- }
- />
- }
- items={variableItems}
- menuId={dialogMenuId}
- className="pointer-events-auto z-[200]"
- />
- <OGDialogClose asChild>
- <Button>{localize('com_ui_done')}</Button>
- </OGDialogClose>
- </div>
- </OGDialogContent>
- </OGDialog>
- </div>
+ )}
+ />
);
}
diff --git a/client/src/components/SidePanel/Schedules/ScheduleCard.tsx b/client/src/components/SidePanel/Schedules/ScheduleCard.tsx
new file mode 100644
index 0000000000..03cad7310a
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/ScheduleCard.tsx
@@ -0,0 +1,271 @@
+import { useId, useRef, useMemo, useState, useCallback } from 'react';
+import * as Ariakit from '@ariakit/react';
+import { useNavigate } from 'react-router-dom';
+import { Trans, useTranslation } from 'react-i18next';
+import { Play, Trash, Pencil, Ellipsis } from 'lucide-react';
+import { PermissionTypes, Permissions } from 'librechat-data-provider';
+import {
+ Label,
+ Chip,
+ Switch,
+ Spinner,
+ OGDialog,
+ DropdownPopup,
+ OGDialogTemplate,
+ useToastContext,
+} from '@librechat/client';
+import type { TSchedule, ScheduleRunStatus, ScheduleDisabledReason } from 'librechat-data-provider';
+import type { TranslationKeys } from '~/hooks';
+import {
+ useGetAgentByIdQuery,
+ useDeleteScheduleMutation,
+ useUpdateScheduleMutation,
+ useRunScheduleNowMutation,
+} from '~/data-provider';
+import { useLocalize, useHasAccess } from '~/hooks';
+import { useAgentsMapContext } from '~/Providers';
+import { getMessageTimestamp } from '~/utils';
+import ScheduleDialog from './ScheduleDialog';
+import { describeCadence } from './cadence';
+
+interface ScheduleCardProps {
+ schedule: TSchedule;
+}
+
+type StatusTone = 'neutral' | 'success' | 'warning' | 'error';
+
+const STATUS_CHIPS: Record<ScheduleRunStatus, { label: TranslationKeys; tone: StatusTone }> = {
+ success: { label: 'com_ui_schedule_last_run', tone: 'success' },
+ error: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },
+ interrupted: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },
+ requires_action: { label: 'com_ui_schedule_needs_approval', tone: 'warning' },
+ started: { label: 'com_ui_schedule_run_started', tone: 'neutral' },
+ skipped_overlap: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },
+ skipped_balance: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },
+};
+
+const DISABLED_REASON_LABELS: Record<ScheduleDisabledReason, TranslationKeys> = {
+ too_many_failures: 'com_ui_schedule_disabled_too_many_failures',
+ agent_deleted: 'com_ui_schedule_disabled_agent_deleted',
+ invalid_schedule: 'com_ui_schedule_disabled_invalid',
+ permission_revoked: 'com_ui_schedule_disabled_permission_revoked',
+ insufficient_balance: 'com_ui_schedule_disabled_insufficient_balance',
+};
+
+export default function ScheduleCard({ schedule }: ScheduleCardProps) {
+ const localize = useLocalize();
+ const navigate = useNavigate();
+ const { i18n } = useTranslation();
+ const { showToast } = useToastContext();
+ const agentsMap = useAgentsMapContext();
+ // Enable/disable, run-now, edit and delete all hit CREATE-gated routes, so a
+ // USE-only viewer sees a read-only card instead of controls that 403.
+ const canWrite = useHasAccess({
+ permissionType: PermissionTypes.SCHEDULES,
+ permission: Permissions.CREATE,
+ });
+
+ const menuId = useId();
+ const [menuOpen, setMenuOpen] = useState(false);
+ const [editOpen, setEditOpen] = useState(false);
+ const [deleteOpen, setDeleteOpen] = useState(false);
+ const editButtonRef = useRef<HTMLButtonElement>(null);
+ const deleteButtonRef = useRef<HTMLButtonElement>(null);
+
+ const mappedAgent = agentsMap?.[schedule.agent_id];
+ const { data: fetchedAgent } = useGetAgentByIdQuery(schedule.agent_id, {
+ enabled: agentsMap !== undefined && mappedAgent == null,
+ });
+ const agentName = mappedAgent?.name || fetchedAgent?.name || schedule.agent_id;
+
+ const updateSchedule = useUpdateScheduleMutation({
+ onError: () => {
+ showToast({ message: localize('com_ui_error'), status: 'error' });
+ },
+ });
+ const deleteSchedule = useDeleteScheduleMutation();
+ const runSchedule = useRunScheduleNowMutation();
+
+ const handleToggle = useCallback(
+ (checked: boolean) => {
+ updateSchedule.mutate({ id: schedule.id, payload: { enabled: checked } });
+ },
+ [schedule.id, updateSchedule],
+ );
+
+ const handleRunNow = useCallback(() => {
+ runSchedule.mutate(schedule.id, {
+ onSuccess: () => {
+ showToast({ message: localize('com_ui_schedule_run_now_started'), status: 'success' });
+ setMenuOpen(false);
+ },
+ onError: () => {
+ showToast({ message: localize('com_ui_error'), status: 'error' });
+ },
+ });
+ }, [schedule.id, runSchedule, showToast, localize]);
+
+ const confirmDelete = useCallback(() => {
+ deleteSchedule.mutate(schedule.id, {
+ onSuccess: () => {
+ showToast({ message: localize('com_ui_deleted'), status: 'success' });
+ setDeleteOpen(false);
+ },
+ onError: () => {
+ showToast({ message: localize('com_ui_error'), status: 'error' });
+ },
+ });
+ }, [schedule.id, deleteSchedule, showToast, localize]);
+
+ const cadenceText = describeCadence(schedule.cadence, localize, i18n.language);
+
+ const nextRunText = useMemo(() => {
+ if (!schedule.enabled || schedule.nextRunAt == null) {
+ return null;
+ }
+ const timestamp = getMessageTimestamp(schedule.nextRunAt, i18n.language);
+ if (!timestamp) {
+ return null;
+ }
+ return localize('com_ui_schedule_next_run', { time: timestamp.relative });
+ }, [schedule.enabled, schedule.nextRunAt, i18n.language, localize]);
+
+ const dropdownItems = useMemo(
+ () => [
+ {
+ label: localize('com_ui_schedule_run_now'),
+ onClick: handleRunNow,
+ hideOnClick: false,
+ disabled: runSchedule.isLoading,
+ icon: runSchedule.isLoading ? (
+ <Spinner className="size-4" />
+ ) : (
+ <Play className="icon-sm mr-2 text-text-primary" aria-hidden="true" />
+ ),
+ },
+ {
+ label: localize('com_ui_edit'),
+ onClick: () => setEditOpen(true),
+ icon: <Pencil className="icon-sm mr-2 text-text-primary" aria-hidden="true" />,
+ ariaHasPopup: 'dialog' as const,
+ hideOnClick: false,
+ ref: editButtonRef,
+ render: (props) => <button {...props} />,
+ },
+ {
+ label: localize('com_ui_delete'),
+ onClick: () => setDeleteOpen(true),
+ icon: <Trash className="icon-sm mr-2 text-text-primary" aria-hidden="true" />,
+ ariaHasPopup: 'dialog' as const,
+ hideOnClick: false,
+ ref: deleteButtonRef,
+ render: (props) => <button {...props} />,
+ },
+ ],
+ [localize, handleRunNow, runSchedule.isLoading],
+ );
+
+ const statusChip = schedule.lastRun ? STATUS_CHIPS[schedule.lastRun.status] : null;
+ const lastRunConvoId = schedule.lastRun?.conversationId;
+
+ return (
+ <div
+ data-testid="schedule-card"
+ className="rounded-lg border border-border-light bg-transparent px-3 py-2.5 transition-colors duration-theme-fast hover:bg-surface-secondary"
+ >
+ <div className="flex items-center gap-2">
+ <span className="min-w-0 flex-1 truncate text-sm font-semibold text-text-primary">
+ {schedule.name}
+ </span>
+ {canWrite && (
+ <>
+ <Switch
+ checked={schedule.enabled}
+ onCheckedChange={handleToggle}
+ disabled={updateSchedule.isLoading}
+ aria-label={`${localize('com_ui_schedule_enabled')}: ${schedule.name}`}
+ className="shrink-0"
+ />
+ <DropdownPopup
+ portal={true}
+ menuId={menuId}
+ focusLoop={true}
+ className="z-[125]"
+ unmountOnHide={true}
+ isOpen={menuOpen}
+ setIsOpen={setMenuOpen}
+ trigger={
+ <Ariakit.MenuButton
+ id={`schedule-menu-${schedule.id}`}
+ aria-label={`${localize('com_ui_schedule_options')}: ${schedule.name}`}
+ className="inline-flex size-7 shrink-0 items-center justify-center rounded-md text-text-secondary hover:bg-surface-tertiary hover:text-text-primary focus:outline-none focus-visible:ring-2 focus-visible:ring-border-heavy"
+ >
+ <Ellipsis className="size-4" aria-hidden={true} />
+ </Ariakit.MenuButton>
+ }
+ items={dropdownItems}
+ />
+ </>
+ )}
+ </div>
+ <p className="mt-0.5 truncate text-xs text-text-secondary" title={agentName}>
+ {agentName}
+ </p>
+ <p className="mt-1 text-sm text-text-primary">{cadenceText}</p>
+ {nextRunText != null && <p className="mt-0.5 text-xs text-text-secondary">{nextRunText}</p>}
+ {(statusChip != null || schedule.disabledReason != null) && (
+ <div className="mt-1.5 flex flex-wrap items-center gap-1.5">
+ {statusChip != null &&
+ (lastRunConvoId != null && lastRunConvoId !== '' ? (
+ <button
+ type="button"
+ className="rounded-full hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary"
+ onClick={() => navigate(`/c/${lastRunConvoId}`)}
+ >
+ <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>
+ </button>
+ ) : (
+ <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>
+ ))}
+ {schedule.disabledReason != null && (
+ <Chip tone="error">{localize(DISABLED_REASON_LABELS[schedule.disabledReason])}</Chip>
+ )}
+ </div>
+ )}
+ {editOpen && (
+ <ScheduleDialog
+ open={editOpen}
+ onOpenChange={setEditOpen}
+ schedule={schedule}
+ triggerRef={editButtonRef as React.MutableRefObject<HTMLButtonElement | null>}
+ />
+ )}
+ <OGDialog
+ open={deleteOpen}
+ onOpenChange={setDeleteOpen}
+ triggerRef={deleteButtonRef as React.MutableRefObject<HTMLButtonElement | null>}
+ >
+ <OGDialogTemplate
+ showCloseButton={false}
+ title={localize('com_ui_schedule_delete')}
+ className="w-11/12 max-w-lg"
+ main={
+ <Label className="text-left text-sm font-medium">
+ <Trans
+ i18nKey="com_ui_delete_confirm_strong"
+ values={{ title: schedule.name }}
+ components={{ strong: <strong /> }}
+ />
+ </Label>
+ }
+ selection={{
+ selectHandler: confirmDelete,
+ selectClasses:
+ 'bg-surface-destructive text-text-on-status hover:bg-surface-destructive-hover',
+ selectText: localize('com_ui_delete'),
+ }}
+ />
+ </OGDialog>
+ </div>
+ );
+}
diff --git a/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx b/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx
new file mode 100644
index 0000000000..5f71c241a6
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx
@@ -0,0 +1,24 @@
+import { Skeleton } from '@librechat/client';
+
+/** Mirrors the schedule card's title controls, agent, cadence, and next-run rows. */
+export default function ScheduleCardSkeleton({ count = 5 }: { count?: number }) {
+ return (
+ <div className="flex flex-col gap-2" aria-hidden="true">
+ {Array.from({ length: count }, (_, index) => (
+ <div
+ key={index}
+ className="rounded-lg border border-border-light bg-transparent px-3 py-2.5"
+ >
+ <div className="flex items-center gap-2">
+ <Skeleton className="h-4 min-w-0 flex-1 rounded" />
+ <Skeleton className="h-5 w-9 shrink-0 rounded-full" />
+ <Skeleton className="size-7 shrink-0 rounded-md" />
+ </div>
+ <Skeleton className="mt-1 h-3 w-2/5 rounded" />
+ <Skeleton className="mt-2 h-4 w-3/4 rounded" />
+ <Skeleton className="mt-1 h-3 w-1/2 rounded" />
+ </div>
+ ))}
+ </div>
+ );
+}
diff --git a/client/src/components/SidePanel/Schedules/ScheduleDialog.tsx b/client/src/components/SidePanel/Schedules/ScheduleDialog.tsx
new file mode 100644
index 0000000000..16d1356e18
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/ScheduleDialog.tsx
@@ -0,0 +1,575 @@
+import { useMemo, useRef } from 'react';
+import { v4 } from 'uuid';
+import { useTranslation } from 'react-i18next';
+import { useForm, Controller } from 'react-hook-form';
+import { PermissionBits, scheduleFrequencies } from 'librechat-data-provider';
+import {
+ Input,
+ Label,
+ Radio,
+ Button,
+ FieldMessage,
+ Spinner,
+ Dropdown,
+ OGDialog,
+ ControlCombobox,
+ OGDialogTemplate,
+ useToastContext,
+} from '@librechat/client';
+import type {
+ TSchedule,
+ TCreateSchedule,
+ TScheduleCadence,
+ ScheduleFrequency,
+} from 'librechat-data-provider';
+import type { TranslationKeys } from '~/hooks';
+import type { Meridiem } from './cadence';
+import {
+ useListAgentsQuery,
+ useCreateScheduleMutation,
+ useUpdateScheduleMutation,
+} from '~/data-provider';
+import { to12Hour, to24Hour, describeCadence, formatScheduleDay } from './cadence';
+import { VariableEditor } from '~/components/Variables';
+import { useLocalize } from '~/hooks';
+import { cn } from '~/utils';
+
+interface ScheduleDialogProps {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ schedule?: TSchedule;
+ triggerRef?: React.MutableRefObject<HTMLButtonElement | null>;
+}
+
+type ScheduleFormValues = {
+ name: string;
+ prompt: string;
+ agent_id: string;
+ frequency: ScheduleFrequency;
+ hour12: number;
+ minute: number;
+ meridiem: Meridiem;
+ dayOfWeek: number;
+};
+
+const FREQUENCY_LABELS: Record<ScheduleFrequency, TranslationKeys> = {
+ hourly: 'com_ui_schedule_hourly',
+ daily: 'com_ui_schedule_daily',
+ weekdays: 'com_ui_schedule_weekdays',
+ weekly: 'com_ui_schedule_weekly',
+};
+
+const BASE_MINUTES = [0, 15, 30, 45];
+
+const FORM_ID = 'schedule-form';
+
+const getDefaultValues = (schedule?: TSchedule): ScheduleFormValues => {
+ if (!schedule) {
+ return {
+ name: '',
+ prompt: '',
+ agent_id: '',
+ frequency: 'daily',
+ hour12: 9,
+ minute: 0,
+ meridiem: 'AM',
+ dayOfWeek: 1,
+ };
+ }
+ const { hour12, meridiem } = to12Hour(schedule.cadence.hour);
+ return {
+ name: schedule.name,
+ prompt: schedule.prompt,
+ agent_id: schedule.agent_id,
+ frequency: schedule.cadence.frequency,
+ hour12,
+ minute: schedule.cadence.minute,
+ meridiem,
+ dayOfWeek: schedule.cadence.daysOfWeek?.[0] ?? 1,
+ };
+};
+
+type CadenceFormValues = Pick<
+ ScheduleFormValues,
+ 'frequency' | 'hour12' | 'minute' | 'meridiem' | 'dayOfWeek'
+>;
+
+const buildCadence = (values: CadenceFormValues, overrideDays?: number[]): TScheduleCadence => {
+ if (values.frequency === 'hourly') {
+ return { frequency: 'hourly', hour: 0, minute: values.minute };
+ }
+ const hour = to24Hour(values.hour12, values.meridiem);
+ if (values.frequency === 'weekly') {
+ // `overrideDays` preserves a stored multi-day set (which this single-day
+ // picker can't represent) when only the time — not the day — was changed.
+ return {
+ frequency: 'weekly',
+ hour,
+ minute: values.minute,
+ daysOfWeek: overrideDays ?? [values.dayOfWeek],
+ };
+ }
+ return { frequency: values.frequency, hour, minute: values.minute };
+};
+
+export default function ScheduleDialog({
+ open,
+ onOpenChange,
+ schedule,
+ triggerRef,
+}: ScheduleDialogProps) {
+ const localize = useLocalize();
+ const { i18n } = useTranslation();
+ const { showToast } = useToastContext();
+ const locale = i18n.language;
+
+ const {
+ control,
+ register,
+ watch,
+ handleSubmit,
+ formState: { dirtyFields, errors },
+ } = useForm<ScheduleFormValues>({
+ defaultValues: getDefaultValues(schedule),
+ mode: 'onChange',
+ });
+ /** The revision the form defaults were built from, captured at the same instant.
+ * The `schedule` prop keeps refreshing while the dialog is open (the schedules
+ * query polls), so fencing with the LIVE prop's revision would stamp a freshly
+ * refreshed revision onto a cadence rebuilt from this stale snapshot — exactly
+ * the overwrite the fence exists to refuse. */
+ const openedConfigRevision = useRef(schedule?.configRevision);
+ const frequency = watch('frequency');
+ const hour12 = watch('hour12');
+ const minute = watch('minute');
+ const meridiem = watch('meridiem');
+ const dayOfWeek = watch('dayOfWeek');
+
+ const { data: agents } = useListAgentsQuery(
+ { requiredPermission: PermissionBits.VIEW },
+ { select: (res) => res.data },
+ );
+
+ const agentItems = useMemo(
+ () => (agents ?? []).map((agent) => ({ label: agent.name || agent.id, value: agent.id })),
+ [agents],
+ );
+
+ const frequencyOptions = useMemo(
+ () =>
+ scheduleFrequencies.map((value) => ({
+ value,
+ label: localize(FREQUENCY_LABELS[value]),
+ })),
+ [localize],
+ );
+
+ const hourOptions = useMemo(
+ () => Array.from({ length: 12 }, (_, index) => String(index + 1)),
+ [],
+ );
+
+ const minuteOptions = useMemo(() => {
+ const minutes = new Set(BASE_MINUTES);
+ if (schedule) {
+ minutes.add(schedule.cadence.minute);
+ }
+ return [...minutes]
+ .sort((a, b) => a - b)
+ .map((minute) => ({ value: String(minute), label: String(minute).padStart(2, '0') }));
+ }, [schedule]);
+
+ const meridiemOptions = useMemo(
+ () => [
+ { value: 'AM', label: localize('com_ui_schedule_am') },
+ { value: 'PM', label: localize('com_ui_schedule_pm') },
+ ],
+ [localize],
+ );
+
+ const dayOptions = useMemo(
+ () =>
+ Array.from({ length: 7 }, (_, day) => ({
+ value: String(day),
+ label: formatScheduleDay(day, locale),
+ })),
+ [locale],
+ );
+
+ const timezone = useMemo(
+ () => schedule?.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
+ [schedule],
+ );
+
+ /** Idempotency key for the create being attempted. Held in a ref so every retry of
+ * the same intent reuses it, and rotated when the attempt SUCCEEDS or when the
+ * form content changes after a failed one (the server pins each key to the exact
+ * payload it first saw, so retrying edited content under the old key answers 409).
+ * `uuid` rather than `crypto.randomUUID`, which secure contexts gate — an HTTP/IP
+ * deployment would throw here and never open the dialog. */
+ const createRequestId = useRef(v4());
+ /** Fingerprint of the payload the current key was last attempted with. */
+ const lastAttemptedPayload = useRef<string | null>(null);
+
+ const createSchedule = useCreateScheduleMutation({
+ onSuccess: () => {
+ createRequestId.current = v4();
+ lastAttemptedPayload.current = null;
+ showToast({ message: localize('com_ui_schedule_created'), status: 'success' });
+ onOpenChange(false);
+ },
+ onError: () => {
+ showToast({ message: localize('com_ui_error'), status: 'error' });
+ },
+ });
+
+ const updateSchedule = useUpdateScheduleMutation({
+ onSuccess: () => {
+ showToast({ message: localize('com_ui_schedule_updated'), status: 'success' });
+ onOpenChange(false);
+ },
+ onError: (error) => {
+ const status = (error as { response?: { status?: number } } | undefined)?.response?.status;
+ showToast({
+ message: localize(status === 409 ? 'com_ui_schedule_conflict' : 'com_ui_error'),
+ status: 'error',
+ });
+ },
+ });
+
+ const isLoading = createSchedule.isLoading || updateSchedule.isLoading;
+
+ /** The stored weekly days an edit deliberately keeps: the form's single-day picker
+ * only holds `daysOfWeek[0]`, so an untouched picker must not collapse a
+ * multi-day (API-created) weekly schedule. ONE rule, used by both the submitted
+ * cadence and the summary — a summary built without it told the user the schedule
+ * runs on one day while the submit preserved (and kept firing) all of them. */
+ const resolvePreservedWeeklyDays = (nextFrequency: ScheduleFormValues['frequency']) =>
+ schedule &&
+ !dirtyFields.dayOfWeek &&
+ !dirtyFields.frequency &&
+ schedule.cadence.frequency === 'weekly' &&
+ nextFrequency === 'weekly'
+ ? schedule.cadence.daysOfWeek
+ : undefined;
+
+ const onSubmit = (values: ScheduleFormValues) => {
+ if (schedule) {
+ // Preserve the stored cadence entirely on a pure rename (no cadence control
+ // touched).
+ const cadenceTouched =
+ dirtyFields.frequency ||
+ dirtyFields.hour12 ||
+ dirtyFields.minute ||
+ dirtyFields.meridiem ||
+ dirtyFields.dayOfWeek;
+ const cadence = buildCadence(values, resolvePreservedWeeklyDays(values.frequency));
+ // PATCH only the fields the user actually touched, like the cadence handling
+ // above: submitting the whole form snapshot silently overwrites fields another
+ // tab or session edited while this dialog sat open (the server's revision fence
+ // reads the CURRENT revision, so it cannot catch a stale full-form write).
+ const payload = {
+ ...(dirtyFields.name ? { name: values.name.trim() } : {}),
+ ...(dirtyFields.prompt ? { prompt: values.prompt.trim() } : {}),
+ ...(dirtyFields.agent_id ? { agent_id: values.agent_id } : {}),
+ ...(cadenceTouched ? { cadence } : {}),
+ };
+ // Nothing touched: a field-less PATCH is refused server-side (it would rotate
+ // the schedule's fencing for a request that changes nothing), so just close.
+ if (Object.keys(payload).length === 0) {
+ onOpenChange(false);
+ return;
+ }
+ updateSchedule.mutate({
+ id: schedule.id,
+ // Fence on the revision this dialog OPENED with (not the live prop, which
+ // polling refreshes under the stale form): cadence is rebuilt whole from
+ // the opening snapshot, so an edit from another tab would otherwise be
+ // silently overwritten — the server answers 409 instead.
+ payload: {
+ ...payload,
+ ...(openedConfigRevision.current != null
+ ? { expectedConfigRevision: openedConfigRevision.current }
+ : {}),
+ },
+ });
+ return;
+ }
+ const content = {
+ name: values.name.trim(),
+ prompt: values.prompt.trim(),
+ agent_id: values.agent_id,
+ cadence: buildCadence(values),
+ timezone,
+ target: 'new' as const,
+ enabled: true,
+ };
+ // STABLE across retries of the SAME intent, which is the whole point: the server
+ // commits the row and arms it in two writes, so a failure between them leaves the
+ // client unable to tell what persisted, and a fresh key per attempt would make
+ // each retry a new schedule. But the server pins each key to the exact payload it
+ // first saw — so once the user EDITS the form after a failed attempt, this is a
+ // new intent and needs a new key, or the retry answers 409 until the dialog is
+ // reopened.
+ const fingerprint = JSON.stringify(content);
+ if (lastAttemptedPayload.current != null && lastAttemptedPayload.current !== fingerprint) {
+ createRequestId.current = v4();
+ }
+ lastAttemptedPayload.current = fingerprint;
+ const payload: TCreateSchedule = {
+ ...content,
+ clientRequestId: createRequestId.current,
+ };
+ createSchedule.mutate(payload);
+ };
+
+ const summaryCadence = buildCadence(
+ { frequency, hour12, minute, meridiem, dayOfWeek },
+ resolvePreservedWeeklyDays(frequency),
+ );
+ const summary = `${describeCadence(summaryCadence, localize, locale)} · ${timezone}`;
+
+ return (
+ <OGDialog open={open} onOpenChange={onOpenChange} triggerRef={triggerRef}>
+ <OGDialogTemplate
+ title={localize(schedule ? 'com_ui_schedule_edit' : 'com_ui_schedule_new')}
+ showCloseButton={false}
+ // The agent and time popovers cannot portal out of a focus-trapping dialog
+ // (see below), so `overflow-visible` keeps them from being clipped. Only from
+ // `md` up: the two-column layout fits well inside 90vh there, while narrow
+ // viewports keep the template's scrolling so the footer stays reachable.
+ className="w-11/12 md:max-w-3xl md:overflow-visible"
+ main={
+ <form id={FORM_ID} onSubmit={handleSubmit(onSubmit)} className="space-y-4">
+ <div className="grid gap-4 md:grid-cols-2">
+ <div className="space-y-2">
+ <Label htmlFor="schedule-name" className="text-sm font-medium text-text-primary">
+ {localize('com_ui_name')}
+ </Label>
+ <Input
+ id="schedule-name"
+ className="w-full"
+ placeholder={localize('com_ui_schedule_name_placeholder')}
+ aria-invalid={errors.name != null}
+ aria-describedby="schedule-name-message"
+ {...register('name', { required: localize('com_ui_field_required') })}
+ />
+ <FieldMessage id="schedule-name-message" message={errors.name?.message} />
+ </div>
+ <div className="space-y-2">
+ <Label htmlFor="schedule-agent" className="text-sm font-medium text-text-primary">
+ {localize('com_ui_agent')}
+ </Label>
+ <Controller
+ name="agent_id"
+ control={control}
+ rules={{ required: localize('com_ui_field_required') }}
+ render={({ field }) => (
+ <ControlCombobox
+ selectedValue={field.value}
+ displayValue={
+ agentItems.find((item) => item.value === field.value)?.label ?? ''
+ }
+ selectPlaceholder={localize('com_ui_select_agent')}
+ searchPlaceholder={localize('com_agents_search_name')}
+ setValue={field.onChange}
+ onBlur={field.onBlur}
+ items={agentItems}
+ ariaLabel={localize('com_ui_agent')}
+ ariaInvalid={errors.agent_id != null}
+ ariaDescribedBy="schedule-agent-message"
+ selectId="schedule-agent"
+ isCollapsed={false}
+ showCarat={true}
+ placement="bottom-start"
+ // Radix traps focus inside the dialog, so a popover portaled to
+ // the body cannot be clicked, tabbed into, or typed in — and the
+ // trap fighting Ariakit for focus locks the page up.
+ portal={false}
+ matchTriggerWidth={true}
+ variant="field"
+ />
+ )}
+ />
+ <FieldMessage id="schedule-agent-message" message={errors.agent_id?.message} />
+ <p className="text-xs text-text-secondary">
+ {localize('com_ui_schedule_target_new_chat')}
+ </p>
+ </div>
+ </div>
+
+ <Controller
+ name="prompt"
+ control={control}
+ rules={{ required: localize('com_ui_field_required') }}
+ render={({ field }) => (
+ <div className="space-y-2">
+ <VariableEditor
+ id="schedule-prompt"
+ label={localize('com_ui_prompt')}
+ value={field.value}
+ onChange={field.onChange}
+ onBlur={field.onBlur}
+ inputRef={field.ref}
+ placeholder={localize('com_ui_schedule_prompt_placeholder')}
+ className="min-h-[120px] resize-none bg-transparent"
+ labelClassName="text-sm font-medium text-text-primary"
+ rows={4}
+ required={true}
+ invalid={errors.prompt != null}
+ describedBy="schedule-prompt-message"
+ portal={false}
+ />
+ <FieldMessage id="schedule-prompt-message" message={errors.prompt?.message} />
+ </div>
+ )}
+ />
+
+ <fieldset className="space-y-2">
+ <legend>
+ <Label
+ id="schedule-frequency-label"
+ className="text-sm font-medium text-text-primary"
+ >
+ {localize('com_ui_schedule_frequency')}
+ </Label>
+ </legend>
+ <Controller
+ name="frequency"
+ control={control}
+ render={({ field }) => (
+ <Radio
+ options={frequencyOptions}
+ value={field.value}
+ onChange={(value) => field.onChange(value as ScheduleFrequency)}
+ fullWidth
+ aria-labelledby="schedule-frequency-label"
+ />
+ )}
+ />
+ </fieldset>
+
+ <div className="grid gap-4 md:grid-cols-2">
+ {frequency === 'weekly' && (
+ <fieldset className="space-y-2">
+ <legend>
+ <Label
+ id="schedule-day-label"
+ className="text-sm font-medium text-text-primary"
+ >
+ {localize('com_ui_schedule_day')}
+ </Label>
+ </legend>
+ <Controller
+ name="dayOfWeek"
+ control={control}
+ render={({ field }) => (
+ <Dropdown
+ value={String(field.value)}
+ onChange={(value) => field.onChange(Number(value))}
+ options={dayOptions}
+ variant="field"
+ portal={false}
+ aria-labelledby="schedule-day-label"
+ testId="schedule-day-select"
+ />
+ )}
+ />
+ </fieldset>
+ )}
+ <fieldset className="space-y-2">
+ <legend>
+ <Label id="schedule-time-label" className="text-sm font-medium text-text-primary">
+ {localize(
+ frequency === 'hourly'
+ ? 'com_ui_schedule_minutes_past_hour'
+ : 'com_ui_schedule_time',
+ )}
+ </Label>
+ </legend>
+ <div
+ className={cn(
+ 'grid gap-2',
+ frequency === 'hourly' ? 'max-w-[8rem] grid-cols-1' : 'grid-cols-3',
+ )}
+ >
+ {frequency !== 'hourly' && (
+ <Controller
+ name="hour12"
+ control={control}
+ render={({ field }) => (
+ <Dropdown
+ value={String(field.value)}
+ onChange={(value) => field.onChange(Number(value))}
+ options={hourOptions}
+ variant="field"
+ portal={false}
+ ariaLabel={localize('com_ui_schedule_hour')}
+ testId="schedule-hour-select"
+ />
+ )}
+ />
+ )}
+ <Controller
+ name="minute"
+ control={control}
+ render={({ field }) => (
+ <Dropdown
+ value={String(field.value)}
+ onChange={(value) => field.onChange(Number(value))}
+ options={minuteOptions}
+ variant="field"
+ portal={false}
+ ariaLabel={localize('com_ui_schedule_minute')}
+ testId="schedule-minute-select"
+ />
+ )}
+ />
+ {frequency !== 'hourly' && (
+ <Controller
+ name="meridiem"
+ control={control}
+ render={({ field }) => (
+ <Dropdown
+ value={field.value}
+ onChange={field.onChange}
+ options={meridiemOptions}
+ variant="field"
+ portal={false}
+ ariaLabel={localize('com_ui_schedule_meridiem')}
+ testId="schedule-meridiem-select"
+ />
+ )}
+ />
+ )}
+ </div>
+ </fieldset>
+ </div>
+
+ <p
+ className="break-words rounded-lg bg-surface-secondary px-3 py-2 text-sm text-text-secondary"
+ data-testid="schedule-summary"
+ >
+ {summary}
+ </p>
+ </form>
+ }
+ buttons={
+ <Button
+ type="submit"
+ form={FORM_ID}
+ variant="submit"
+ disabled={isLoading}
+ aria-label={localize(schedule ? 'com_ui_save' : 'com_ui_create')}
+ >
+ {isLoading ? (
+ <Spinner className="size-4" />
+ ) : (
+ localize(schedule ? 'com_ui_save' : 'com_ui_create')
+ )}
+ </Button>
+ }
+ />
+ </OGDialog>
+ );
+}
diff --git a/client/src/components/SidePanel/Schedules/SchedulePanel.tsx b/client/src/components/SidePanel/Schedules/SchedulePanel.tsx
new file mode 100644
index 0000000000..006eae48c3
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/SchedulePanel.tsx
@@ -0,0 +1,105 @@
+import { useState, type ReactNode } from 'react';
+import { Plus } from 'lucide-react';
+import { Button, TooltipAnchor } from '@librechat/client';
+import { PermissionTypes, Permissions } from 'librechat-data-provider';
+import { PanelContent, PanelFooter } from '~/components/ui';
+import ScheduleCardSkeleton from './ScheduleCardSkeleton';
+import { useSchedulesQuery } from '~/data-provider';
+import { useLocalize, useHasAccess } from '~/hooks';
+import ScheduleDialog from './ScheduleDialog';
+import ScheduleCard from './ScheduleCard';
+
+export default function SchedulePanel() {
+ const localize = useLocalize();
+ const { data, isLoading, isError, refetch } = useSchedulesQuery();
+ const [createOpen, setCreateOpen] = useState(false);
+
+ const hasCreateAccess = useHasAccess({
+ permissionType: PermissionTypes.SCHEDULES,
+ permission: Permissions.CREATE,
+ });
+
+ const schedules = data?.schedules ?? [];
+ const maxPerUser = data?.limits.maxPerUser;
+ const atLimit = maxPerUser !== undefined && schedules.length >= maxPerUser;
+ let panelContent: ReactNode;
+
+ if (isError) {
+ panelContent = (
+ <div className="flex min-h-40 flex-col items-center justify-center gap-2 p-4">
+ <p className="text-center text-sm text-text-secondary">
+ {localize('com_ui_schedules_error')}
+ </p>
+ <Button type="button" variant="outline" size="sm" onClick={() => refetch()}>
+ {localize('com_ui_retry')}
+ </Button>
+ </div>
+ );
+ } else if (schedules.length === 0) {
+ panelContent = (
+ <div className="rounded-lg border border-dashed border-border-light p-4 text-center">
+ <p className="text-sm text-text-secondary">{localize('com_ui_schedules_empty')}</p>
+ </div>
+ );
+ } else {
+ panelContent = (
+ <div className="space-y-2" role="list" aria-label={localize('com_ui_schedules')}>
+ {schedules.map((schedule) => (
+ <div key={schedule.id} role="listitem">
+ <ScheduleCard schedule={schedule} />
+ </div>
+ ))}
+ </div>
+ );
+ }
+
+ return (
+ <div
+ role="region"
+ aria-label={localize('com_ui_schedules')}
+ className="flex h-full w-full flex-col overflow-hidden pt-2"
+ >
+ <div className="shrink-0 px-3 pb-2">
+ <div className="flex items-center justify-between gap-2">
+ <span className="text-sm font-medium text-text-primary">
+ {localize('com_ui_schedules')}
+ </span>
+ {hasCreateAccess && (
+ <TooltipAnchor
+ description={localize('com_ui_schedule_new')}
+ side="bottom"
+ render={
+ <Button
+ variant="outline"
+ size="icon"
+ className="size-9 shrink-0 bg-transparent"
+ aria-label={localize('com_ui_schedule_new')}
+ disabled={atLimit || isError || isLoading}
+ onClick={() => setCreateOpen(true)}
+ >
+ <Plus className="size-4" aria-hidden="true" />
+ </Button>
+ }
+ />
+ )}
+ </div>
+ </div>
+
+ <PanelContent isLoading={isLoading} skeleton={<ScheduleCardSkeleton />} className="px-3 pb-3">
+ {/* A failed query must not masquerade as an empty list or enable creation
+ against limits that could not be loaded. */}
+ {panelContent}
+ </PanelContent>
+
+ {!isLoading && !isError && maxPerUser !== undefined && (
+ <PanelFooter className="justify-start">
+ <p className="text-xs text-text-secondary">
+ {localize('com_ui_schedules_used', { used: schedules.length, max: maxPerUser })}
+ </p>
+ </PanelFooter>
+ )}
+
+ {createOpen && <ScheduleDialog open={createOpen} onOpenChange={setCreateOpen} />}
+ </div>
+ );
+}
diff --git a/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsx b/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsx
new file mode 100644
index 0000000000..1facc9e110
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/__tests__/ScheduleDialog.spec.tsx
@@ -0,0 +1,109 @@
+import { createElement } from 'react';
+import { ToastProvider } from '@librechat/client';
+import userEvent from '@testing-library/user-event';
+import { render, screen, within } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import type { ReactNode } from 'react';
+import ScheduleDialog from '../ScheduleDialog';
+
+jest.mock('~/hooks', () => ({
+ useLocalize: () => (key: string) => key,
+}));
+
+/** `@librechat/client` primitives localize through their own `useLocalize`, so the
+ * shared `t` has to resolve too — not just the `i18n` this dialog reads for locale. */
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key, i18n: { language: 'en-US' } }),
+}));
+
+const mockMutate = jest.fn();
+
+jest.mock('~/data-provider', () => ({
+ useListAgentsQuery: () => ({
+ data: [
+ { id: 'agent-1', name: 'Research Agent' },
+ { id: 'agent-2', name: 'Digest Agent' },
+ ],
+ }),
+ useCreateScheduleMutation: () => ({ mutate: mockMutate, isLoading: false }),
+ useUpdateScheduleMutation: () => ({ mutate: mockMutate, isLoading: false }),
+}));
+
+const renderDialog = () => {
+ const queryClient = new QueryClient({
+ defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
+ });
+ function Wrapper({ children }: { children: ReactNode }) {
+ return createElement(
+ QueryClientProvider,
+ { client: queryClient },
+ createElement(ToastProvider, null, children),
+ );
+ }
+ return render(<ScheduleDialog open={true} onOpenChange={jest.fn()} />, { wrapper: Wrapper });
+};
+
+describe('ScheduleDialog', () => {
+ afterEach(() => jest.clearAllMocks());
+
+ /**
+ * The agent list must render INSIDE the dialog. Portaled to the body it lands
+ * outside Radix's focus trap, where it cannot be clicked, tabbed into, or typed
+ * in — and the trap fighting Ariakit for focus locks the page up.
+ */
+ it('opens the agent list inside the dialog rather than portaling it away', async () => {
+ const user = userEvent.setup();
+ renderDialog();
+
+ const dialog = screen.getByRole('dialog');
+ await user.click(within(dialog).getByRole('combobox', { name: 'com_ui_agent' }));
+
+ expect(within(dialog).getByPlaceholderText('com_agents_search_name')).toBeInTheDocument();
+ });
+
+ it('exposes frequency as a radiogroup with the default selected', () => {
+ renderDialog();
+
+ const group = screen.getByRole('radiogroup', { name: 'com_ui_schedule_frequency' });
+ expect(within(group).getByRole('radio', { name: 'com_ui_schedule_daily' })).toBeChecked();
+ expect(within(group).getByRole('radio', { name: 'com_ui_schedule_weekly' })).not.toBeChecked();
+ });
+
+ it('switches the day picker on only for weekly schedules', async () => {
+ const user = userEvent.setup();
+ renderDialog();
+
+ expect(screen.queryByTestId('schedule-day-select')).not.toBeInTheDocument();
+ await user.click(screen.getByRole('radio', { name: 'com_ui_schedule_weekly' }));
+ expect(screen.getByTestId('schedule-day-select')).toBeInTheDocument();
+ });
+
+ /** The submit button sits in the dialog footer, outside the <form>, so it has to
+ * reach it by id — that association is what lets Enter submit from a text field. */
+ it('wires the footer submit button to the form', () => {
+ renderDialog();
+
+ const form = document.getElementById('schedule-form');
+ expect(form).toBeInstanceOf(HTMLFormElement);
+ const submit = screen.getByRole('button', { name: 'com_ui_create' });
+ expect(submit).toHaveAttribute('type', 'submit');
+ expect(submit).toHaveAttribute('form', 'schedule-form');
+ });
+
+ it('describes both text fields with placeholders', () => {
+ renderDialog();
+
+ expect(screen.getByPlaceholderText('com_ui_schedule_name_placeholder')).toBeInTheDocument();
+ expect(screen.getByPlaceholderText('com_ui_schedule_prompt_placeholder')).toBeInTheDocument();
+ });
+
+ it('offers the special-variable menu on the prompt field', async () => {
+ const user = userEvent.setup();
+ renderDialog();
+
+ await user.click(screen.getByRole('button', { name: 'com_ui_variables' }));
+ expect(
+ await screen.findByRole('menuitem', { name: 'com_ui_special_var_current_date' }),
+ ).toBeInTheDocument();
+ });
+});
diff --git a/client/src/components/SidePanel/Schedules/cadence.ts b/client/src/components/SidePanel/Schedules/cadence.ts
new file mode 100644
index 0000000000..cbcb3dde22
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/cadence.ts
@@ -0,0 +1,58 @@
+import type { TScheduleCadence } from 'librechat-data-provider';
+import type { LocalizeFunction } from '~/common';
+
+export type Meridiem = 'AM' | 'PM';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Mirrors the server's default weekly day (Monday) when a weekly cadence omits
+ * daysOfWeek, so an API-created/migrated `frequency: 'weekly'` renders as weekly. */
+const WEEKLY_DEFAULT_DAY = 1;
+
+/** August 1st, 2021 was a Sunday; anchors day-of-week indices 0-6 to real dates */
+const SUNDAY_UTC = Date.UTC(2021, 7, 1);
+
+export const to12Hour = (hour: number): { hour12: number; meridiem: Meridiem } => ({
+ hour12: hour % 12 === 0 ? 12 : hour % 12,
+ meridiem: hour >= 12 ? 'PM' : 'AM',
+});
+
+export const to24Hour = (hour12: number, meridiem: Meridiem): number =>
+ meridiem === 'PM' ? (hour12 % 12) + 12 : hour12 % 12;
+
+export const formatScheduleTime = (hour: number, minute: number, locale?: string): string =>
+ new Intl.DateTimeFormat(locale, { hour: 'numeric', minute: '2-digit' }).format(
+ new Date(2000, 0, 1, hour, minute),
+ );
+
+export const formatScheduleDay = (day: number, locale?: string): string =>
+ new Intl.DateTimeFormat(locale, { weekday: 'long', timeZone: 'UTC' }).format(
+ new Date(SUNDAY_UTC + day * DAY_MS),
+ );
+
+export const describeCadence = (
+ cadence: TScheduleCadence,
+ localize: LocalizeFunction,
+ locale?: string,
+): string => {
+ const { frequency, hour, minute, daysOfWeek } = cadence;
+ if (frequency === 'hourly') {
+ return localize('com_ui_schedule_runs_hourly', {
+ minute: String(minute).padStart(2, '0'),
+ });
+ }
+
+ const time = formatScheduleTime(hour, minute, locale);
+ if (frequency === 'weekdays') {
+ return localize('com_ui_schedule_runs_weekdays', { time });
+ }
+ if (frequency === 'weekly') {
+ // A weekly cadence with no daysOfWeek is valid — the server fires it on the
+ // default weekly day — so render it as weekly (not daily) using that same day.
+ const effectiveDays =
+ daysOfWeek != null && daysOfWeek.length > 0 ? daysOfWeek : [WEEKLY_DEFAULT_DAY];
+ const days = effectiveDays.map((day) => formatScheduleDay(day, locale)).join(', ');
+ return localize('com_ui_schedule_runs_weekly', { days, time });
+ }
+ return localize('com_ui_schedule_runs_daily', { time });
+};
diff --git a/client/src/components/SidePanel/Schedules/index.ts b/client/src/components/SidePanel/Schedules/index.ts
new file mode 100644
index 0000000000..4b0457b900
--- /dev/null
+++ b/client/src/components/SidePanel/Schedules/index.ts
@@ -0,0 +1,3 @@
+export { default as SchedulePanel } from './SchedulePanel';
+export { default as ScheduleCard } from './ScheduleCard';
+export { default as ScheduleDialog } from './ScheduleDialog';
diff --git a/client/src/components/Variables/Editor.tsx b/client/src/components/Variables/Editor.tsx
new file mode 100644
index 0000000000..27b5820bcb
--- /dev/null
+++ b/client/src/components/Variables/Editor.tsx
@@ -0,0 +1,203 @@
+import { useId, useState } from 'react';
+import * as Menu from '@ariakit/react/menu';
+import { PlusCircle, Maximize2 } from 'lucide-react';
+import { specialVariables } from 'librechat-data-provider';
+import {
+ Label,
+ Button,
+ OGDialog,
+ Textarea,
+ DropdownPopup,
+ OGDialogClose,
+ TooltipAnchor,
+ OGDialogTitle,
+ OGDialogHeader,
+ OGDialogContent,
+} from '@librechat/client';
+import type { TSpecialVarLabel } from 'librechat-data-provider';
+import { useLocalize } from '~/hooks';
+import { cn } from '~/utils';
+
+interface VariableOption {
+ label: TSpecialVarLabel;
+ value: string;
+}
+
+const variableOptions: VariableOption[] = Object.keys(specialVariables).map((key) => ({
+ label: `com_ui_special_var_${key}` as TSpecialVarLabel,
+ value: `{{${key}}}`,
+}));
+
+interface VariableEditorProps {
+ id: string;
+ /** Already-localized heading rendered above the field. */
+ label: string;
+ value: string;
+ onChange: (value: string) => void;
+ onBlur?: () => void;
+ inputRef?: React.Ref<HTMLTextAreaElement>;
+ placeholder?: string;
+ rows?: number;
+ required?: boolean;
+ invalid?: boolean;
+ describedBy?: string;
+ className?: string;
+ labelClassName?: string;
+ containerClassName?: string;
+ /**
+ * Ariakit popovers portal to the body by default, which puts them outside a Radix
+ * dialog's focus trap. Pass `false` from inside a dialog, and give that dialog
+ * `overflow-visible` so the inline menu is not clipped.
+ */
+ portal?: boolean;
+}
+
+/**
+ * A textarea for model-facing text with a menu that appends special variables
+ * (`{{current_date}}` and friends) and an expand-to-fullscreen editor. Controlled so
+ * each caller binds it to its own form field — insertions go through `onChange`, which
+ * keeps them visible to react-hook-form's dirty tracking.
+ */
+export default function VariableEditor({
+ id,
+ label,
+ value,
+ onChange,
+ onBlur,
+ inputRef,
+ placeholder,
+ rows = 3,
+ required,
+ invalid,
+ describedBy,
+ className,
+ labelClassName,
+ containerClassName,
+ portal = true,
+}: VariableEditorProps) {
+ const menuId = useId();
+ const dialogMenuId = useId();
+ const localize = useLocalize();
+
+ const [isMenuOpen, setIsMenuOpen] = useState(false);
+ const [isDialogMenuOpen, setIsDialogMenuOpen] = useState(false);
+ const [isDialogOpen, setIsDialogOpen] = useState(false);
+
+ const handleAddVariable = (variableLabel: TSpecialVarLabel, variableValue: string) => {
+ const spacer = value.length > 0 ? '\n' : '';
+ onChange(value + spacer + localize(variableLabel) + ': ' + variableValue);
+ setIsMenuOpen(false);
+ setIsDialogMenuOpen(false);
+ };
+
+ const variableItems = variableOptions.map((option) => ({
+ label: localize(option.label) || option.label,
+ onClick: () => handleAddVariable(option.label, option.value),
+ }));
+
+ return (
+ <div className={cn('flex flex-col', containerClassName)}>
+ <div className="mb-1 flex items-center justify-between">
+ <Label className={labelClassName} htmlFor={id}>
+ {label}
+ </Label>
+ <div className="flex items-center gap-0.5">
+ <DropdownPopup
+ portal={portal}
+ mountByState={true}
+ unmountOnHide={true}
+ preserveTabOrder={true}
+ isOpen={isMenuOpen}
+ setIsOpen={setIsMenuOpen}
+ trigger={
+ <Menu.MenuButton
+ id={`${id}-variables-menu-button`}
+ aria-label={localize('com_ui_variables')}
+ title={localize('com_ui_variables')}
+ className="inline-flex h-7 w-7 items-center justify-center rounded-lg text-text-secondary transition-colors hover:bg-surface-secondary hover:text-text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring-primary"
+ >
+ <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
+ </Menu.MenuButton>
+ }
+ items={variableItems}
+ menuId={menuId}
+ className="pointer-events-auto z-30"
+ />
+ <TooltipAnchor
+ description={localize('com_ui_expand_editor')}
+ render={
+ <Button
+ type="button"
+ variant="ghost"
+ onClick={() => setIsDialogOpen(true)}
+ aria-label={localize('com_ui_expand_editor')}
+ className="h-7 w-7 p-0 text-text-secondary hover:bg-surface-secondary hover:text-text-primary"
+ >
+ <Maximize2 className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
+ </Button>
+ }
+ />
+ </div>
+ </div>
+ <Textarea
+ id={id}
+ ref={inputRef}
+ value={value}
+ onChange={(event) => onChange(event.target.value)}
+ onBlur={onBlur}
+ className={className}
+ placeholder={placeholder}
+ rows={rows}
+ aria-label={label}
+ aria-required={required}
+ aria-invalid={invalid}
+ aria-describedby={describedBy}
+ />
+
+ <OGDialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>
+ <OGDialogContent
+ className="flex h-[85vh] max-h-[85vh] w-11/12 max-w-6xl flex-col gap-4 p-6"
+ showCloseButton={false}
+ >
+ <OGDialogHeader className="mb-2 pr-14">
+ <OGDialogTitle className="text-left text-2xl font-semibold">{label}</OGDialogTitle>
+ </OGDialogHeader>
+ <Textarea
+ value={value}
+ onChange={(event) => onChange(event.target.value)}
+ className="min-h-0 flex-1 resize-none text-base leading-relaxed"
+ placeholder={placeholder}
+ aria-label={label}
+ />
+ <div className="flex items-center justify-between">
+ <DropdownPopup
+ portal={portal}
+ mountByState={true}
+ unmountOnHide={true}
+ preserveTabOrder={true}
+ isOpen={isDialogMenuOpen}
+ setIsOpen={setIsDialogMenuOpen}
+ trigger={
+ <Menu.MenuButton
+ id={`${id}-variables-menu-button-dialog`}
+ render={
+ <Button type="button" variant="outline" className="gap-1.5">
+ <PlusCircle className="h-4 w-4" strokeWidth={1.75} aria-hidden={true} />
+ {localize('com_ui_variables')}
+ </Button>
+ }
+ />
+ }
+ items={variableItems}
+ menuId={dialogMenuId}
+ className="pointer-events-auto z-[200]"
+ />
+ <OGDialogClose asChild>
+ <Button type="button">{localize('com_ui_done')}</Button>
+ </OGDialogClose>
+ </div>
+ </OGDialogContent>
+ </OGDialog>
+ </div>
+ );
+}
diff --git a/client/src/components/Variables/index.ts b/client/src/components/Variables/index.ts
new file mode 100644
index 0000000000..59c4636d36
--- /dev/null
+++ b/client/src/components/Variables/index.ts
@@ -0,0 +1 @@
+export { default as VariableEditor } from './Editor';
diff --git a/client/src/data-provider/Schedules/__tests__/mutations.spec.tsx b/client/src/data-provider/Schedules/__tests__/mutations.spec.tsx
new file mode 100644
index 0000000000..d193acfa10
--- /dev/null
+++ b/client/src/data-provider/Schedules/__tests__/mutations.spec.tsx
@@ -0,0 +1,48 @@
+import { createElement } from 'react';
+import { QueryKeys } from 'librechat-data-provider';
+import { act, renderHook } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import type { ReactNode } from 'react';
+import { useDeleteScheduleMutation } from '../mutations';
+
+const mockDeleteSchedule = jest.fn();
+
+jest.mock('librechat-data-provider', () => {
+ const actual = jest.requireActual('librechat-data-provider');
+ return {
+ ...actual,
+ dataService: {
+ ...actual.dataService,
+ deleteSchedule: (...args: unknown[]) => mockDeleteSchedule(...args),
+ },
+ };
+});
+
+const createWrapper = (queryClient: QueryClient) =>
+ function Wrapper({ children }: { children: ReactNode }) {
+ return createElement(QueryClientProvider, { client: queryClient }, children);
+ };
+
+describe('useDeleteScheduleMutation', () => {
+ it('invalidates the schedules list when deletion returns an error after mutating state', async () => {
+ const queryClient = new QueryClient({
+ logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
+ defaultOptions: {
+ queries: { retry: false },
+ mutations: { retry: false },
+ },
+ });
+ const invalidateQueries = jest.spyOn(queryClient, 'invalidateQueries');
+ mockDeleteSchedule.mockRejectedValueOnce(new Error('Could not confirm active run stopped'));
+ const { result } = renderHook(() => useDeleteScheduleMutation(), {
+ wrapper: createWrapper(queryClient),
+ });
+
+ await act(async () => {
+ await result.current.mutateAsync('schedule-1').catch(() => undefined);
+ });
+
+ expect(invalidateQueries).toHaveBeenCalledWith([QueryKeys.schedules]);
+ queryClient.clear();
+ });
+});
diff --git a/client/src/data-provider/Schedules/index.ts b/client/src/data-provider/Schedules/index.ts
new file mode 100644
index 0000000000..53e6ebc9a8
--- /dev/null
+++ b/client/src/data-provider/Schedules/index.ts
@@ -0,0 +1,3 @@
+/* Scheduled chats */
+export * from './queries';
+export * from './mutations';
diff --git a/client/src/data-provider/Schedules/mutations.ts b/client/src/data-provider/Schedules/mutations.ts
new file mode 100644
index 0000000000..a16e48e2ef
--- /dev/null
+++ b/client/src/data-provider/Schedules/mutations.ts
@@ -0,0 +1,87 @@
+/* Scheduled chats */
+import { useMutation, useQueryClient } from '@tanstack/react-query';
+import { QueryKeys, MutationKeys, dataService } from 'librechat-data-provider';
+import type {
+ TSchedule,
+ TCreateSchedule,
+ TUpdateSchedule,
+ TScheduleRunNowResponse,
+} from 'librechat-data-provider';
+import type { UseMutationOptions } from '@tanstack/react-query';
+
+export const useCreateScheduleMutation = (
+ options?: UseMutationOptions<TSchedule, Error, TCreateSchedule>,
+) => {
+ const queryClient = useQueryClient();
+ return useMutation<TSchedule, Error, TCreateSchedule>(
+ [MutationKeys.createSchedule],
+ (payload: TCreateSchedule) => dataService.createSchedule(payload),
+ {
+ ...options,
+ onSuccess: (...args) => {
+ queryClient.invalidateQueries([QueryKeys.schedules]);
+ options?.onSuccess?.(...args);
+ },
+ },
+ );
+};
+
+export type UpdateScheduleParams = { id: string; payload: TUpdateSchedule };
+export const useUpdateScheduleMutation = (
+ options?: UseMutationOptions<TSchedule, Error, UpdateScheduleParams>,
+) => {
+ const queryClient = useQueryClient();
+ return useMutation<TSchedule, Error, UpdateScheduleParams>(
+ [MutationKeys.updateSchedule],
+ ({ id, payload }: UpdateScheduleParams) => dataService.updateSchedule(id, payload),
+ {
+ ...options,
+ onSuccess: (...args) => {
+ queryClient.invalidateQueries([QueryKeys.schedules]);
+ options?.onSuccess?.(...args);
+ },
+ },
+ );
+};
+
+export const useDeleteScheduleMutation = (
+ options?: UseMutationOptions<{ id: string }, Error, string>,
+) => {
+ const queryClient = useQueryClient();
+ return useMutation<{ id: string }, Error, string>(
+ [MutationKeys.deleteSchedule],
+ (id: string) => dataService.deleteSchedule(id),
+ {
+ ...options,
+ // An `unconfirmed` 503 is still a server-side mutation: the schedule has
+ // already been disabled, marked deleting, and hidden from list reads.
+ // Refresh on either outcome so the card cannot remain actionable in cache.
+ onSettled: (...args) => {
+ queryClient.invalidateQueries([QueryKeys.schedules]);
+ options?.onSettled?.(...args);
+ },
+ },
+ );
+};
+
+export const useRunScheduleNowMutation = (
+ options?: UseMutationOptions<TScheduleRunNowResponse, Error, string>,
+) => {
+ const queryClient = useQueryClient();
+ return useMutation<TScheduleRunNowResponse, Error, string>(
+ [MutationKeys.runSchedule],
+ (id: string) => dataService.runScheduleNow(id),
+ {
+ ...options,
+ // Invalidate on SETTLED, not just success: several run-now 409 paths are still
+ // server-side mutations (a balance skip updates lastRun/counters and can
+ // auto-disable; agent/permission/invalid-schedule skips disable the schedule
+ // before returning), so the card must refresh on those errors too rather than
+ // wait for the polling interval.
+ onSettled: (...args) => {
+ queryClient.invalidateQueries([QueryKeys.schedules]);
+ options?.onSettled?.(...args);
+ },
+ },
+ );
+};
diff --git a/client/src/data-provider/Schedules/queries.ts b/client/src/data-provider/Schedules/queries.ts
new file mode 100644
index 0000000000..a5fd8fee9e
--- /dev/null
+++ b/client/src/data-provider/Schedules/queries.ts
@@ -0,0 +1,18 @@
+/* Scheduled chats */
+import { useQuery } from '@tanstack/react-query';
+import { QueryKeys, dataService } from 'librechat-data-provider';
+import type { UseQueryOptions, QueryObserverResult } from '@tanstack/react-query';
+import type { TSchedulesResponse } from 'librechat-data-provider';
+
+export const useSchedulesQuery = (
+ config?: UseQueryOptions<TSchedulesResponse>,
+): QueryObserverResult<TSchedulesResponse> => {
+ return useQuery<TSchedulesResponse>([QueryKeys.schedules], () => dataService.getSchedules(), {
+ // Automatic runs mutate nextRunAt/lastRun/auto-disable server-side while the
+ // panel is open; refresh on focus and on a modest interval so it stays current.
+ refetchOnWindowFocus: true,
+ refetchOnReconnect: true,
+ refetchInterval: 60_000,
+ ...config,
+ });
+};
diff --git a/client/src/data-provider/index.ts b/client/src/data-provider/index.ts
index a9f6799033..8285caa905 100644
--- a/client/src/data-provider/index.ts
+++ b/client/src/data-provider/index.ts
@@ -10,6 +10,8 @@ export * from './Memories';
export * from './Messages';
export * from './Misc';
export * from './Projects';
+/* Scheduled chats */
+export * from './Schedules';
export * from './Tools';
export * from './connection';
export * from './Favorites';
diff --git a/client/src/hooks/Nav/useSideNavLinks.ts b/client/src/hooks/Nav/useSideNavLinks.ts
index 04cf146f7a..20ccf3c321 100644
--- a/client/src/hooks/Nav/useSideNavLinks.ts
+++ b/client/src/hooks/Nav/useSideNavLinks.ts
@@ -6,6 +6,7 @@ import {
Bookmark,
NotebookPen,
ScrollText,
+ CalendarClock,
ArrowRightToLine,
SlidersHorizontal,
} from 'lucide-react';
@@ -29,6 +30,7 @@ import MCPBuilderPanel from '~/components/SidePanel/MCPBuilder/MCPBuilderPanel';
import AgentPanelSwitch from '~/components/SidePanel/Agents/AgentPanelSwitch';
import BookmarkPanel from '~/components/SidePanel/Bookmarks/BookmarkPanel';
import PanelSwitch from '~/components/SidePanel/Builder/PanelSwitch';
+import { SchedulePanel } from '~/components/SidePanel/Schedules';
import Parameters from '~/components/SidePanel/Parameters/Panel';
import { MemoryPanel } from '~/components/SidePanel/Memories';
import FilesPanel from '~/components/SidePanel/Files/Panel';
@@ -88,6 +90,10 @@ export default function useSideNavLinks({
permissionType: PermissionTypes.MCP_SERVERS,
permission: Permissions.CREATE,
});
+ const hasAccessToSchedules = useHasAccess({
+ permissionType: PermissionTypes.SCHEDULES,
+ permission: Permissions.USE,
+ });
const { availableMCPServers } = useMCPServerManager();
const { agentsConfig } = useGetAgentsConfig({ endpointsConfig });
@@ -140,6 +146,26 @@ export default function useSideNavLinks({
});
}
+ // Scheduled chats are EXPERIMENTAL and default-OFF: the server enables them only
+ // when an admin opts in explicitly, so ABSENT config means disabled here too.
+ // Mirrors getLimits exactly — absent/null/`false` are all off, `true` is on, and the
+ // object form is on unless it sets `use: false`. Any mismatch would show an entry
+ // whose create/run operations the backend rejects.
+ const schedulesConfig = interfaceConfig.schedules;
+ const schedulesEnabled =
+ schedulesConfig != null &&
+ schedulesConfig !== false &&
+ !(typeof schedulesConfig === 'object' && schedulesConfig.use === false);
+ if (hasAccessToSchedules && schedulesEnabled) {
+ links.push({
+ title: 'com_ui_schedules',
+ label: '',
+ icon: CalendarClock,
+ id: 'schedules',
+ Component: SchedulePanel,
+ });
+ }
+
if (hasAccessToPrompts) {
links.push({
title: 'com_ui_prompts',
@@ -228,6 +254,8 @@ export default function useSideNavLinks({
skillsEnabled,
hasAccessToMemories,
hasAccessToReadMemories,
+ hasAccessToSchedules,
+ interfaceConfig.schedules,
interfaceConfig.parameters,
endpointType,
hasAccessToBookmarks,
diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json
index 3f2405e8a5..7a47979387 100644
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@@ -1852,6 +1852,51 @@
"com_ui_save_message_error": "The message could not be saved. Your changes are still in the editor.",
"com_ui_saved": "Saved!",
"com_ui_saving": "Saving...",
+ "com_ui_schedule_am": "AM",
+ "com_ui_schedule_conflict": "This schedule was changed elsewhere. Reopen it to edit the latest version.",
+ "com_ui_schedule_created": "Schedule created",
+ "com_ui_schedule_daily": "Daily",
+ "com_ui_schedule_day": "Day of week",
+ "com_ui_schedule_delete": "Delete schedule",
+ "com_ui_schedule_disabled_agent_deleted": "Disabled: agent deleted",
+ "com_ui_schedule_disabled_insufficient_balance": "Disabled: insufficient balance",
+ "com_ui_schedule_disabled_invalid": "Disabled: invalid schedule",
+ "com_ui_schedule_disabled_permission_revoked": "Disabled: permission revoked",
+ "com_ui_schedule_disabled_too_many_failures": "Disabled: too many failures",
+ "com_ui_schedule_edit": "Edit schedule",
+ "com_ui_schedule_enabled": "Enabled",
+ "com_ui_schedule_frequency": "Frequency",
+ "com_ui_schedule_hour": "Hour",
+ "com_ui_schedule_hourly": "Hourly",
+ "com_ui_schedule_last_run": "Last run",
+ "com_ui_schedule_last_run_failed": "Last run failed",
+ "com_ui_schedule_meridiem": "AM or PM",
+ "com_ui_schedule_minute": "Minute",
+ "com_ui_schedule_minutes_past_hour": "Minutes past the hour",
+ "com_ui_schedule_name_placeholder": "Morning news briefing",
+ "com_ui_schedule_needs_approval": "Needs approval",
+ "com_ui_schedule_new": "New schedule",
+ "com_ui_schedule_next_run": "Next run {{time}}",
+ "com_ui_schedule_options": "Schedule options",
+ "com_ui_schedule_pm": "PM",
+ "com_ui_schedule_prompt_placeholder": "Summarize the top AI news from the last 24 hours and highlight anything worth acting on.",
+ "com_ui_schedule_run_now": "Run now",
+ "com_ui_schedule_run_now_started": "Run started",
+ "com_ui_schedule_run_skipped": "Skipped",
+ "com_ui_schedule_run_started": "Running",
+ "com_ui_schedule_runs_daily": "Runs daily at {{time}}",
+ "com_ui_schedule_runs_hourly": "Runs every hour at :{{minute}}",
+ "com_ui_schedule_runs_weekdays": "Runs weekdays at {{time}}",
+ "com_ui_schedule_runs_weekly": "Runs weekly on {{days}} at {{time}}",
+ "com_ui_schedule_target_new_chat": "Each run starts a new chat with this agent",
+ "com_ui_schedule_time": "Time",
+ "com_ui_schedule_updated": "Schedule updated",
+ "com_ui_schedule_weekdays": "Weekdays",
+ "com_ui_schedule_weekly": "Weekly",
+ "com_ui_schedules": "Scheduled chats",
+ "com_ui_schedules_error": "Couldn't load your scheduled chats",
+ "com_ui_schedules_empty": "No scheduled chats yet",
+ "com_ui_schedules_used": "{{used}} of {{max}} schedules used",
"com_ui_schema": "Schema",
"com_ui_scope": "Scope",
"com_ui_scroll_left": "Scroll left",
diff --git a/config/__tests__/delete-user.spec.js b/config/__tests__/delete-user.spec.js
new file mode 100644
index 0000000000..282fe37767
--- /dev/null
+++ b/config/__tests__/delete-user.spec.js
@@ -0,0 +1,216 @@
+const mockModelRegistry = {};
+const mockModelFor = (name) => {
+ if (!mockModelRegistry[name]) {
+ mockModelRegistry[name] = {
+ findOne: jest.fn(async () => null),
+ deleteMany: jest.fn(async () => ({ deletedCount: 0 })),
+ updateMany: jest.fn(async () => ({ modifiedCount: 0 })),
+ };
+ }
+ return mockModelRegistry[name];
+};
+
+const mockMethods = {
+ beginAgentTriggerUserDeletion: jest.fn(),
+ recoverStaleAgentTriggerUserDeletion: jest.fn(),
+ prepareAgentTriggerUserPurge: jest.fn(),
+ suspendUserSchedulesForDeletion: jest.fn(),
+ restoreUserSchedulesFromDeletion: jest.fn(),
+ countActiveAgentTriggerDeliveriesByUser: jest.fn(),
+ deleteSchedulesByUser: jest.fn(),
+ deleteUserById: jest.fn(),
+ deleteAgentTriggerDeliveriesByUser: jest.fn(),
+ cancelAgentTriggerUserPurge: jest.fn(),
+ cancelAgentTriggerUserDeletion: jest.fn(),
+};
+const mockGetCleanupBlockingJobIdsForUser = jest.fn();
+const mockAbortJob = jest.fn();
+const mockDestroy = jest.fn();
+const mockSilentExit = jest.fn();
+const mockAskQuestion = jest.fn();
+
+jest.mock('../connect', () => jest.fn(async () => undefined));
+jest.mock('mongoose', () => ({ disconnect: jest.fn(async () => undefined) }));
+jest.mock('@librechat/data-schemas', () => ({
+ createModels: () => new Proxy({}, { get: (_target, prop) => mockModelFor(prop) }),
+ createMethods: () => mockMethods,
+ runAsSystem: (operation) => operation(),
+}));
+jest.mock('@librechat/api', () => ({
+ waitForKeyvRedisClient: jest.fn(async () => undefined),
+ createStreamServices: jest.fn(() => ({ isRedis: true })),
+ GenerationJobManager: {
+ configure: jest.fn(),
+ initialize: jest.fn(),
+ getCleanupBlockingJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),
+ abortJob: (...args) => mockAbortJob(...args),
+ destroy: (...args) => mockDestroy(...args),
+ },
+}));
+jest.mock('~/cache/getLogStores', () => jest.fn());
+jest.mock('../helpers', () => ({
+ ...jest.requireActual('../helpers'),
+ askQuestion: mockAskQuestion,
+ silentExit: mockSilentExit,
+}));
+
+const USER_ID = 'user-being-deleted';
+
+const runCli = () =>
+ new Promise((resolve, reject) => {
+ mockSilentExit.mockImplementation((code = 0) => resolve(code));
+ jest.spyOn(process, 'exit').mockImplementation((code) => {
+ resolve(code);
+ });
+ jest.isolateModules(() => {
+ try {
+ require('../delete-user');
+ } catch (error) {
+ reject(error);
+ }
+ });
+ });
+
+describe('Delete user CLI', () => {
+ const originalArgv = process.argv;
+ let logSpy;
+
+ beforeEach(() => {
+ process.argv = ['node', 'delete-user.js', '[email protected]'];
+ logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined);
+ jest.spyOn(console, 'error').mockImplementation(() => undefined);
+ for (const model of Object.values(mockModelRegistry)) {
+ for (const method of Object.values(model)) {
+ method.mockClear();
+ }
+ }
+ mockModelFor('User').findOne.mockResolvedValue({
+ _id: { toString: () => USER_ID },
+ email: '[email protected]',
+ tenantId: 'tenant-1',
+ });
+ mockMethods.beginAgentTriggerUserDeletion.mockReset().mockResolvedValue('acquired');
+ mockMethods.recoverStaleAgentTriggerUserDeletion.mockReset().mockResolvedValue('acquired');
+ mockMethods.prepareAgentTriggerUserPurge.mockReset().mockResolvedValue(undefined);
+ mockMethods.suspendUserSchedulesForDeletion.mockReset().mockResolvedValue(undefined);
+ mockMethods.restoreUserSchedulesFromDeletion.mockReset().mockResolvedValue(undefined);
+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(0);
+ mockMethods.deleteSchedulesByUser.mockReset().mockResolvedValue(undefined);
+ mockMethods.deleteUserById.mockReset().mockResolvedValue({ deletedCount: 1 });
+ mockMethods.deleteAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(undefined);
+ mockMethods.cancelAgentTriggerUserPurge.mockReset().mockResolvedValue(true);
+ mockMethods.cancelAgentTriggerUserDeletion.mockReset().mockResolvedValue(true);
+ mockGetCleanupBlockingJobIdsForUser.mockReset().mockResolvedValue([]);
+ mockAbortJob.mockReset().mockResolvedValue({ success: true });
+ mockDestroy.mockReset().mockResolvedValue(undefined);
+ mockAskQuestion.mockReset().mockResolvedValueOnce('y').mockResolvedValueOnce('n');
+ });
+
+ afterEach(() => {
+ process.argv = originalArgv;
+ logSpy.mockRestore();
+ jest.restoreAllMocks();
+ });
+
+ it('fences triggers and schedules before inspecting active work', async () => {
+ expect(await runCli()).toBe(0);
+
+ expect(mockMethods.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
+ USER_ID,
+ expect.any(Date),
+ );
+ expect(mockMethods.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
+ USER_ID,
+ expect.any(Date),
+ 'tenant-1',
+ );
+ expect(mockMethods.suspendUserSchedulesForDeletion).toHaveBeenCalledWith(
+ USER_ID,
+ expect.any(String),
+ );
+ expect(mockMethods.beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ mockMethods.prepareAgentTriggerUserPurge.mock.invocationCallOrder[0],
+ );
+ expect(mockMethods.prepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(
+ mockMethods.suspendUserSchedulesForDeletion.mock.invocationCallOrder[0],
+ );
+ expect(mockMethods.suspendUserSchedulesForDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mock.invocationCallOrder[0],
+ );
+ expect(
+ mockMethods.countActiveAgentTriggerDeliveriesByUser.mock.invocationCallOrder[0],
+ ).toBeLessThan(mockGetCleanupBlockingJobIdsForUser.mock.invocationCallOrder[0]);
+ });
+
+ it('aborts provider work before deleting account-owned records', async () => {
+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);
+
+ expect(await runCli()).toBe(0);
+
+ expect(mockGetCleanupBlockingJobIdsForUser).toHaveBeenCalledWith(USER_ID, 'tenant-1');
+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });
+ expect(mockAbortJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockModelFor('Message').deleteMany.mock.invocationCallOrder[0],
+ );
+ expect(mockMethods.deleteSchedulesByUser).toHaveBeenCalledWith(USER_ID);
+ expect(mockMethods.deleteSchedulesByUser.mock.invocationCallOrder[0]).toBeLessThan(
+ mockMethods.deleteUserById.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('deletes nothing and releases both fences when schedule quiescing fails', async () => {
+ mockMethods.suspendUserSchedulesForDeletion.mockRejectedValueOnce(
+ new Error('schedule write failed'),
+ );
+
+ expect(await runCli()).toBe(1);
+
+ expect(mockModelFor('Message').deleteMany).not.toHaveBeenCalled();
+ expect(mockMethods.deleteSchedulesByUser).not.toHaveBeenCalled();
+ expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
+ const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
+ expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
+ expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
+ });
+
+ /**
+ * The suspension is reversible, so a CLI attempt that does not commit must restore the
+ * rows it suspended — and must do so while the user-deletion fence still refuses new
+ * schedule writes, or an owner edit (or a second attempt re-suspending under a new token)
+ * can race the restore and strand the disabled snapshot.
+ */
+ it('restores suspended schedules BEFORE releasing the deletion fence on failure', async () => {
+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);
+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain failed'));
+
+ expect(await runCli()).toBe(1);
+
+ const token = mockMethods.suspendUserSchedulesForDeletion.mock.calls[0][1];
+ expect(token).toEqual(expect.any(String));
+ expect(mockMethods.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(USER_ID, token);
+ expect(mockMethods.restoreUserSchedulesFromDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ mockMethods.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('never restores schedules after a committed deletion', async () => {
+ expect(await runCli()).toBe(0);
+
+ expect(mockMethods.deleteSchedulesByUser).toHaveBeenCalledWith(USER_ID);
+ expect(mockMethods.restoreUserSchedulesFromDeletion).not.toHaveBeenCalled();
+ });
+
+ it('deletes nothing and releases both fences when provider drain fails', async () => {
+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);
+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain failed'));
+
+ expect(await runCli()).toBe(1);
+
+ expect(mockModelFor('Message').deleteMany).not.toHaveBeenCalled();
+ expect(mockMethods.deleteSchedulesByUser).not.toHaveBeenCalled();
+ expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
+ const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
+ expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
+ expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
+ });
+});
diff --git a/config/delete-user.js b/config/delete-user.js
index fec8df7590..533a239f2f 100644
--- a/config/delete-user.js
+++ b/config/delete-user.js
@@ -3,6 +3,7 @@
// @ts-nocheck
const path = require('path');
const mongoose = require('mongoose');
+const { randomUUID } = require('node:crypto');
const { createModels, createMethods, runAsSystem } = require('@librechat/data-schemas');
const {
Key,
@@ -118,6 +119,7 @@ async function gracefulExit(code = 0) {
}
let deletionFence;
+ let scheduleSuspensionToken;
let userDeleted = false;
try {
@@ -158,6 +160,13 @@ async function gracefulExit(code = 0) {
await runAsSystem(() =>
methods.prepareAgentTriggerUserPurge(uid, deletionFence, user.tenantId),
);
+ // Reversible, token-fenced suspension (the same protocol the HTTP controller uses):
+ // an attempt that does not commit restores exactly these rows in the finally block,
+ // rather than leaving a surviving account with disabled, erasure-eligible schedules.
+ scheduleSuspensionToken = randomUUID();
+ await runAsSystem(() =>
+ methods.suspendUserSchedulesForDeletion(uid, scheduleSuspensionToken),
+ );
if (hasSharedGenerationStore) {
const deadline = Date.now() + TRIGGER_DRAIN_TIMEOUT_MS;
while (
@@ -214,6 +223,7 @@ async function gracefulExit(code = 0) {
}
await Promise.all(tasks);
+ await runAsSystem(() => methods.deleteSchedulesByUser(uid));
// 6) Remove user from all groups
await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });
@@ -226,6 +236,22 @@ async function gracefulExit(code = 0) {
userDeleted = true;
await runAsSystem(() => methods.deleteAgentTriggerDeliveriesByUser(uid));
} finally {
+ // RESTORE BEFORE RELEASING THE FENCE. While the user-deletion fence is still armed, new
+ // schedule writes/claims are refused, so this restore cannot race an owner PATCH nor be
+ // superseded by a second deletion attempt re-suspending these rows under a new token.
+ if (scheduleSuspensionToken != null && !userDeleted) {
+ // Retried inside the method; the fence is still released below on purpose, since
+ // retaining it would block the retry that is the convergence path. Print the token
+ // so a restore that never converges stays recoverable by hand.
+ await runAsSystem(() =>
+ methods.restoreUserSchedulesFromDeletion(uid, scheduleSuspensionToken),
+ ).catch((error) =>
+ console.error(
+ `Failed to restore suspended schedules; they remain disabled for user ${uid} under suspension token ${scheduleSuspensionToken}:`,
+ error,
+ ),
+ );
+ }
if (deletionFence != null && !userDeleted) {
await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch(
(error) => console.error('Failed to disarm trigger purge recovery:', error),
diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml
index 7b8dc5a218..17421f502b 100644
--- a/e2e/config/librechat.e2e.yaml
+++ b/e2e/config/librechat.e2e.yaml
@@ -13,6 +13,13 @@ interface:
# value the permission falls through to the seeded role default and
# `handlePlusCommand` returns before opening the popover.
multiConvo: true
+ # Scheduled chats are EXPERIMENTAL and default-off, so the e2e harness must opt in
+ # explicitly before schedule writes and fires are accepted.
+ # minIntervalMinutes is lowered so the spec can create a schedule without waiting
+ # out the production 60-minute floor.
+ schedules:
+ use: true
+ minIntervalMinutes: 1
# Enables the memory feature so the MEMORIES.USE permission is granted and the
# ephemeral memory badge (inline set_memory/delete_memory tools) is available.
diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts
index d215f09256..0230984726 100644
--- a/e2e/playwright.config.mock.ts
+++ b/e2e/playwright.config.mock.ts
@@ -241,6 +241,12 @@ export default defineConfig({
// network fixtures so inspection and persistent connections agree.
command: `node ${serverPath}`,
cwd: rootPath,
+ // Only the one-replica harness may assert the scheduler's single-process topology.
+ // The two-replica MCP suite must leave scheduled writes disabled.
+ env: {
+ ...process.env,
+ ...(replicaCount === 1 ? { SCHEDULES_SINGLE_PROCESS: 'true' } : {}),
+ },
url: baseURL,
stdout: 'pipe',
ignoreHTTPSErrors: true,
diff --git a/e2e/specs/mock/schedules-execution.spec.ts b/e2e/specs/mock/schedules-execution.spec.ts
new file mode 100644
index 0000000000..bfad84e4fb
--- /dev/null
+++ b/e2e/specs/mock/schedules-execution.spec.ts
@@ -0,0 +1,301 @@
+import { expect, test } from '@playwright/test';
+import type { Page } from '@playwright/test';
+import { getAccessToken, requestJson, replyPrompt, replyText } from './helpers';
+
+const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
+
+type AgentSummary = { id: string; name?: string };
+type AgentList = { data?: AgentSummary[] };
+type Schedule = {
+ id: string;
+ name: string;
+ nextRunAt?: string;
+ enabled?: boolean;
+ cadence?: { frequency: string; hour?: number; minute?: number };
+ lastRun?: { status: string; conversationId?: string };
+ runCount?: number;
+};
+type ScheduleList = { schedules: Schedule[] };
+type RunNowResult = { scheduleId: string; conversationId?: string; status?: string };
+
+async function ensureAgent(page: Page, token: string): Promise<AgentSummary> {
+ const list = await requestJson<AgentList>(page, { path: '/api/agents?limit=1', token }).catch(
+ () => ({}) as AgentList,
+ );
+ const existing = list.data?.[0];
+ if (existing?.id) {
+ return existing;
+ }
+ const agent = await requestJson<AgentSummary>(page, {
+ path: '/api/agents',
+ token,
+ method: 'POST',
+ body: { name: uniqueName('E2E Agent'), provider: 'Mock Provider A', model: 'mock-model-a' },
+ });
+ expect(agent.id).toBeTruthy();
+ return agent;
+}
+
+async function createSchedule(
+ page: Page,
+ token: string,
+ body: Record<string, unknown>,
+): Promise<Schedule> {
+ const schedule = await requestJson<Schedule>(page, {
+ path: '/api/schedules',
+ token,
+ method: 'POST',
+ body,
+ });
+ expect(schedule.id).toBeTruthy();
+ return schedule;
+}
+
+async function readSchedule(page: Page, token: string, id: string): Promise<Schedule | undefined> {
+ const list = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });
+ return list.schedules.find((s) => s.id === id);
+}
+
+async function openSchedulesPanel(page: Page) {
+ const navButton = page.getByRole('button', { name: 'Scheduled chats' });
+ await expect(navButton).toBeVisible();
+ if ((await navButton.getAttribute('aria-pressed')) !== 'true') {
+ await navButton.click();
+ }
+ const panel = page.getByRole('region', { name: 'Scheduled chats' });
+ await expect(panel).toBeVisible({ timeout: 15000 });
+ return panel;
+}
+
+const scheduleBody = (agentId: string, over: Record<string, unknown> = {}) => ({
+ name: uniqueName('E2E Schedule'),
+ prompt: 'Summarize what happened today',
+ agent_id: agentId,
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ clientRequestId: uniqueName('e2e-intent'),
+ ...over,
+});
+
+test.describe('scheduled chat execution', () => {
+ /**
+ * Run Now is the one path that dispatches a real generation on demand, so it is
+ * where a broken loopback URL, a rejected fire token, or a lost schedule identity
+ * surfaces. The smoke spec only proves the card renders.
+ */
+ test('Run Now generates a conversation and records it on the schedule', async ({ page }) => {
+ test.setTimeout(120000);
+ await page.goto('/c/new', { timeout: 15000 });
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+ const label = `sched-${Date.now()}`;
+ const schedule = await createSchedule(
+ page,
+ token,
+ scheduleBody(agent.id, { prompt: replyPrompt(label) }),
+ );
+
+ // A skip/throttle answers 409/429, which requestJson surfaces as a throw.
+ const result = await requestJson<RunNowResult>(page, {
+ path: `/api/schedules/${schedule.id}/run`,
+ token,
+ method: 'POST',
+ });
+ expect(result.status).toBe('started');
+ expect(result.conversationId).toBeTruthy();
+
+ // Poll the schedule until the run's own completion hook records its outcome.
+ await expect
+ .poll(async () => (await readSchedule(page, token, schedule.id))?.lastRun?.status, {
+ timeout: 60000,
+ intervals: [1000],
+ })
+ .toBe('success');
+
+ const settled = await readSchedule(page, token, schedule.id);
+ expect(settled?.runCount).toBe(1);
+ const conversationId = settled?.lastRun?.conversationId;
+ expect(conversationId).toBeTruthy();
+
+ // The generated chat is real and reachable: the agent's reply is persisted.
+ await page.goto(`/c/${conversationId}`, { timeout: 15000 });
+ await expect(page.getByTestId('messages-view')).toContainText(replyText(label), {
+ timeout: 20000,
+ });
+ });
+
+ /**
+ * A due schedule must fire on the engine's own tick — no user action — exactly once,
+ * and then advance past the occurrence. TICK_MS is 30s, so budget for one tick.
+ */
+ test('a due schedule fires automatically, once, and advances', async ({ page }) => {
+ test.setTimeout(600000);
+ await page.goto('/c/new', { timeout: 15000 });
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+ const label = `auto-${Date.now()}`;
+ // Hourly ignores `hour` (cron `m * * * *`) but the payload schema still requires it.
+ const schedule = await createSchedule(
+ page,
+ token,
+ scheduleBody(agent.id, {
+ prompt: replyPrompt(label),
+ cadence: { frequency: 'hourly', hour: 0, minute: (new Date().getUTCMinutes() + 1) % 60 },
+ timezone: 'UTC',
+ }),
+ );
+ const before = await readSchedule(page, token, schedule.id);
+ expect(before?.nextRunAt).toBeTruthy();
+
+ // Budget from the server's OWN nextRunAt rather than a guessed constant: it already
+ // includes this schedule's deterministic jitter (up to SCHEDULE_JITTER_WINDOW_MS,
+ // 120s), which no fixed timeout can safely assume away. Add the engine tick
+ // (30s + 2s jitter) plus room for the generation.
+ const dueIn = Math.max(new Date(before!.nextRunAt!).getTime() - Date.now(), 0);
+ const budget = dueIn + 120000;
+
+ await expect
+ .poll(async () => (await readSchedule(page, token, schedule.id))?.lastRun?.status, {
+ timeout: budget,
+ intervals: [2000],
+ })
+ .toBe('success');
+
+ const after = await readSchedule(page, token, schedule.id);
+ // Exactly one run, and the occurrence was advanced rather than re-fired.
+ expect(after?.runCount).toBe(1);
+ expect(new Date(after!.nextRunAt!).getTime()).toBeGreaterThan(
+ new Date(before!.nextRunAt!).getTime(),
+ );
+ });
+
+ test('rejects an invalid timezone before persisting anything', async ({ page }) => {
+ await page.goto('/c/new', { timeout: 15000 });
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+
+ const before = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });
+ const rejected = await requestJson<unknown>(page, {
+ path: '/api/schedules',
+ token,
+ method: 'POST',
+ body: scheduleBody(agent.id, { timezone: 'Not/AZone' }),
+ }).then(
+ () => null,
+ (err: Error) => err.message,
+ );
+
+ expect(rejected).toMatch(/400/);
+ const after = await requestJson<ScheduleList>(page, { path: '/api/schedules', token });
+ expect(after.schedules).toHaveLength(before.schedules.length);
+ });
+
+ /**
+ * Edits through the real dialog and proves the change round-trips the backend.
+ *
+ * Seeded over the API rather than created through the UI on purpose: creation
+ * requires the agent picker, whose list comes from a React Query cache that an
+ * API-created agent does not invalidate, and which renders virtualized. That made
+ * the create half brittle for reasons that have nothing to do with schedules. UI
+ * CREATION is therefore still uncovered — worth a follow-up that seeds the agent
+ * before first paint.
+ */
+ test('edits a schedule through the UI with the cadence persisted', async ({ page }) => {
+ test.setTimeout(120000);
+ await page.goto('/c/new', { timeout: 15000 });
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+ const name = uniqueName('UI Schedule');
+ await createSchedule(
+ page,
+ token,
+ scheduleBody(agent.id, { name, cadence: { frequency: 'weekly', hour: 8, minute: 0 } }),
+ );
+
+ await openSchedulesPanel(page);
+ const persisted = page.getByTestId('schedule-card').filter({ hasText: name });
+ await expect(persisted).toContainText(/Runs weekly/i, { timeout: 15000 });
+
+ // EDIT through the dialog: rename and move the cadence to daily. The dialog
+ // pre-populates the agent from the schedule, so no picker interaction is needed.
+ await persisted.getByRole('button', { name: 'Schedule options' }).click();
+ await page.getByRole('menuitem', { name: 'Edit' }).click();
+ const editDialog = page.getByRole('dialog');
+ const renamed = `${name} edited`;
+ await editDialog.locator('#schedule-name').fill(renamed);
+ await editDialog.getByRole('radio', { name: 'Daily' }).click();
+ await editDialog.getByRole('button', { name: 'Save' }).click();
+
+ await page.reload();
+ await openSchedulesPanel(page);
+ const edited = page.getByTestId('schedule-card').filter({ hasText: renamed });
+ await expect(edited).toBeVisible({ timeout: 15000 });
+ await expect(edited).toContainText(/Runs daily/i);
+ });
+
+ /**
+ * Deleting a schedule mid-run must quiesce it: the in-flight generation is aborted and
+ * the run settles, rather than the row lingering `started` and holding a global
+ * capacity slot until the orphan sweep.
+ */
+ test('deleting a schedule while its run is active aborts the generation', async ({ page }) => {
+ test.setTimeout(180000);
+ await page.goto('/c/new', { timeout: 15000 });
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+ const schedule = await createSchedule(
+ page,
+ token,
+ // A slow reply keeps the generation in flight long enough to delete underneath it.
+ scheduleBody(agent.id, { prompt: `E2E_SLOW_REPLY:del-${Date.now()}` }),
+ );
+
+ const started = await requestJson<RunNowResult>(page, {
+ path: `/api/schedules/${schedule.id}/run`,
+ token,
+ method: 'POST',
+ });
+ const conversationId = started.conversationId!;
+ expect(conversationId).toBeTruthy();
+
+ // PROVE the run is actually generating before deleting. The schedule row is hidden
+ // from the owner the instant it is soft-deleted, so its disappearance is no evidence
+ // that the abort was delivered, the run settled, or the row was erased.
+ await expect
+ .poll(
+ async () =>
+ (
+ await requestJson<{ active?: boolean }>(page, {
+ path: `/api/agents/chat/status/${conversationId}`,
+ token,
+ })
+ ).active,
+ { timeout: 60000, intervals: [500] },
+ )
+ .toBe(true);
+
+ await requestJson<unknown>(page, {
+ path: `/api/schedules/${schedule.id}`,
+ token,
+ method: 'DELETE',
+ });
+
+ // The delete has to reach the loopback generation, not just hide the row.
+ await expect
+ .poll(
+ async () =>
+ (
+ await requestJson<{ active?: boolean }>(page, {
+ path: `/api/agents/chat/status/${conversationId}`,
+ token,
+ })
+ ).active,
+ { timeout: 60000, intervals: [1000] },
+ )
+ .toBe(false);
+
+ expect(await readSchedule(page, token, schedule.id)).toBeUndefined();
+ });
+});
diff --git a/e2e/specs/mock/schedules.spec.ts b/e2e/specs/mock/schedules.spec.ts
new file mode 100644
index 0000000000..309e828176
--- /dev/null
+++ b/e2e/specs/mock/schedules.spec.ts
@@ -0,0 +1,112 @@
+import { expect, test } from '@playwright/test';
+import type { Page } from '@playwright/test';
+import { getAccessToken, requestJson } from './helpers';
+
+const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
+
+type AgentSummary = { id: string; name?: string };
+type AgentList = { data?: AgentSummary[] };
+type Schedule = { id: string; name: string };
+
+/**
+ * Ensures an agent exists (reusing one if present, else creating). Uses a real
+ * Bearer token because the app authenticates with an access token, not a plain
+ * cookie. Requires the page to be on the app origin. Returns id + name.
+ */
+async function ensureAgent(page: Page, token: string): Promise<AgentSummary> {
+ const list = await requestJson<AgentList>(page, { path: '/api/agents?limit=1', token }).catch(
+ () => ({}) as AgentList,
+ );
+ const existing = list.data?.[0];
+ if (existing?.id) {
+ return existing;
+ }
+ const name = uniqueName('E2E Agent');
+ const agent = await requestJson<AgentSummary>(page, {
+ path: '/api/agents',
+ token,
+ method: 'POST',
+ body: { name, provider: 'Mock Provider A', model: 'mock-model-a' },
+ });
+ expect(agent.id).toBeTruthy();
+ return agent;
+}
+
+/** Seeds a schedule via the API so the UI test can exercise its lifecycle. */
+async function seedSchedule(page: Page, token: string, agentId: string): Promise<Schedule> {
+ const name = uniqueName('E2E Schedule');
+ const schedule = await requestJson<Schedule>(page, {
+ path: '/api/schedules',
+ token,
+ method: 'POST',
+ body: {
+ name,
+ prompt: 'Summarize what happened today',
+ agent_id: agentId,
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ clientRequestId: uniqueName('e2e-intent'),
+ },
+ });
+ expect(schedule.id).toBeTruthy();
+ return schedule;
+}
+
+async function openSchedulesPanel(page: Page) {
+ const navButton = page.getByRole('button', { name: 'Scheduled chats' });
+ await expect(navButton).toBeVisible();
+ // The nav button TOGGLES: clicking it while schedules is the active, expanded
+ // panel collapses the sidebar. The active panel persists to localStorage, so
+ // after a reload schedules may already be active but still loading its query
+ // (a spinner, no region yet). Clicking then would collapse it. Branch on
+ // aria-pressed: only click to activate when it isn't already the active panel,
+ // then wait for the query to resolve and render the region.
+ const isActive = (await navButton.getAttribute('aria-pressed')) === 'true';
+ if (!isActive) {
+ await navButton.click();
+ }
+ const panel = page.getByRole('region', { name: 'Scheduled chats' });
+ await expect(panel).toBeVisible({ timeout: 15000 });
+ return panel;
+}
+
+test.describe('scheduled chats', () => {
+ test('lists a schedule and toggles + deletes it through the panel', async ({ page }) => {
+ test.setTimeout(120000);
+ await page.goto('/c/new', { timeout: 15000 });
+
+ const token = await getAccessToken(page);
+ const agent = await ensureAgent(page, token);
+ const schedule = await seedSchedule(page, token, agent.id);
+
+ // The seeded schedule renders as a card in the panel (full backend round-trip).
+ await openSchedulesPanel(page);
+ const card = page.getByTestId('schedule-card').filter({ hasText: schedule.name });
+ await expect(card).toBeVisible({ timeout: 15000 });
+
+ // It survives a full reload (persisted through the real backend + DB).
+ await page.reload();
+ await openSchedulesPanel(page);
+ const reloadedCard = page.getByTestId('schedule-card').filter({ hasText: schedule.name });
+ await expect(reloadedCard).toBeVisible({ timeout: 15000 });
+
+ // Toggling the enabled switch round-trips and re-reads from the server.
+ const toggle = reloadedCard.getByRole('switch', { name: 'Enabled' });
+ await expect(toggle).toBeChecked();
+ await toggle.click();
+ await expect(toggle).not.toBeChecked();
+
+ // Delete via the kebab menu + confirmation dialog.
+ await reloadedCard.getByRole('button', { name: 'Schedule options' }).click();
+ await page.getByRole('menuitem', { name: 'Delete' }).click();
+ const confirm = page.getByRole('dialog', { name: /delete schedule/i });
+ await confirm.getByRole('button', { name: 'Delete', exact: true }).click();
+
+ await expect(page.getByTestId('schedule-card').filter({ hasText: schedule.name })).toHaveCount(
+ 0,
+ { timeout: 15000 },
+ );
+ });
+});
diff --git a/librechat.example.yaml b/librechat.example.yaml
index 53f0955429..b6b4f3183c 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -196,6 +196,17 @@ interface:
create: true
share: false
public: false
+ # Scheduled chats are experimental and disabled unless explicitly enabled.
+ # A deployment with Redis-backed resumable streams can run this on every replica.
+ # A single-process deployment without Redis must also set
+ # `SCHEDULES_SINGLE_PROCESS=true`; unsafe multi-replica writes fail closed.
+ # schedules:
+ # use: true
+ # create: true
+ # maxPerUser: 10
+ # minIntervalMinutes: 60
+ # autoDisableAfterFailures: 5
+ # fireConcurrency: 5
peoplePicker:
users: true
groups: true
diff --git a/package-lock.json b/package-lock.json
index 7f4c6826a7..887ae7adfc 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -23269,6 +23269,25 @@
"resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz",
"integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="
},
+ "node_modules/croner": {
+ "version": "10.0.1",
+ "resolved": "https://registry.npmjs.org/croner/-/croner-10.0.1.tgz",
+ "integrity": "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==",
+ "funding": [
+ {
+ "type": "other",
+ "url": "https://paypal.me/hexagonpp"
+ },
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/hexagon"
+ }
+ ],
+ "license": "MIT",
+ "engines": {
+ "node": ">=18.0"
+ }
+ },
"node_modules/cross-env": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
@@ -42822,6 +42841,7 @@
"@langchain/langgraph-checkpoint": "^1.1.2",
"@langchain/langgraph-checkpoint-mongodb": "^1.4.0",
"cluster-key-slot": "^1.1.2",
+ "croner": "^10.0.1",
"proxy-from-env": "^2.1.0"
},
"devDependencies": {
diff --git a/packages/api/package.json b/packages/api/package.json
index 210f8572ad..fbef1b1a81 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -169,6 +169,7 @@
"@langchain/langgraph-checkpoint": "^1.1.2",
"@langchain/langgraph-checkpoint-mongodb": "^1.4.0",
"cluster-key-slot": "^1.1.2",
+ "croner": "^10.0.1",
"proxy-from-env": "^2.1.0"
}
}
diff --git a/packages/api/src/admin/config.handler.spec.ts b/packages/api/src/admin/config.handler.spec.ts
index 7d44f65a12..2b037bda76 100644
--- a/packages/api/src/admin/config.handler.spec.ts
+++ b/packages/api/src/admin/config.handler.spec.ts
@@ -521,6 +521,25 @@ describe('createAdminConfigHandlers', () => {
expect(savedOverrides.interface).toEqual({ modelSelect: false });
});
+ it('collapses an explicit schedules disable to the boolean form in overrides', async () => {
+ const { handlers, deps } = createHandlers({
+ upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }),
+ });
+ const req = mockReq({
+ params: { principalType: 'role', principalId: 'admin' },
+ body: {
+ overrides: { interface: { schedules: { use: false, maxPerUser: 2 } } },
+ },
+ });
+ const res = mockRes();
+
+ await handlers.upsertConfigOverrides(req, res);
+
+ expect(res.statusCode).toBe(201);
+ const savedOverrides = deps.upsertConfig.mock.calls[0][3];
+ expect(savedOverrides.interface).toEqual({ schedules: false });
+ });
+
it('preserves skillSync sections in admin overrides', async () => {
const { handlers, deps } = createHandlers({
upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }),
@@ -1046,6 +1065,43 @@ describe('createAdminConfigHandlers', () => {
expect(patchedFields['interface.prompts']).toBeUndefined();
});
+ /** `use` is both a permission bit and the runtime disable for dual-purpose fields.
+ * Stripping it alone leaves an object, which `getLimits` reads as ENABLED — so an
+ * override meant to stop scheduled billing would start it. */
+ it('collapses an explicit schedules disable to the boolean form in patches', async () => {
+ const { handlers, deps } = createHandlers();
+ const req = mockReq({
+ params: { principalType: 'role', principalId: 'admin' },
+ body: {
+ entries: [{ fieldPath: 'interface.schedules', value: { use: false, maxPerUser: 2 } }],
+ },
+ });
+ const res = mockRes();
+
+ await handlers.patchConfigField(req, res);
+
+ expect(res.statusCode).toBe(200);
+ const patchedFields = deps.patchConfigFields.mock.calls[0][3];
+ expect(patchedFields['interface.schedules']).toBe(false);
+ });
+
+ it('keeps a schedules object that only narrows limits', async () => {
+ const { handlers, deps } = createHandlers();
+ const req = mockReq({
+ params: { principalType: 'role', principalId: 'admin' },
+ body: {
+ entries: [{ fieldPath: 'interface.schedules', value: { maxPerUser: 2 } }],
+ },
+ });
+ const res = mockRes();
+
+ await handlers.patchConfigField(req, res);
+
+ expect(res.statusCode).toBe(200);
+ const patchedFields = deps.patchConfigFields.mock.calls[0][3];
+ expect(patchedFields['interface.schedules']).toEqual({ maxPerUser: 2 });
+ });
+
it('preserves skillSync field entries in patches', async () => {
const { handlers, deps } = createHandlers();
const req = mockReq({
diff --git a/packages/api/src/admin/config.ts b/packages/api/src/admin/config.ts
index 20852cc1a3..979aaafa19 100644
--- a/packages/api/src/admin/config.ts
+++ b/packages/api/src/admin/config.ts
@@ -5,6 +5,7 @@ import {
PrincipalType,
PrincipalModel,
INTERFACE_PERMISSION_FIELDS,
+ RUNTIME_CONFIG_INTERFACE_FIELDS,
PERMISSION_SUB_KEYS,
hasProcessMCPServerConfig,
isProcessMCPServerConfig,
@@ -124,14 +125,47 @@ function isInterfacePermissionPath(fieldPath: string): boolean {
if (!INTERFACE_PERMISSION_FIELDS.has(parts[1])) {
return false;
}
- // "interface.<permField>" with no sub-key → permission (blocks the whole field)
+ // "interface.<permField>" with no sub-key → permission (blocks the whole field),
+ // EXCEPT dual-purpose runtime fields (e.g. schedules) whose bare top-level value
+ // is a runtime enable toggle, not a permission — those must pass through so admin
+ // field patches/tombstones can set or clear them (their .use/.create permission
+ // sub-keys are still blocked below).
if (parts.length === 2) {
- return true;
+ return !RUNTIME_CONFIG_INTERFACE_FIELDS.has(parts[1]);
}
// "interface.<permField>.<subKey>" → only block if sub-key is a permission bit
return PERMISSION_SUB_KEYS.has(parts[2]);
}
+/**
+ * Collapses an explicit disable on a dual-purpose runtime interface field (e.g. `schedules`)
+ * to its boolean form.
+ *
+ * For these fields `use` is BOTH a permission bit — stripped from DB overrides — and the
+ * runtime disable signal. Stripping it alone would leave `{ maxPerUser: 2 }`, which
+ * `getLimits` reads as ENABLED because an object opts in unless it sets `use: false`. An
+ * override written to stop scheduled billing for a principal would therefore start it.
+ * Other object forms are left alone so a principal can still narrow limits.
+ */
+function normalizeRuntimeInterfaceValue(field: string, value: unknown): unknown {
+ if (!RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {
+ return value;
+ }
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ return value;
+ }
+ return (value as Record<string, unknown>).use === false ? false : value;
+}
+
+/** Applies {@link normalizeRuntimeInterfaceValue} to a bare `interface.<field>` patch. */
+function normalizeInterfaceFieldPatch(fieldPath: string, value: unknown): unknown {
+ const parts = fieldPath.split('.');
+ if (parts[0] !== 'interface' || parts.length !== 2) {
+ return value;
+ }
+ return normalizeRuntimeInterfaceValue(parts[1], value);
+}
+
export interface AdminConfigDeps {
listAllConfigs: (filter?: { isActive?: boolean }, session?: ClientSession) => Promise<IConfig[]>;
findConfigByPrincipal: (
@@ -612,7 +646,8 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
const iface = (overrides as Record<string, unknown>).interface;
if (iface != null && typeof iface === 'object' && !Array.isArray(iface)) {
const filteredIface: Record<string, unknown> = {};
- for (const [field, val] of Object.entries(iface as Record<string, unknown>)) {
+ for (const [field, rawVal] of Object.entries(iface as Record<string, unknown>)) {
+ const val = normalizeRuntimeInterfaceValue(field, rawVal);
if (!INTERFACE_PERMISSION_FIELDS.has(field)) {
filteredIface[field] = val;
} else if (val != null && typeof val === 'object' && !Array.isArray(val)) {
@@ -631,6 +666,10 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
if (Object.keys(uiOnly).length > 0) {
filteredIface[field] = uiOnly;
}
+ } else if (RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {
+ // Dual-purpose field: the boolean form is a runtime disable, not a
+ // permission toggle, so preserve it (e.g. schedules: false).
+ filteredIface[field] = val;
} else {
logger.warn(
`[adminConfig] Stripping interface permission field "${field}" — use role permissions instead`,
@@ -794,27 +833,32 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
return res.status(401).json({ error: 'Authentication required' });
}
- const validEntries = entries.filter((entry) => {
- if (isBaseOnlyFieldPath(entry.fieldPath)) {
- logger.warn(
- `[adminConfig] Stripping base-only config field "${entry.fieldPath}" - configure it in librechat.yaml instead`,
- );
- return false;
- }
- if (BASE_PRINCIPAL_OVERRIDE_SECTIONS.has(getTopLevelSection(entry.fieldPath))) {
- logger.warn(
- `[adminConfig] Stripping dedicated tenant-wide config field "${entry.fieldPath}" from the generic config API`,
- );
- return false;
- }
- if (isInterfacePermissionPath(entry.fieldPath)) {
- logger.warn(
- `[adminConfig] Stripping interface permission field "${entry.fieldPath}" — use role permissions instead`,
- );
- return false;
- }
- return true;
- });
+ const validEntries = entries
+ .map((entry) => ({
+ ...entry,
+ value: normalizeInterfaceFieldPatch(entry.fieldPath, entry.value),
+ }))
+ .filter((entry) => {
+ if (isBaseOnlyFieldPath(entry.fieldPath)) {
+ logger.warn(
+ `[adminConfig] Stripping base-only config field "${entry.fieldPath}" - configure it in librechat.yaml instead`,
+ );
+ return false;
+ }
+ if (BASE_PRINCIPAL_OVERRIDE_SECTIONS.has(getTopLevelSection(entry.fieldPath))) {
+ logger.warn(
+ `[adminConfig] Stripping dedicated tenant-wide config field "${entry.fieldPath}" from the generic config API`,
+ );
+ return false;
+ }
+ if (isInterfacePermissionPath(entry.fieldPath)) {
+ logger.warn(
+ `[adminConfig] Stripping interface permission field "${entry.fieldPath}" — use role permissions instead`,
+ );
+ return false;
+ }
+ return true;
+ });
const hasBroadManage = await hasConfigCapability(user, null, 'manage');
diff --git a/packages/api/src/agents/triggers/envelope.spec.ts b/packages/api/src/agents/triggers/envelope.spec.ts
index da869e1c63..80c1980d73 100644
--- a/packages/api/src/agents/triggers/envelope.spec.ts
+++ b/packages/api/src/agents/triggers/envelope.spec.ts
@@ -19,6 +19,12 @@ describe('createAgentTriggerEnvelope', () => {
tenantId: 'tenant-1',
},
target: { agentId: 'agent-1' },
+ run: {
+ conversationId: 'conversation-1',
+ timezone: 'America/New_York',
+ files: [{ file_id: 'file-1' }],
+ metadata: { adapter: 'test' },
+ },
event: {
id: 'event-1',
type: 'work.ready',
@@ -49,6 +55,12 @@ describe('createAgentTriggerEnvelope', () => {
receivedAt: 1_725_000_000_010,
principal: { userId: 'user-1', role: 'USER', tenantId: 'tenant-1' },
target: { agentId: 'agent-1' },
+ run: {
+ conversationId: 'conversation-1',
+ timezone: 'America/New_York',
+ files: [{ file_id: 'file-1' }],
+ metadata: { adapter: 'test' },
+ },
event: {
id: 'event-1',
type: 'work.ready',
@@ -208,6 +220,33 @@ describe('createAgentTriggerEnvelope', () => {
).toThrow('target.agentId must be a non-empty string');
});
+ it('validates and detaches trusted fire run context', () => {
+ const input = createFireInput();
+ const envelope = createAgentTriggerEnvelope(input);
+ const files = input.mode === 'fire' ? input.run?.files : undefined;
+ if (files != null) {
+ (files[0] as { file_id: string }).file_id = 'changed';
+ }
+
+ expect(envelope.mode).toBe('fire');
+ if (envelope.mode === 'fire') {
+ expect(envelope.run?.files).toEqual([{ file_id: 'file-1' }]);
+ }
+
+ expect(() =>
+ createAgentTriggerEnvelope({
+ ...createFireInput(),
+ run: { files: 'not-an-array' },
+ } as unknown as CreateAgentTriggerEnvelopeInput),
+ ).toThrow('run.files must be an array');
+ expect(() =>
+ createAgentTriggerEnvelope({
+ ...createFireInput(),
+ run: { metadata: { callback: () => undefined } },
+ } as unknown as CreateAgentTriggerEnvelopeInput),
+ ).toThrow('run.metadata.callback contains a non-JSON function value');
+ });
+
it('rejects non-JSON and circular event payloads', () => {
expect(() =>
createAgentTriggerEnvelope({
diff --git a/packages/api/src/agents/triggers/envelope.ts b/packages/api/src/agents/triggers/envelope.ts
index 2391fb0293..c599df6b80 100644
--- a/packages/api/src/agents/triggers/envelope.ts
+++ b/packages/api/src/agents/triggers/envelope.ts
@@ -30,6 +30,20 @@ interface AgentTriggerTarget {
agentId: string;
}
+/**
+ * Trusted host controls for a new fire. These values shape LibreChat's own
+ * generation request; they are never sourced directly from model output.
+ * `metadata` is opaque to the trigger core and lets an authenticated adapter
+ * identify its delivery to host-side lifecycle hooks without widening the
+ * public chat request surface.
+ */
+export interface AgentFireRunContext {
+ conversationId?: string;
+ timezone?: string;
+ files?: JsonValue[];
+ metadata?: JsonValue;
+}
+
/**
* One fire delivery represents one new conversation for this agent; retries
* reuse the delivery id instead of starting another conversation.
@@ -68,6 +82,7 @@ interface AgentTriggerEnvelopeBase {
export interface AgentFireTriggerEnvelope extends AgentTriggerEnvelopeBase {
mode: 'fire';
target: AgentFireTarget;
+ run?: AgentFireRunContext;
}
export interface AgentContinueTriggerEnvelope extends AgentTriggerEnvelopeBase {
@@ -98,6 +113,7 @@ export type CreateAgentTriggerEnvelopeInput =
| (CreateAgentTriggerEnvelopeBase & {
mode: 'fire';
target: AgentFireTarget;
+ run?: AgentFireRunContext;
})
| (CreateAgentTriggerEnvelopeBase & {
mode: 'continue';
@@ -168,6 +184,32 @@ function createEvent(input: AgentTriggerEvent | null | undefined): AgentTriggerE
};
}
+function createFireRunContext(
+ input: AgentFireRunContext | null | undefined,
+): AgentFireRunContext | undefined {
+ if (input == null) {
+ return undefined;
+ }
+ const run = requireRecord(input, 'run');
+ const context: AgentFireRunContext = {};
+ if (run.conversationId != null) {
+ context.conversationId = requireString(run.conversationId, 'run.conversationId');
+ }
+ if (run.timezone != null) {
+ context.timezone = requireString(run.timezone, 'run.timezone');
+ }
+ if (run.files != null) {
+ if (!Array.isArray(run.files)) {
+ throw error('run.files must be an array');
+ }
+ context.files = cloneJsonValue(run.files, 'run.files', error) as JsonValue[];
+ }
+ if (run.metadata !== undefined) {
+ context.metadata = cloneJsonValue(run.metadata, 'run.metadata', error) as JsonValue;
+ }
+ return context;
+}
+
export function createAgentTriggerEnvelope(
input: CreateAgentTriggerEnvelopeInput,
): AgentTriggerEnvelope {
@@ -183,10 +225,12 @@ export function createAgentTriggerEnvelope(
};
if (input.mode === 'fire') {
+ const run = createFireRunContext(input.run);
return {
...base,
mode: input.mode,
target: { agentId: requireString(input.target?.agentId, 'target.agentId') },
+ ...(run != null && { run }),
};
}
@@ -276,10 +320,14 @@ export function parseAgentTriggerEnvelope(input: unknown): AgentTriggerEnvelope
const target = requireRecord(envelope.target, 'target');
if (mode === 'fire') {
+ const run = createFireRunContext(
+ envelope.run == null ? undefined : (envelope.run as AgentFireRunContext),
+ );
return {
...base,
mode,
target: { agentId: requireString(target.agentId, 'target.agentId') },
+ ...(run != null && { run }),
};
}
diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.ts
index c9c9624064..cf97319b5d 100644
--- a/packages/api/src/agents/triggers/host.spec.ts
+++ b/packages/api/src/agents/triggers/host.spec.ts
@@ -12,6 +12,12 @@ const createFireEnvelope = () =>
receivedAt: 20,
principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },
target: { agentId: 'agent-1' },
+ run: {
+ conversationId: 'scheduled-conversation-1',
+ timezone: 'Europe/Paris',
+ files: [{ file_id: 'file-1' }],
+ metadata: { manual: false },
+ },
event: {
id: 'event-1',
type: 'resource.ready',
@@ -154,7 +160,20 @@ describe('createAgentTriggerExecutionHost fire adapter', () => {
isRegenerate: false,
clientRequestId: idempotencyKey,
generationProtocolVersion: 2,
- timezone: 'America/New_York',
+ agentTrigger: {
+ version: envelope.version,
+ deliveryId: envelope.deliveryId,
+ event: {
+ id: envelope.event.id,
+ type: envelope.event.type,
+ occurredAt: envelope.event.occurredAt,
+ source: envelope.event.source,
+ },
+ metadata: { manual: false },
+ },
+ newConversationId: 'scheduled-conversation-1',
+ files: [{ file_id: 'file-1' }],
+ timezone: 'Europe/Paris',
});
expect(mintToken).toHaveBeenCalledWith(envelope.principal, envelope);
});
diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts
index c3a33464d7..869d326b86 100644
--- a/packages/api/src/agents/triggers/host.ts
+++ b/packages/api/src/agents/triggers/host.ts
@@ -556,7 +556,7 @@ async function startRun(
}
const input = preparation?.status === 'ready' ? preparation.input : envelope.input;
const parentMessageId = resolveParentMessageId(preparation, envelope);
- const [token, timezone, baseUrl] = await Promise.all([
+ const [token, resolvedTimezone, baseUrl] = await Promise.all([
setupValue(
() => deps.mintToken(envelope.principal, envelope),
mode,
@@ -574,6 +574,8 @@ async function startRun(
scope.abort();
throw error;
});
+ const run = envelope.mode === 'fire' ? envelope.run : undefined;
+ const timezone = run?.timezone ?? resolvedTimezone;
const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);
const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;
let response: Response;
@@ -602,6 +604,25 @@ async function startRun(
isRegenerate: false,
clientRequestId: context.idempotencyKey,
generationProtocolVersion: 2,
+ ...(envelope.mode === 'fire' && {
+ agentTrigger: {
+ version: envelope.version,
+ deliveryId: envelope.deliveryId,
+ event: {
+ id: envelope.event.id,
+ type: envelope.event.type,
+ occurredAt: envelope.event.occurredAt,
+ source: envelope.event.source,
+ },
+ ...(run?.metadata !== undefined && {
+ metadata: run.metadata,
+ }),
+ },
+ }),
+ ...(run?.conversationId != null && {
+ newConversationId: run.conversationId,
+ }),
+ ...(run?.files != null && { files: run.files }),
...(typeof timezone === 'string' && timezone.trim().length > 0
? { timezone: timezone.trim() }
: {}),
diff --git a/packages/api/src/app/permissions.spec.ts b/packages/api/src/app/permissions.spec.ts
index b93efe64d3..d12fe0e3fe 100644
--- a/packages/api/src/app/permissions.spec.ts
+++ b/packages/api/src/app/permissions.spec.ts
@@ -118,6 +118,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -177,6 +181,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -300,6 +308,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -359,6 +371,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -468,6 +484,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -527,6 +547,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -649,6 +673,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -708,6 +736,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -817,6 +849,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -876,6 +912,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -990,6 +1030,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -1037,6 +1081,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -1168,6 +1216,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
const expectedPermissionsForAdmin = {
@@ -1219,6 +1271,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
};
expect(mockUpdateAccessPermissions).toHaveBeenCalledTimes(2);
@@ -2777,6 +2833,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
},
});
@@ -2814,6 +2874,10 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE]: false,
[Permissions.SHARE_PUBLIC]: false,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
},
});
@@ -2842,4 +2906,66 @@ describe('updateInterfacePermissions - permissions', () => {
[Permissions.SHARE_PUBLIC]: true,
});
});
+
+ it('does not re-enable a DB-disabled schedules permission when only runtime limits are configured', async () => {
+ // The role already has schedules USE disabled in the DB.
+ mockGetRoleByName.mockResolvedValue({
+ permissions: {
+ [PermissionTypes.SCHEDULES]: { [Permissions.USE]: false, [Permissions.CREATE]: false },
+ },
+ });
+ // Runtime-only limits — NOT a permission config; must not re-derive USE.
+ const config = {
+ interface: {
+ schedules: { maxPerUser: 20, fireConcurrency: 2 },
+ },
+ };
+ const configDefaults = { interface: {} } as TConfigDefaults;
+ const interfaceConfig = await loadDefaultInterface({ config, configDefaults });
+ const appConfig = { config, interfaceConfig } as unknown as AppConfig;
+
+ await updateInterfacePermissions({
+ appConfig,
+ getRoleByName: mockGetRoleByName,
+ updateAccessPermissions: mockUpdateAccessPermissions,
+ });
+
+ // Schedules must be preserved (omitted from every role update), not silently
+ // re-enabled to USE:true from defaults just because limits were tuned.
+ for (const call of mockUpdateAccessPermissions.mock.calls) {
+ expect(call[1][PermissionTypes.SCHEDULES]).toBeUndefined();
+ }
+ });
+
+ it('treats a boolean schedules kill switch as runtime-only (does not touch the permission)', async () => {
+ // The role currently has schedules enabled.
+ mockGetRoleByName.mockResolvedValue({
+ permissions: {
+ [PermissionTypes.SCHEDULES]: { [Permissions.USE]: true, [Permissions.CREATE]: true },
+ },
+ });
+ // `schedules: false` is the RUNTIME kill switch read by getLimits, NOT a permission
+ // config: it must not write SCHEDULES into the role docs, so removing it later can
+ // never leave USE stuck false (forbidden) until manual repair.
+ const config = {
+ interface: {
+ schedules: false,
+ },
+ };
+ const configDefaults = { interface: {} } as TConfigDefaults;
+ const interfaceConfig = await loadDefaultInterface({ config, configDefaults });
+ const appConfig = { config, interfaceConfig } as unknown as AppConfig;
+
+ await updateInterfacePermissions({
+ appConfig,
+ getRoleByName: mockGetRoleByName,
+ updateAccessPermissions: mockUpdateAccessPermissions,
+ });
+
+ // The kill switch is runtime-only: SCHEDULES is omitted from every role update,
+ // leaving the existing USE:true untouched.
+ for (const call of mockUpdateAccessPermissions.mock.calls) {
+ expect(call[1][PermissionTypes.SCHEDULES]).toBeUndefined();
+ }
+ });
});
diff --git a/packages/api/src/app/permissions.ts b/packages/api/src/app/permissions.ts
index dcc626143c..c4ea6ed610 100644
--- a/packages/api/src/app/permissions.ts
+++ b/packages/api/src/app/permissions.ts
@@ -49,6 +49,19 @@ function hasExplicitConfig(
return interfaceConfig?.skills !== undefined;
case PermissionTypes.SHARED_LINKS:
return interfaceConfig?.sharedLinks !== undefined;
+ case PermissionTypes.SCHEDULES: {
+ // `schedules` is dual-purpose. The BOOLEAN form is the RUNTIME kill switch read
+ // by getLimits, NOT a permission config: treating it as explicit would write
+ // SCHEDULES.USE into the role docs, and removing the kill switch later would
+ // leave that disabled permission stuck (forbidden) until manual repair. Only an
+ // OBJECT carrying explicit use/create is permission intent; runtime-only limits
+ // (maxPerUser, fireConcurrency, …) are not.
+ const schedules = interfaceConfig?.schedules;
+ if (typeof schedules !== 'object' || schedules == null) {
+ return false;
+ }
+ return schedules.use !== undefined || schedules.create !== undefined;
+ }
default:
return false;
}
@@ -199,6 +212,12 @@ export async function updateInterfacePermissions({
typeof defaults.agents === 'object' ? defaults.agents?.public : undefined;
const skillsDefaultPublic =
typeof defaults.skills === 'object' ? defaults.skills?.public : undefined;
+ // `schedules` is intentionally absent from the interface DEFAULTS (it is
+ // experimental/default-off at runtime), so the PERMISSION defaults are stated here.
+ // Runtime availability and the role permission are separate concerns: a user may hold
+ // the permission while the feature stays off until an admin enables it.
+ const schedulesDefaultUse = true;
+ const schedulesDefaultCreate = true;
const sharedLinksDefaultCreate =
typeof defaults.sharedLinks === 'boolean' ? undefined : defaults.sharedLinks?.create;
const sharedLinksDefaultShare =
@@ -528,6 +547,29 @@ export async function updateInterfacePermissions({
}
: {}),
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: getPermissionValue(
+ // Only an OBJECT `use` drives the permission; the boolean form is the runtime
+ // kill switch and must not seed SCHEDULES.USE (see hasExplicitConfig), so a
+ // removed kill switch can never leave USE stuck false.
+ typeof loadedInterface.schedules === 'object'
+ ? loadedInterface.schedules?.use
+ : undefined,
+ defaultPerms[PermissionTypes.SCHEDULES]?.[Permissions.USE],
+ schedulesDefaultUse,
+ ),
+ ...((typeof interfaceConfig?.schedules === 'object' &&
+ 'create' in interfaceConfig.schedules) ||
+ !existingPermissions?.[PermissionTypes.SCHEDULES]
+ ? {
+ [Permissions.CREATE]: getPermissionValue(
+ getConfigCreate(loadedInterface.schedules),
+ defaultPerms[PermissionTypes.SCHEDULES]?.[Permissions.CREATE],
+ schedulesDefaultCreate ?? true,
+ ),
+ }
+ : {}),
+ },
};
// Check and add each permission type if needed
diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts
index eb54509115..f1e99a01bf 100644
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@@ -58,6 +58,8 @@ export * from './projects';
/* Conversations */
export * from './conversations';
/* Skills */
+export * from './schedules';
+export * from './schedules/service';
export * from './skills';
export * from './favorites';
/* User */
diff --git a/packages/api/src/middleware/balance.ts b/packages/api/src/middleware/balance.ts
index 453d7c66b0..c6e8978b9e 100644
--- a/packages/api/src/middleware/balance.ts
+++ b/packages/api/src/middleware/balance.ts
@@ -55,7 +55,7 @@ async function runBalanceUpdate(userId: string, task: () => Promise<void>): Prom
* @param userId - The user's ID
* @returns Fields that need updating
*/
-function buildUpdateFields(
+export function buildBalanceUpdateFields(
config: BalanceConfig,
userRecord: IBalance | null,
userId: string,
@@ -144,7 +144,7 @@ export function createSetBalanceConfig({
const userId = typeof user._id === 'string' ? user._id : user._id.toString();
await runBalanceUpdate(userId, async () => {
const userBalanceRecord = await findBalanceByUser(userId);
- const updateFields = buildUpdateFields(balanceConfig, userBalanceRecord, userId);
+ const updateFields = buildBalanceUpdateFields(balanceConfig, userBalanceRecord, userId);
if (Object.keys(updateFields).length === 0) {
balanceLocals.balanceData = userBalanceRecord;
diff --git a/packages/api/src/schedules/FOLLOWUPS.md b/packages/api/src/schedules/FOLLOWUPS.md
new file mode 100644
index 0000000000..4ebeee3368
--- /dev/null
+++ b/packages/api/src/schedules/FOLLOWUPS.md
@@ -0,0 +1,32 @@
+# Scheduled chats: deferred scope
+
+Scheduled chats remain experimental and default-off.
+
+## Must-have product follow-ups
+
+- Add one-time and monthly schedules; current cadences are hourly, daily, weekdays, and weekly.
+- Add non-agent chat targets and existing-conversation continuation; current schedules require
+ `agent_id` and always target `new`.
+
+## Client/API parity
+
+- Return `minIntervalMinutes` in schedule limits and surface it in the dialog.
+- Add active-run and run-history discovery. The list response currently projects only `lastRun`,
+ which is written on pause, skip, or terminal outcomes, so the client's `started` chip is
+ unreachable.
+- Add multi-day weekly, timezone, and attachment controls. The API supports these, but the dialog
+ does not.
+
+## Runtime scope
+
+- The standard server supports multi-replica scheduling only with a confirmed Redis-backed
+ `GenerationJobManager`; in-memory mode requires `SCHEDULES_SINGLE_PROCESS`. The legacy
+ `experimental.js` clustered entrypoint does not arm the engine and rejects schedule writes.
+- Deferred account deletion blocks new remote API admission but cannot yet drain already-admitted
+ OpenAI-compatible or Responses requests; see
+ [#14594](https://github.com/danny-avila/LibreChat/issues/14594).
+
+## Test debt
+
+- Add Playwright coverage for creating a schedule through the UI. The execution E2E currently
+ seeds schedules through the API.
diff --git a/packages/api/src/schedules/access.ts b/packages/api/src/schedules/access.ts
new file mode 100644
index 0000000000..83cb793124
--- /dev/null
+++ b/packages/api/src/schedules/access.ts
@@ -0,0 +1,77 @@
+import { logger, ResourceCapabilityMap } from '@librechat/data-schemas';
+import {
+ ResourceType,
+ Permissions,
+ PermissionBits,
+ PermissionTypes,
+} from 'librechat-data-provider';
+import type { SystemCapability } from '@librechat/data-schemas';
+import type { Types } from 'mongoose';
+import type { ScheduleUserContext } from './types';
+
+type AgentAccess = 'ok' | 'missing' | 'forbidden';
+
+export interface AgentFireAccessDeps {
+ /** Resolves an agent's internal `_id` by its custom id, or null when it doesn't exist. */
+ findAgentObjectId: (agentId: string) => Promise<{ _id: Types.ObjectId } | null>;
+ /** Loads a role's permission map by name. */
+ getRoleByName: (
+ role?: string,
+ ) => Promise<{ permissions?: Record<string, Record<string, boolean | undefined>> } | null>;
+ /** Whether the user's role grants a system capability (the manage:agents bypass). */
+ hasCapability: (user: ScheduleUserContext, capability: SystemCapability) => Promise<boolean>;
+ /** Resource ACL check for a specific permission bit. */
+ checkPermission: (params: {
+ userId: string;
+ role?: string;
+ resourceType: ResourceType;
+ resourceId: Types.ObjectId;
+ requiredPermission: PermissionBits;
+ }) => Promise<boolean>;
+}
+
+/**
+ * Resolves a user's live access to a schedule's target agent, mirroring the loopback
+ * chat route's authorization EXACTLY so the create/update precheck and the fire-time
+ * precheck accept a schedule iff the actual fire would be accepted:
+ * 1) role-level AGENTS:USE (checkAccess on the route; admins do NOT bypass)
+ * 2) resource VIEW with the manage:agents capability bypass
+ * The two prechecks must never diverge — a create-time VIEW-only check would let a
+ * role without AGENTS:USE schedule runs that every fire then rejects, burning failures.
+ */
+export function createResolveAgentFireAccess(deps: AgentFireAccessDeps) {
+ return async function resolveAgentFireAccess(
+ agentId: string,
+ user: ScheduleUserContext,
+ ): Promise<AgentAccess> {
+ const agent = await deps.findAgentObjectId(agentId);
+ if (agent == null) {
+ return 'missing';
+ }
+ // Mirror the chat route's checkAccess, which reads the role's AGENTS:USE directly
+ // and does NOT special-case admins: an admin whose role has AGENTS:USE disabled is
+ // rejected there, so the precheck must reject too — otherwise every fire 403s.
+ const role = await deps.getRoleByName(user.role);
+ if (role?.permissions?.[PermissionTypes.AGENTS]?.[Permissions.USE] !== true) {
+ return 'forbidden';
+ }
+ const capability = ResourceCapabilityMap[ResourceType.AGENT];
+ try {
+ if (capability != null && (await deps.hasCapability(user, capability))) {
+ return 'ok';
+ }
+ } catch (err) {
+ logger.warn(
+ `[schedules] agent capability check failed, denying bypass: ${(err as Error).message}`,
+ );
+ }
+ const allowed = await deps.checkPermission({
+ userId: user.id,
+ role: user.role,
+ resourceType: ResourceType.AGENT,
+ resourceId: agent._id,
+ requiredPermission: PermissionBits.VIEW,
+ });
+ return allowed ? 'ok' : 'forbidden';
+ };
+}
diff --git a/packages/api/src/schedules/cadence.spec.ts b/packages/api/src/schedules/cadence.spec.ts
new file mode 100644
index 0000000000..9b7011f35b
--- /dev/null
+++ b/packages/api/src/schedules/cadence.spec.ts
@@ -0,0 +1,322 @@
+import type { TScheduleCadence } from 'librechat-data-provider';
+import {
+ cadenceToCron,
+ computeNextRunAt,
+ isValidTimezone,
+ scheduleJitterMs,
+ cadenceIntervalMinutes,
+ SCHEDULE_JITTER_WINDOW_MS,
+} from './cadence';
+
+const NEW_YORK = 'America/New_York';
+
+function cadence(overrides: Partial<TScheduleCadence>): TScheduleCadence {
+ return { frequency: 'daily', hour: 0, minute: 0, ...overrides };
+}
+
+function wallClock(date: Date, timeZone: string): string {
+ const parts = new Intl.DateTimeFormat('en-CA', {
+ timeZone,
+ weekday: 'short',
+ year: 'numeric',
+ month: '2-digit',
+ day: '2-digit',
+ hour: '2-digit',
+ minute: '2-digit',
+ hourCycle: 'h23',
+ }).formatToParts(date);
+ const get = (type: Intl.DateTimeFormatPartTypes): string =>
+ parts.find((part) => part.type === type)?.value ?? '';
+ return `${get('weekday')} ${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}`;
+}
+
+function nextRun(params: {
+ cadence: TScheduleCadence;
+ timezone: string;
+ after: Date;
+ scheduleId?: string;
+}): Date {
+ const result = computeNextRunAt({
+ cadence: params.cadence,
+ timezone: params.timezone,
+ scheduleId: params.scheduleId ?? 'sched-test',
+ after: params.after,
+ disableJitter: true,
+ });
+ if (result == null) {
+ throw new Error('expected computeNextRunAt to return a Date');
+ }
+ return result;
+}
+
+describe('cadenceToCron', () => {
+ it('maps hourly to a minute-only pattern', () => {
+ expect(cadenceToCron(cadence({ frequency: 'hourly', minute: 15 }))).toBe('15 * * * *');
+ });
+
+ it('maps daily to minute + hour', () => {
+ expect(cadenceToCron(cadence({ frequency: 'daily', hour: 8, minute: 0 }))).toBe('0 8 * * *');
+ });
+
+ it('maps weekdays to Monday through Friday', () => {
+ expect(cadenceToCron(cadence({ frequency: 'weekdays', hour: 9, minute: 30 }))).toBe(
+ '30 9 * * 1-5',
+ );
+ });
+
+ it('maps weekly multi-day with days sorted ascending', () => {
+ expect(
+ cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45, daysOfWeek: [5, 1, 3] })),
+ ).toBe('45 7 * * 1,3,5');
+ });
+
+ it('defaults weekly to Monday when daysOfWeek is missing or empty', () => {
+ expect(cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45 }))).toBe('45 7 * * 1');
+ expect(
+ cadenceToCron(cadence({ frequency: 'weekly', hour: 7, minute: 45, daysOfWeek: [] })),
+ ).toBe('45 7 * * 1');
+ });
+});
+
+describe('computeNextRunAt', () => {
+ const daily8amNewYork = {
+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),
+ timezone: NEW_YORK,
+ };
+
+ it('fires later the same day when the occurrence is still ahead', () => {
+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T09:00:00Z') });
+ expect(wallClock(next, NEW_YORK)).toBe('Wed 2026-07-15 08:00');
+ expect(next.toISOString()).toBe('2026-07-15T12:00:00.000Z');
+ });
+
+ it('rolls to the next day once the occurrence has passed', () => {
+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T13:00:00Z') });
+ expect(wallClock(next, NEW_YORK)).toBe('Thu 2026-07-16 08:00');
+ expect(next.toISOString()).toBe('2026-07-16T12:00:00.000Z');
+ });
+
+ it('returns null for a malformed stored timezone instead of throwing', () => {
+ // A bad schedule reaching the engine must not throw out of a tick — null is
+ // the "uncomputable" signal the engine disables (invalid_schedule) on.
+ expect(() =>
+ computeNextRunAt({
+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),
+ timezone: 'Not/AZone',
+ scheduleId: 'sched_bad_tz',
+ }),
+ ).not.toThrow();
+ expect(
+ computeNextRunAt({
+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),
+ timezone: 'Not/AZone',
+ scheduleId: 'sched_bad_tz',
+ }),
+ ).toBeNull();
+ });
+
+ it('skips an occurrence exactly at `after` (strictly-after semantics)', () => {
+ const next = nextRun({ ...daily8amNewYork, after: new Date('2026-07-15T12:00:00Z') });
+ expect(wallClock(next, NEW_YORK)).toBe('Thu 2026-07-16 08:00');
+ });
+
+ describe('DST spring-forward (US 2026-03-08)', () => {
+ const daily230amNewYork = {
+ cadence: cadence({ frequency: 'daily', hour: 2, minute: 30 }),
+ timezone: NEW_YORK,
+ };
+
+ it('fires the gap occurrence shifted to 03:30 EDT rather than dropping March 8', () => {
+ const next = nextRun({ ...daily230amNewYork, after: new Date('2026-03-07T17:00:00Z') });
+ expect(next.toISOString()).toBe('2026-03-08T07:30:00.000Z');
+ expect(wallClock(next, NEW_YORK)).toBe('Sun 2026-03-08 03:30');
+ });
+
+ it('resumes at 02:30 EDT on March 9 after the gap occurrence', () => {
+ const march8 = nextRun({ ...daily230amNewYork, after: new Date('2026-03-07T17:00:00Z') });
+ const next = nextRun({ ...daily230amNewYork, after: march8 });
+ expect(next.toISOString()).toBe('2026-03-09T06:30:00.000Z');
+ expect(wallClock(next, NEW_YORK)).toBe('Mon 2026-03-09 02:30');
+ });
+ });
+
+ describe('DST fall-back (US 2026-11-01)', () => {
+ it('fires the repeated 01:30 exactly once, at the first (EDT) occurrence', () => {
+ const daily130amNewYork = {
+ cadence: cadence({ frequency: 'daily', hour: 1, minute: 30 }),
+ timezone: NEW_YORK,
+ };
+ const runs: Date[] = [];
+ let after = new Date('2026-10-31T16:00:00Z');
+ for (let i = 0; i < 3; i++) {
+ const next = nextRun({ ...daily130amNewYork, after });
+ runs.push(next);
+ after = next;
+ }
+ expect(runs.map((run) => run.toISOString())).toEqual([
+ '2026-11-01T05:30:00.000Z',
+ '2026-11-02T06:30:00.000Z',
+ '2026-11-03T06:30:00.000Z',
+ ]);
+ expect(runs.map((run) => wallClock(run, NEW_YORK))).toEqual([
+ 'Sun 2026-11-01 01:30',
+ 'Mon 2026-11-02 01:30',
+ 'Tue 2026-11-03 01:30',
+ ]);
+ });
+ });
+
+ it('weekdays cadence skips weekends', () => {
+ const weekdays9amNewYork = {
+ cadence: cadence({ frequency: 'weekdays', hour: 9, minute: 0 }),
+ timezone: NEW_YORK,
+ };
+ const fromFriday = nextRun({ ...weekdays9amNewYork, after: new Date('2026-07-17T15:00:00Z') });
+ expect(wallClock(fromFriday, NEW_YORK)).toBe('Mon 2026-07-20 09:00');
+ const fromSaturday = nextRun({
+ ...weekdays9amNewYork,
+ after: new Date('2026-07-18T12:00:00Z'),
+ });
+ expect(wallClock(fromSaturday, NEW_YORK)).toBe('Mon 2026-07-20 09:00');
+ });
+
+ it('hourly cadence fires at the next :MM', () => {
+ const hourly45 = { cadence: cadence({ frequency: 'hourly', minute: 45 }), timezone: 'UTC' };
+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:30:00Z') }).toISOString()).toBe(
+ '2026-07-15T10:45:00.000Z',
+ );
+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:50:00Z') }).toISOString()).toBe(
+ '2026-07-15T11:45:00.000Z',
+ );
+ expect(nextRun({ ...hourly45, after: new Date('2026-07-15T10:45:00Z') }).toISOString()).toBe(
+ '2026-07-15T11:45:00.000Z',
+ );
+ });
+
+ it('weekly multi-day cadence fires only on the configured days', () => {
+ const weekly = {
+ cadence: cadence({ frequency: 'weekly', hour: 10, minute: 0, daysOfWeek: [3, 6] }),
+ timezone: 'UTC',
+ };
+ const first = nextRun({ ...weekly, after: new Date('2026-07-16T00:00:00Z') });
+ const second = nextRun({ ...weekly, after: first });
+ const third = nextRun({ ...weekly, after: second });
+ expect([first, second, third].map((run) => run.toISOString())).toEqual([
+ '2026-07-18T10:00:00.000Z',
+ '2026-07-22T10:00:00.000Z',
+ '2026-07-25T10:00:00.000Z',
+ ]);
+ expect([first, second, third].map((run) => wallClock(run, 'UTC').slice(0, 3))).toEqual([
+ 'Sat',
+ 'Wed',
+ 'Sat',
+ ]);
+ });
+});
+
+describe('jitter', () => {
+ it('is deterministic for the same schedule id', () => {
+ expect(scheduleJitterMs('sched-abc')).toBe(scheduleJitterMs('sched-abc'));
+ });
+
+ it('stays within the jitter window', () => {
+ for (let i = 0; i < 100; i++) {
+ const value = scheduleJitterMs(`schedule-${i}`);
+ expect(value).toBeGreaterThanOrEqual(0);
+ expect(value).toBeLessThan(SCHEDULE_JITTER_WINDOW_MS);
+ }
+ });
+
+ it('respects a custom window', () => {
+ for (let i = 0; i < 20; i++) {
+ const value = scheduleJitterMs(`schedule-${i}`, 1000);
+ expect(value).toBeGreaterThanOrEqual(0);
+ expect(value).toBeLessThan(1000);
+ }
+ });
+
+ it('spreads different ids across the window', () => {
+ const values = new Set(Array.from({ length: 50 }, (_, i) => scheduleJitterMs(`schedule-${i}`)));
+ expect(values.size).toBeGreaterThan(40);
+ });
+
+ it('computeNextRunAt offsets the cron instant by the schedule jitter', () => {
+ const params = {
+ cadence: cadence({ frequency: 'daily', hour: 8, minute: 0 }),
+ timezone: NEW_YORK,
+ scheduleId: 'sched-jitter',
+ after: new Date('2026-07-15T09:00:00Z'),
+ };
+ const jittered = computeNextRunAt(params);
+ const bare = computeNextRunAt({ ...params, disableJitter: true });
+ expect(jittered).not.toBeNull();
+ expect(bare).not.toBeNull();
+ expect((jittered?.getTime() ?? 0) - (bare?.getTime() ?? 0)).toBe(
+ scheduleJitterMs('sched-jitter'),
+ );
+ });
+
+ it('does not skip a still-future occurrence inside the jitter window', () => {
+ const scheduleId = 'sched-window';
+ const jitter = scheduleJitterMs(scheduleId);
+ expect(jitter).toBeGreaterThan(1000); // ensure the window is meaningful
+ const cadenceHourly = cadence({ frequency: 'hourly', hour: 0, minute: 0 });
+ const cronInstant = new Date('2026-07-15T12:00:00Z'); // an hourly :00 boundary (UTC)
+ // `after` is just past the unjittered instant but before the jittered one:
+ // the jittered occurrence (cronInstant + jitter) is still in the future.
+ const after = new Date(cronInstant.getTime() + 1000);
+ const next = computeNextRunAt({ cadence: cadenceHourly, timezone: 'UTC', scheduleId, after });
+ // Must return THIS hour's jittered instant, not next hour's.
+ expect(next?.getTime()).toBe(cronInstant.getTime() + jitter);
+ });
+});
+
+describe('isValidTimezone', () => {
+ it('accepts IANA zone names', () => {
+ expect(isValidTimezone('America/New_York')).toBe(true);
+ expect(isValidTimezone('Europe/Stockholm')).toBe(true);
+ expect(isValidTimezone('UTC')).toBe(true);
+ });
+
+ it('rejects invalid zones', () => {
+ expect(isValidTimezone('Not/AZone')).toBe(false);
+ expect(isValidTimezone('EST5EDT-Bogus')).toBe(false);
+ expect(isValidTimezone('')).toBe(false);
+ });
+});
+
+describe('cadenceIntervalMinutes', () => {
+ it('returns the minimum minutes between occurrences per frequency', () => {
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'hourly' }))).toBe(60);
+ // Day-and-longer gaps report the DST-compressed minimum (largest real-world
+ // transition is 2h — Antarctica/Troll): a floor set at the nominal value would
+ // admit a schedule that genuinely violates it once a year.
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'daily' }))).toBe(24 * 60 - 120);
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekdays' }))).toBe(24 * 60 - 120);
+ });
+
+ it('uses the shortest gap between weekly days, incl. the week wrap-around', () => {
+ // single day → full week
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly' }))).toBe(7 * 24 * 60 - 120);
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [3] }))).toBe(
+ 7 * 24 * 60 - 120,
+ );
+ // adjacent days fire 24h apart — the floor must reflect that, not the average
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 2] }))).toBe(
+ 24 * 60 - 120,
+ );
+ // evenly spaced Mon/Wed/Fri → 2-day min gap
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 3, 5] }))).toBe(
+ 2 * 24 * 60 - 120,
+ );
+ // duplicates are one selection, not a zero-day gap: [1, 1] fires once a week,
+ // and reading it as 0 minutes rejected the schedule against every valid floor
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [1, 1] }))).toBe(
+ 7 * 24 * 60 - 120,
+ );
+ // wrap-around: Sun + Sat are 1 day apart across the week boundary
+ expect(cadenceIntervalMinutes(cadence({ frequency: 'weekly', daysOfWeek: [0, 6] }))).toBe(
+ 24 * 60 - 120,
+ );
+ });
+});
diff --git a/packages/api/src/schedules/cadence.ts b/packages/api/src/schedules/cadence.ts
new file mode 100644
index 0000000000..f03563c67f
--- /dev/null
+++ b/packages/api/src/schedules/cadence.ts
@@ -0,0 +1,133 @@
+import { Cron } from 'croner';
+import type { TScheduleCadence } from 'librechat-data-provider';
+
+export const SCHEDULE_JITTER_WINDOW_MS = 120_000;
+
+/**
+ * Spring-forward compresses consecutive wall-clock occurrences, so the ENFORCEABLE
+ * minimum for day-and-longer gaps is the nominal gap minus the largest real-world
+ * transition: two hours (Antarctica/Troll; every other zone shifts at most one).
+ * A floor set exactly at the nominal value would otherwise admit a schedule that
+ * genuinely violates it once a year. Hourly gaps are unaffected (the skipped hours
+ * lengthen, never shorten, the gap between occurrences).
+ */
+const DST_COMPRESSION_MINUTES = 120;
+
+const WEEKLY_DEFAULT_DAY = 1;
+
+/**
+ * Compiles a structured cadence to a 5-field cron expression. The cadence
+ * object stays canonical (UI-native, no cron round-tripping); cron exists only
+ * as croner's input.
+ */
+export function cadenceToCron(cadence: TScheduleCadence): string {
+ const { frequency, hour, minute } = cadence;
+ if (frequency === 'hourly') {
+ return `${minute} * * * *`;
+ }
+ if (frequency === 'daily') {
+ return `${minute} ${hour} * * *`;
+ }
+ if (frequency === 'weekdays') {
+ return `${minute} ${hour} * * 1-5`;
+ }
+ const days = cadence.daysOfWeek?.length ? cadence.daysOfWeek : [WEEKLY_DEFAULT_DAY];
+ return `${minute} ${hour} * * ${[...days].sort((a, b) => a - b).join(',')}`;
+}
+
+export function isValidTimezone(timezone: string): boolean {
+ try {
+ new Intl.DateTimeFormat('en-US', { timeZone: timezone });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Minimum minutes between occurrences, for the admin interval floor. */
+export function cadenceIntervalMinutes(cadence: TScheduleCadence): number {
+ if (cadence.frequency === 'hourly') {
+ return 60;
+ }
+ if (cadence.frequency === 'daily' || cadence.frequency === 'weekdays') {
+ return 24 * 60 - DST_COMPRESSION_MINUTES;
+ }
+ // Deduped defensively: the payload schema normalizes new writes, but a legacy
+ // stored [1, 1] would otherwise read as a zero-day gap and fail every floor.
+ const days = cadence.daysOfWeek?.length
+ ? Array.from(new Set(cadence.daysOfWeek))
+ : [WEEKLY_DEFAULT_DAY];
+ if (days.length <= 1) {
+ return 7 * 24 * 60 - DST_COMPRESSION_MINUTES;
+ }
+ // The interval floor must reflect the SHORTEST gap between selected days
+ // (incl. the week wrap-around), not the average — e.g. [Mon, Tue] fires 24h
+ // apart, so it must be rejected against a >1440-minute floor.
+ const sorted = [...days].sort((a, b) => a - b);
+ let minGapDays = 7;
+ for (let i = 0; i < sorted.length; i++) {
+ const gap = i + 1 < sorted.length ? sorted[i + 1] - sorted[i] : 7 - sorted[i] + sorted[0];
+ minGapDays = Math.min(minGapDays, gap);
+ }
+ return minGapDays * 24 * 60 - DST_COMPRESSION_MINUTES;
+}
+
+/**
+ * Deterministic per-schedule jitter so fleet-wide fire spikes (everyone at
+ * 9:00) spread across a window while each schedule's displayed next-run time
+ * stays stable across recomputations.
+ */
+export function scheduleJitterMs(
+ scheduleId: string,
+ windowMs: number = SCHEDULE_JITTER_WINDOW_MS,
+): number {
+ let hash = 5381;
+ for (let i = 0; i < scheduleId.length; i++) {
+ hash = ((hash << 5) + hash + scheduleId.charCodeAt(i)) | 0;
+ }
+ return Math.abs(hash) % windowMs;
+}
+
+export interface ComputeNextRunParams {
+ cadence: TScheduleCadence;
+ timezone: string;
+ scheduleId: string;
+ /** Occurrences at or before this instant are skipped (misfire skip-forward). */
+ after?: Date;
+ disableJitter?: boolean;
+}
+
+/**
+ * Next fire instant strictly after `after` (default now), jitter applied.
+ * DST semantics are croner's behavior (locked by cadence.spec.ts): a
+ * spring-forward gap occurrence fires shifted to the first valid instant
+ * (02:30 → 03:30); fall-back ambiguity fires the first occurrence only.
+ */
+export function computeNextRunAt(params: ComputeNextRunParams): Date | null {
+ // A malformed stored schedule (e.g. an invalid timezone inserted before
+ // validation or by an admin script) makes croner throw at construction. Treat
+ // an uncomputable next run as null — the same signal as "no future occurrence"
+ // — so the engine disables it (`invalid_schedule`) rather than throwing out of
+ // a tick every time the lease expires and starving other due schedules.
+ try {
+ const cron = new Cron(cadenceToCron(params.cadence), {
+ timezone: params.timezone,
+ paused: true,
+ });
+ const base = params.after ?? new Date();
+ const jitter = params.disableJitter === true ? 0 : scheduleJitterMs(params.scheduleId);
+ // The jittered instant is `cronOccurrence + jitter`. To return the first one
+ // strictly after `base`, find the first cron occurrence after `base - jitter`
+ // (occurrence O > base - jitter ⇒ O + jitter > base). Querying from `base`
+ // directly would skip an occurrence whose jittered time is still in the future
+ // but whose unjittered time already passed (e.g. an hourly :00 created at
+ // 12:00:30 with 90s jitter must fire at 12:01:30, not 13:01:30).
+ const next = cron.nextRun(new Date(base.getTime() - jitter));
+ if (next == null) {
+ return null;
+ }
+ return new Date(next.getTime() + jitter);
+ } catch {
+ return null;
+ }
+}
diff --git a/packages/api/src/schedules/capacity.ts b/packages/api/src/schedules/capacity.ts
new file mode 100644
index 0000000000..79770b5e5d
--- /dev/null
+++ b/packages/api/src/schedules/capacity.ts
@@ -0,0 +1,53 @@
+/** Occupancy of the global scheduled-run capacity slots. `unslotted` counts legacy
+ * `started` rows written before slots existed; they shrink the effective cap so the
+ * bound stays conservative during rollout instead of transiently overshooting. */
+export interface CapacityOccupancy {
+ takenSlots: number[];
+ unslotted: number;
+}
+
+/** A claim either succeeded (carrying the caller's own result) or lost the slot race. */
+export type SlotClaimResult<T> = { claimed: T } | 'slot-taken';
+
+/**
+ * Allocates the lowest free global capacity slot and hands it to `claim`, retrying the
+ * next free slot when the DB rejects a collision on the unique partial index.
+ *
+ * This replaces "count active runs, compare to the cap, then insert": the count is a
+ * read-then-write race, so two admissions of DIFFERENT schedules could both observe
+ * cap-1 and both proceed. Here the slot itself is the contended resource and the
+ * unique index is the arbiter, so the cap is enforced by the database.
+ *
+ * Bounded by cap+1 attempts: every collision advances to a strictly higher free slot.
+ */
+export async function withCapacitySlot<T>(
+ cap: number,
+ readOccupancy: () => Promise<CapacityOccupancy>,
+ claim: (slot: number) => Promise<SlotClaimResult<T>>,
+): Promise<{ claimed: T } | 'capacity'> {
+ if (cap <= 0) {
+ return 'capacity';
+ }
+ for (let attempt = 0; attempt <= cap; attempt++) {
+ const { takenSlots, unslotted } = await readOccupancy();
+ if (takenSlots.length + unslotted >= cap) {
+ return 'capacity';
+ }
+ const taken = new Set(takenSlots);
+ let slot = -1;
+ for (let candidate = 0; candidate < cap; candidate++) {
+ if (!taken.has(candidate)) {
+ slot = candidate;
+ break;
+ }
+ }
+ if (slot < 0) {
+ return 'capacity';
+ }
+ const result = await claim(slot);
+ if (result !== 'slot-taken') {
+ return result;
+ }
+ }
+ return 'capacity';
+}
diff --git a/packages/api/src/schedules/engine.spec.ts b/packages/api/src/schedules/engine.spec.ts
new file mode 100644
index 0000000000..4e1a582712
--- /dev/null
+++ b/packages/api/src/schedules/engine.spec.ts
@@ -0,0 +1,737 @@
+import type { ScheduleEngineDeps, ScheduleLimits, ScheduleUserContext } from './types';
+import type { FireableSchedule } from './types';
+import { startScheduleEngine } from './engine';
+
+const LEASE_MS = 5 * 60_000;
+const OWNER: ScheduleUserContext = { id: 'user-1', tenantId: 't1', role: 'USER' };
+const LIMITS: ScheduleLimits = {
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+};
+
+/**
+ * A schedule due 60s ago whose lease was just taken, so the tick reads it as due
+ * rather than misfired (`dbNow` is derived from `leaseUntil - LEASE_MS`).
+ */
+function makeClaimedSchedule(overrides: Partial<FireableSchedule> = {}): FireableSchedule {
+ const now = Date.now();
+ return {
+ id: 'sched-1',
+ user: 'user-1' as never,
+ tenantId: 't1',
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ claimToken: 'ct-1',
+ leaseBy: 'inst-1',
+ nextRunAt: new Date(now - 60_000),
+ leaseUntil: new Date(now + LEASE_MS),
+ runCount: 0,
+ failureCount: 0,
+ balanceSkipCount: 0,
+ ...overrides,
+ } as FireableSchedule;
+}
+
+function makeMethods(schedule: FireableSchedule) {
+ let claims = 0;
+ return {
+ // One claimable occurrence per tick, then the claim scan comes up empty.
+ claimDueSchedule: jest.fn(async () => (claims++ === 0 ? schedule : null)),
+ countActiveRuns: jest.fn(async () => 0),
+ advanceSchedule: jest.fn(
+ async (_id: string, _next: Date | null, _from?: Date, _token?: string) => true,
+ ),
+ releaseLeaseByHolder: jest.fn(async () => undefined),
+ disableSchedule: jest.fn(async (_id: string, _reason: string, _token?: string) => undefined),
+ releaseLease: jest.fn(async () => true),
+ holdsLease: jest.fn(async () => true),
+ scheduleExists: jest.fn(async () => true),
+ deleteScheduleRun: jest.fn(async () => undefined),
+ getRunsForReconciliation: jest.fn(async () => []),
+ markRunsReconciled: jest.fn(async () => undefined),
+ getUnbookkeptRuns: jest.fn(async () => []),
+ getDeletingSchedules: jest.fn(async () => []),
+ markEraseAttempted: jest.fn(async () => undefined),
+ recordRunOutcome: jest.fn(async () => undefined),
+ getUnarmedSchedules: jest.fn(async () => []),
+ armSchedule: jest.fn(async () => undefined),
+ eraseScheduleIfDrained: jest.fn(async () => true),
+ finalizeBookkeeping: jest.fn(async () => undefined),
+ };
+}
+
+function makeDeps(
+ methods: ReturnType<typeof makeMethods>,
+ over: Partial<ScheduleEngineDeps> = {},
+): ScheduleEngineDeps {
+ return {
+ methods: methods as unknown as ScheduleEngineDeps['methods'],
+ getLimits: async () => LIMITS,
+ getUserContext: async () => OWNER,
+ isOutOfBalance: async () => false,
+ agentAccess: async () => 'ok',
+ hasScheduleAccess: async () => true,
+ resolveFiles: async () => [],
+ enqueueTrigger: jest.fn(async () => undefined),
+ getTriggerDelivery: async () => null,
+ runInTenantContext: (_user, fn) => fn(),
+ getJobStatus: async () => null,
+ abortScheduledJob: async () => undefined,
+ clearReconciledJob: async () => undefined,
+ isOwnerDeleting: async () => false,
+ isGloballyDisabled: async () => false,
+ countActiveRunsGlobal: async () => 0,
+ withGlobalCapacitySlot: (_cap: number, claim: (slot: number) => Promise<unknown>) => claim(0),
+ ...over,
+ } as ScheduleEngineDeps;
+}
+
+/** Builds the engine and immediately parks its timer; tests drive `runTick` directly. */
+async function tickOnce(deps: ScheduleEngineDeps): Promise<void> {
+ const engine = startScheduleEngine(deps);
+ engine.stop();
+ await engine.runTick();
+}
+
+/** Drives one AWAITED reconciliation pass (startup's own is fire-and-forget). */
+async function reconcileOnce(deps: ScheduleEngineDeps): Promise<void> {
+ const engine = startScheduleEngine(deps);
+ engine.stop();
+ await engine.reconcile();
+}
+
+afterEach(() => jest.restoreAllMocks());
+
+/** Overdue past MISFIRE_GRACE_MS (15m), so the tick skips it forward instead of firing. */
+const staleAt = () => new Date(Date.now() - 20 * 60_000);
+
+describe('runTick misfire skip-forward', () => {
+ it('advances a stale occurrence to the next future one', async () => {
+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt() });
+ const methods = makeMethods(schedule);
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as never;
+ await tickOnce(makeDeps(methods));
+ // Skipping forward is the whole point of the branch. Leaving nextRunAt on the
+ // stale occurrence would make every later tick reclaim and skip the same one,
+ // so a schedule overdue past an outage would never fire again.
+ expect(methods.advanceSchedule).toHaveBeenCalledTimes(1);
+ const [id, next, expectedFrom, token] = methods.advanceSchedule.mock.calls[0];
+ expect(id).toBe('sched-1');
+ expect(next).toBeInstanceOf(Date);
+ expect(next?.getTime()).toBeGreaterThan(Date.now());
+ // Fenced on the claimed occurrence and the claim token, so a re-claim or an
+ // owner edit is not clobbered.
+ expect(expectedFrom).toBe(schedule.nextRunAt);
+ expect(token).toBe('ct-1');
+ // Being overdue is not a fault: the schedule stays enabled and is not fired.
+ expect(methods.disableSchedule).not.toHaveBeenCalled();
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
+ it('releases its old holder when an owner edit fences the misfire advance', async () => {
+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt() });
+ const methods = makeMethods(schedule);
+ // The edit rotated claimToken/nextRunAt after this worker claimed but preserved
+ // its unique lease holder, so the token-fenced advance no longer matches.
+ methods.advanceSchedule.mockResolvedValueOnce(false);
+
+ await tickOnce(makeDeps(methods));
+
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
+ });
+
+ it('disables and clears a stale occurrence whose cadence is uncomputable', async () => {
+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt(), timezone: 'Not/AZone' });
+ const methods = makeMethods(schedule);
+ await tickOnce(makeDeps(methods));
+ expect(methods.disableSchedule).toHaveBeenCalledWith('sched-1', 'invalid_schedule', 'ct-1');
+ expect(methods.advanceSchedule).toHaveBeenCalledWith(
+ 'sched-1',
+ null,
+ schedule.nextRunAt,
+ 'ct-1',
+ );
+ });
+
+ it('leaves a stale occurrence alone when its disable failed', async () => {
+ const schedule = makeClaimedSchedule({ nextRunAt: staleAt(), timezone: 'Not/AZone' });
+ const methods = makeMethods(schedule);
+ methods.disableSchedule.mockRejectedValue(new Error('mongo unavailable') as never);
+ await tickOnce(makeDeps(methods));
+ // Clearing nextRunAt without the disable landing would leave `enabled: true`
+ // with no nextRunAt and no disabledReason: unclaimable and invisible.
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ });
+});
+
+describe('runTick error handling', () => {
+ it('retains the due occurrence when a preflight query throws', async () => {
+ const schedule = makeClaimedSchedule();
+ const methods = makeMethods(schedule);
+ await tickOnce(
+ makeDeps(methods, {
+ // A transient infrastructure failure on one of the preflight reads.
+ getUserContext: async () => {
+ throw new Error('mongo unavailable');
+ },
+ }),
+ );
+ // Advancing schedules the NEXT recurrence, so the due one would be discarded
+ // permanently with no ScheduleRun row and no evidence it was ever attempted.
+ // The lease is left to expire instead, which re-claims THIS occurrence.
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ // A transient read failure says nothing about the schedule's validity.
+ expect(methods.disableSchedule).not.toHaveBeenCalled();
+ });
+
+ it('disables and clears an uncomputable cadence whose in-fire disable threw', async () => {
+ // An unparseable timezone makes the next run uncomputable, so fireSchedule
+ // disables the schedule up front — and throws out to the tick when that write
+ // fails. Retrying is pointless here: the occurrence can never be computed, and
+ // leaving the lease would re-claim it forever.
+ const schedule = makeClaimedSchedule({ timezone: 'Not/AZone' });
+ const methods = makeMethods(schedule);
+ methods.disableSchedule
+ .mockRejectedValueOnce(new Error('mongo unavailable') as never)
+ .mockResolvedValueOnce(undefined as never);
+ await tickOnce(makeDeps(methods));
+ expect(methods.disableSchedule).toHaveBeenCalledTimes(2);
+ // null clears nextRunAt, so the disabled schedule is no longer due.
+ expect(methods.advanceSchedule).toHaveBeenCalledWith(
+ 'sched-1',
+ null,
+ schedule.nextRunAt,
+ 'ct-1',
+ );
+ });
+
+ it('does not clear nextRunAt when the disable itself keeps failing', async () => {
+ const schedule = makeClaimedSchedule({ timezone: 'Not/AZone' });
+ const methods = makeMethods(schedule);
+ methods.disableSchedule.mockRejectedValue(new Error('mongo unavailable') as never);
+ await tickOnce(makeDeps(methods));
+ // Advancing on a failed disable would leave `enabled: true` with no nextRunAt
+ // and no disabledReason: permanently unclaimable and invisible to its owner.
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ });
+});
+
+describe('reconciliation consults the durable trigger delivery', () => {
+ const YOUNG = () => new Date(Date.now() - 5 * 60_000); // past reconcile min, before orphan cutoff
+ const OLD = () => new Date(Date.now() - 60 * 60_000); // past the 30-minute orphan cutoff
+
+ function joblessRun(firedAt: Date, deliveryKey: string | undefined = 'dk1') {
+ return {
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ user: 'u1',
+ status: 'started',
+ conversationId: 'conv-1',
+ firedAt,
+ ...(deliveryKey != null ? { deliveryKey } : {}),
+ };
+ }
+
+ async function runReconcile(
+ run: ReturnType<typeof joblessRun>,
+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'],
+ ) {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([run]);
+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null, getTriggerDelivery }));
+ return methods;
+ }
+
+ it('settles a DEAD delivery as error promptly, before the 30-minute orphan age', async () => {
+ const methods = await runReconcile(joblessRun(YOUNG()), async () => ({
+ status: 'dead',
+ // The durable record's lastError is an AgentTriggerDeliveryFailure OBJECT, not a
+ // string — the recorded outcome must carry its `message`, or the String-typed
+ // Mongoose field rejects the cast and the run keeps its capacity slot.
+ lastError: {
+ code: 'rate_limited',
+ message: 'rate limited',
+ certainty: 'definite' as const,
+ retryable: false,
+ attemptedAt: new Date(),
+ },
+ }));
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'error', error: 'rate limited' }),
+ );
+ });
+
+ it('does NOT orphan a PENDING delivery past the cutoff (Retry-After may still fire it)', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'pending' }));
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('does NOT orphan a LEASED delivery past the cutoff', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'leased' }));
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('does NOT orphan a STAGING delivery past the cutoff', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'staging' }));
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('falls back to the legacy interrupted orphan for a SUCCEEDED delivery past the cutoff', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => ({ status: 'succeeded' }));
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
+ );
+ });
+
+ it('does not prematurely orphan a SUCCEEDED delivery before the cutoff', async () => {
+ const methods = await runReconcile(joblessRun(YOUNG()), async () => ({ status: 'succeeded' }));
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('uses the legacy orphan policy when the delivery record is gone (null)', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => null);
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'interrupted' }),
+ );
+ });
+
+ it('DEFERS (stays reconcilable) when the delivery lookup fails', async () => {
+ const methods = await runReconcile(joblessRun(OLD()), async () => {
+ throw new Error('delivery store unavailable');
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+});
+
+describe('reconciliation is isolated per row', () => {
+ /**
+ * getRunsForReconciliation returns the OLDEST rows first, so a row that always throws
+ * came back every tick and starved every run behind it — in the one component whose
+ * entire job is settling states nothing else will.
+ */
+ it('keeps reconciling after a row throws', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ const rows = [
+ { scheduleId: 'poison', scheduledFor: new Date(0), user: 'u1', status: 'started' },
+ { scheduleId: 'healthy', scheduledFor: new Date(0), user: 'u1', status: 'started' },
+ ];
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue(rows);
+ (methods.recordRunOutcome as jest.Mock).mockImplementation(async (params) => {
+ if (params.scheduleId === 'poison') {
+ throw new Error('permanently broken row');
+ }
+ });
+
+ await tickOnce(
+ makeDeps(methods, {
+ // Both rows are orphans well past the cutoff, so both reach recordRunOutcome.
+ getJobStatus: async () => null,
+ }),
+ );
+
+ const settled = (methods.recordRunOutcome as jest.Mock).mock.calls.map(
+ ([params]) => params.scheduleId,
+ );
+ expect(settled).toContain('poison');
+ expect(settled).toContain('healthy');
+ });
+});
+
+describe('unarmed recovery sweep', () => {
+ it('disables an unarmed schedule whose cadence is uncomputable', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ methods.claimDueSchedule.mockResolvedValue(null);
+ const broken = makeClaimedSchedule({
+ id: 'sched-broken',
+ timezone: 'Not/AZone',
+ claimToken: 'ct-9',
+ configRevision: 4,
+ nextRunAt: undefined,
+ });
+ const healthy = makeClaimedSchedule({
+ id: 'sched-ok',
+ claimToken: 'ct-10',
+ nextRunAt: undefined,
+ });
+ (methods.getUnarmedSchedules as jest.Mock).mockResolvedValue([broken, healthy]);
+
+ await reconcileOnce(makeDeps(methods));
+
+ // Skipping the broken row left it enabled-but-unarmed forever: holding the
+ // owner's slot, never firing, and re-filling the bounded window ahead of
+ // valid crash-left rows. Fenced on token + revision so a concurrent edit
+ // that repairs the cadence wins over this observation of the broken one.
+ expect(methods.disableSchedule).toHaveBeenCalledWith(
+ 'sched-broken',
+ 'invalid_schedule',
+ 'ct-9',
+ 4,
+ );
+ expect(methods.armSchedule).toHaveBeenCalledWith('sched-ok', expect.any(Date));
+ });
+});
+
+describe('bookkeeping replay rotation', () => {
+ it('stamps every replayed row so persistent failures rotate out of the window', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ methods.claimDueSchedule.mockResolvedValue(null);
+ const poison = {
+ _id: 'row-1',
+ scheduleId: 'poison',
+ scheduledFor: new Date(0),
+ user: 'u1',
+ status: 'success',
+ };
+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([poison]);
+ (methods.finalizeBookkeeping as jest.Mock).mockRejectedValue(new Error('permanently broken'));
+
+ await reconcileOnce(makeDeps(methods));
+
+ // getUnbookkeptRuns reads least-recently-attempted first; without the stamp a
+ // batch of failing rows re-fills the bounded window and every later terminal
+ // row's counters never land.
+ expect(methods.markRunsReconciled).toHaveBeenCalledWith([poison]);
+ });
+
+ /**
+ * A terminal run reaches this pass only because its OWNER crashed before bookkeeping —
+ * which is also before it could release the job it retained for exactly this recovery.
+ * A preserved job is kept WITHOUT `completedAt` so the store's finished-job sweep cannot
+ * reap it early, so if this pass does not clear it, nothing ever does.
+ */
+ it('releases the retained job once replayed bookkeeping is durable', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ methods.claimDueSchedule.mockResolvedValue(null);
+ const unbookkept = {
+ _id: 'row-2',
+ scheduleId: 'sched-crashed',
+ scheduledFor: new Date(0),
+ user: 'u1',
+ status: 'success',
+ conversationId: 'convo-crashed',
+ };
+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([unbookkept]);
+ const clearReconciledJob = jest.fn(async () => undefined);
+
+ await reconcileOnce(makeDeps(methods, { clearReconciledJob }));
+
+ expect(methods.finalizeBookkeeping).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 'sched-crashed' }),
+ );
+ expect(clearReconciledJob).toHaveBeenCalledWith('convo-crashed', {
+ scheduleId: 'sched-crashed',
+ scheduledFor: new Date(0),
+ });
+ });
+
+ /** The retained job is the only surviving evidence when bookkeeping fails, so it must
+ * outlive a failed replay rather than be cleared alongside it. */
+ it('keeps the retained job when the bookkeeping replay itself fails', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ methods.claimDueSchedule.mockResolvedValue(null);
+ (methods.getUnbookkeptRuns as jest.Mock).mockResolvedValue([
+ {
+ _id: 'row-3',
+ scheduleId: 'sched-broken-bk',
+ scheduledFor: new Date(0),
+ user: 'u1',
+ status: 'success',
+ conversationId: 'convo-broken',
+ },
+ ]);
+ (methods.finalizeBookkeeping as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const clearReconciledJob = jest.fn(async () => undefined);
+
+ await reconcileOnce(makeDeps(methods, { clearReconciledJob }));
+
+ expect(clearReconciledJob).not.toHaveBeenCalled();
+ });
+});
+
+describe('reconciliation abort fence', () => {
+ const scheduledFor = new Date(0);
+ const abortedRun = (abortAgeMs: number, status = 'started', jobGone = false) => ({
+ scheduleId: 'sched-1',
+ scheduledFor,
+ user: 'u1',
+ status,
+ conversationId: jobGone ? 'c1' : 'c1',
+ firedAt: new Date(Date.now() - 60 * 60_000),
+ abortRequestedAt: new Date(Date.now() - abortAgeMs),
+ });
+ const abortedJob = async () => ({
+ status: 'aborted',
+ scheduleId: 'sched-1',
+ scheduledFor: scheduledFor.toISOString(),
+ });
+
+ /**
+ * A job reads `aborted` the moment abortJob wins its status CAS — BEFORE the
+ * generation owner has unwound and persisted (partial response, user message).
+ * Settling here releases the run mid-persistence: an account-deletion drain then
+ * observes zero active runs and destroys data a pending save recreates. The owner's
+ * own outcome write is the only settlement while its abort is in flight.
+ */
+ it('defers an aborted job while its abort is in flight', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(60_000)]);
+ const clearReconciledJob = jest.fn(async () => undefined);
+ await tickOnce(makeDeps(methods, { getJobStatus: abortedJob, clearReconciledJob }));
+
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ expect(clearReconciledJob).not.toHaveBeenCalled();
+ });
+
+ it('finalizes an aborted job once its owner is presumed dead', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);
+ const clearReconciledJob = jest.fn(async () => undefined);
+ await tickOnce(makeDeps(methods, { getJobStatus: abortedJob, clearReconciledJob }));
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 'sched-1', status: 'interrupted' }),
+ );
+ expect(clearReconciledJob).toHaveBeenCalled();
+ });
+
+ it('omits the conversation link when the aborted job never started', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);
+ const preStartAbortedJob = async () => ({
+ status: 'aborted',
+ scheduleId: 'sched-1',
+ scheduledFor: scheduledFor.toISOString(),
+ createdEventEmitted: false,
+ });
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: preStartAbortedJob,
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ // A pre-start abort reserved an id but no conversation ever came to exist;
+ // projecting it hands the card a link to a missing chat, and the row's
+ // reserved id is erased so the crash-retry replay cannot restore it either.
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ status: 'interrupted',
+ conversationId: undefined,
+ clearConversationId: true,
+ }),
+ );
+ });
+
+ /** Account-deletion quiesce DELETES the aborted job, so post-abort absence carries
+ * the same fence: the orphan branch must not settle a run whose owner is still
+ * unwinding its persistence. */
+ it('defers a vanished job while its abort is in flight', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(60_000)]);
+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null }));
+
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('still reaps a vanished job once the abort fence lapses', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([abortedRun(31 * 60_000)]);
+ await tickOnce(makeDeps(methods, { getJobStatus: async () => null }));
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'interrupted' }),
+ );
+ });
+});
+
+describe('reconciliation resume hand-off fence', () => {
+ const scheduledFor = new Date(0);
+ const resumedRun = (claimAgeMs: number) => ({
+ scheduleId: 'sched-1',
+ scheduledFor,
+ user: 'u1',
+ status: 'started',
+ conversationId: 'c1',
+ firedAt: new Date(Date.now() - 60 * 60_000),
+ resumeClaimedAt: new Date(Date.now() - claimAgeMs),
+ });
+ const pausedJob = async () => ({
+ status: 'requires_action',
+ scheduleId: 'sched-1',
+ scheduledFor: scheduledFor.toISOString(),
+ });
+
+ it('does not release freshly reacquired capacity while approval claiming is in flight', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(60_000)]);
+
+ await reconcileOnce(makeDeps(methods, { getJobStatus: pausedJob }));
+
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('does not treat a temporarily missing hand-off job as an orphan', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(60_000)]);
+
+ await reconcileOnce(makeDeps(methods, { getJobStatus: async () => null }));
+
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('re-pauses a crashed resume after the hand-off fence expires', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([resumedRun(11 * 60_000)]);
+
+ await reconcileOnce(makeDeps(methods, { getJobStatus: pausedJob }));
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 'sched-1', status: 'requires_action' }),
+ );
+ });
+});
+
+describe('reconciliation preserves the intended outcome', () => {
+ const scheduledFor = new Date(0);
+ /** A run whose inline outcome write never landed: old enough to reconcile, still
+ * `started`, with the owner's retained terminal job as the only evidence. */
+ const unsettledRun = () => ({
+ scheduleId: 'sched-1',
+ scheduledFor,
+ user: 'u1',
+ status: 'started',
+ conversationId: 'c1',
+ firedAt: new Date(Date.now() - 60 * 60_000),
+ });
+ const retainedComplete =
+ (extra: Record<string, string> = {}) =>
+ async () => ({
+ status: 'complete',
+ scheduleId: 'sched-1',
+ scheduledFor: scheduledFor.toISOString(),
+ ...extra,
+ });
+
+ /**
+ * A terminal `complete` is generic: it covers a clean finish, a mid-run balance
+ * refusal, and a provider failure the client swallowed into an error part. Deriving
+ * `success` from it turned a transient outcome-write failure into a reset of the very
+ * streaks that drive insufficient_balance and too_many_failures auto-disable.
+ */
+ it('recovers a balance refusal as skipped_balance, not success', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ const clearReconciledJob = jest.fn(async () => undefined);
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: retainedComplete({ scheduleOutcome: 'skipped_balance' }),
+ clearReconciledJob,
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 'sched-1', status: 'skipped_balance' }),
+ );
+ expect(clearReconciledJob).toHaveBeenCalled();
+ });
+
+ it('preserves an interrupted owner outcome instead of converting it to success', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: retainedComplete({
+ scheduleOutcome: 'interrupted',
+ scheduleOutcomeError: 'Stopped by owner',
+ }),
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'interrupted', error: 'Stopped by owner' }),
+ );
+ });
+
+ it('recovers a balance refusal that claimed an ERROR terminal as skipped_balance', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ await tickOnce(
+ makeDeps(methods, {
+ // A mid-continuation balance refusal finalizes through completeJob('error'),
+ // so the terminal is `error` — but the stamp still routes it to the
+ // insufficient_balance streak instead of too_many_failures.
+ getJobStatus: async () => ({
+ status: 'error',
+ scheduleId: 'sched-1',
+ scheduledFor: scheduledFor.toISOString(),
+ scheduleOutcome: 'skipped_balance',
+ }),
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'skipped_balance' }),
+ );
+ });
+
+ it('recovers a swallowed generation failure as error, with the owner’s message', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: retainedComplete({
+ scheduleOutcome: 'error',
+ scheduleOutcomeError: 'upstream 503',
+ }),
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', error: 'upstream 503' }),
+ );
+ });
+
+ it('still records success when the owner left no stamp', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: retainedComplete(),
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'success' }),
+ );
+ });
+
+ // The stamp crosses a serialization boundary, and recordRunOutcome would reject a
+ // status outside its union: degrade to success rather than fail the recovery write.
+ it('degrades an unrecognized stamp to success', async () => {
+ const methods = makeMethods(makeClaimedSchedule());
+ (methods.getRunsForReconciliation as jest.Mock).mockResolvedValue([unsettledRun()]);
+ await tickOnce(
+ makeDeps(methods, {
+ getJobStatus: retainedComplete({ scheduleOutcome: 'not_a_status' }),
+ clearReconciledJob: jest.fn(async () => undefined),
+ }),
+ );
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'success' }),
+ );
+ });
+});
diff --git a/packages/api/src/schedules/engine.ts b/packages/api/src/schedules/engine.ts
new file mode 100644
index 0000000000..92a884447a
--- /dev/null
+++ b/packages/api/src/schedules/engine.ts
@@ -0,0 +1,630 @@
+import { logger, runAsSystem } from '@librechat/data-schemas';
+import type { IScheduleRun } from '@librechat/data-schemas';
+import type { ScheduleEngineDeps, JobState } from './types';
+import { hasAbortInFlight, hasResumeHandoffInFlight, retainedOutcome } from './types';
+import { isShutdownInProgress, registerShutdownTask } from '~/app/shutdown';
+import { fireSchedule, BALANCE_SKIP_DISABLE_THRESHOLD } from './fire';
+import { computeNextRunAt } from './cadence';
+
+const TICK_MS = 30_000;
+const TICK_JITTER_MS = 2_000;
+const LEASE_MS = 5 * 60_000;
+const RECONCILE_MIN_RUN_AGE_MS = 2 * 60_000;
+const ORPHAN_RUN_AGE_MS = 30 * 60_000;
+const ABANDONED_PAUSE_AGE_MS = 25 * 60 * 60_000;
+const RECONCILE_BATCH = 100;
+// Occurrences due more than this long ago (server downtime / paused engine) are
+// skipped forward instead of fired, so a restart doesn't burst stale chats.
+const MISFIRE_GRACE_MS = 15 * 60_000;
+
+/**
+ * Whether the job currently at a run's conversationId is THIS occurrence's
+ * generation. A replacement turn reuses the conversationId but strips the
+ * scheduleId/scheduledFor metadata, so an identity mismatch means the original
+ * job is gone and its status must not be attributed to (or its hash deleted for)
+ * this scheduled run.
+ */
+function jobIdentityMatches(jobState: JobState | null, run: IScheduleRun): boolean {
+ if (jobState == null || jobState.scheduleId !== run.scheduleId || jobState.scheduledFor == null) {
+ return false;
+ }
+ const jobFor = new Date(jobState.scheduledFor).getTime();
+ return jobFor === run.scheduledFor.getTime();
+}
+
+export type ScheduleEngine = {
+ stop: () => void;
+ /** Exposed for tests and the run-now handler: one full claim/fire pass. */
+ runTick: () => Promise<number>;
+ /** Exposed for tests: one awaitable reconciliation pass (the tick loop's is fire-and-forget). */
+ reconcile: () => Promise<void>;
+};
+
+export function startScheduleEngine(deps: ScheduleEngineDeps): ScheduleEngine {
+ let stopped = false;
+ let timer: NodeJS.Timeout | undefined;
+ let ticks = 0;
+ const instanceId = `${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
+
+ /**
+ * Job-store-aware run reconciliation: pauses (`requires_action`) surface on
+ * the run doc so overlap-skip ignores them; crashed runs become
+ * `interrupted`; runs resumed elsewhere become `success` when the job store
+ * still shows completion. Long-running generations are left alone.
+ */
+ async function reconcile() {
+ try {
+ const limits = await deps.getLimits();
+ const runs = await runAsSystem(() =>
+ deps.methods.getRunsForReconciliation(
+ new Date(Date.now() - RECONCILE_MIN_RUN_AGE_MS),
+ RECONCILE_BATCH,
+ ),
+ );
+ await runAsSystem(async () => {
+ for (const run of runs) {
+ // PER-ROW isolation. A single throwing row used to abort the whole pass, and
+ // since this query returns the OLDEST rows first, that row came back every
+ // tick and starved every run behind it indefinitely. Reconciliation is the
+ // backstop for exactly the states nothing else settles, so it has to make
+ // progress on the rest.
+ try {
+ // Identity-fence the job lookup: a replacement user turn reuses this
+ // conversationId but sheds the scheduleId/scheduledFor metadata. Only
+ // trust the job's status when it still carries THIS occurrence's identity;
+ // otherwise treat the job as gone (null) so a replacement generation's
+ // status can never finalize — or its hash be deleted for — this run.
+ const jobState = run.conversationId
+ ? await deps.getJobStatus(run.conversationId)
+ : null;
+ const jobStatus = jobIdentityMatches(jobState, run) ? jobState!.status : null;
+ const ageMs = Date.now() - (run.firedAt?.getTime() ?? 0);
+ // Resolve the run owner's limits so crash-reconciled auto-disable uses
+ // the same per-principal threshold as an inline completion. Must run in
+ // the OWNER's tenant context: getLimits resolves config via the ALS
+ // tenant, and this loop is under runAsSystem (system tenant).
+ const owner = await deps.getUserContext(run.user);
+ const runLimits = owner
+ ? await deps.runInTenantContext(owner, () => deps.getLimits(owner))
+ : limits;
+ // All transitions go through recordRunOutcome so the schedule's lastRun
+ // (and the card's status chip) tracks the run, including the pause.
+ const finalize = (
+ status: 'success' | 'interrupted' | 'error' | 'requires_action' | 'skipped_balance',
+ error?: string,
+ opts?: { omitConversationId?: boolean },
+ ) =>
+ deps.methods.recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status,
+ // Pre-start aborts have a reserved id but no conversation was ever
+ // created; projecting it gives the card a link to a missing chat.
+ conversationId: opts?.omitConversationId ? undefined : run.conversationId,
+ clearConversationId: opts?.omitConversationId,
+ error,
+ autoDisableAfterFailures: runLimits.autoDisableAfterFailures,
+ });
+ // The clear runs AFTER finalize (the retained job is the only evidence if
+ // the finalize write fails), which means a clear that keeps failing has no
+ // natural retry: the now-terminal run never rescans, so nothing else would
+ // reap the retained job. Retry the transient case inline; the store's own
+ // aged-retained-job backstop bounds anything more persistent.
+ const clearRetainedJob = async () => {
+ for (let attempt = 1; attempt <= 2; attempt++) {
+ try {
+ await deps.clearReconciledJob(run.conversationId as string, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ });
+ return;
+ } catch (clearError) {
+ logger.warn(
+ `[schedules] failed to clear retained job for ${run.scheduleId} (attempt ${attempt}/2):`,
+ clearError,
+ );
+ }
+ }
+ };
+ if (jobStatus === 'running') {
+ continue;
+ }
+ // Surface a pause on the card (lastRun → requires_action). Also re-invoked for
+ // a row ALREADY `requires_action`: recordRunOutcome flips the row before
+ // projecting the card, so a crash between the two leaves the pause invisible
+ // until this replays it. Both writes are idempotent.
+ if (
+ (run.status === 'started' || run.status === 'requires_action') &&
+ jobStatus === 'requires_action'
+ ) {
+ // A resumed run claims capacity in Mongo BEFORE the approval CAS flips
+ // the job back to `running`. During that short hand-off, the old paused
+ // job is expected and must not re-pause the run (which would release the
+ // newly claimed capacity slot while the continuation is about to run).
+ // A stale stamp still falls through so a crashed resume can recover.
+ if (run.status === 'started' && hasResumeHandoffInFlight(run, Date.now())) {
+ continue;
+ }
+ await finalize('requires_action');
+ continue;
+ }
+ // A retained terminal job whose inline outcome hook failed transiently —
+ // finalize the run from the retained status, then delete the job. The
+ // delete is the cleanup path for `preserveForReconcile` jobs (kept
+ // without `completedAt`, so the store's finished-job sweep never reaps
+ // them); `conversationId` is guaranteed here since jobStatus was fetched.
+ if (jobStatus === 'complete') {
+ // Finalize either a paused OR a still-started run so it stops consuming
+ // capacity / blocking overlap — but from the OWNER'S intended outcome when
+ // it left one. A terminal `complete` covers a clean finish, a mid-run
+ // balance refusal, and a provider failure the client swallowed into an
+ // error part alike, so re-deriving `success` here turned a transient
+ // outcome-write failure into a reset of the very streaks that drive
+ // insufficient_balance and too_many_failures auto-disable.
+ const intended = retainedOutcome(jobState, 'success');
+ await finalize(intended.status, intended.error);
+ await clearRetainedJob();
+ continue;
+ }
+ if (jobStatus === 'error') {
+ // Honor the stamp here too: a mid-continuation balance refusal claims an
+ // `error` terminal but must walk the insufficient_balance streak.
+ const intended = retainedOutcome(jobState, 'error');
+ await finalize(intended.status, intended.error);
+ await clearRetainedJob();
+ continue;
+ }
+ if (jobStatus === 'aborted') {
+ // ABORT FENCE, same rule as the deletion drains: a job flips `aborted`
+ // the moment abortJob wins its status CAS, BEFORE the generation owner
+ // has unwound and persisted (partial response, user message). Settling
+ // here would release the run mid-persistence — the exact drain-mid-write
+ // hazard the deletion paths defer on — so while the abort is in flight
+ // the owner's own outcome write is the only settlement. Past the window
+ // the owner is presumed dead and this backstop finalizes the run.
+ if (hasAbortInFlight(run, Date.now())) {
+ continue;
+ }
+ await finalize('interrupted', undefined, {
+ omitConversationId: jobState?.createdEventEmitted !== true,
+ });
+ await clearRetainedJob();
+ continue;
+ }
+ // A `started` run whose job is gone (jobStatus null). Every run records its
+ // conversationId up front, so getJobStatus above already liveness-checked it: a
+ // live long-running fire reads as `running` and is left alone. A fresh abort
+ // also DELETES the job (account-deletion quiesce), so absence carries the same
+ // abort fence as `aborted` above.
+ //
+ // But a jobless run is NOT automatically an orphan: fireSchedule reserves the
+ // run and its capacity BEFORE the durable trigger delivery reaches the chat
+ // route, and that delivery can be deferred (a `Retry-After` up to 24h, far past
+ // the 30-minute cutoff) or dead-lettered by a pre-generation rejection (an
+ // interactive limiter, PII, or moderation). Consult the durable delivery keyed
+ // by the reservation before deciding.
+ if (
+ run.status === 'started' &&
+ jobStatus == null &&
+ !hasAbortInFlight(run, Date.now()) &&
+ !hasResumeHandoffInFlight(run, Date.now())
+ ) {
+ let delivery: Awaited<ReturnType<typeof deps.getTriggerDelivery>> | null = null;
+ try {
+ delivery = run.deliveryKey ? await deps.getTriggerDelivery(run.deliveryKey) : null;
+ } catch (deliveryError) {
+ // Unknown is not gone: defer rather than orphan a possibly-live delivery.
+ logger.warn(
+ `[schedules] delivery lookup failed for ${run.scheduleId}@${run.scheduledFor?.toISOString?.() ?? run.scheduledFor}; deferring:`,
+ deliveryError,
+ );
+ continue;
+ }
+ const deliveryStatus = delivery?.status;
+ if (
+ deliveryStatus === 'staging' ||
+ deliveryStatus === 'pending' ||
+ deliveryStatus === 'leased'
+ ) {
+ // Admission is still live; the delivery will fire or dead-letter later.
+ continue;
+ }
+ if (deliveryStatus === 'dead') {
+ // Definitively failed before a generation ever existed. Record the error
+ // PROMPTLY from the durable lastError and release capacity through the
+ // ordinary outcome path — no need to wait the 30-minute orphan age.
+ await finalize(
+ 'error',
+ delivery?.lastError?.message ?? 'Scheduled delivery failed before running',
+ );
+ continue;
+ }
+ // `succeeded` (a generation ran; its own job/outcome evidence is the
+ // authority) or NO delivery record (a legacy fire): only a genuinely aged
+ // jobless run is an orphan.
+ if (ageMs > ORPHAN_RUN_AGE_MS) {
+ await finalize('interrupted');
+ }
+ continue;
+ }
+ if (
+ run.status === 'requires_action' &&
+ jobStatus == null &&
+ ageMs > ABANDONED_PAUSE_AGE_MS &&
+ !hasAbortInFlight(run, Date.now())
+ ) {
+ await finalize('interrupted');
+ }
+ } catch (rowError) {
+ logger.error(
+ `[schedules] reconciliation failed for run ${run.scheduleId}@${run.scheduledFor?.toISOString?.() ?? run.scheduledFor}:`,
+ rowError,
+ );
+ }
+ }
+ // Stamp EVERY row this pass looked at, including the ones that threw — a row
+ // that keeps failing must still rotate to the back, or it re-fills the window
+ // and starves the rest, which is the same starvation the ordering exists to
+ // prevent (and the per-row try/catch above already exists because of).
+ await deps.methods
+ .markRunsReconciled(runs)
+ .catch((err) => logger.warn('[schedules] failed to stamp reconciled runs:', err));
+ });
+
+ // Catch terminal runs whose schedule bookkeeping never landed (a crash
+ // between the run-row terminalization and the schedule counter update).
+ const unbookkept = await runAsSystem(() =>
+ deps.methods.getUnbookkeptRuns(
+ new Date(Date.now() - RECONCILE_MIN_RUN_AGE_MS),
+ RECONCILE_BATCH,
+ ),
+ );
+ await runAsSystem(async () => {
+ for (const run of unbookkept) {
+ try {
+ const owner = await deps.getUserContext(run.user);
+ const runLimits = owner
+ ? await deps.runInTenantContext(owner, () => deps.getLimits(owner))
+ : limits;
+ await deps.methods.finalizeBookkeeping({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: run.status as
+ | 'success'
+ | 'error'
+ | 'interrupted'
+ | 'skipped_balance'
+ | 'skipped_overlap',
+ conversationId: run.conversationId,
+ error: run.error,
+ autoDisableAfterFailures: runLimits.autoDisableAfterFailures,
+ balanceSkipDisableThreshold: BALANCE_SKIP_DISABLE_THRESHOLD,
+ });
+ // The run is terminal but its OWNER crashed before bookkeeping, which is also
+ // before it could release the job it retained for exactly this recovery. The
+ // active-run pass clears its own; this one never did, and a preserved job is
+ // kept WITHOUT `completedAt` precisely so the store's finished-job sweep
+ // cannot reap it early — so nothing else would ever have. Identity-guarded,
+ // and a no-op when no retained job is there.
+ if (run.conversationId) {
+ await deps
+ .clearReconciledJob(run.conversationId, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ })
+ .catch((clearError) =>
+ logger.warn(
+ `[schedules] failed to clear retained job after bookkeeping replay for ${run.scheduleId}:`,
+ clearError,
+ ),
+ );
+ }
+ } catch (rowError) {
+ logger.error(
+ `[schedules] bookkeeping replay failed for run ${run.scheduleId}:`,
+ rowError,
+ );
+ }
+ }
+ // Same rotation as the active-run pass above: stamp every row this pass
+ // looked at so persistent failures rotate to the back of the bounded
+ // window instead of starving the rows behind them. Successful replays
+ // left the query via bookkept:true; re-stamping them is harmless.
+ await deps.methods
+ .markRunsReconciled(unbookkept)
+ .catch((err) => logger.warn('[schedules] failed to stamp replayed runs:', err));
+ });
+
+ // Erase soft-deleted schedules once their active runs have drained. Delete
+ // disabled + marked them `deleting` and aborted in-flight jobs; the run
+ // reconciliation above finalizes those runs, so here we erase only the
+ // schedules with no run still active — their evidence is preserved until
+ // settled. eraseScheduleIfDrained is idempotent, so concurrent workers are safe.
+ await runAsSystem(async () => {
+ const deleting = await deps.methods.getDeletingSchedules(RECONCILE_BATCH);
+ for (const schedule of deleting) {
+ await deps.methods.eraseScheduleIfDrained(schedule.id).catch(() => undefined);
+ }
+ // Rotate the window (never-attempted first) so a batch of undrainable rows
+ // cannot re-fill it every pass and starve the rows behind them.
+ await deps.methods
+ .markEraseAttempted(deleting.map((schedule) => schedule.id))
+ .catch((err) => logger.warn('[schedules] failed to stamp erase attempts:', err));
+ });
+
+ // Re-arm schedules that are enabled but carry no nextRunAt. Creation arms in a
+ // second write, so a crash or a failed arm leaves a row that LOOKS enabled to its
+ // owner and occupies a slot while claimDueSchedule (which sorts on nextRunAt) can
+ // never select it. Recovering here means the owner does not have to notice and
+ // edit it; armSchedule is conditional on still being unarmed, so it cannot
+ // disturb one that armed itself meanwhile.
+ await runAsSystem(async () => {
+ const unarmed = await deps.methods.getUnarmedSchedules(RECONCILE_BATCH);
+ for (const schedule of unarmed) {
+ const nextRunAt = computeNextRunAt({
+ cadence: schedule.cadence,
+ timezone: schedule.timezone,
+ scheduleId: schedule.id,
+ });
+ if (nextRunAt == null) {
+ // Uncomputable cadence is NOT transient (same policy as the claim path):
+ // skipping left the row enabled-but-unarmed forever — holding the owner's
+ // slot while never firing, and re-filling this bounded window until valid
+ // crash-left rows behind it could never be recovered. Fenced on both the
+ // claim token and revision so a concurrent edit that repairs the cadence
+ // wins over this observation of the broken one.
+ await deps.methods
+ .disableSchedule(
+ schedule.id,
+ 'invalid_schedule',
+ schedule.claimToken,
+ schedule.configRevision,
+ )
+ .catch((err) => {
+ logger.warn(`[schedules] failed to disable unarmed ${schedule.id}:`, err);
+ });
+ continue;
+ }
+ await deps.methods.armSchedule(schedule.id, nextRunAt).catch((err) => {
+ logger.warn(`[schedules] failed to re-arm ${schedule.id}:`, err);
+ });
+ }
+ });
+ } catch (error) {
+ logger.error('[schedules] run reconciliation failed:', error);
+ }
+ }
+
+ /**
+ * One claim/fire pass. Claims and cross-tenant reconciliation run in a system
+ * tenant context (the claim scans all tenants; strict tenant isolation would
+ * otherwise throw on the unscoped query). The fire itself re-enters the
+ * owner's context via `runInTenantContext`.
+ */
+ async function runTick(): Promise<number> {
+ // GLOBAL kill switch: stop claiming entirely. This is the operator's hard stop
+ // (SCHEDULES_DISABLED, or `interface.schedules: false` in the BASE config), which
+ // no principal override can widen — distinct from per-principal availability below.
+ // Deliberately gates CLAIMS only; reconcile() is never gated, because in-flight
+ // runs must still be settled while the feature is off or they strand `started`
+ // rows and leak capacity forever.
+ if (await deps.isGloballyDisabled()) {
+ return 0;
+ }
+ // Do NOT gate claims on the per-principal `enabled`: schedules can be enabled per
+ // user/role/tenant, so gating here would silently never fire those users'
+ // occurrences. The fire path re-resolves the OWNER's limits and skips ('disabled')
+ // any occurrence whose owner has the feature off, so an owner-scoped disable is
+ // still honored. The base config only supplies the per-tick claim budget.
+ const limits = await deps.getLimits();
+ let fired = 0;
+ // Cap on ACTIVE scheduled runs, not just per-tick starts: the loopback chat
+ // endpoint returns as soon as the generation starts and scheduled fires
+ // bypass the interactive limiter, so without this the in-flight count would
+ // grow by fireConcurrency every tick. Only claim up to the free headroom.
+ const active = await runAsSystem(() => deps.methods.countActiveRuns());
+ const budget = Math.max(0, limits.fireConcurrency - active);
+ for (let i = 0; i < budget; i++) {
+ // Claim + the fire's pre-owner-context bookkeeping (disable/advance,
+ // cross-tenant reads) run as system; fireSchedule re-enters owner context
+ // internally for the run-specific work.
+ const done = await runAsSystem(async () => {
+ const schedule = await deps.methods.claimDueSchedule({ instanceId, leaseMs: LEASE_MS });
+ if (schedule == null) {
+ return true;
+ }
+ const scheduledFor = schedule.nextRunAt ?? new Date();
+ /**
+ * Advances this claimed occurrence without preserving a superseded holder.
+ * Owner edits rotate the token/next occurrence but intentionally leave the old
+ * lease in place; a fenced miss therefore releases only this worker's unique
+ * holder. Infrastructure errors keep the lease as retry backoff, as before.
+ */
+ const advanceClaim = async (nextRunAt: Date | null): Promise<void> => {
+ try {
+ const advanced = await deps.methods.advanceSchedule(
+ schedule.id,
+ nextRunAt,
+ scheduledFor,
+ schedule.claimToken,
+ );
+ if (!advanced && schedule.leaseBy != null) {
+ await deps.methods.releaseLeaseByHolder(schedule.id, schedule.leaseBy);
+ }
+ } catch {
+ // Leave the claim lease as bounded backoff for transient storage failures.
+ }
+ };
+ // Use the CLAIM's clock for the misfire cutoff: the claim wrote
+ // leaseUntil = now + LEASE_MS from the claiming worker's clock, so
+ // leaseUntil - LEASE_MS is that worker's "now" at claim — usually this very
+ // process, making cutoff and claim self-consistent. (DocumentDB rules out the
+ // server-clock `$$NOW` CAS; skew between REPLICAS shifts fire timing by at
+ // most the skew and can never double-fire — the lease CAS and the unique
+ // occurrence index arbitrate that regardless of clocks.)
+ const dbNow = schedule.leaseUntil ? schedule.leaseUntil.getTime() - LEASE_MS : Date.now();
+ // Misfire skip-forward: an occurrence overdue past the grace window (the
+ // engine was down/paused) is advanced to the next FUTURE occurrence
+ // without firing, so a restart doesn't launch stale or bursty chats.
+ if (dbNow - scheduledFor.getTime() > MISFIRE_GRACE_MS) {
+ const next = computeNextRunAt({
+ cadence: schedule.cadence,
+ timezone: schedule.timezone,
+ scheduleId: schedule.id,
+ // The claim's own clock (see dbNow above): a clock-behind worker would
+ // otherwise compute another already-due occurrence and reclaim the row.
+ after: new Date(dbNow),
+ });
+ if (next == null) {
+ // Uncomputable cadence is NOT transient, so this occurrence can never run.
+ // The advance below would clear nextRunAt AND the lease; doing that after a
+ // transiently failed disable would leave `enabled: true` with no nextRunAt
+ // and no disabledReason, permanently unclaimable and invisible. Bail
+ // instead: the lease expires and the occurrence is retried.
+ const disabled = await deps.methods
+ .disableSchedule(schedule.id, 'invalid_schedule', schedule.claimToken)
+ .then(() => true)
+ .catch(() => false);
+ if (!disabled) {
+ return false;
+ }
+ }
+ // The SKIP-FORWARD itself, and the whole point of this branch: move to the
+ // next FUTURE occurrence. Without it nextRunAt keeps pointing at the stale
+ // one, so every later tick reclaims and skips the same occurrence forever and
+ // a schedule overdue past an outage never fires again.
+ await advanceClaim(next);
+ logger.info(`[schedules] skipped stale occurrence for ${schedule.id} (misfire grace)`);
+ return false;
+ }
+ try {
+ const result = await fireSchedule(
+ // The dispatch boundary observes shutdown from BOTH signals: the
+ // coordinator flag flips before the listener starts closing (ahead of
+ // any pre-drain task ordering), and the engine's own stop covers direct
+ // runTick callers outside a coordinated shutdown.
+ { ...deps, isShuttingDown: () => stopped || isShutdownInProgress() },
+ schedule,
+ limits,
+ scheduledFor,
+ {
+ dbNow: new Date(dbNow),
+ },
+ );
+ if (result.fired) {
+ fired += 1;
+ }
+ } catch (error) {
+ // A transient preflight throw (user/config/permission/balance/capacity query)
+ // must NOT advance past this occurrence: advancing schedules the NEXT
+ // recurrence, so the due one is discarded permanently with no ScheduleRun row
+ // and no evidence it was ever attempted. Leave nextRunAt alone and keep the
+ // claim lease as backoff — the lease expires and the SAME occurrence is
+ // re-claimed and retried, matching how the fire path already treats a
+ // transient file-resolution failure.
+ logger.error(`[schedules] unexpected fire error for ${schedule.id} (will retry):`, error);
+ // An UNCOMPUTABLE cadence is the one non-transient case: retrying can never
+ // make progress, and leaving the lease would re-claim the same occurrence
+ // forever. Disable so it stops being due, and only then clear nextRunAt —
+ // advancing on a failed disable would leave `enabled: true` with no nextRunAt
+ // and no disabledReason (permanently unclaimable and invisible). Everything
+ // else falls through untouched: nextRunAt and the lease both stand.
+ const next = computeNextRunAt({
+ cadence: schedule.cadence,
+ timezone: schedule.timezone,
+ scheduleId: schedule.id,
+ // The claim's clock (see the misfire branch) so a skewed worker doesn't
+ // reschedule to an already-due occurrence and reclaim the same row.
+ after: new Date(dbNow),
+ });
+ if (next == null) {
+ const disabled = await deps.methods
+ .disableSchedule(schedule.id, 'invalid_schedule', schedule.claimToken)
+ .then(() => true)
+ .catch(() => false);
+ if (disabled) {
+ await advanceClaim(null);
+ }
+ }
+ }
+ return false;
+ });
+ if (done) {
+ break;
+ }
+ }
+ return fired;
+ }
+
+ const scheduleNext = () => {
+ if (stopped) {
+ return;
+ }
+ timer = setTimeout(tick, TICK_MS + Math.floor(Math.random() * TICK_JITTER_MS));
+ timer.unref?.();
+ };
+
+ let activePass: Promise<void> | null = null;
+
+ async function tick() {
+ if (stopped) {
+ return;
+ }
+ const pass = (async () => {
+ try {
+ if (ticks % 4 === 0) {
+ await reconcile();
+ }
+ ticks += 1;
+ await runTick();
+ } catch (error) {
+ logger.error('[schedules] tick failed:', error);
+ }
+ })().finally(() => {
+ activePass = null;
+ });
+ activePass = pass;
+ await pass;
+ scheduleNext();
+ }
+
+ void reconcile();
+ scheduleNext();
+
+ const engine: ScheduleEngine = {
+ stop: () => {
+ stopped = true;
+ if (timer) {
+ clearTimeout(timer);
+ timer = undefined;
+ }
+ },
+ runTick,
+ reconcile,
+ };
+
+ // PRE-DRAIN: the default post-drain phase leaves this timer armed while the HTTP
+ // listener is closing and the generation manager has already begun refusing new jobs.
+ // A tick in that window claims a due occurrence, fails its loopback POST against a
+ // server that is shutting down, and books the failure against the schedule — walking a
+ // healthy schedule toward auto-disable for nothing more than a restart. Stopping the
+ // timer alone only narrowed the window: a pass ALREADY in flight could still claim a
+ // due occurrence and lose its loopback POST against the closing listener, so the
+ // shutdown also AWAITS the active pass — the coordinator's drain then holds the
+ // listener open until that pass's fire completes or its claim is released.
+ // Occurrences skipped by stopping early are simply still due at restart, within the
+ // misfire grace.
+ registerShutdownTask(
+ 'schedule engine',
+ async () => {
+ engine.stop();
+ if (activePass) {
+ await activePass;
+ }
+ },
+ { phase: 'pre-drain' },
+ );
+
+ logger.info('[schedules] engine started');
+ return engine;
+}
diff --git a/packages/api/src/schedules/erasure.spec.ts b/packages/api/src/schedules/erasure.spec.ts
new file mode 100644
index 0000000000..0617201ad6
--- /dev/null
+++ b/packages/api/src/schedules/erasure.spec.ts
@@ -0,0 +1,402 @@
+import type { IScheduleRun, ScheduleMethods } from '@librechat/data-schemas';
+import type { JobState } from './types';
+import { startScheduleErasureSweep } from './erasure';
+
+const oldRun = (overrides: Partial<IScheduleRun> = {}): IScheduleRun =>
+ ({
+ scheduleId: 'schedule-1',
+ user: 'user-1',
+ scheduledFor: new Date('2026-08-17T12:00:00.000Z'),
+ firedAt: new Date(Date.now() - 31 * 60_000),
+ conversationId: 'conversation-1',
+ status: 'started',
+ ...overrides,
+ }) as unknown as IScheduleRun;
+
+async function sweepOnce(options: {
+ canInferOwnerDeathFromMissingJob: boolean;
+ job?: JobState | null;
+}) {
+ const methods = {
+ getDeletingSchedules: jest.fn(async () => [{ id: 'schedule-1' }]),
+ getActiveRunsForSchedule: jest.fn(async () => [oldRun()]),
+ recordRunOutcome: jest.fn(async () => undefined),
+ eraseScheduleIfDrained: jest.fn(async () => false),
+ markEraseAttempted: jest.fn(async () => undefined),
+ getRunsForReconciliation: jest.fn(async () => []),
+ };
+ const sweep = startScheduleErasureSweep({
+ methods: methods as unknown as ScheduleMethods,
+ getJobStatus: jest.fn(async () => options.job ?? null),
+ getTriggerDelivery: jest.fn(async () => null),
+ clearReconciledJob: jest.fn(async () => undefined),
+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,
+ });
+ await jest.advanceTimersByTimeAsync(5 * 60_000);
+ sweep.stop();
+ return methods;
+}
+
+describe('schedule erasure fallback owner-death evidence', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.spyOn(Math, 'random').mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+ });
+
+ it('does not settle a peer-owned run from process-local job absence', async () => {
+ const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: false });
+
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('schedule-1');
+ });
+
+ it('retains owner-death recovery when job absence is deployment-authoritative', async () => {
+ const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: true });
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scheduleId: 'schedule-1',
+ status: 'interrupted',
+ error: 'Schedule deleted',
+ }),
+ );
+ });
+
+ it('still trusts an identity-matched terminal job in an unsafe topology', async () => {
+ const methods = await sweepOnce({
+ canInferOwnerDeathFromMissingJob: false,
+ job: {
+ status: 'complete',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ },
+ });
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 'schedule-1', status: 'success' }),
+ );
+ });
+});
+
+const DEFINITE = {
+ code: 'rejected',
+ message: 'rejected before running',
+ certainty: 'definite' as const,
+};
+
+describe('topology-safe dead-delivery convergence', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.spyOn(Math, 'random').mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+ });
+
+ async function convergeOnce(options: {
+ run?: Partial<IScheduleRun>;
+ delivery?: {
+ status: string;
+ lastError?: { code: string; message: string; certainty: 'definite' | 'ambiguous' };
+ } | null;
+ job?: JobState | null;
+ canInferOwnerDeathFromMissingJob?: boolean;
+ }) {
+ const methods = {
+ getDeletingSchedules: jest.fn(async () => []),
+ getActiveRunsForSchedule: jest.fn(async () => []),
+ recordRunOutcome: jest.fn(async () => undefined),
+ eraseScheduleIfDrained: jest.fn(async () => false),
+ markEraseAttempted: jest.fn(async () => undefined),
+ getRunsForReconciliation: jest.fn(async () => [
+ oldRun({ deliveryKey: 'dk-1', ...options.run } as Partial<IScheduleRun>),
+ ]),
+ };
+ const getTriggerDelivery = jest.fn(async () => options.delivery ?? null);
+ const clearReconciledJob = jest.fn(async () => undefined);
+ const sweep = startScheduleErasureSweep({
+ methods: methods as unknown as ScheduleMethods,
+ getJobStatus: jest.fn(async () => options.job ?? null),
+ getTriggerDelivery: getTriggerDelivery as never,
+ clearReconciledJob,
+ // Defaults to the UNSAFE topology to prove this path never depends on it.
+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob ?? false,
+ });
+ await jest.advanceTimersByTimeAsync(5 * 60_000);
+ sweep.stop();
+ return { methods, getTriggerDelivery, clearReconciledJob };
+ }
+
+ it('settles a dead delivery as error even in an unsafe topology (positive evidence)', async () => {
+ const { methods } = await convergeOnce({
+ delivery: {
+ status: 'dead',
+ lastError: { code: 'blocked', message: 'blocked by moderation', certainty: 'definite' },
+ },
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scheduleId: 'schedule-1',
+ status: 'error',
+ error: 'blocked by moderation',
+ }),
+ );
+ });
+
+ it('leaves a live delivery alone', async () => {
+ const { methods } = await convergeOnce({ delivery: { status: 'pending' } });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('never settles while an identity-matched generation still owns the reservation', async () => {
+ const { methods, getTriggerDelivery } = await convergeOnce({
+ delivery: { status: 'dead', lastError: DEFINITE },
+ job: {
+ status: 'running',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ } as unknown as JobState,
+ });
+ expect(getTriggerDelivery).not.toHaveBeenCalled();
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('defers a run whose abort is still in flight', async () => {
+ const { methods } = await convergeOnce({
+ run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,
+ delivery: { status: 'dead', lastError: DEFINITE },
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ /**
+ * The gap this closes: a generation that FINISHED but whose owner exhausted every Mongo
+ * outcome retry leaves a preserved terminal job as the only evidence. The armed engine's
+ * reconciler replays that; the clustered entrypoint arms no engine, so before this the run
+ * kept its `started` row — and its global capacity slot — until the job store expired it.
+ */
+ it('settles a live schedule from its retained terminal job, then releases the evidence', async () => {
+ const { methods, clearReconciledJob } = await convergeOnce({
+ job: {
+ status: 'complete',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scheduleId: 'schedule-1',
+ status: 'success',
+ conversationId: 'conversation-1',
+ }),
+ );
+ expect(clearReconciledJob).toHaveBeenCalledWith('conversation-1', {
+ scheduleId: 'schedule-1',
+ scheduledFor: new Date('2026-08-17T12:00:00.000Z'),
+ });
+ });
+
+ /** Presence of an identity-matched job is positive evidence in EVERY topology, so this
+ * must not depend on the owner-death inference the unsafe fallback refuses. */
+ it('honors the owner-stamped outcome over the generic terminal status', async () => {
+ const { methods } = await convergeOnce({
+ job: {
+ status: 'complete',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleOutcome: 'skipped_balance',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'skipped_balance' }),
+ );
+ });
+
+ it('carries the stamped failure reason from a retained error job', async () => {
+ const { methods } = await convergeOnce({
+ job: {
+ status: 'error',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleOutcomeError: 'provider exploded',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', error: 'provider exploded' }),
+ );
+ });
+
+ /** A pre-start abort reserves a conversationId but never creates the conversation, so
+ * projecting it would point the schedule card at a chat that does not exist. */
+ it('clears the reserved conversationId when the generation never emitted its created event', async () => {
+ const { methods } = await convergeOnce({
+ job: {
+ status: 'aborted',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ createdEventEmitted: false,
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'interrupted', clearConversationId: true }),
+ );
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.not.objectContaining({ conversationId: 'conversation-1' }),
+ );
+ });
+
+ /** A terminal job belonging to a REPLACEMENT turn (same conversationId, no scheduled
+ * identity) must never finalize this run, nor have its evidence deleted for it. */
+ it("ignores a terminal job that lost this occurrence's identity", async () => {
+ const { methods, clearReconciledJob } = await convergeOnce({
+ job: { status: 'complete' } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ expect(clearReconciledJob).not.toHaveBeenCalled();
+ });
+
+ /**
+ * A pause whose `requires_action` projection failed every retry leaves the row `started`
+ * while the job is durably paused. The armed reconciler replays that; with no engine the
+ * row held its GLOBAL capacity slot forever, since a paused job is not terminal and the
+ * dead-delivery path never inspects an identity-matched job.
+ */
+ it('projects a pause the owner never managed to record, freeing the slot', async () => {
+ const { methods, clearReconciledJob } = await convergeOnce({
+ job: {
+ status: 'requires_action',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scheduleId: 'schedule-1',
+ status: 'requires_action',
+ conversationId: 'conversation-1',
+ // Every replica runs this sweep, so the write itself must be fenced against a
+ // resume that claimed the row after this snapshot was taken — as a staleness
+ // CUTOFF, so an abandoned claim still recovers.
+ resumeClaimStaleBefore: expect.any(Date),
+ }),
+ );
+ // The job is still LIVE awaiting approval — its evidence must not be released.
+ expect(clearReconciledJob).not.toHaveBeenCalled();
+ });
+
+ /** A resume claims capacity before flipping the job back to `running`; re-projecting the
+ * pause during that hand-off would release the slot the continuation just took. */
+ it('defers a paused job while a resume hand-off is in flight', async () => {
+ const { methods } = await convergeOnce({
+ run: { resumeClaimedAt: new Date() } as Partial<IScheduleRun>,
+ job: {
+ status: 'requires_action',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('never settles a terminal job while its abort is still in flight', async () => {
+ const { methods, clearReconciledJob } = await convergeOnce({
+ run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,
+ job: {
+ status: 'aborted',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ } as unknown as JobState,
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ expect(clearReconciledJob).not.toHaveBeenCalled();
+ });
+
+ it('ignores a legacy reservation carrying no deliveryKey', async () => {
+ const { methods, getTriggerDelivery } = await convergeOnce({
+ run: { deliveryKey: undefined } as Partial<IScheduleRun>,
+ delivery: { status: 'dead', lastError: DEFINITE },
+ });
+ expect(getTriggerDelivery).not.toHaveBeenCalled();
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+});
+
+describe('dead-delivery certainty fence', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.spyOn(Math, 'random').mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+ });
+
+ async function sweepWithDelivery(options: {
+ certainty: 'definite' | 'ambiguous';
+ canInferOwnerDeathFromMissingJob: boolean;
+ }) {
+ const methods = {
+ getDeletingSchedules: jest.fn(async () => []),
+ getActiveRunsForSchedule: jest.fn(async () => []),
+ recordRunOutcome: jest.fn(async () => undefined),
+ eraseScheduleIfDrained: jest.fn(async () => false),
+ markEraseAttempted: jest.fn(async () => undefined),
+ getRunsForReconciliation: jest.fn(async () => [oldRun({ deliveryKey: 'dk-1' } as never)]),
+ };
+ const sweep = startScheduleErasureSweep({
+ methods: methods as unknown as ScheduleMethods,
+ getJobStatus: jest.fn(async () => null),
+ getTriggerDelivery: (async () => ({
+ status: 'dead',
+ lastError: { code: 'x', message: 'timed out', certainty: options.certainty },
+ })) as never,
+ clearReconciledJob: jest.fn(async () => undefined),
+ canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,
+ });
+ await jest.advanceTimersByTimeAsync(5 * 60_000);
+ sweep.stop();
+ return methods;
+ }
+
+ /**
+ * `dead` is not proof of rejection: the trigger host marks response timeouts and invalid
+ * success responses `ambiguous`, and the engine dead-letters those after exhausting
+ * retries — so an ambiguous dead letter can sit over a generation a peer accepted and is
+ * still running. Settling it here would release capacity mid-run.
+ */
+ it('refuses an AMBIGUOUS dead letter when absence is not deployment-authoritative', async () => {
+ const methods = await sweepWithDelivery({
+ certainty: 'ambiguous',
+ canInferOwnerDeathFromMissingJob: false,
+ });
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('settles an ambiguous dead letter once job absence IS deployment-authoritative', async () => {
+ const methods = await sweepWithDelivery({
+ certainty: 'ambiguous',
+ canInferOwnerDeathFromMissingJob: true,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', error: 'timed out' }),
+ );
+ });
+
+ it('settles a DEFINITE rejection even in an unsafe topology', async () => {
+ const methods = await sweepWithDelivery({
+ certainty: 'definite',
+ canInferOwnerDeathFromMissingJob: false,
+ });
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', error: 'timed out' }),
+ );
+ });
+});
diff --git a/packages/api/src/schedules/erasure.ts b/packages/api/src/schedules/erasure.ts
new file mode 100644
index 0000000000..37dc92ea01
--- /dev/null
+++ b/packages/api/src/schedules/erasure.ts
@@ -0,0 +1,376 @@
+import { logger, runAsSystem } from '@librechat/data-schemas';
+import type { ScheduleMethods, IScheduleRun } from '@librechat/data-schemas';
+import type { JobState, ScheduleEngineDeps } from './types';
+import {
+ hasResumeHandoffInFlight,
+ hasAbortInFlight,
+ retainedOutcome,
+ RESUME_HANDOFF_STALE_MS,
+} from './types';
+import { registerShutdownTask } from '~/app/shutdown';
+
+const SWEEP_MS = 5 * 60_000;
+const SWEEP_JITTER_MS = 30_000;
+const SWEEP_BATCH = 100;
+/** Runs with no readable job older than this are presumed owner-dead (matches the
+ * engine reconciler's orphan cutoff). */
+const ABANDONED_RUN_AGE_MS = 30 * 60_000;
+/** Grace before a live schedule's reservation is converged, so an accepted delivery still
+ * creating its generation — or an owner still writing its terminal outcome — is never
+ * settled mid-handoff. */
+const STRANDED_RUN_MIN_AGE_MS = 2 * 60_000;
+
+/** Terminal job status → the run outcome it proves (mirror of the quiesce map). */
+const TERMINAL_JOB_OUTCOMES: Record<string, 'success' | 'error' | 'interrupted' | undefined> = {
+ complete: 'success',
+ error: 'error',
+ aborted: 'interrupted',
+};
+
+export interface ScheduleErasureSweep {
+ stop: () => void;
+}
+
+export interface ScheduleErasureDeps {
+ methods: Pick<
+ ScheduleMethods,
+ | 'getDeletingSchedules'
+ | 'eraseScheduleIfDrained'
+ | 'markEraseAttempted'
+ | 'getActiveRunsForSchedule'
+ | 'getRunsForReconciliation'
+ | 'recordRunOutcome'
+ >;
+ /** Job state at a run's conversationId; null = confirmed absent, throw = unknown. */
+ getJobStatus: (conversationId: string) => Promise<JobState | null>;
+ /** Durable trigger delivery for a reservation's `deliveryKey`. A `dead` delivery is
+ * POSITIVE shared evidence that no generation owns the reservation, so settling on it
+ * is topology-safe (unlike absence). */
+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'];
+ /** Deletes a run's retained terminal job once its outcome is durable. Identity-guarded,
+ * so a replacement generation reusing the conversationId is never destroyed. */
+ clearReconciledJob: ScheduleEngineDeps['clearReconciledJob'];
+ /** Whether absence in this process's job store proves absence deployment-wide.
+ * False for the process-local fallback whose scheduler refused unsafe topology. */
+ canInferOwnerDeathFromMissingJob: boolean;
+}
+
+/** Whether the observed job still carries THIS occurrence's scheduled identity. */
+function jobMatchesRun(job: JobState | null, run: IScheduleRun): boolean {
+ if (job == null || job.scheduleId !== run.scheduleId || job.scheduledFor == null) {
+ return false;
+ }
+ return new Date(job.scheduledFor).getTime() === run.scheduledFor.getTime();
+}
+
+/**
+ * Settles the abandoned active runs of a DELETING schedule so the erase below can
+ * proceed. The clustered entrypoint runs no engine reconciler, and the run TTL now
+ * (correctly) never expires active rows — so a deleting schedule whose generation
+ * owner died would otherwise retain the run and the owner's prompt indefinitely.
+ * Same evidence discipline as the quiesce paths: settle only on positive evidence
+ * (a terminal identity-matched job, or a confirmed-absent job past the owner-death
+ * cutoff), and defer anything fenced by an in-flight abort or resume hand-off.
+ */
+async function settleAbandonedRuns(deps: ScheduleErasureDeps, scheduleId: string): Promise<void> {
+ const runs = await deps.methods.getActiveRunsForSchedule(scheduleId);
+ const now = Date.now();
+ for (const run of runs) {
+ try {
+ if (hasAbortInFlight(run, now) || hasResumeHandoffInFlight(run, now)) {
+ continue;
+ }
+ const job = run.conversationId
+ ? await deps.getJobStatus(run.conversationId).then(
+ (state) => ({ known: true, state }),
+ () => ({ known: false, state: null }),
+ )
+ : { known: true, state: null };
+ if (!job.known) {
+ continue;
+ }
+ const identity = jobMatchesRun(job.state, run);
+ if (identity && job.state!.status === 'running') {
+ continue;
+ }
+ if (identity && job.state!.status === 'requires_action') {
+ // A paused run of a DELETING schedule: its approval can never be consumed,
+ // but a fresh pause hand-off may still be writing — the started-row gate
+ // and the resume fence above already deferred those; a settled-state
+ // paused row is safe to interrupt.
+ if (run.status === 'started') {
+ continue;
+ }
+ }
+ const settledPause =
+ identity && job.state!.status === 'requires_action' && run.status === 'requires_action';
+ const retained = identity ? TERMINAL_JOB_OUTCOMES[job.state!.status] : undefined;
+ if (retained == null) {
+ // In the unsafe-topology fallback, a peer-owned live job is indistinguishable
+ // from an absent one. Never turn that local absence (or identity mismatch) into
+ // owner-death evidence and free its globally visible run/capacity slot.
+ // An identity-matched, durably paused row is positive local evidence, so it
+ // remains eligible for the existing age-based deleting-schedule cleanup.
+ if (!settledPause && !deps.canInferOwnerDeathFromMissingJob) {
+ continue;
+ }
+ // No terminal evidence: only presume the owner dead past the cutoff.
+ const age = now - (run.firedAt?.getTime() ?? 0);
+ if (age < ABANDONED_RUN_AGE_MS) {
+ continue;
+ }
+ }
+ await deps.methods.recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: retained ?? 'interrupted',
+ conversationId: run.conversationId,
+ ...(retained == null ? { error: 'Schedule deleted' } : {}),
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ });
+ } catch (err) {
+ logger.warn(`[schedules] abandoned-run settle failed for ${scheduleId}:`, err);
+ }
+ }
+}
+
+/**
+ * Converges a run from its OWN identity-matched job — the clustered mirror of the engine
+ * reconciler's retained-job and pause branches.
+ */
+async function settleFromObservedJob(
+ deps: ScheduleErasureDeps,
+ run: IScheduleRun,
+ job: JobState,
+ now: number,
+): Promise<void> {
+ // A PAUSE the owner never managed to project. `recordRunOutcome('requires_action')`
+ // moves the row off `started`, which is what frees its global capacity slot; the job
+ // itself stays live awaiting approval, so its evidence is NOT released here. Without
+ // this the row held a slot forever wherever no engine is armed, since a paused job is
+ // not terminal and the dead-delivery path never looks at an identity-matched job.
+ if (job.status === 'requires_action') {
+ if (run.status !== 'started') {
+ return;
+ }
+ await deps.methods.recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: 'requires_action',
+ conversationId: run.conversationId,
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ // Every clustered replica runs this sweep, so N sweepers can observe the same
+ // unprojected pause. The `run` above is a SNAPSHOT: once one sweeper projects the
+ // pause, the owner's approval can claim a fresh slot, and a peer still holding the
+ // pre-projection snapshot would pass hasResumeHandoffInFlight and unset that slot
+ // and claim stamp under the running continuation. Fence it in the write itself —
+ // on the SAME staleness bound as the caller's check above, so an abandoned claim
+ // (worker died between claiming and resuming) still recovers rather than pinning
+ // the row `started` forever with its approval unresumable.
+ resumeClaimStaleBefore: new Date(now - RESUME_HANDOFF_STALE_MS),
+ });
+ return;
+ }
+ const terminal = TERMINAL_JOB_OUTCOMES[job.status];
+ if (terminal == null) {
+ // `running`: the owner is alive and owns the settlement.
+ return;
+ }
+ // An `aborted` job flips the moment abortJob wins its status CAS — BEFORE its owner
+ // unwinds and persists — so it is the abort fence, not this status, that decides. The
+ // in-flight check in the caller already deferred those; reaching here means the owner
+ // is past its presumed-alive window and this is the backstop.
+ const intended =
+ terminal === 'interrupted'
+ ? { status: 'interrupted' as const, error: undefined }
+ : retainedOutcome(job, terminal === 'error' ? 'error' : 'success');
+ await deps.methods.recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: intended.status,
+ // A pre-start abort reserved a conversationId but never created the conversation;
+ // projecting it would point the card at a chat that does not exist.
+ ...(terminal === 'interrupted' && job.createdEventEmitted !== true
+ ? { clearConversationId: true }
+ : { conversationId: run.conversationId }),
+ error: intended.error,
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ });
+ // AFTER the outcome write, never before: the retained job is the only surviving
+ // evidence if that write fails, and a preserved job carries no `completedAt`, so the
+ // store's finished-job sweep can never reap it on its own.
+ await deps.clearReconciledJob(run.conversationId as string, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ });
+}
+
+/**
+ * Converges the stranded reservations of LIVE (non-deleting) schedules, on POSITIVE
+ * EVIDENCE ONLY.
+ *
+ * `fireSchedule` reserves the run and its global capacity slot before the delivery reaches
+ * the chat route, and the clustered entrypoint arms no engine — so three states would
+ * otherwise hold that slot indefinitely:
+ *
+ * - A RETAINED TERMINAL JOB. The generation finished, but its owner's Mongo outcome write
+ * exhausted every retry (`recordScheduleOutcome` returns false) and left the preserved
+ * job as the only surviving evidence. The armed engine's reconciler replays exactly
+ * this; with no engine nothing did, so the run stayed `started` and its retained
+ * generation evidence lived until store expiry.
+ * - AN UNPROJECTED PAUSE. The generation paused for approval, but the owner's
+ * `requires_action` projection failed every retry, so the row never left `started` even
+ * though the pause itself is durable in the job.
+ * - A DEAD DELIVERY. A pre-generation rejection (interactive limiter, PII, moderation)
+ * dead-letters the delivery while the run stays `started` with no job at all.
+ *
+ * Both are safe in EVERY topology because both read positive, durable evidence instead of
+ * inferring owner death from absence: an identity-matched job is authoritative wherever it
+ * is observed (a shared store shows the real generation; a process-local store can only be
+ * showing this process's own), and a `dead` delivery is shared state proving no generation
+ * owns the reservation. This never claims, fires, or advances, and defers anything fenced
+ * by an in-flight abort or resume hand-off. Auto-disable policy is deliberately NOT applied
+ * (the armed engine owns that): the run settles and frees its slot, the streak is untouched.
+ */
+async function settleStrandedRuns(deps: ScheduleErasureDeps): Promise<void> {
+ const runs = await deps.methods.getRunsForReconciliation(
+ new Date(Date.now() - STRANDED_RUN_MIN_AGE_MS),
+ SWEEP_BATCH,
+ );
+ const now = Date.now();
+ for (const run of runs) {
+ try {
+ // `started` is the capacity-consuming state this pass exists to release. A paused
+ // row holds no slot, and its approval-expiry path owns its own durable retry.
+ if (run.status !== 'started') {
+ continue;
+ }
+ if (hasAbortInFlight(run, now) || hasResumeHandoffInFlight(run, now)) {
+ continue;
+ }
+ const job = run.conversationId
+ ? await deps.getJobStatus(run.conversationId).then(
+ (state) => ({ known: true, state }),
+ () => ({ known: false, state: null }),
+ )
+ : { known: true, state: null };
+ // Unknown is not gone: never settle on a failed job-store read.
+ if (!job.known) {
+ continue;
+ }
+ if (jobMatchesRun(job.state, run)) {
+ await settleFromObservedJob(deps, run, job.state as JobState, now);
+ continue;
+ }
+ // No job of THIS occurrence's identity, so the durable delivery is the authority.
+ if (!run.deliveryKey) {
+ continue;
+ }
+ const delivery = await deps.getTriggerDelivery(run.deliveryKey);
+ if (delivery?.status !== 'dead') {
+ continue;
+ }
+ // `dead` alone is NOT proof the request was rejected. The trigger host marks
+ // response timeouts and invalid success responses `ambiguous`, and the engine
+ // dead-letters those once retries are exhausted — so an ambiguous dead letter can
+ // sit over a generation a peer accepted and is still running. Settle only on a
+ // DEFINITE rejection, unless this process can observe generation absence
+ // deployment-wide (safe topology), where the confirmed-absent job above is itself
+ // authoritative evidence.
+ if (delivery.lastError?.certainty !== 'definite' && !deps.canInferOwnerDeathFromMissingJob) {
+ continue;
+ }
+ await deps.methods.recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: 'error',
+ conversationId: run.conversationId,
+ // `lastError` is an AgentTriggerDeliveryFailure, not a string: passing the object
+ // into the String-typed run/schedule error fields fails the Mongoose cast, and the
+ // per-row catch would swallow it while the run kept its capacity slot.
+ error: delivery.lastError?.message ?? 'Scheduled delivery failed before running',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ });
+ } catch (err) {
+ logger.warn(`[schedules] stranded-run settle failed for ${run.scheduleId}:`, err);
+ }
+ }
+}
+
+/**
+ * Erases soft-deleted schedules once they drain — and NOTHING else.
+ *
+ * A `deleting` row is normally erased by whichever actor first observes it drained: the
+ * delete request, or the terminal outcome write (erase-on-settle). Both are single
+ * best-effort attempts, so one transient failure — or a lease that outlived the delete —
+ * leaves a hidden row holding the user's prompt, with no TTL and no way for the owner to
+ * retry it (the row is hidden from their list). In the standard entrypoint the
+ * reconciler retries it; the clustered entrypoint runs no engine, so nothing does.
+ *
+ * This is deliberately NOT the engine: it never claims, leases, fires, advances, or
+ * reconciles a run, so running it in every replica of a clustered deployment is safe and
+ * changes nothing about v1's single-process scheduling. It only re-drives
+ * `eraseScheduleIfDrained`, which re-checks drained-ness itself (no active run, no live
+ * lease) and is idempotent — concurrent sweepers race harmlessly.
+ */
+export function startScheduleErasureSweep(deps: ScheduleErasureDeps): ScheduleErasureSweep {
+ let stopped = false;
+ let timer: NodeJS.Timeout | undefined;
+
+ async function sweep(): Promise<void> {
+ try {
+ await runAsSystem(async () => {
+ const deleting = await deps.methods.getDeletingSchedules(SWEEP_BATCH);
+ for (const schedule of deleting) {
+ await settleAbandonedRuns(deps, schedule.id).catch((err) => {
+ logger.warn(`[schedules] abandoned-run pass failed for ${schedule.id}:`, err);
+ });
+ await deps.methods.eraseScheduleIfDrained(schedule.id).catch((err) => {
+ logger.warn(`[schedules] erasure sweep failed for ${schedule.id}:`, err);
+ });
+ }
+ // Rotate the window (never-attempted first) so a batch of undrainable rows
+ // cannot re-fill it every sweep and starve the rows behind them.
+ await deps.methods
+ .markEraseAttempted(deleting.map((schedule) => schedule.id))
+ .catch((err) => logger.warn('[schedules] failed to stamp erase attempts:', err));
+ // Live schedules too: converge reservations left stranded by a retained terminal
+ // job or a dead delivery, so neither can hold a global capacity slot — and no
+ // generation evidence can outlive its run — where no engine is armed.
+ await settleStrandedRuns(deps).catch((err) =>
+ logger.warn('[schedules] stranded-run convergence pass failed:', err),
+ );
+ });
+ } catch (err) {
+ logger.error('[schedules] erasure sweep failed:', err);
+ }
+ }
+
+ function schedule(): void {
+ if (stopped) {
+ return;
+ }
+ // Jittered so replicas of a clustered deployment do not sweep in lockstep.
+ const delay = SWEEP_MS + Math.floor(Math.random() * SWEEP_JITTER_MS);
+ timer = setTimeout(() => {
+ void sweep().finally(schedule);
+ }, delay);
+ timer.unref?.();
+ }
+
+ schedule();
+
+ const engineSweep: ScheduleErasureSweep = {
+ stop: () => {
+ stopped = true;
+ if (timer) {
+ clearTimeout(timer);
+ timer = undefined;
+ }
+ },
+ };
+
+ registerShutdownTask('schedule erasure sweep', () => engineSweep.stop(), { phase: 'pre-drain' });
+
+ return engineSweep;
+}
diff --git a/packages/api/src/schedules/fire.spec.ts b/packages/api/src/schedules/fire.spec.ts
new file mode 100644
index 0000000000..8284546901
--- /dev/null
+++ b/packages/api/src/schedules/fire.spec.ts
@@ -0,0 +1,802 @@
+import type { ScheduleEngineDeps, ScheduleLimits, ScheduleUserContext } from './types';
+import type { FireableSchedule } from './types';
+import { AgentTriggerServiceUnavailableError } from '../agents/triggers/service';
+import { getAgentTriggerIdempotencyKey } from '../agents/triggers/envelope';
+import { AgentTriggerDeliveryError } from '../agents/triggers/delivery';
+import { buildFireClientRequestId, fireSchedule } from './fire';
+import { withCapacitySlot } from './capacity';
+
+const OWNER: ScheduleUserContext = { id: 'user-1', tenantId: 't1', role: 'USER' };
+const LIMITS: ScheduleLimits = {
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+};
+
+function makeSchedule(overrides: Partial<FireableSchedule> = {}): FireableSchedule {
+ return {
+ id: 'sched-1',
+ user: 'user-1' as never,
+ tenantId: 't1',
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ claimToken: 'ct-1',
+ leaseBy: 'inst-1',
+ runCount: 0,
+ failureCount: 0,
+ balanceSkipCount: 0,
+ ...overrides,
+ } as FireableSchedule;
+}
+
+/** In-memory run store exercising the real insert/count/delete/idempotency interplay. */
+function makeMethods() {
+ const runs = new Map<
+ string,
+ { status: string; conversationId?: string; capacitySlot?: number }
+ >();
+ const calls = {
+ advance: 0,
+ releaseLease: 0,
+ disable: [] as string[],
+ recordOutcome: [] as { status: string }[],
+ skipped: [] as string[],
+ setFireDetails: 0,
+ };
+ const key = (id: string, when: Date) => `${id}:${when.toISOString()}`;
+ const methods = {
+ releaseLease: jest.fn(async () => {
+ calls.releaseLease += 1;
+ return true;
+ }),
+ advanceSchedule: jest.fn(async () => {
+ calls.advance += 1;
+ return true;
+ }),
+ disableSchedule: jest.fn(async (_id: string, reason: string) => {
+ calls.disable.push(reason);
+ }),
+ hasActiveRun: jest.fn(async (id: string) =>
+ [...runs.entries()].some(([k, r]) => k.startsWith(`${id}:`) && r.status === 'started'),
+ ),
+ countActiveRuns: jest.fn(
+ async () => [...runs.values()].filter((r) => r.status === 'started').length,
+ ),
+ insertScheduleRun: jest.fn(
+ async (data: { scheduleId: string; scheduledFor: Date; conversationId?: string }) => {
+ const k = key(data.scheduleId, data.scheduledFor);
+ if (runs.has(k)) {
+ return null; // unique {scheduleId, scheduledFor}
+ }
+ runs.set(k, { status: 'started', conversationId: data.conversationId });
+ return { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor };
+ },
+ ),
+ // Mirrors the partial-unique-index semantics: same-occurrence row => 'duplicate';
+ // any OTHER started run for the schedule => 'overlap'; else reserve the slot.
+ reserveStartedRun: jest.fn(
+ async (data: {
+ scheduleId: string;
+ scheduledFor: Date;
+ conversationId?: string;
+ capacitySlot?: number;
+ deliveryKey?: string;
+ }) => {
+ const k = key(data.scheduleId, data.scheduledFor);
+ if (runs.has(k)) {
+ // Mirrors the real method: a duplicate reports the EXISTING row's status so
+ // the caller can tell "still running" from "already finished".
+ return { conflict: 'duplicate' as const, existingStatus: runs.get(k)!.status };
+ }
+ // Mirrors the unique {capacitySlot} partial index (status:'started').
+ if (
+ data.capacitySlot != null &&
+ [...runs.values()].some(
+ (r) => r.status === 'started' && r.capacitySlot === data.capacitySlot,
+ )
+ ) {
+ return { conflict: 'slot-taken' as const };
+ }
+ const overlap = [...runs.entries()].some(
+ ([rk, r]) => rk.startsWith(`${data.scheduleId}:`) && r.status === 'started',
+ );
+ if (overlap) {
+ return { conflict: 'overlap' as const };
+ }
+ runs.set(k, {
+ status: 'started',
+ conversationId: data.conversationId,
+ capacitySlot: data.capacitySlot,
+ });
+ return { run: { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor } };
+ },
+ ),
+ getCapacityOccupancy: jest.fn(async () => {
+ const takenSlots: number[] = [];
+ let unslotted = 0;
+ for (const r of runs.values()) {
+ if (r.status !== 'started') {
+ continue;
+ }
+ if (typeof r.capacitySlot === 'number') {
+ takenSlots.push(r.capacitySlot);
+ } else {
+ unslotted += 1;
+ }
+ }
+ return { takenSlots, unslotted };
+ }),
+ revalidateClaim: jest.fn(async () => true),
+ releaseLeaseByHolder: jest.fn(async () => undefined),
+ deleteScheduleRun: jest.fn(
+ async (id: string, when: Date, _status?: string, expectedConversationId?: string) => {
+ const k = key(id, when);
+ const row = runs.get(k) as { conversationId?: string } | undefined;
+ // Mirrors the conversationId fence: a fire only deletes the row IT inserted.
+ if (expectedConversationId != null && row?.conversationId !== expectedConversationId) {
+ return;
+ }
+ runs.delete(k);
+ },
+ ),
+ setRunFireDetails: jest.fn(async () => {
+ calls.setFireDetails += 1;
+ }),
+ recordSkippedRun: jest.fn(async (data: { status: string }) => {
+ calls.skipped.push(data.status);
+ }),
+ recordRunOutcome: jest.fn(
+ async (data: { scheduleId: string; scheduledFor: Date; status: string }) => {
+ const k = key(data.scheduleId, data.scheduledFor);
+ if (runs.has(k)) {
+ runs.set(k, { status: data.status });
+ }
+ calls.recordOutcome.push({ status: data.status });
+ },
+ ),
+ };
+ return { methods, runs, calls };
+}
+
+function makeDeps(
+ methods: ReturnType<typeof makeMethods>['methods'],
+ over: Partial<ScheduleEngineDeps> = {},
+): ScheduleEngineDeps {
+ return {
+ methods: methods as unknown as ScheduleEngineDeps['methods'],
+ getLimits: async () => LIMITS,
+ getUserContext: async () => OWNER,
+ isOutOfBalance: async () => false,
+ agentAccess: async () => 'ok',
+ hasScheduleAccess: async () => true,
+ resolveFiles: async () => [],
+ enqueueTrigger: jest.fn(async () => undefined),
+ runInTenantContext: (_user, fn) => fn(),
+ getJobStatus: async () => null,
+ abortScheduledJob: async () => undefined,
+ clearReconciledJob: async () => undefined,
+ isOwnerDeleting: async () => false,
+ isGloballyDisabled: async () => false,
+ countActiveRunsGlobal: async () => methods.countActiveRuns(),
+ withGlobalCapacitySlot: (cap: number, claim: (slot: number) => Promise<unknown>) =>
+ withCapacitySlot(
+ cap,
+ () => methods.getCapacityOccupancy(),
+ claim as Parameters<typeof withCapacitySlot>[2],
+ ),
+ ...over,
+ } as ScheduleEngineDeps;
+}
+
+function mockFetch(impl: () => Promise<unknown> | never) {
+ global.fetch = jest.fn(impl as never) as never;
+}
+
+const okResponse = (conversationId = 'convo-1') =>
+ ({
+ ok: true,
+ status: 200,
+ // The accept path answers with JSON; fire reads the body as text and JSON-parses it.
+ text: async () => JSON.stringify({ conversationId }),
+ }) as Response;
+
+const dueAt = () => new Date(Date.now() - 60_000);
+
+afterEach(() => jest.restoreAllMocks());
+
+/**
+ * The chat route validates `clientRequestId` against `/^[A-Za-z0-9:_-]{1,128}$/`
+ * (CLIENT_REQUEST_ID_PATTERN in api/server/controllers/agents/request.js). An id
+ * outside that charset makes the route answer 400 INVALID_CLIENT_REQUEST_ID, which
+ * fails EVERY fire — so the encoding is a contract, not a formatting choice.
+ */
+describe('buildFireClientRequestId', () => {
+ const ROUTE_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
+
+ it('stays within the charset the chat route accepts', () => {
+ const id = buildFireClientRequestId(
+ 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',
+ new Date('2026-08-01T14:05:55.132Z'),
+ );
+
+ // A raw ISO instant carries a '.' in its milliseconds, which the route rejects.
+ expect(id).not.toContain('.');
+ expect(id).toMatch(ROUTE_PATTERN);
+ });
+
+ it('is deterministic per occurrence and distinct across occurrences', () => {
+ const scheduleId = 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b';
+ const first = new Date('2026-08-01T14:05:55.132Z');
+ const second = new Date('2026-08-01T15:05:55.132Z');
+
+ expect(buildFireClientRequestId(scheduleId, first)).toBe(
+ buildFireClientRequestId(scheduleId, new Date(first.getTime())),
+ );
+ expect(buildFireClientRequestId(scheduleId, first)).not.toBe(
+ buildFireClientRequestId(scheduleId, second),
+ );
+ });
+
+ it('stays inside the 128-character cap', () => {
+ expect(
+ buildFireClientRequestId(
+ 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',
+ new Date('2026-08-01T14:05:55.132Z'),
+ ).length,
+ ).toBeLessThanOrEqual(128);
+ });
+});
+
+describe('fireSchedule', () => {
+ it('fires the happy path and records fire details', async () => {
+ const { methods, runs } = makeMethods();
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.fired).toBe(true);
+ // The conversation id is pre-generated and recorded on the run row up front
+ // (so reconciliation can always find the occurrence's job), not read back from
+ // the POST response.
+ expect(result.conversationId).toMatch(/^[0-9a-f-]{36}$/);
+ expect([...runs.values()][0].conversationId).toBe(result.conversationId);
+ expect(methods.setRunFireDetails).toHaveBeenCalledTimes(1);
+ expect([...runs.values()][0].status).toBe('started');
+ });
+
+ it('releases the old holder when a post-enqueue owner edit fences the advance', async () => {
+ const { methods } = makeMethods();
+ const enqueueTrigger = jest.fn(async () => undefined);
+ methods.advanceSchedule.mockResolvedValueOnce(false);
+
+ const result = await fireSchedule(
+ makeDeps(methods, { enqueueTrigger }),
+ makeSchedule({ leaseBy: 'claim-holder-1' }),
+ LIMITS,
+ dueAt(),
+ );
+
+ expect(result.fired).toBe(true);
+ expect(enqueueTrigger).toHaveBeenCalledTimes(1);
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'claim-holder-1');
+ });
+
+ it('stamps the deterministic deliveryKey on the reservation (== the enqueued key)', async () => {
+ const { methods } = makeMethods();
+ const enqueueTrigger = jest.fn<
+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
+ Parameters<ScheduleEngineDeps['enqueueTrigger']>
+ >(async () => undefined);
+
+ const result = await fireSchedule(
+ makeDeps(methods, { enqueueTrigger }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+
+ expect(result.fired).toBe(true);
+ const reservation = (methods.reserveStartedRun as jest.Mock).mock.calls[0][0];
+ const envelope = enqueueTrigger.mock.calls[0][0];
+ // Stored BEFORE enqueue and equal to the enqueued delivery's idempotency key, so
+ // reconciliation can read the durable delivery for THIS occurrence even if the enqueue
+ // committed ambiguously.
+ expect(reservation.deliveryKey).toEqual(expect.any(String));
+ expect(reservation.deliveryKey).toBe(getAgentTriggerIdempotencyKey(envelope));
+ });
+
+ it('marks a Run Now trigger as manual for downstream limiter policy', async () => {
+ const { methods } = makeMethods();
+ const enqueueTrigger = jest.fn<
+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
+ Parameters<ScheduleEngineDeps['enqueueTrigger']>
+ >(async () => undefined);
+ await fireSchedule(makeDeps(methods, { enqueueTrigger }), makeSchedule(), LIMITS, dueAt(), {
+ manual: true,
+ });
+ const envelope = enqueueTrigger.mock.calls[0][0];
+ expect(envelope).toMatchObject({
+ mode: 'fire',
+ principal: { userId: 'user-1', role: 'USER', tenantId: 't1' },
+ event: {
+ type: 'schedule.occurrence',
+ source: { id: 'sched-1', type: 'schedule' },
+ },
+ run: { metadata: { manual: true } },
+ });
+ expect(enqueueTrigger).toHaveBeenCalledWith(envelope, { orderingKey: 'sched-1' });
+ });
+
+ it('marks an automatic occurrence as non-manual', async () => {
+ const { methods } = makeMethods();
+ const enqueueTrigger = jest.fn<
+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
+ Parameters<ScheduleEngineDeps['enqueueTrigger']>
+ >(async () => undefined);
+ await fireSchedule(makeDeps(methods, { enqueueTrigger }), makeSchedule(), LIMITS, dueAt());
+
+ expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({
+ run: { metadata: { manual: false } },
+ });
+ });
+
+ it('carries the claimed config revision and fire inputs on the durable trigger', async () => {
+ const { methods } = makeMethods();
+ const enqueueTrigger = jest.fn<
+ ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
+ Parameters<ScheduleEngineDeps['enqueueTrigger']>
+ >(async () => undefined);
+ await fireSchedule(
+ makeDeps(methods, {
+ enqueueTrigger,
+ resolveFiles: async () => [{ file_id: 'file-1' }],
+ }),
+ makeSchedule({ configRevision: 7, file_ids: ['file-1'] } as never),
+ LIMITS,
+ dueAt(),
+ );
+ expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({
+ input: 'Summarize',
+ target: { agentId: 'agent-1' },
+ run: {
+ timezone: 'America/New_York',
+ files: [{ file_id: 'file-1' }],
+ metadata: { manual: false, configRevision: 7 },
+ },
+ });
+ });
+
+ it('records a definite trigger admission rejection as an error', async () => {
+ const { methods, calls } = makeMethods();
+ const result = await fireSchedule(
+ makeDeps(methods, {
+ enqueueTrigger: async () => {
+ throw new AgentTriggerServiceUnavailableError('not initialized');
+ },
+ }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+ expect(result.fired).toBe(false);
+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
+ });
+
+ it('leaves an ambiguous persistence failure reconcilable', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const result = await fireSchedule(
+ makeDeps(methods, {
+ enqueueTrigger: async () => {
+ throw new Error('Mongo result unknown');
+ },
+ }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+ expect(result.fired).toBe(false);
+ expect(calls.recordOutcome).toHaveLength(0);
+ expect([...runs.values()][0].status).toBe('started');
+ expect(calls.advance).toBe(1);
+ });
+
+ it('records an invalid durable delivery as a definite error', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const result = await fireSchedule(
+ makeDeps(methods, {
+ enqueueTrigger: async () => {
+ throw new AgentTriggerDeliveryError('invalid delivery');
+ },
+ }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+ expect(result.fired).toBe(false);
+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
+ expect([...runs.values()][0].status).toBe('error');
+ });
+
+ it('does not orphan a run when file resolution fails', async () => {
+ const { methods, runs, calls } = makeMethods();
+ mockFetch(async () => okResponse());
+ const deps = makeDeps(methods, {
+ resolveFiles: async () => {
+ throw new Error('db down');
+ },
+ });
+ const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt());
+ expect(result.fired).toBe(false);
+ expect(runs.size).toBe(0); // no run row created
+ // Automatic fire KEEPS the claim lease as a backoff (nextRunAt untouched → the
+ // occurrence retries when the lease expires) so a transient file error can't get
+ // this row re-claimed every tick and starve others.
+ expect(calls.releaseLease).toBe(0);
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ it('releases the lease on a manual run-now when file resolution fails', async () => {
+ const { methods, runs } = makeMethods();
+ // An owner edit can rotate the token while the file lookup is in flight while
+ // deliberately preserving this fire's unique lease holder.
+ methods.releaseLease.mockResolvedValueOnce(false);
+ mockFetch(async () => okResponse());
+ const deps = makeDeps(methods, {
+ resolveFiles: async () => {
+ throw new Error('db down');
+ },
+ });
+ const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt(), {
+ manual: true,
+ });
+ expect(result.fired).toBe(false);
+ expect(runs.size).toBe(0);
+ // Run-now releases so the user can retry immediately (no misleading lease-held 409).
+ expect(methods.releaseLease).toHaveBeenCalledWith('sched-1', 'ct-1');
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ it('refuses the fire at the global capacity cap WITHOUT inserting a run', async () => {
+ const { methods, runs } = makeMethods();
+ // All 5 slots taken → the allocator finds no free slot and never inserts.
+ for (let i = 0; i < 5; i++) {
+ runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });
+ }
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('capacity');
+ expect(global.fetch).not.toHaveBeenCalled();
+ // Slot-based capacity is decided BEFORE the write, so there is nothing to roll back.
+ expect([...runs.values()].filter((r) => r.status === 'started')).toHaveLength(5);
+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();
+ expect(methods.deleteScheduleRun).not.toHaveBeenCalled();
+ });
+
+ it('does not let a principal override widen the global capacity cap', async () => {
+ const { methods, runs } = makeMethods();
+ // The single deployment-wide slot is already occupied.
+ runs.set('other-0:x', { status: 'started', capacitySlot: 0 });
+ mockFetch(async () => okResponse());
+ const deps = makeDeps(methods, {
+ // Base allows 1 concurrent scheduled generation; this owner's role/tenant
+ // override raises their limit to 5.
+ getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 5 : 1 }),
+ });
+ // Run Now resolves the OWNER's limits and skips the tick's base-config budget, so
+ // without the clamp the override would hand out slots 1-4 and run five billed
+ // generations against a deployment that advertises a cap of one.
+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('capacity');
+ expect(global.fetch).not.toHaveBeenCalled();
+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();
+ });
+
+ it('still honors an owner override that is STRICTER than the deployment cap', async () => {
+ const { methods, runs } = makeMethods();
+ runs.set('other-0:x', { status: 'started', capacitySlot: 0 });
+ mockFetch(async () => okResponse());
+ const deps = makeDeps(methods, {
+ getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 1 : 5 }),
+ });
+ // Only WIDENING is the defect; a tenant that wants less concurrency than the
+ // deployment allows must still get it.
+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('capacity');
+ });
+
+ it('claims a free slot and never exceeds the cap when slots collide', async () => {
+ const { methods, runs } = makeMethods();
+ // Slots 0 and 2 are taken; the allocator must land the fire on slot 1.
+ runs.set('other-a:x', { status: 'started', capacitySlot: 0 });
+ runs.set('other-b:x', { status: 'started', capacitySlot: 2 });
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.fired).toBe(true);
+ const own = [...runs.entries()].find(([k]) => k.startsWith('sched-1:'));
+ expect(own?.[1].capacitySlot).toBe(1);
+ });
+
+ it('re-fires cleanly next tick once capacity frees, exactly once', async () => {
+ const { methods, runs } = makeMethods();
+ for (let i = 0; i < 5; i++) {
+ runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });
+ }
+ const enqueueTrigger = jest.fn(async () => undefined);
+ const deps = makeDeps(methods, { enqueueTrigger });
+ const schedule = makeSchedule();
+ const when = dueAt();
+ // Tick 1: every slot taken → refused before any insert.
+ const first = await fireSchedule(deps, schedule, LIMITS, when);
+ expect(first.skipped).toBe('capacity');
+ // Capacity frees up before the next tick.
+ runs.delete('other-0:x');
+ // Tick 2: same occurrence re-claimed → now fires, exactly one live run.
+ const second = await fireSchedule(deps, schedule, LIMITS, when);
+ expect(second.fired).toBe(true);
+ expect(methods.reserveStartedRun).toHaveBeenCalledTimes(1); // only the successful tick inserts
+ expect(
+ [...runs.entries()].filter(([k, r]) => k.startsWith('sched-1:') && r.status === 'started'),
+ ).toHaveLength(1);
+ expect(enqueueTrigger).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not reserve a run when the claim already lapsed during preflight', async () => {
+ const { methods, runs } = makeMethods();
+ // The preflight (user/config/permission/balance/attachment queries) outlasted the
+ // 5-minute lease and another worker re-claimed the occurrence.
+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('superseded');
+ // Reserving here would win the occurrence's unique row: the FRESH claimer would then
+ // see `duplicate` and advance without firing, while this worker's own revalidation
+ // fails and rollbackReservation deliberately retains the row (leaseBy changed). The
+ // occurrence would be lost with its capacity slot held until the orphan sweep.
+ expect(methods.reserveStartedRun).not.toHaveBeenCalled();
+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
+ expect(global.fetch).not.toHaveBeenCalled();
+ // The lease is handed back by holder so the fresh claimer is not left waiting.
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
+ });
+
+ /**
+ * A failed revalidation must NEVER advance. In every true supersession (takeover,
+ * owner edit) the claim token rotated, so an advance would no-op anyway — the only
+ * case where it can land is a PURE lease expiry with no takeover, where the token
+ * never rotated. advanceSchedule checks no lease, so that advance moved nextRunAt
+ * past an occurrence nothing had fired: a slow preflight silently DROPPED it
+ * instead of leaving it due for the next claim to retry.
+ */
+ it('leaves the occurrence due when only the lease expired (no takeover)', async () => {
+ const { methods } = makeMethods();
+ // Same claim token, merely past leaseUntil: revalidateClaim fails on the lease
+ // predicate alone while the token-fenced advance filter would still MATCH.
+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('superseded');
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ it('never advances from any superseded branch, including post-reserve', async () => {
+ const { methods } = makeMethods();
+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('superseded');
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ });
+
+ /**
+ * The lease can expire between the pre-reserve revalidation and the reservation
+ * itself: the capacity allocator reads occupancy in between, and the preflight
+ * before it can already have outlasted the 5-minute lease. Gating the rollback on
+ * lease ownership stranded the row — the new holder's own reserve saw `duplicate`
+ * and advanced past the occurrence without firing it, while this row held a global
+ * capacity slot until the 30-minute orphan sweep.
+ */
+ it('deletes its own undispatched reservation even after a lease takeover', async () => {
+ const { methods, runs } = makeMethods();
+ // Valid at the pre-reserve check, superseded by the pre-POST one: that ordering IS
+ // the scenario, since a claim already dead before reserving never reserves at all.
+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('superseded');
+ // Nothing was dispatched for this row, so it is unambiguously this fire's garbage.
+ expect(methods.deleteScheduleRun).toHaveBeenCalledWith(
+ 'sched-1',
+ expect.any(Date),
+ 'started',
+ expect.any(String),
+ );
+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ it('rechecks shutdown immediately before dispatch and rolls back the reservation', async () => {
+ const { methods, runs } = makeMethods();
+ // The coordinator flips AFTER the pre-reservation gate: the deployment-limit
+ // read, capacity allocation, and claim revalidation all run in between and can
+ // overlap SIGTERM. Without the recheck the POST lands on a closing listener —
+ // booked as an error (or a refused connect that advances the occurrence).
+ const isShuttingDown = jest.fn().mockReturnValueOnce(false).mockReturnValue(true);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(
+ makeDeps(methods, { isShuttingDown }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+ expect(result.skipped).toBe('superseded');
+ expect(global.fetch).not.toHaveBeenCalled();
+ // Rolled back, not advanced: the occurrence stays due for the restarted process.
+ expect(methods.advanceSchedule).not.toHaveBeenCalled();
+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
+ });
+
+ it('deletes the reserved run when the schedule was hard-deleted mid-fire', async () => {
+ const { methods, runs } = makeMethods();
+ // Account deletion hard-deleted the schedule after this fire reserved its run.
+ (methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('superseded');
+ // The orphaned reservation (no schedule left to own it) is deleted, not leaked.
+ expect(methods.deleteScheduleRun).toHaveBeenCalledWith(
+ 'sched-1',
+ expect.any(Date),
+ 'started',
+ expect.any(String),
+ );
+ expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
+ });
+
+ it('records an unavailable trigger service as a definite error', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const result = await fireSchedule(
+ makeDeps(methods, {
+ enqueueTrigger: async () => {
+ throw new AgentTriggerServiceUnavailableError('not ready');
+ },
+ }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+ expect(result.fired).toBe(false);
+ expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
+ expect([...runs.values()][0].status).toBe('error');
+ });
+
+ it('skips overlap when a prior run is still active', async () => {
+ const { methods, runs, calls } = makeMethods();
+ runs.set('sched-1:prior', { status: 'started' });
+ mockFetch(async () => okResponse());
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('overlap');
+ expect(calls.skipped).toEqual(['skipped_overlap']);
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ it('disables and stops firing when agent VIEW access was revoked', async () => {
+ const { methods, calls } = makeMethods();
+ const deps = makeDeps(methods, { agentAccess: async () => 'forbidden' });
+ const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
+ expect(result.skipped).toBe('permission_revoked');
+ expect(calls.disable).toEqual(['permission_revoked']);
+ });
+
+ it('skips a duplicate occurrence (idempotency claim already held)', async () => {
+ const { methods, runs } = makeMethods();
+ runs.set(`sched-1:${dueAt().toISOString()}`, { status: 'requires_action' });
+ mockFetch(async () => okResponse());
+ const when = new Date(dueAt().getTime());
+ runs.set(`sched-1:${when.toISOString()}`, { status: 'requires_action' });
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
+ expect(result.skipped).toBe('duplicate');
+ expect(global.fetch).not.toHaveBeenCalled();
+ });
+
+ /**
+ * A balance skip is not a no-op: it stamps the card and walks the balance-skip streak
+ * toward auto-disable. The preflight above it (user, config, permission and balance
+ * lookups) can outlast the 5-minute lease, so writing it under a dead claim is a write
+ * on behalf of a fire that no longer owns the occurrence.
+ */
+ it('does not record a balance skip under a superseded claim', async () => {
+ const { methods, calls } = makeMethods();
+ (methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
+ mockFetch(async () => okResponse());
+
+ const result = await fireSchedule(
+ makeDeps(methods, { isOutOfBalance: async () => true }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+
+ expect(result.skipped).toBe('superseded');
+ expect(methods.recordSkippedRun).not.toHaveBeenCalled();
+ expect(calls.skipped).toEqual([]);
+ });
+
+ it('records the balance skip normally while the claim is still valid', async () => {
+ const { methods, calls } = makeMethods();
+ mockFetch(async () => okResponse());
+
+ const result = await fireSchedule(
+ makeDeps(methods, { isOutOfBalance: async () => true }),
+ makeSchedule(),
+ LIMITS,
+ dueAt(),
+ );
+
+ expect(result.skipped).toBe('balance');
+ expect(calls.skipped).toEqual(['skipped_balance']);
+ });
+
+ /**
+ * A `duplicate` means ANOTHER worker holds this occurrence's row — not that the
+ * occurrence is done. Advancing past it hands the occurrence away: if that other
+ * worker is a stale lease holder whose own revalidation then fails, it rolls its
+ * undispatched row back and NOTHING ever fires the occurrence. Leaving nextRunAt
+ * alone keeps it claimable; the worker that actually dispatches is the one that
+ * advances, and the claim's lease provides the retry backoff.
+ */
+ /**
+ * A settled-but-unadvanced occurrence (its fire was accepted, the post-accept advance
+ * failed) leaves nextRunAt pointing at it. Refusing to advance on `duplicate` then
+ * makes every future claim re-pick the same finished occurrence — a permanent stall.
+ */
+ it('advances past a duplicate whose run already settled', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const when = new Date(dueAt().getTime());
+ runs.set(`sched-1:${when.toISOString()}`, { status: 'success', conversationId: 'done' });
+ mockFetch(async () => okResponse());
+
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
+
+ expect(result.skipped).toBe('duplicate');
+ expect(global.fetch).not.toHaveBeenCalled();
+ expect(calls.advance).toBe(1);
+ });
+
+ it('does not advance past an occurrence another worker is holding', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const when = new Date(dueAt().getTime());
+ // A peer already reserved this occurrence's row.
+ runs.set(`sched-1:${when.toISOString()}`, {
+ status: 'started',
+ conversationId: 'peer-convo',
+ });
+ mockFetch(async () => okResponse());
+
+ const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
+
+ expect(result.skipped).toBe('duplicate');
+ expect(global.fetch).not.toHaveBeenCalled();
+ expect(calls.advance).toBe(0);
+ });
+
+ it('releases the lease on a duplicate for run-now so the user can retry', async () => {
+ const { methods, runs, calls } = makeMethods();
+ const when = new Date(dueAt().getTime());
+ runs.set(`sched-1:${when.toISOString()}`, { status: 'started' });
+ mockFetch(async () => okResponse());
+
+ await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when, { manual: true });
+
+ // Automatic claims keep the lease as backoff; a manual click must not be told
+ // "already in progress" for the full lease TTL.
+ expect(calls.releaseLease).toBe(1);
+ expect(calls.advance).toBe(0);
+ });
+});
diff --git a/packages/api/src/schedules/fire.ts b/packages/api/src/schedules/fire.ts
new file mode 100644
index 0000000000..de23f83455
--- /dev/null
+++ b/packages/api/src/schedules/fire.ts
@@ -0,0 +1,604 @@
+import { randomUUID } from 'node:crypto';
+import { logger } from '@librechat/data-schemas';
+import type { ScheduleEngineDeps, ScheduleLimits, FireResult, FireableSchedule } from './types';
+import type { AgentTriggerEnvelope } from '../agents/triggers/envelope';
+import type { JsonValue } from '../agents/json';
+import {
+ AgentTriggerEnvelopeError,
+ createAgentTriggerEnvelope,
+ getAgentTriggerIdempotencyKey,
+} from '../agents/triggers/envelope';
+import { AgentTriggerServiceUnavailableError } from '../agents/triggers/service';
+import { AgentTriggerDeliveryError } from '../agents/triggers/delivery';
+import { computeNextRunAt, cadenceIntervalMinutes } from './cadence';
+
+/** Consecutive balance skips (pre-fire or mid-generation) before auto-disable. */
+export const BALANCE_SKIP_DISABLE_THRESHOLD: number = 5;
+
+/**
+ * Stable source delivery identity for one scheduled occurrence. The trigger
+ * service hashes this into the generation-compatible idempotency key.
+ */
+export function buildFireClientRequestId(scheduleId: string, scheduledFor: Date): string {
+ return `sched:${scheduleId}:${scheduledFor.toISOString().replace('.', '-')}`;
+}
+
+/**
+ * `ambiguous` means Mongo may have committed the idempotent trigger delivery even
+ * though enqueue returned an error. Those must remain reconcilable rather than be
+ * terminalized as a definite failure.
+ */
+class ScheduleFireError extends Error {
+ constructor(
+ message: string,
+ readonly ambiguous: boolean,
+ ) {
+ super(message);
+ }
+}
+
+/**
+ * Builds the durable trigger envelope for one occurrence. Pure and deterministic in the
+ * fields that {@link getAgentTriggerIdempotencyKey} hashes (only `requestId`/`receivedAt`
+ * vary), so the `deliveryKey` derived from it is stable across retries and is stored on
+ * the reservation BEFORE enqueue — that lets schedule reconciliation read the durable
+ * delivery state even for an ambiguously-committed enqueue.
+ */
+function buildScheduleTriggerEnvelope(
+ schedule: FireableSchedule,
+ user: NonNullable<Awaited<ReturnType<ScheduleEngineDeps['getUserContext']>>>,
+ scheduledFor: Date,
+ files: Awaited<ReturnType<ScheduleEngineDeps['resolveFiles']>>,
+ conversationId: string,
+ manual: boolean,
+): AgentTriggerEnvelope {
+ const occurrenceId = buildFireClientRequestId(schedule.id, scheduledFor);
+ const triggerFiles: JsonValue[] = files.map((file) => {
+ const value: { [key: string]: JsonValue } = { file_id: file.file_id };
+ if (file.filepath != null) value.filepath = file.filepath;
+ if (file.filename != null) value.filename = file.filename;
+ if (file.type != null) value.type = file.type;
+ if (file.height != null) value.height = file.height;
+ if (file.width != null) value.width = file.width;
+ if (file.source != null) value.source = file.source;
+ return value;
+ });
+ return createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId: randomUUID(),
+ deliveryId: occurrenceId,
+ receivedAt: Date.now(),
+ principal: {
+ id: user.id,
+ ...(user.role != null && { role: user.role }),
+ ...(user.tenantId != null && { tenantId: user.tenantId }),
+ },
+ target: { agentId: schedule.agent_id },
+ event: {
+ id: occurrenceId,
+ type: 'schedule.occurrence',
+ occurredAt: scheduledFor.getTime(),
+ source: { id: schedule.id, type: 'schedule' },
+ },
+ input: schedule.prompt,
+ run: {
+ conversationId,
+ timezone: schedule.timezone,
+ ...(triggerFiles.length > 0 && { files: triggerFiles }),
+ metadata: {
+ manual,
+ ...(typeof schedule.configRevision === 'number' && {
+ configRevision: schedule.configRevision,
+ }),
+ },
+ },
+ });
+}
+
+async function enqueueScheduleTrigger(
+ deps: ScheduleEngineDeps,
+ envelope: AgentTriggerEnvelope,
+ orderingKey: string,
+): Promise<void> {
+ try {
+ await deps.enqueueTrigger(envelope, { orderingKey });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ const definite =
+ error instanceof AgentTriggerEnvelopeError ||
+ error instanceof AgentTriggerDeliveryError ||
+ error instanceof AgentTriggerServiceUnavailableError;
+ // A storage error can be returned after Mongo committed the idempotent delivery.
+ // Leave that occurrence reconcilable; a later worker may already own it.
+ throw new ScheduleFireError(`Trigger enqueue failed: ${message}`, !definite);
+ }
+}
+
+/**
+ * Fires one claimed occurrence. The caller owns the lease; this function owns
+ * the run-doc idempotency insert, the skip checks, durable trigger enqueue, and
+ * advancing `nextRunAt` (every path advances so a schedule can never wedge).
+ */
+export async function fireSchedule(
+ deps: ScheduleEngineDeps,
+ schedule: FireableSchedule,
+ limits: ScheduleLimits,
+ scheduledFor: Date,
+ options?: { manual?: boolean; dbNow?: Date },
+): Promise<FireResult> {
+ const { methods } = deps;
+ // Compute the NEXT occurrence relative to the CLAIM's clock (the engine passes
+ // the claim time derived from leaseUntil), keeping the advance self-consistent
+ // with the claim that selected this occurrence. Falls back to the process clock
+ // when no claim time is provided (e.g. manual run-now, which never reschedules
+ // and so ignores the result anyway). Inter-replica skew shifts WHEN an occurrence
+ // fires by at most the skew; it can never fire one twice (lease CAS + the unique
+ // occurrence index) — see LEASE_SKEW_MARGIN_MS in data-schemas for the takeover
+ // margin that protects the only cross-worker clock comparison.
+ const now = options?.dbNow ?? new Date();
+ const nextRunAt = computeNextRunAt({
+ cadence: schedule.cadence,
+ timezone: schedule.timezone,
+ scheduleId: schedule.id,
+ after: new Date(Math.max(now.getTime(), scheduledFor.getTime())),
+ });
+ // Every worker-side schedule write is fenced on the claim token so a stale
+ // worker (lease expired + re-claimed, or the schedule edited/re-enabled/deleted
+ // — all of which rotate the token) cannot clobber the newer authoritative state.
+ const claimToken = schedule.claimToken;
+
+ /**
+ * Rolls back the `started` row this fire reserved. Fenced on the conversation id it
+ * generated rather than on lease ownership: every call site here is a path where
+ * NOTHING was dispatched, and the row can only carry this id if this fire inserted
+ * it (the single-active partial index admits one). Lease state is the wrong fence —
+ * the preflight and capacity allocator can outlast the 5-minute lease, and gating on
+ * a takeover left the undispatched row (and its global capacity slot) stranded until
+ * the 30-minute orphan sweep while the new holder saw `duplicate` and advanced past
+ * the occurrence without firing it.
+ */
+ const rollbackReservation = (conversationId: string) =>
+ methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', conversationId);
+
+ /**
+ * Steps aside from a superseded fire (owner edit/delete, or a lease-expiry re-claim).
+ * `advance()` is fenced on the OLD claim token, which the edit rotated, so it no-ops
+ * and would leave this worker's lease held until its TTL — reporting the edited
+ * schedule / Run now as "already in progress" though no run was dispatched. Releasing
+ * by HOLDER makes it immediately re-claimable, and correctly no-ops after a takeover
+ * (leaseBy changed) so the new holder's lease is never stripped.
+ */
+ const releaseSupersededLease = async () => {
+ if (schedule.leaseBy != null) {
+ await methods.releaseLeaseByHolder(schedule.id, schedule.leaseBy);
+ }
+ };
+
+ /**
+ * Releases a manual run's lease without leaving its old holder behind when an owner
+ * edit rotates the claim token during an asynchronous preflight. The token fence is
+ * authoritative when it still matches; the unique holder fallback can only clear this
+ * fire's lease and safely no-ops if another worker has taken it over.
+ */
+ const releaseManualLease = async () => {
+ const released = await methods.releaseLease(schedule.id, claimToken);
+ if (!released) {
+ await releaseSupersededLease();
+ }
+ return released;
+ };
+
+ // A manual run-now must never reschedule the next automatic occurrence; it only
+ // releases its serialization lease. Both writes are token-fenced, so an owner edit
+ // racing after durable enqueue can make them miss while deliberately preserving the
+ // old holder fields. Release by the unique OLD holder on a miss: an edit is unwedged
+ // immediately, while a takeover changed leaseBy and remains untouched.
+ const advance = async () => {
+ if (options?.manual) {
+ return releaseManualLease();
+ }
+ const advanced = await methods.advanceSchedule(
+ schedule.id,
+ nextRunAt,
+ scheduledFor,
+ claimToken,
+ );
+ if (!advanced) {
+ await releaseSupersededLease();
+ }
+ return advanced;
+ };
+
+ /**
+ * Steps aside after a failed revalidation WITHOUT advancing. `advance()` here is a
+ * designed no-op in every true supersession (takeover, owner edit — both rotate the
+ * claim token, so its token-fenced filter misses), which means the only case where it
+ * actually LANDS is the one it must not: a PURE lease expiry with no takeover, where
+ * the token never rotated. advanceSchedule checks no lease, so the advance moved
+ * nextRunAt past an occurrence nothing had fired — a preflight outlasting the lease
+ * silently dropped it instead of leaving it due for the next claim to retry.
+ * Manual run-now still releases its serialization lease (release-only, no advance)
+ * so a repeat click isn't met with a stale "already in progress".
+ */
+ const stepAsideSuperseded = async () => {
+ if (options?.manual) {
+ await releaseManualLease();
+ } else {
+ await releaseSupersededLease();
+ }
+ return { fired: false, skipped: 'superseded' as const };
+ };
+
+ if (nextRunAt == null) {
+ await methods.disableSchedule(schedule.id, 'invalid_schedule', claimToken);
+ await advance();
+ return { fired: false, error: 'No next occurrence computable' };
+ }
+
+ const user = await deps.getUserContext(schedule.user);
+ if (user == null) {
+ await methods.disableSchedule(schedule.id, 'permission_revoked', claimToken);
+ await advance();
+ return { fired: false, skipped: 'user_missing' };
+ }
+
+ return deps.runInTenantContext(user, async () => {
+ // Re-resolve limits for the OWNER (per-principal role/user + tenant config):
+ // a tenant- or role-specific config (disabled schedules, different
+ // auto-disable threshold) must win over the base config the engine read.
+ const ownerLimits = await deps.getLimits(user);
+ if (!ownerLimits.enabled) {
+ await advance();
+ return { fired: false, skipped: 'disabled' as const };
+ }
+
+ // Enforce a raised interval floor at fire time: create/update reject too-frequent
+ // cadences, but an admin raising the floor later must also stop an already-enabled
+ // schedule that now runs more often than policy allows.
+ if (cadenceIntervalMinutes(schedule.cadence) < ownerLimits.minIntervalMinutes) {
+ await methods.disableSchedule(schedule.id, 'invalid_schedule', claimToken);
+ await advance();
+ return { fired: false, skipped: 'disabled' as const };
+ }
+
+ // Account-deletion barrier, re-checked at the DISPATCH boundary. Admission (the
+ // create/update/run-now handlers) is the primary gate, but there is always a window
+ // between admission and persistence, so the owner is re-checked immediately before a
+ // billed generation is dispatched. Skips silently: the deletion cascade owns the row.
+ if (await deps.isOwnerDeleting(user.id)) {
+ await advance();
+ return { fired: false, skipped: 'user_deleting' as const };
+ }
+
+ // Re-check the owner's live schedule permission: a role that lost
+ // SCHEDULES access after the schedule was created must stop firing.
+ if (!(await deps.hasScheduleAccess(user))) {
+ await methods.disableSchedule(schedule.id, 'permission_revoked', claimToken);
+ await advance();
+ return { fired: false, skipped: 'permission_revoked' as const };
+ }
+
+ // The owner-config generation this occurrence was CLAIMED under, stamped onto every
+ // run row (fired or skipped) so bookkeeping can derive its fence from the row.
+ const claimedRevision =
+ typeof schedule.configRevision === 'number'
+ ? { configRevision: schedule.configRevision }
+ : {};
+
+ const baseRun = {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ tenantId: schedule.tenantId,
+ scheduledFor,
+ };
+
+ const agentAccess = await deps.agentAccess(schedule.agent_id, user);
+ if (agentAccess !== 'ok') {
+ // 'missing' → deleted; 'forbidden' → the owner's VIEW access was revoked.
+ // Disable immediately instead of letting the loopback chat reject the run
+ // and burn attempts toward the failure threshold.
+ const reason = agentAccess === 'missing' ? 'agent_deleted' : 'permission_revoked';
+ await methods.disableSchedule(schedule.id, reason, claimToken);
+ await advance();
+ return { fired: false, skipped: reason };
+ }
+
+ if (await deps.isOutOfBalance(user)) {
+ // Revalidate BEFORE writing the skip. Everything above (user, config, permission
+ // and balance lookups) can outlast the 5-minute lease, and a skip is not a no-op:
+ // it stamps the card and walks the balance-skip streak toward auto-disable. Under
+ // a dead claim — lease taken over, schedule deleted, account deletion begun — that
+ // is a write on behalf of a fire that no longer owns this occurrence. The reserve
+ // path below has its own revalidation for the same reason; this one guards the
+ // branch that returns before ever reaching it.
+ if (
+ claimToken != null &&
+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))
+ ) {
+ return stepAsideSuperseded();
+ }
+ // Skip rows carry no `bookkept:false` marker, so the reconciler has no path to
+ // repair a half-applied skip. If the schedule-side bookkeeping throws, do NOT
+ // advance: leave the occurrence due so the next claim retries it (the insert is
+ // duplicate-guarded and the same-skip retry path is idempotent).
+ try {
+ await methods.recordSkippedRun(
+ { ...baseRun, status: 'skipped_balance', ...claimedRevision },
+ BALANCE_SKIP_DISABLE_THRESHOLD,
+ );
+ } catch (skipError) {
+ logger.error(`[schedules] balance-skip bookkeeping failed for ${schedule.id}:`, skipError);
+ return { fired: false, skipped: 'balance' as const };
+ }
+ await advance();
+ return { fired: false, skipped: 'balance' as const };
+ }
+
+ // Resolve attachments BEFORE claiming the run row: a transient file-query
+ // failure here must not orphan a `started` run that consumes capacity.
+ const requestedFileIds = schedule.file_ids ?? [];
+ let files: Awaited<ReturnType<ScheduleEngineDeps['resolveFiles']>>;
+ try {
+ files = requestedFileIds.length ? await deps.resolveFiles(requestedFileIds, user) : [];
+ } catch (fileError) {
+ logger.error(
+ `[schedules] file resolution failed for ${schedule.id} (will retry):`,
+ fileError,
+ );
+ // nextRunAt is untouched so the occurrence retries. Automatic claims keep the
+ // lease as a backoff (releasing it would let the nextRunAt-sorted claimer
+ // re-pick this failing row every tick and starve others / hammer the file
+ // lookup); manual run-now releases so the user can retry immediately.
+ if (options?.manual) {
+ await releaseManualLease();
+ }
+ return { fired: false, error: 'File resolution failed' };
+ }
+ const droppedFileIds = requestedFileIds.filter(
+ (id) => !files.some((file) => file.file_id === id),
+ );
+
+ // Revalidate BEFORE reserving, not only before the POST below. The preflight above
+ // (user, config, permission, balance, attachment queries) can outlast the 5-minute
+ // lease, and a stale worker that reserves anyway wins the occurrence's unique row:
+ // the fresh claimer then sees `duplicate` and advances without firing, while this
+ // worker's own revalidation fails and rollbackReservation deliberately RETAINS the
+ // row (the lease takeover changed leaseBy). The occurrence is lost and its global
+ // capacity slot stays held until the 30-minute orphan sweep. Nothing is reserved yet
+ // here, so a superseded fire simply steps aside.
+ if (
+ claimToken != null &&
+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))
+ ) {
+ return stepAsideSuperseded();
+ }
+
+ // SHUTDOWN gate at the dispatch boundary: the coordinator closes the listener
+ // BEFORE the engine's pre-drain task runs, so a pass already past its preflight
+ // would POST at a refusing socket — a definite connect failure that terminalizes
+ // as `error` and walks a healthy schedule toward auto-disable for nothing more
+ // than a restart. Nothing is reserved yet, so stepping aside leaves the
+ // occurrence due for the restarted process (within the misfire grace).
+ if (deps.isShuttingDown?.() === true) {
+ logger.info(`[schedules] shutdown in progress; releasing claim on ${schedule.id}`);
+ return stepAsideSuperseded();
+ }
+
+ // Pre-generate the conversation id and reserve the run row up front. The
+ // loopback POST reuses it (streamId === conversationId), so reconciliation can
+ // ALWAYS locate this occurrence's job — even if the post-accept detail write
+ // fails — instead of mislabeling an accepted (or preserved) run as an orphan.
+ // reserveStartedRun is the atomic overlap guard: the single-active partial index
+ // rejects a second `started` run for the schedule, so a concurrent occurrence
+ // surfaces as 'overlap' with no read-then-insert race.
+ const conversationId = randomUUID();
+ // Build the durable trigger envelope NOW so the deterministic deliveryKey can be
+ // stored on the reservation below — before enqueue. That way schedule reconciliation
+ // can read the delivery state (live vs dead) even when the enqueue commits ambiguously,
+ // instead of orphaning a still-live delivery after the 30-minute cutoff. A schedule
+ // whose envelope cannot be built can never fire, so skip WITHOUT reserving a phantom
+ // `started` run (a definite, retry-invariant failure).
+ let triggerEnvelope: AgentTriggerEnvelope;
+ try {
+ triggerEnvelope = buildScheduleTriggerEnvelope(
+ schedule,
+ user,
+ scheduledFor,
+ files,
+ conversationId,
+ options?.manual === true,
+ );
+ } catch (envelopeError) {
+ logger.error(`[schedules] trigger envelope build failed for ${schedule.id}:`, envelopeError);
+ if (options?.manual) {
+ await releaseManualLease();
+ } else {
+ await advance();
+ }
+ return {
+ fired: false,
+ error: envelopeError instanceof Error ? envelopeError.message : String(envelopeError),
+ };
+ }
+ const deliveryKey = getAgentTriggerIdempotencyKey(triggerEnvelope);
+ // The GLOBAL fireConcurrency cap is enforced by claiming a unique capacity slot in
+ // the SAME insert that reserves the run, so it is decided by the DB rather than by
+ // a count read before the write. The allocator advances to the next free slot when
+ // another admission wins one, and reports 'capacity' only when genuinely saturated.
+ // Occupancy is read system-scoped so the cap stays global across tenants.
+ // CLAMPED to the deployment-wide cap. The slots are global across every owner, so a
+ // role/user/tenant override must never be able to WIDEN them: manual Run Now resolves
+ // the owner's limits and bypasses the engine tick's base-config budget entirely, so
+ // an override of 5 against a base of 1 would otherwise let concurrent clicks occupy
+ // slots 0-4 and run five billed generations at once. Re-read without a principal for
+ // the base value (the same read the tick budgets from); a STRICTER owner value still
+ // applies, since only widening is the defect.
+ const deploymentLimits = await deps.getLimits();
+ const allocation = await deps.withGlobalCapacitySlot(
+ Math.min(ownerLimits.fireConcurrency, deploymentLimits.fireConcurrency),
+ async (capacitySlot) => {
+ const attempt = await methods.reserveStartedRun({
+ ...baseRun,
+ conversationId,
+ firedAt: new Date(),
+ capacitySlot,
+ deliveryKey,
+ ...(typeof schedule.configRevision === 'number'
+ ? { configRevision: schedule.configRevision }
+ : {}),
+ });
+ return 'conflict' in attempt && attempt.conflict === 'slot-taken'
+ ? 'slot-taken'
+ : { claimed: attempt };
+ },
+ );
+ if (allocation === 'capacity') {
+ // Automatic claims keep the claim's lease as a backoff so the nextRunAt-sorted
+ // claimer doesn't immediately re-pick this row and starve others; nextRunAt is
+ // untouched, so the occurrence retries once the lease expires. A manual run-now
+ // MUST release its lease, or repeated Run-now clicks hit a misleading "already
+ // in progress" 409 for the full manual-lease TTL even after capacity frees.
+ if (options?.manual) {
+ await releaseManualLease();
+ }
+ return { fired: false, skipped: 'capacity' as const };
+ }
+ const reservation = allocation.claimed;
+ if ('conflict' in reservation) {
+ if (reservation.conflict === 'overlap') {
+ // Another occurrence of this schedule is already active. Record the skip
+ // (its own occurrence row) and advance past this one.
+ try {
+ await methods.recordSkippedRun({
+ ...baseRun,
+ status: 'skipped_overlap',
+ ...claimedRevision,
+ });
+ } catch (skipError) {
+ // Same reasoning as the balance skip: no reconciliation path repairs a
+ // half-applied skip, so leave the occurrence due rather than advancing past it.
+ logger.error(
+ `[schedules] overlap-skip bookkeeping failed for ${schedule.id}:`,
+ skipError,
+ );
+ return { fired: false, skipped: 'overlap' as const };
+ }
+ await advance();
+ return { fired: false, skipped: 'overlap' as const };
+ }
+ // A duplicate means another fire already holds this occurrence's row — but
+ // "still running it" and "already finished with it" need OPPOSITE handling.
+ //
+ // TERMINAL: the occurrence is done and merely never advanced past (its fire was
+ // accepted but the post-accept advance failed). nextRunAt still points here, so
+ // refusing to advance makes every future claim re-pick the same settled
+ // occurrence — a permanent stall. Advance past it.
+ //
+ // ACTIVE: another worker owns it. Advancing hands the occurrence away — if that
+ // worker is a stale lease holder whose own revalidation then fails, it rolls its
+ // undispatched row back and nothing ever fires this occurrence. Leave nextRunAt
+ // alone so it stays claimable; whichever worker actually dispatches is the one
+ // that advances, and the claim's lease is the retry backoff (the same shape as
+ // `capacity` above). A crashed holder's row is cleared by the orphan sweep, after
+ // which the occurrence reserves cleanly.
+ const settledAlready =
+ reservation.existingStatus != null &&
+ reservation.existingStatus !== 'started' &&
+ reservation.existingStatus !== 'requires_action';
+ if (settledAlready) {
+ await advance();
+ return { fired: false, skipped: 'duplicate' as const };
+ }
+ // Manual run-now still releases its lease so repeated clicks aren't met with a
+ // stale "already in progress".
+ if (options?.manual) {
+ await releaseManualLease();
+ }
+ return { fired: false, skipped: 'duplicate' as const };
+ }
+
+ // Last check before the point of no return: re-verify this fire still holds an
+ // authoritative claim (same claim token, lease unexpired, not deleting; and for
+ // an automatic fire, still enabled). An owner delete/edit or a lease-expiry
+ // re-claim landing AFTER the lease was taken but before here supersedes this
+ // fire — roll the reservation back (status-fenced) and skip WITHOUT dispatching
+ // a billed generation. Manual run-now still revalidates (a delete/edit can land
+ // after acquireManualRunLease); it only relaxes the `enabled` check since the
+ // user explicitly triggered it.
+ if (
+ claimToken != null &&
+ !(await methods.revalidateClaim(schedule.id, claimToken, !options?.manual))
+ ) {
+ await rollbackReservation(conversationId);
+ return stepAsideSuperseded();
+ }
+
+ // SHUTDOWN recheck immediately before dispatch: the gate before the reservation
+ // is not the last dispatch-boundary operation — the deployment-limit read,
+ // capacity allocation, and claim revalidation all run after it and can overlap
+ // SIGTERM, leaving the listener closing by the time the POST goes out. Roll the
+ // reservation back (status-fenced) and step aside without advancing: the
+ // occurrence stays due for the restarted process within the misfire grace.
+ if (deps.isShuttingDown?.() === true) {
+ logger.info(
+ `[schedules] shutdown in progress; rolling back reserved dispatch of ${schedule.id}`,
+ );
+ await rollbackReservation(conversationId);
+ return stepAsideSuperseded();
+ }
+
+ try {
+ await enqueueScheduleTrigger(deps, triggerEnvelope, schedule.id);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ const ambiguous = error instanceof ScheduleFireError && error.ambiguous;
+ if (ambiguous) {
+ // Mongo may have accepted the durable delivery, which can later start a
+ // generation and call recordScheduleOutcome. Leave the run `started` (a
+ // reconcilable, non-terminal state) so that completion can finalize it and
+ // overlap/capacity keep seeing it; the orphan sweep settles it otherwise.
+ // Do NOT terminalize here — `interrupted` would block the real outcome.
+ logger.warn(
+ `[schedules] trigger enqueue ambiguously failed for ${schedule.id} (left reconcilable):`,
+ error,
+ );
+ await advance();
+ return { fired: false, error: message };
+ }
+ // Definite enqueue rejection: no durable delivery exists and nothing started.
+ logger.error(`[schedules] trigger enqueue rejected for ${schedule.id}:`, error);
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ error: message,
+ autoDisableAfterFailures: ownerLimits.autoDisableAfterFailures,
+ });
+ await advance();
+ return { fired: false, error: message };
+ }
+
+ // The trigger delivery is durable: the eventual chat reports its own terminal
+ // outcome via the completion hook. Post-enqueue bookkeeping failures must NOT
+ // flip the run to `error` (that would block the real completion, which only
+ // matches started/paused).
+ try {
+ await advance();
+ await methods.setRunFireDetails(schedule.id, scheduledFor, {
+ conversationId,
+ ...(droppedFileIds.length > 0 ? { droppedFileIds } : {}),
+ });
+ if (droppedFileIds.length > 0) {
+ logger.warn(
+ `[schedules] ${schedule.id} fired without ${droppedFileIds.length} missing attachment(s)`,
+ );
+ }
+ } catch (bookkeepingError) {
+ logger.error(
+ `[schedules] post-enqueue bookkeeping failed for ${schedule.id} (run continues):`,
+ bookkeepingError,
+ );
+ }
+ return { fired: true, conversationId };
+ });
+}
diff --git a/packages/api/src/schedules/gate.spec.ts b/packages/api/src/schedules/gate.spec.ts
new file mode 100644
index 0000000000..6d2199707f
--- /dev/null
+++ b/packages/api/src/schedules/gate.spec.ts
@@ -0,0 +1,162 @@
+import type { SchedulesServiceDeps } from './service';
+import { createSchedulesService } from './service';
+
+/** Swappable per test: whether the stream store is SHARED across replicas. */
+let mockIsRedis = false;
+
+jest.mock('../stream/GenerationJobManager', () => ({
+ GenerationJobManager: {
+ getJobStore: () => null,
+ abortJob: jest.fn(),
+ get isRedis() {
+ return mockIsRedis;
+ },
+ },
+}));
+
+type Cfg = { interfaceConfig?: { schedules?: unknown } };
+
+function makeService(base: Cfg, merged: Cfg = base) {
+ const deps = {
+ methods: {
+ countActiveRuns: jest.fn(async () => 0),
+ getCapacityOccupancy: jest.fn(async () => ({ takenSlots: [], unslotted: 0 })),
+ ensureScheduleIndexes: jest.fn(async () => undefined),
+ acquireManualRunLease: jest.fn(async () => null),
+ },
+ getAppConfig: jest.fn(async (options?: { baseOnly?: boolean }) =>
+ options?.baseOnly === true ? base : merged,
+ ),
+ findUserById: jest.fn(async () => null),
+ findBalance: jest.fn(async () => null),
+ upsertBalance: jest.fn(async () => null),
+ initializeNullBalance: jest.fn(async () => null),
+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
+ isUserDeleting: jest.fn(async () => false),
+ enqueueAgentTrigger: jest.fn(async () => undefined),
+ getTriggerDelivery: jest.fn(async () => null),
+ } as unknown as SchedulesServiceDeps;
+ return createSchedulesService(deps);
+}
+
+const schedule = { id: 's1', user: 'u1' } as never;
+const limits = {
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+};
+
+describe('v1 experimental gate, asserted at real entry points', () => {
+ beforeEach(() => {
+ // The harness mocks an in-memory job store, so every arming assertion below needs
+ // the single-replica assertion the topology gate demands (see isTopologySafeToArm).
+ process.env.SCHEDULES_SINGLE_PROCESS = 'true';
+ });
+
+ afterEach(() => {
+ mockIsRedis = false;
+ delete process.env.SCHEDULES_DISABLED;
+ delete process.env.SCHEDULES_SINGLE_PROCESS;
+ });
+
+ it('REFUSES to arm on a process-local job store with no single-replica assertion', async () => {
+ delete process.env.SCHEDULES_SINGLE_PROCESS;
+ // The standard entrypoint arms the scheduler in EVERY replica. With a process-local
+ // store a peer sees the globally visible `started` row but not its job, and after the
+ // orphan cutoff marks a still-running generation interrupted and frees its capacity.
+ // A process cannot count its own replicas, so unproven means refuse.
+ const service = makeService({ interfaceConfig: { schedules: true } });
+ expect(await service.initializeScheduleEngine()).toBeUndefined();
+ });
+
+ it('arms on a SHARED job store without any assertion, since replicas see each other', async () => {
+ delete process.env.SCHEDULES_SINGLE_PROCESS;
+ mockIsRedis = true;
+ const service = makeService({ interfaceConfig: { schedules: true } });
+ expect(await service.initializeScheduleEngine()).toBeDefined();
+ });
+
+ it('is OFF for limits when the admin never opted in', async () => {
+ expect((await makeService({}).getLimits()).enabled).toBe(false);
+ });
+
+ it('STILL arms the engine when globally stopped, so reconciliation can settle prior state', async () => {
+ // The engine owns firing AND reconciliation. Refusing to start would strand `started`
+ // rows and preserved jobs left by a previous process until scheduling is re-enabled.
+ // Firing is gated separately (runTick + getLimits), so nothing fires.
+ const service = makeService({ interfaceConfig: { schedules: false } });
+ expect(await service.initializeScheduleEngine()).toBeDefined();
+ expect((await service.getLimits()).enabled).toBe(false);
+ });
+
+ it('reports limits DISABLED under the SCHEDULES_DISABLED lever, so writes and fires refuse', async () => {
+ process.env.SCHEDULES_DISABLED = 'true';
+ // Even with an explicit opt-in, the env stop must be visible wherever limits are
+ // consulted — not only at the engine tick.
+ const service = makeService({ interfaceConfig: { schedules: true } });
+ expect((await service.getLimits()).enabled).toBe(false);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
+ });
+
+ it('DOES arm the engine when the base is merely absent, so principal-scoped enables work', async () => {
+ // Gating engine start on the base config would never start it for a role/user that
+ // enables schedules via override, and would leave schedulesReady false so their
+ // writes are rejected outright. Absent base is not a global stop; the fire path and
+ // write handlers still refuse owners who do not have it enabled.
+ const service = makeService({}, { interfaceConfig: { schedules: { use: true } } });
+ expect(await service.initializeScheduleEngine()).toBeDefined();
+ });
+
+ it('stays OFF for an explicit false', async () => {
+ expect((await makeService({ interfaceConfig: { schedules: false } }).getLimits()).enabled).toBe(
+ false,
+ );
+ });
+
+ it('turns ON for an explicit opt-in', async () => {
+ expect((await makeService({ interfaceConfig: { schedules: true } }).getLimits()).enabled).toBe(
+ true,
+ );
+ const tuned = makeService({ interfaceConfig: { schedules: { maxPerUser: 3 } } });
+ const resolved = await tuned.getLimits();
+ expect(resolved.enabled).toBe(true);
+ expect(resolved.maxPerUser).toBe(3);
+ });
+
+ it('REFUSES a manual run-now while the global kill switch is on', async () => {
+ // Run Now dispatches the same billed generation as an automatic fire, so gating only
+ // the engine tick would leave this path open.
+ process.env.SCHEDULES_DISABLED = 'true';
+ const service = makeService({ interfaceConfig: { schedules: true } });
+ const result = await service.fireScheduleNow(schedule, limits);
+ expect(result).toEqual({ fired: false, skipped: 'disabled' });
+ });
+
+ it('refuses run-now when the BASE config disables it, even if a principal re-enables', async () => {
+ const service = makeService(
+ { interfaceConfig: { schedules: false } },
+ { interfaceConfig: { schedules: true } },
+ );
+ const result = await service.fireScheduleNow(schedule, limits);
+ expect(result).toEqual({ fired: false, skipped: 'disabled' });
+ });
+
+ it('constructing the service without a required dep fails LOUDLY at boot', () => {
+ // The JS adapter is not typechecked against SchedulesServiceDeps, which is how the
+ // deletion-barrier probe shipped unwired twice. A missing dep must not surface as a
+ // cryptic per-fire "is not a function".
+ expect(() =>
+ createSchedulesService({
+ methods: {},
+ getAppConfig: jest.fn(),
+ findUserById: jest.fn(),
+ findBalance: jest.fn(),
+ upsertBalance: jest.fn(),
+ initializeNullBalance: jest.fn(),
+ resolveAgentFireAccess: jest.fn(),
+ } as unknown as SchedulesServiceDeps),
+ ).toThrow(/isUserDeleting/);
+ });
+});
diff --git a/packages/api/src/schedules/handlers.spec.ts b/packages/api/src/schedules/handlers.spec.ts
new file mode 100644
index 0000000000..0c32909a1c
--- /dev/null
+++ b/packages/api/src/schedules/handlers.spec.ts
@@ -0,0 +1,858 @@
+import type { ISchedule } from '@librechat/data-schemas';
+import type { Response } from 'express';
+import type { SchedulesHandlersDeps } from './handlers';
+import type { ServerRequest } from '~/types';
+import { createSchedulesHandlers, toWireSchedule, computeCreateDigest } from './handlers';
+
+/** A lean schedule doc carrying both public fields and internal bookkeeping. */
+function fullScheduleDoc(): ISchedule {
+ return {
+ _id: 'mongo-id',
+ __v: 0,
+ id: 'sched-1',
+ user: 'user-1',
+ tenantId: 't1',
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ file_ids: ['file-1'],
+ enabled: true,
+ disabledReason: undefined,
+ nextRunAt: new Date('2026-07-21T12:00:00Z'),
+ lastRun: { conversationId: 'c1', status: 'success', firedAt: new Date() },
+ runCount: 3,
+ failureCount: 0,
+ // Internal bookkeeping that must NEVER reach the browser.
+ claimToken: 'ct-secret',
+ leaseUntil: new Date(),
+ leaseBy: 'inst-1',
+ slot: 2,
+ deleting: false,
+ countedFor: [new Date()],
+ balanceSkipCount: 1,
+ bookkept: true,
+ createdAt: new Date('2026-07-01T00:00:00Z'),
+ updatedAt: new Date('2026-07-10T00:00:00Z'),
+ } as unknown as ISchedule;
+}
+
+describe('toWireSchedule', () => {
+ const INTERNAL_FIELDS = [
+ '_id',
+ '__v',
+ 'tenantId',
+ 'claimToken',
+ 'leaseUntil',
+ 'leaseBy',
+ 'slot',
+ 'deleting',
+ 'countedFor',
+ 'balanceSkipCount',
+ 'bookkept',
+ ];
+
+ it('emits exactly the public TSchedule fields', () => {
+ const wire = toWireSchedule(fullScheduleDoc());
+ expect(Object.keys(wire).sort()).toEqual(
+ [
+ 'agent_id',
+ 'cadence',
+ // Public so the edit dialog can fence its PATCH on the revision it opened
+ // with (updateSchedulePayloadSchema.expectedConfigRevision).
+ 'configRevision',
+ 'createdAt',
+ 'disabledReason',
+ 'enabled',
+ 'failureCount',
+ 'file_ids',
+ 'id',
+ 'lastRun',
+ 'name',
+ 'nextRunAt',
+ 'prompt',
+ 'runCount',
+ 'target',
+ 'timezone',
+ 'updatedAt',
+ 'user',
+ ].sort(),
+ );
+ });
+
+ it('leaks no internal bookkeeping field', () => {
+ const wire = toWireSchedule(fullScheduleDoc()) as Record<string, unknown>;
+ for (const field of INTERNAL_FIELDS) {
+ expect(wire).not.toHaveProperty(field);
+ }
+ });
+
+ it('preserves the public field values', () => {
+ const wire = toWireSchedule(fullScheduleDoc());
+ expect(wire.id).toBe('sched-1');
+ expect(wire.name).toBe('Digest');
+ expect(wire.runCount).toBe(3);
+ expect(wire.cadence).toEqual({ frequency: 'daily', hour: 8, minute: 0 });
+ });
+});
+
+/** Minimal Express double capturing the status/body the handler settled on. */
+function makeRes() {
+ const captured: { status?: number; body?: unknown; headers: Record<string, string> } = {
+ headers: {},
+ };
+ const res = {
+ status(code: number) {
+ captured.status = code;
+ return this;
+ },
+ json(payload: unknown) {
+ captured.body = payload;
+ return this;
+ },
+ set(name: string, value: string) {
+ captured.headers[name] = value;
+ return this;
+ },
+ };
+ return { res: res as unknown as Response, captured };
+}
+
+const CREATE_BODY = {
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily' as const, hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ clientRequestId: 'intent-1',
+};
+
+function makeCreateReq(): ServerRequest {
+ return {
+ body: { ...CREATE_BODY },
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ } as unknown as ServerRequest;
+}
+
+/** The digest a genuine retry of CREATE_BODY carries (zod defaults applied). */
+function createBodyDigest(): string {
+ return computeCreateDigest({ ...CREATE_BODY, target: 'new', enabled: true });
+}
+
+function makeCreateDeps(over: Partial<SchedulesHandlersDeps> = {}): SchedulesHandlersDeps {
+ const methods = {
+ countSchedulesByUser: jest.fn(async () => 0),
+ createScheduleWithSlot: jest.fn(async (data: Partial<ISchedule>) => data as ISchedule),
+ getScheduleByClientRequestId: jest.fn(async () => null),
+ getScheduleById: jest.fn(async () => null),
+ deleteScheduleById: jest.fn(async () => true),
+ deleteUnarmedSchedule: jest.fn(async () => 'deleted'),
+ markScheduleDeleting: jest.fn(async () => ({ id: 'sched-1' }) as ISchedule),
+ updateScheduleById: jest.fn(async () => ({ id: 'sched-1' }) as ISchedule),
+ armSchedule: jest.fn(async () => undefined),
+ };
+ return {
+ methods: methods as unknown as SchedulesHandlersDeps['methods'],
+ getLimits: async () => ({
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+ }),
+ canViewAgent: async () => true,
+ filterOwnedFileIds: async (ids: string[]) => ids,
+ markFilesUsed: async () => undefined,
+ fireNow: async () => null,
+ deleteSchedule: async () => true,
+ // Passes admission, then the barrier rises before the insert lands.
+ isUserDeleting: jest.fn().mockResolvedValueOnce(false).mockResolvedValue(true),
+ ...over,
+ } as SchedulesHandlersDeps;
+}
+
+describe('createSchedule late-create compensation', () => {
+ it('answers 410 when the guarded delete removes the unarmed revision', async () => {
+ const deps = makeCreateDeps();
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+ // Guarded to the exact unarmed revision this attempt inserted — never the
+ // unconditional hard delete, which could erase a row a concurrent replay armed
+ // (and the engine claimed) before the barrier re-check ran.
+ expect(deps.methods.deleteUnarmedSchedule).toHaveBeenCalledWith(
+ expect.any(String),
+ 'user-1',
+ 0,
+ );
+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();
+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();
+ expect(captured.status).toBe(410);
+ });
+
+ it('soft-deletes instead when the row moved past the inserted revision', async () => {
+ const deps = makeCreateDeps();
+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('kept');
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+ // An armed/claimed (or edited) row enters the ordinary drain-then-erase teardown
+ // rather than being hard-deleted out from under a possible live fire.
+ expect(deps.methods.markScheduleDeleting).toHaveBeenCalledWith(expect.any(String), 'user-1');
+ expect(captured.status).toBe(410);
+ });
+
+ it('falls back to the durable soft-delete when the guarded delete fails', async () => {
+ const deps = makeCreateDeps();
+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+ // Non-claimable at once, and erased by the reconciler's `deleting` sweep.
+ expect(deps.methods.markScheduleDeleting).toHaveBeenCalledWith(expect.any(String), 'user-1');
+ expect(captured.status).toBe(410);
+ });
+
+ it('refuses to report a clean 410 when no cleanup succeeded', async () => {
+ const deps = makeCreateDeps();
+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ (deps.methods.markScheduleDeleting as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+ // 410 would claim the row is gone while the deleted user's prompt and
+ // attachments stay in a live, non-TTL schedule.
+ expect(captured.status).toBe(500);
+ });
+
+ /**
+ * The compensation is best-effort by nature — when it runs, Mongo is usually the thing
+ * failing, so no durable marker can be written either. Durability has to come from the
+ * insert instead: an UNARMED row (no nextRunAt) is never claimed by the engine, so
+ * even a total compensation failure cannot leave a schedule firing billed generations
+ * for an account already being erased.
+ */
+ it('never inserts an armed row, so a failed compensation leaves an inert one', async () => {
+ const deps = makeCreateDeps();
+ (deps.methods.deleteScheduleById as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ (deps.methods.markScheduleDeleting as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const { res } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];
+ expect(inserted.nextRunAt).toBeUndefined();
+ // And it is never armed afterwards, because the barrier refused the create.
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('arms the schedule only after the barrier re-check clears', async () => {
+ const armedRow = {
+ ...CREATE_BODY,
+ id: 'sched-fresh',
+ target: 'new',
+ enabled: true,
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ } as unknown as ISchedule;
+ const deps = makeCreateDeps({
+ isUserDeleting: jest.fn(async () => false),
+ });
+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(true);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armedRow);
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status).toBe(201);
+ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];
+ expect(inserted.nextRunAt).toBeUndefined();
+ // The ONE shared arming CAS, fenced on the revision stamped at insert — never
+ // updateScheduleById, whose token rotation would fence an engine claim off a row
+ // a concurrent replay armed first.
+ expect(deps.methods.armSchedule).toHaveBeenCalledWith(expect.any(String), expect.any(Date), 0);
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('stamps the immutable create digest on the inserted row', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const { res } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ const inserted = (deps.methods.createScheduleWithSlot as jest.Mock).mock.calls[0][0];
+ expect(inserted.clientRequestDigest).toBe(createBodyDigest());
+ });
+
+ it('answers 201 with the current row when a concurrent PATCH wins the arming CAS', async () => {
+ const patched = {
+ ...CREATE_BODY,
+ id: 'sched-fresh',
+ target: 'new',
+ enabled: true,
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ } as unknown as ISchedule;
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(false);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(patched);
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ // The revision moved: the PATCH's own arming governs, and this POST must not
+ // overwrite it — it reports the row as the edit left it.
+ expect(captured.status).toBe(201);
+ expect((captured.body as { nextRunAt?: Date }).nextRunAt).toEqual(patched.nextRunAt);
+ });
+
+ /**
+ * updateScheduleById filters out rows marked `deleting`, so a null arming result means
+ * the deletion cascade claimed this row between the barrier re-check and the arming
+ * write. Reporting the pre-delete snapshot as a 201 tells the client a schedule exists
+ * that is already hidden and pending erasure.
+ */
+ it('does not report success when the arming write loses a delete race', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.updateScheduleById as jest.Mock).mockResolvedValue(null);
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status).toBe(410);
+ expect(captured.status).not.toBe(201);
+ });
+
+ /**
+ * A thrown arming write rolls the committed row back — but ONLY while it is still
+ * the unarmed, unedited revision this attempt inserted: an ambiguously-committed
+ * arm, a concurrent replay of this key that armed the row and already answered
+ * 201, or a concurrent PATCH that edited it all leave a row that must survive.
+ * deleteUnarmedSchedule carries exactly that guard (revision-fenced).
+ */
+ it('rolls back the committed row when the arming write throws', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status).toBe(500);
+ expect(deps.methods.deleteUnarmedSchedule).toHaveBeenCalledWith(
+ expect.any(String),
+ 'user-1',
+ 0,
+ );
+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('answers 201 instead of deleting when the ambiguous arm actually committed', async () => {
+ const armedRow = {
+ ...CREATE_BODY,
+ id: 'sched-fresh',
+ target: 'new',
+ enabled: true,
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ } as unknown as ISchedule;
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('socket reset'));
+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('kept');
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armedRow);
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ // The write landed before the error surfaced: the schedule exists exactly as
+ // requested, and deleting it would erase a row a concurrent replay may already
+ // have confirmed with its own 201.
+ expect(captured.status).toBe(201);
+ });
+});
+
+describe('create idempotency', () => {
+ /** The row the FIRST attempt committed for CREATE_BODY, digest-less (legacy shape). */
+ const originalRow = (): ISchedule =>
+ ({
+ id: 'sched-original',
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ clientRequestId: 'intent-1',
+ configRevision: 0,
+ }) as unknown as ISchedule;
+
+ /**
+ * The retry is the whole point: the first attempt committed a row and then failed to
+ * arm, so the client cannot know what persisted. Resolving to the ORIGINAL row is
+ * what keeps one user intent from becoming two recurring schedules.
+ */
+ it('re-arms the row a previous attempt committed instead of creating another', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const original = originalRow();
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(original);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue({
+ ...original,
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ });
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status ?? 201).toBe(201);
+ // Nothing was inserted, and the arm is the GUARDED one keyed by the ORIGINAL row's
+ // id (same deterministic jitter as the first attempt) — never updateScheduleById,
+ // whose claim-token rotation would fence off an active run of this schedule and
+ // whose revision bump would break a concurrent PATCH's CAS.
+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();
+ expect(deps.methods.armSchedule).toHaveBeenCalledWith('sched-original', expect.any(Date));
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('leaves an already-armed row completely untouched on replay', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(armed);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status ?? 201).toBe(201);
+ expect(deps.methods.armSchedule).not.toHaveBeenCalled();
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ expect(deps.methods.deleteUnarmedSchedule).not.toHaveBeenCalled();
+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();
+ });
+
+ /**
+ * A retry can arrive while the user sits AT their schedule cap — with the first
+ * attempt's row occupying the final slot. The key must resolve before any capacity
+ * refusal, or the client is denied the very row it is trying to confirm.
+ */
+ it('resolves a retry occupying the final slot before the capacity pre-check', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.countSchedulesByUser as jest.Mock).mockResolvedValue(10);
+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(armed);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status ?? 201).toBe(201);
+ });
+
+ it('resolves a concurrent retry that hits the allocator limit to the committed row', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const armed = { ...originalRow(), nextRunAt: new Date('2026-07-22T12:00:00Z') };
+ // Absent at the pre-insert lookup, present after the allocator reports 'limit':
+ // the concurrent first attempt committed in between and took the last slot.
+ (deps.methods.getScheduleByClientRequestId as jest.Mock)
+ .mockResolvedValueOnce(null)
+ .mockResolvedValue(armed);
+ (deps.methods.createScheduleWithSlot as jest.Mock).mockResolvedValue('limit');
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status ?? 201).toBe(201);
+ });
+
+ it('refuses a key reused for a different schedule', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ // Same key, but the row it resolves to is a different intent entirely.
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue({
+ id: 'sched-original',
+ name: 'Something else',
+ prompt: 'Different prompt',
+ agent_id: 'agent-9',
+ } as ISchedule);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ // A 201 here would describe a schedule the caller never asked for.
+ expect(captured.status).toBe(409);
+ expect(deps.methods.armSchedule).not.toHaveBeenCalled();
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ /**
+ * The digest is the IMMUTABLE record of the create intent: a PATCH that reshaped the
+ * row between the first attempt and its retry must not make a genuine replay read as
+ * key reuse — the mutable row no longer matches, but the stamped digest still does.
+ */
+ it('matches a replay by digest even after a PATCH reshaped the row', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const patched = {
+ ...originalRow(),
+ name: 'Renamed by a PATCH',
+ prompt: 'Edited prompt',
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ clientRequestDigest: createBodyDigest(),
+ };
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(patched);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(patched);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status ?? 201).toBe(201);
+ });
+
+ it('refuses by digest when the key was reused for different content', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ // Field-identical row (a legacy comparison would pass) whose stamped digest says
+ // the ORIGINAL create asked for something else — e.g. different attachments.
+ const differentIntent = {
+ ...originalRow(),
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ clientRequestDigest: 'not-this-request',
+ };
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(differentIntent);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ expect(captured.status).toBe(409);
+ });
+
+ it('never compensates on the replay path, even when its arm fails', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.getScheduleByClientRequestId as jest.Mock).mockResolvedValue(originalRow());
+ (deps.methods.armSchedule as jest.Mock).mockRejectedValue(new Error('mongo down'));
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ // The established row survives for the next retry; deleting it here is data loss.
+ expect(captured.status).toBe(500);
+ expect(deps.methods.deleteScheduleById).not.toHaveBeenCalled();
+ expect(deps.methods.deleteUnarmedSchedule).not.toHaveBeenCalled();
+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();
+ });
+
+ it('refuses a create without an idempotency key', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const req = makeCreateReq() as unknown as { body: Record<string, unknown> };
+ delete req.body.clientRequestId;
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(req as unknown as ServerRequest, res);
+
+ // Optional keys preserved the keyless duplicate path — the exact failure the key
+ // exists to close — so the payload schema requires one.
+ expect(captured.status).toBe(400);
+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();
+ });
+});
+
+describe('deferred deletion retry', () => {
+ it('re-drives the full deletion of the caller’s soft-deleted schedules on list', async () => {
+ const deps = makeCreateDeps({
+ deleteSchedule: jest.fn(async () => 'deleted'),
+ } as Partial<SchedulesHandlersDeps>);
+ (deps.methods.getSchedulesByUser as jest.Mock) = jest.fn(async () => []);
+ (deps.methods.getDeletingScheduleIds as jest.Mock) = jest.fn(async () => ['stranded-1']);
+ (deps.methods.markEraseAttempted as jest.Mock) = jest.fn(async () => undefined);
+ const { res } = makeRes();
+
+ await createSchedulesHandlers(deps).listSchedules(
+ { user: { id: 'user-1' } } as unknown as ServerRequest,
+ res,
+ );
+ // Fire-and-forget, so let the microtask chain settle before asserting.
+ await new Promise((resolve) => setImmediate(resolve));
+
+ // The service delete (abort + settle + erase), not a bare erase probe: a schedule
+ // stranded mid-drain with a still-active run needs the abort re-driven too.
+ expect(deps.deleteSchedule).toHaveBeenCalledWith('stranded-1', 'user-1');
+ // Stamped attempted so the bounded window rotates past rows that stay unconfirmed.
+ expect(deps.methods.markEraseAttempted).toHaveBeenCalledWith(['stranded-1']);
+ });
+
+ it('keeps listing even when a stranded deletion re-drive rejects', async () => {
+ const deps = makeCreateDeps({
+ deleteSchedule: jest.fn(async () => {
+ throw new Error('still draining');
+ }),
+ } as Partial<SchedulesHandlersDeps>);
+ (deps.methods.getSchedulesByUser as jest.Mock) = jest.fn(async () => []);
+ (deps.methods.getDeletingScheduleIds as jest.Mock) = jest.fn(async () => ['stranded-1']);
+ (deps.methods.markEraseAttempted as jest.Mock) = jest.fn(async () => undefined);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).listSchedules(
+ { user: { id: 'user-1' } } as unknown as ServerRequest,
+ res,
+ );
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(captured.body).toEqual(expect.objectContaining({ schedules: [] }));
+ });
+});
+
+describe('deleteSchedule result mapping', () => {
+ function makeDeleteReq(): ServerRequest {
+ return {
+ params: { id: 'sched-1' },
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ } as unknown as ServerRequest;
+ }
+
+ const withResult = (result: string) =>
+ makeCreateDeps({
+ deleteSchedule: jest.fn(async () => result),
+ } as Partial<SchedulesHandlersDeps>);
+
+ it('404s when the schedule is not found', async () => {
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(withResult('not_found')).deleteSchedule(makeDeleteReq(), res);
+ expect(captured.status).toBe(404);
+ });
+
+ it('answers 200 when drained and erased', async () => {
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(withResult('deleted')).deleteSchedule(makeDeleteReq(), res);
+ expect(captured.status ?? 200).toBe(200);
+ expect(captured.body).toEqual({ id: 'sched-1' });
+ });
+
+ it('answers 202 while a delivered abort is still settling', async () => {
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(withResult('draining')).deleteSchedule(makeDeleteReq(), res);
+ expect(captured.status).toBe(202);
+ expect(captured.body).toEqual({ id: 'sched-1' });
+ });
+
+ it('refuses honestly when the active run could not be confirmed stopped', async () => {
+ // Reporting success would claim a possibly still-billing generation was stopped.
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(withResult('unconfirmed')).deleteSchedule(makeDeleteReq(), res);
+ expect(captured.status).toBe(503);
+ });
+});
+
+describe('capacity pre-check vs concurrent same-key insert', () => {
+ it('re-checks the key before refusing an over-limit create', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.countSchedulesByUser as jest.Mock).mockResolvedValue(10);
+ const armed = {
+ ...CREATE_BODY,
+ id: 'sched-original',
+ target: 'new',
+ enabled: true,
+ configRevision: 0,
+ nextRunAt: new Date('2026-07-22T12:00:00Z'),
+ } as unknown as ISchedule;
+ // Absent at the replay lookup, present by the capacity pre-check: the concurrent
+ // first attempt committed in between — and may itself be the row at the cap.
+ (deps.methods.getScheduleByClientRequestId as jest.Mock)
+ .mockResolvedValueOnce(null)
+ .mockResolvedValue(armed);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(armed);
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+
+ // A 400 here would refuse the client the very row it is trying to confirm.
+ expect(captured.status ?? 201).toBe(201);
+ expect(deps.methods.createScheduleWithSlot).not.toHaveBeenCalled();
+ });
+});
+
+describe('updateSchedule refuses field-less payloads', () => {
+ it('rejects {} before touching claim-token or revision fencing', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).updateSchedule(
+ {
+ params: { id: 'sched-1' },
+ body: {},
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ } as unknown as ServerRequest,
+ res,
+ );
+ // An empty update still rotates claimToken and bumps configRevision, fencing a
+ // legitimate in-flight occurrence for a request that changed nothing.
+ expect(captured.status).toBe(400);
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+});
+
+describe('updateSchedule client revision fence', () => {
+ const existingRow = () =>
+ ({
+ id: 'sched-1',
+ enabled: true,
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'UTC',
+ nextRunAt: new Date('2026-07-31T09:00:00Z'),
+ configRevision: 7,
+ }) as unknown as ISchedule;
+
+ const makePatchReq = (body: Record<string, unknown>) =>
+ ({
+ params: { id: 'sched-1' },
+ body,
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ }) as unknown as ServerRequest;
+
+ it('refuses a PATCH computed from a superseded revision', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(existingRow());
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(
+ makePatchReq({ name: 'renamed', expectedConfigRevision: 6 }),
+ res,
+ );
+
+ // The dialog rebuilds cadence whole from the snapshot it opened with, so the
+ // server's fresh-read fence alone cannot see that another tab edited the row.
+ expect(captured.status).toBe(409);
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('applies the PATCH and strips the fence field when the revision matches', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(existingRow());
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(
+ makePatchReq({ name: 'renamed', expectedConfigRevision: 7 }),
+ res,
+ );
+
+ expect(captured.status ?? 200).toBe(200);
+ const [, , update] = (deps.methods.updateScheduleById as jest.Mock).mock.calls[0];
+ // The fence input is not a schedule field; writing it would corrupt the row.
+ expect(update).not.toHaveProperty('expectedConfigRevision');
+ expect(update).toHaveProperty('name', 'renamed');
+ });
+
+ it('still refuses a PATCH that carries only the fence field', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(
+ makePatchReq({ expectedConfigRevision: 7 }),
+ res,
+ );
+
+ expect(captured.status).toBe(400);
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+});
+
+describe('attachment id deduplication', () => {
+ it('accepts a create whose payload repeats an owned file id', async () => {
+ const deps = makeCreateDeps({ isUserDeleting: jest.fn(async () => false) });
+ (deps.methods.armSchedule as jest.Mock).mockResolvedValue(true);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue({
+ ...CREATE_BODY,
+ id: 'sched-1',
+ file_ids: ['file-a'],
+ } as unknown as ISchedule);
+ const req = makeCreateReq() as unknown as { body: Record<string, unknown> };
+ req.body.file_ids = ['file-a', 'file-a'];
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).createSchedule(req as unknown as ServerRequest, res);
+
+ // The ownership query returns one doc per UNIQUE id, so an un-deduped list
+ // read as a missing file and 400'd a valid request. The schema dedupes.
+ expect(captured.status ?? 201).toBe(201);
+ expect(deps.methods.createScheduleWithSlot).toHaveBeenCalledWith(
+ expect.objectContaining({ file_ids: ['file-a'] }),
+ expect.any(Number),
+ );
+ });
+});
+
+describe('updateSchedule re-enable attachment revalidation', () => {
+ const disabledWithFiles = () =>
+ ({
+ id: 'sched-1',
+ enabled: false,
+ agent_id: 'agent-1',
+ cadence: { type: 'daily', hour: 9, minute: 0 },
+ timezone: 'UTC',
+ nextRunAt: new Date('2026-07-31T09:00:00Z'),
+ configRevision: 3,
+ file_ids: ['file-a', 'file-b'],
+ }) as unknown as ISchedule;
+
+ const makeReEnableReq = () =>
+ ({
+ params: { id: 'sched-1' },
+ body: { enabled: true },
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ }) as unknown as ServerRequest;
+
+ it('refuses re-enabling when a stored attachment is no longer owned', async () => {
+ const deps = makeCreateDeps({
+ isUserDeleting: jest.fn(async () => false),
+ filterOwnedFileIds: jest.fn(async () => ['file-a']),
+ } as Partial<SchedulesHandlersDeps>);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(makeReEnableReq(), res);
+
+ // The bounded upload hold only renews while the schedule fires, so a long-
+ // disabled schedule can have lost its uploads; silently firing without them
+ // is worse than telling the owner to replace the attachments.
+ expect(captured.status).toBe(400);
+ expect(deps.methods.updateScheduleById).not.toHaveBeenCalled();
+ });
+
+ it('renews the retention hold on the stored attachments before committing', async () => {
+ const markFilesUsed = jest.fn(async () => undefined);
+ const deps = makeCreateDeps({
+ isUserDeleting: jest.fn(async () => false),
+ markFilesUsed,
+ } as Partial<SchedulesHandlersDeps>);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(makeReEnableReq(), res);
+
+ expect(markFilesUsed).toHaveBeenCalledWith(['file-a', 'file-b'], 'user-1');
+ expect(captured.status ?? 200).toBe(200);
+ expect(deps.methods.updateScheduleById).toHaveBeenCalled();
+ });
+
+ it('skips the stored-attachment recheck when the edit replaces file_ids', async () => {
+ const filterOwnedFileIds = jest.fn(async (ids: string[]) => ids);
+ const deps = makeCreateDeps({
+ isUserDeleting: jest.fn(async () => false),
+ filterOwnedFileIds,
+ } as Partial<SchedulesHandlersDeps>);
+ (deps.methods.getScheduleById as jest.Mock).mockResolvedValue(disabledWithFiles());
+ const { res, captured } = makeRes();
+
+ await createSchedulesHandlers(deps).updateSchedule(
+ {
+ params: { id: 'sched-1' },
+ body: { enabled: true, file_ids: ['file-new'] },
+ user: { id: 'user-1', tenantId: 't1', role: 'USER' },
+ } as unknown as ServerRequest,
+ res,
+ );
+
+ // Supplied file_ids are validated by validatePayload; the stored list is
+ // about to be overwritten, so rechecking it would refuse a valid replacement.
+ expect(captured.status ?? 200).toBe(200);
+ expect(filterOwnedFileIds).toHaveBeenCalledTimes(1);
+ expect(filterOwnedFileIds).toHaveBeenCalledWith(['file-new'], 'user-1');
+ });
+});
+
+describe('late-create compensation with a live manual run', () => {
+ it('treats a draining rollback as compensated (the teardown owns it)', async () => {
+ const deps = makeCreateDeps();
+ (deps.methods.deleteUnarmedSchedule as jest.Mock).mockResolvedValue('draining');
+ const { res, captured } = makeRes();
+ await createSchedulesHandlers(deps).createSchedule(makeCreateReq(), res);
+ // Soft-claimed and draining: hidden, unclaimable, erased once its live manual
+ // run settles — never hard-deleted out from under that generation.
+ expect(deps.methods.markScheduleDeleting).not.toHaveBeenCalled();
+ expect(captured.status).toBe(410);
+ });
+});
diff --git a/packages/api/src/schedules/handlers.ts b/packages/api/src/schedules/handlers.ts
new file mode 100644
index 0000000000..daec7c7304
--- /dev/null
+++ b/packages/api/src/schedules/handlers.ts
@@ -0,0 +1,800 @@
+import { logger } from '@librechat/data-schemas';
+import { createHash, randomUUID } from 'node:crypto';
+import { createSchedulePayloadSchema, updateSchedulePayloadSchema } from 'librechat-data-provider';
+import type { TCreateSchedule, TUpdateSchedule } from 'librechat-data-provider';
+import type { ScheduleMethods, ISchedule } from '@librechat/data-schemas';
+import type { Response } from 'express';
+import type {
+ ScheduleDeleteResult,
+ ScheduleUserContext,
+ FireableSchedule,
+ ScheduleLimits,
+ FireResult,
+} from './types';
+import type { ServerRequest } from '~/types';
+import { isValidTimezone, cadenceIntervalMinutes, computeNextRunAt } from './cadence';
+
+export interface SchedulesHandlersDeps {
+ methods: ScheduleMethods;
+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;
+ /** Agent existence + VIEW access for the requesting user. */
+ canViewAgent: (agentId: string, req: ServerRequest) => Promise<boolean>;
+ /** Filters to file ids owned by the user. */
+ filterOwnedFileIds: (fileIds: string[], userId: string) => Promise<string[]>;
+ /** Extends a bounded renewable upload hold on attached files so they survive to the
+ * first fire, which consumes them permanently; a schedule that dies first lets the
+ * hold lapse instead of retaining the upload forever. Throws when any file is gone. */
+ markFilesUsed: (fileIds: string[], userId: string) => Promise<void>;
+ /** Serialized manual fire (acquires the schedule lease); null if already leased. */
+ fireNow: (schedule: FireableSchedule, limits: ScheduleLimits) => Promise<FireResult | null>;
+ /**
+ * Soft-deletes a schedule with quiescing: stops new claims, aborts in-flight
+ * runs, and erases once drained. See ScheduleDeleteResult for the honest states.
+ */
+ deleteSchedule: (id: string, userId: string) => Promise<ScheduleDeleteResult>;
+ /** Whether this user's account deletion has begun. Fail-closed (unknown == true). */
+ isUserDeleting: (userId: string) => Promise<boolean>;
+}
+
+/**
+ * Rolls back a create that raced past the account-deletion cascade. Hard-deletes only
+ * the exact unarmed, unedited revision this attempt inserted: a concurrent same-key
+ * replay can have ARMED the row — and the engine can have CLAIMED it — before the
+ * post-insert barrier re-check ran, and hard deletion there would bypass run
+ * quiescence and erase evidence out from under a live fire. Anything past that
+ * revision is soft-deleted instead, which makes it non-claimable at once and hands it
+ * to the ordinary drain-then-erase teardown. Returns false only when BOTH fail, which
+ * the caller must surface rather than answering a clean 410.
+ */
+async function compensateLateCreate(
+ deps: SchedulesHandlersDeps,
+ id: string,
+ userId: string,
+ expectedConfigRevision: number,
+): Promise<boolean> {
+ const rolled = await deps.methods
+ .deleteUnarmedSchedule(id, userId, expectedConfigRevision)
+ .catch((err) => {
+ logger.error(`[schedules] compensating delete failed for late create ${id}`, err);
+ return null;
+ });
+ // 'draining' is compensated too: the row is soft-claimed (hidden, unclaimable) and
+ // the ordinary erase-on-settle / sweep teardown finishes it once its live manual
+ // run — the reason it could not erase immediately — settles.
+ if (rolled === 'deleted' || rolled === 'missing' || rolled === 'draining') {
+ return true;
+ }
+ const marked = await deps.methods.markScheduleDeleting(id, userId).catch((err) => {
+ logger.error(`[schedules] compensating soft-delete failed for late create ${id}`, err);
+ return null;
+ });
+ return marked != null;
+}
+
+/**
+ * Refuses a scheduling WRITE once the owner's account deletion has begun. A one-shot
+ * disable scan can never close this race (a create landing after the scan is simply not
+ * in it), so admission consults the durable user-level barrier instead. Fail-closed.
+ */
+async function rejectIfUserDeleting(
+ deps: SchedulesHandlersDeps,
+ userId: string,
+ res: Response,
+): Promise<boolean> {
+ if (!(await deps.isUserDeleting(userId))) {
+ return false;
+ }
+ res.status(410).json({ error: 'This account is being deleted' });
+ return true;
+}
+
+/** Bounded attempts to extend the upload hold on a schedule's attachments. */
+const FILE_RETAIN_ATTEMPTS = 3;
+
+/** Soft-deleted rows re-checked for erasure per list request. Small: it is a retry for
+ * a rare stranded row, not a sweep, and it rides a user-facing read. */
+const DEFERRED_ERASE_RETRY_LIMIT = 5;
+
+/**
+ * Digest of a create payload's FULL intent, stamped on the row at insert and never
+ * edited. Replay matching compares this immutable record instead of mutable schedule
+ * state, so a PATCH (or a policy auto-disable flipping `enabled`) landing between the
+ * first attempt and its retry cannot make a genuine replay read as key reuse.
+ */
+export function computeCreateDigest(payload: TCreateSchedule): string {
+ const canonical = JSON.stringify({
+ name: payload.name,
+ prompt: payload.prompt,
+ agent_id: payload.agent_id,
+ timezone: payload.timezone,
+ target: payload.target,
+ enabled: payload.enabled,
+ cadence: {
+ frequency: payload.cadence.frequency,
+ hour: payload.cadence.hour,
+ minute: payload.cadence.minute,
+ daysOfWeek: payload.cadence.daysOfWeek ?? null,
+ },
+ file_ids: payload.file_ids ?? null,
+ });
+ return createHash('sha256').update(canonical).digest('hex');
+}
+
+function sameList<T>(left: T[] | undefined, right: T[] | undefined): boolean {
+ if (left == null || right == null) {
+ return (left?.length ?? 0) === (right?.length ?? 0);
+ }
+ return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
+/**
+ * Legacy replay matching for rows stamped before `clientRequestDigest` existed.
+ * Deliberately omits `enabled` — a policy auto-disable mutates it, and this
+ * comparison exists precisely because mutable state makes a poor replay record.
+ */
+function matchesCreatedSchedule(existing: ISchedule, payload: TCreateSchedule): boolean {
+ return (
+ existing.name === payload.name &&
+ existing.prompt === payload.prompt &&
+ existing.agent_id === payload.agent_id &&
+ existing.timezone === payload.timezone &&
+ existing.target === payload.target &&
+ existing.cadence?.frequency === payload.cadence.frequency &&
+ existing.cadence?.hour === payload.cadence.hour &&
+ existing.cadence?.minute === payload.cadence.minute &&
+ sameList(existing.cadence?.daysOfWeek, payload.cadence.daysOfWeek) &&
+ sameList(existing.file_ids, payload.file_ids)
+ );
+}
+
+/** Whether an existing row is the schedule this create attempt is asking for: the
+ * immutable digest when the row carries one, the legacy field comparison otherwise. */
+function matchesCreateIntent(
+ existing: ISchedule,
+ payload: TCreateSchedule,
+ digest: string,
+): boolean {
+ if (existing.clientRequestDigest != null) {
+ return existing.clientRequestDigest === digest;
+ }
+ return matchesCreatedSchedule(existing, payload);
+}
+
+/** Public projection of a schedule — an allowlist of the `TSchedule` fields, so internal
+ * bookkeeping (_id, tenantId, __v, claimToken, lease*, slot, deleting, countedFor,
+ * balanceSkipCount, bookkept, ...) never reaches the browser. */
+export type WireSchedule = Pick<
+ ISchedule,
+ | 'id'
+ | 'user'
+ | 'name'
+ | 'prompt'
+ | 'agent_id'
+ | 'cadence'
+ | 'timezone'
+ | 'target'
+ | 'file_ids'
+ | 'enabled'
+ | 'disabledReason'
+ | 'nextRunAt'
+ | 'lastRun'
+ | 'runCount'
+ | 'failureCount'
+ | 'configRevision'
+ | 'createdAt'
+ | 'updatedAt'
+>;
+
+export function toWireSchedule(schedule: ISchedule): WireSchedule {
+ return {
+ id: schedule.id,
+ user: schedule.user,
+ name: schedule.name,
+ prompt: schedule.prompt,
+ agent_id: schedule.agent_id,
+ cadence: schedule.cadence,
+ timezone: schedule.timezone,
+ target: schedule.target,
+ file_ids: schedule.file_ids,
+ enabled: schedule.enabled,
+ disabledReason: schedule.disabledReason,
+ nextRunAt: schedule.nextRunAt,
+ lastRun: schedule.lastRun,
+ runCount: schedule.runCount,
+ failureCount: schedule.failureCount,
+ configRevision: schedule.configRevision,
+ createdAt: schedule.createdAt,
+ updatedAt: schedule.updatedAt,
+ };
+}
+
+function requestUser(req: ServerRequest): { id: string; tenantId?: string; role?: string } {
+ const user = req.user as { id: string; tenantId?: string; role?: string };
+ return { id: user.id, tenantId: user.tenantId, role: user.role };
+}
+
+type ScheduleHandler = (req: ServerRequest, res: Response) => Promise<void>;
+
+export interface SchedulesHandlers {
+ listSchedules: ScheduleHandler;
+ getSchedule: ScheduleHandler;
+ createSchedule: ScheduleHandler;
+ updateSchedule: ScheduleHandler;
+ deleteSchedule: ScheduleHandler;
+ runScheduleNow: ScheduleHandler;
+}
+
+export function createSchedulesHandlers(deps: SchedulesHandlersDeps): SchedulesHandlers {
+ async function validatePayload(
+ req: ServerRequest,
+ res: Response,
+ payload: TCreateSchedule | TUpdateSchedule,
+ limits: ScheduleLimits,
+ ): Promise<boolean> {
+ if (payload.timezone != null && !isValidTimezone(payload.timezone)) {
+ res.status(400).json({ error: 'Invalid IANA timezone' });
+ return false;
+ }
+ if (
+ payload.cadence != null &&
+ cadenceIntervalMinutes(payload.cadence) < limits.minIntervalMinutes
+ ) {
+ res.status(400).json({
+ error: `Schedule interval must be at least ${limits.minIntervalMinutes} minutes`,
+ });
+ return false;
+ }
+ if (payload.agent_id != null && !(await deps.canViewAgent(payload.agent_id, req))) {
+ res.status(400).json({ error: 'Agent not found or not accessible' });
+ return false;
+ }
+ if (payload.file_ids != null && payload.file_ids.length > 0) {
+ const owned = await deps.filterOwnedFileIds(payload.file_ids, requestUser(req).id);
+ if (owned.length !== payload.file_ids.length) {
+ res.status(400).json({ error: 'One or more attached files were not found' });
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Extends the bounded upload hold on attached files (so they survive to the first
+ * fire, which consumes them permanently), with bounded retry. Returns false when it
+ * exhausts retries — the caller then compensates (roll back the create / revert the
+ * edit) so a persisted schedule never references files the upload sweep is about to
+ * reap.
+ */
+ async function retainFiles(fileIds: string[], userId: string): Promise<boolean> {
+ for (let attempt = 1; attempt <= FILE_RETAIN_ATTEMPTS; attempt++) {
+ try {
+ await deps.markFilesUsed(fileIds, userId);
+ return true;
+ } catch (err) {
+ logger.error(
+ `[schedules] attachment retention failed (attempt ${attempt}/${FILE_RETAIN_ATTEMPTS}):`,
+ err,
+ );
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Re-drives the FULL deletion of the caller's soft-deleted schedules, off the
+ * response path.
+ *
+ * A `deleting` row is settled and erased by the delete request itself, or by the
+ * terminal outcome write (erase-on-settle). Both are best-effort, and the
+ * reconciler that would otherwise retry does not exist in the clustered
+ * entrypoint. Meanwhile the row is HIDDEN from the owner's list, so once a delete
+ * answers 503-unconfirmed there is no UI/API-list handle left to retry the drain
+ * with — the "please retry" the response asks for has nothing to click.
+ *
+ * A read the owner performs anyway is therefore the re-driver: bounded, scoped to
+ * their own rows, a no-op when nothing is deleting. Re-driving the WHOLE delete
+ * (abort, settle, erase) rather than only the erase is what un-strands a row whose
+ * active run never settled; every step is idempotent and evidence-guarded, so
+ * repeated polls race harmlessly.
+ */
+ function retryDeferredDeletions(userId: string): void {
+ void deps.methods
+ .getDeletingScheduleIds(userId, DEFERRED_ERASE_RETRY_LIMIT)
+ .then(async (ids) => {
+ if (ids.length === 0) {
+ return;
+ }
+ // Stamp BEFORE attempting: the read window is least-recently-attempted
+ // first, so rows this pass touches rotate to the back and an owner with
+ // more stuck rows than the limit reaches all of them across successive
+ // lists instead of re-driving the same unconfirmable few forever.
+ await deps.methods.markEraseAttempted(ids);
+ await Promise.all(
+ ids.map((id) => deps.deleteSchedule(id, userId).catch(() => 'unconfirmed')),
+ );
+ })
+ .catch((err) => logger.warn('[schedules] deferred deletion retry failed', err));
+ }
+
+ async function listSchedules(req: ServerRequest, res: Response): Promise<void> {
+ const [schedules, limits] = await Promise.all([
+ deps.methods.getSchedulesByUser(requestUser(req).id),
+ deps.getLimits(requestUser(req)),
+ ]);
+ retryDeferredDeletions(requestUser(req).id);
+ res.json({
+ schedules: schedules.map(toWireSchedule),
+ limits: { maxPerUser: limits.maxPerUser },
+ });
+ }
+
+ async function getSchedule(req: ServerRequest, res: Response): Promise<void> {
+ const { id } = req.params as { id: string };
+ const schedule = await deps.methods.getScheduleById(id, requestUser(req).id);
+ if (schedule == null) {
+ res.status(404).json({ error: 'Schedule not found' });
+ return;
+ }
+ res.json(toWireSchedule(schedule));
+ }
+
+ async function createSchedule(req: ServerRequest, res: Response): Promise<void> {
+ const parsed = createSchedulePayloadSchema.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({ error: 'Invalid schedule payload', issues: parsed.error.issues });
+ return;
+ }
+ const user = requestUser(req);
+ if (await rejectIfUserDeleting(deps, user.id, res)) {
+ return;
+ }
+ const limits = await deps.getLimits(user);
+ if (!limits.enabled) {
+ res.status(403).json({ error: 'Scheduled chats are disabled' });
+ return;
+ }
+ if (!(await validatePayload(req, res, parsed.data, limits))) {
+ return;
+ }
+ const digest = computeCreateDigest(parsed.data);
+
+ /**
+ * Answers a retry with the row its FIRST attempt committed — an ESTABLISHED
+ * schedule this attempt must never reshape. Re-drives only the arm (guarded:
+ * only-if-still-unarmed, jitter derived from the EXISTING row's id, no claim-token
+ * rotation or revision bump, so an active run is never fenced off), and never
+ * compensates: rollback belongs exclusively to the attempt that inserted a row.
+ */
+ const respondToReplay = async (existing: ISchedule): Promise<void> => {
+ // A key identifies one create INTENT. Reusing it for different content is a
+ // client bug, and silently returning the first row would hide it behind a 201
+ // describing a schedule the caller did not ask for — so say so instead.
+ if (!matchesCreateIntent(existing, parsed.data, digest)) {
+ res.status(409).json({
+ error: 'clientRequestId was already used to create a different schedule',
+ });
+ return;
+ }
+ // The key stays claimed while its row drains (the unique index spans deleting
+ // rows), so a retry of a create whose schedule was since deleted resolves here.
+ if (existing.deleting === true) {
+ res.status(410).json({ error: 'Schedule no longer exists' });
+ return;
+ }
+ if (await deps.isUserDeleting(user.id)) {
+ res.status(410).json({ error: 'This account is being deleted' });
+ return;
+ }
+ if (existing.enabled && existing.nextRunAt == null) {
+ const next = computeNextRunAt({
+ cadence: existing.cadence,
+ timezone: existing.timezone,
+ scheduleId: existing.id,
+ });
+ if (next != null) {
+ try {
+ await deps.methods.armSchedule(existing.id, next);
+ } catch (armError) {
+ logger.error(`[schedules] replay arming failed for ${existing.id}`, armError);
+ res.status(500).json({ error: 'Failed to create schedule. Please retry.' });
+ return;
+ }
+ }
+ }
+ const fresh = await deps.methods.getScheduleById(existing.id, user.id);
+ if (fresh == null) {
+ res.status(410).json({ error: 'Schedule no longer exists' });
+ return;
+ }
+ logger.info(`[schedules] create retry resolved to ${existing.id} for user ${user.id}`);
+ res.status(201).json(toWireSchedule(fresh));
+ };
+
+ // Resolve a retry BEFORE the capacity pre-check: a retry whose first attempt
+ // already occupies the final slot is a replay of THAT row, not a request for a
+ // new one — refusing it as over-limit would deny the client the very row it is
+ // trying to confirm.
+ const replayed = await deps.methods.getScheduleByClientRequestId(
+ user.id,
+ parsed.data.clientRequestId,
+ );
+ if (replayed != null) {
+ await respondToReplay(replayed);
+ return;
+ }
+ // Fail fast on an obvious over-limit BEFORE retaining attachments, so the common
+ // case never clears an upload TTL it then can't use. The {user, slot} partial
+ // unique index below is the atomic arbiter for the concurrent-create race.
+ if ((await deps.methods.countSchedulesByUser(user.id)) >= limits.maxPerUser) {
+ // A concurrent first attempt with this key can have landed between the replay
+ // lookup above and this count — and can itself be what filled the last slot.
+ // The retry must resolve to that row, not be refused for being at capacity.
+ const raced = await deps.methods.getScheduleByClientRequestId(
+ user.id,
+ parsed.data.clientRequestId,
+ );
+ if (raced != null) {
+ await respondToReplay(raced);
+ return;
+ }
+ res.status(400).json({
+ error: `Schedule limit reached (${limits.maxPerUser}). Delete a schedule to add another.`,
+ });
+ return;
+ }
+ // Retain attachments BEFORE creating, so a persisted (claimable) schedule never
+ // references uploads still eligible for TTL expiry — there is no create-then-
+ // retain window where a crash or a failed rollback leaves the two inconsistent.
+ if (parsed.data.file_ids?.length && !(await retainFiles(parsed.data.file_ids, user.id))) {
+ res.status(500).json({ error: 'Failed to retain schedule attachments' });
+ return;
+ }
+ const id = `sched_${randomUUID()}`;
+ const nextRunAt = parsed.data.enabled
+ ? computeNextRunAt({
+ cadence: parsed.data.cadence,
+ timezone: parsed.data.timezone,
+ scheduleId: id,
+ })
+ : undefined;
+ // Atomic cap: createScheduleWithSlot claims a free per-user slot via the
+ // {user, slot} partial unique index, so concurrent creates can never exceed
+ // maxPerUser. 'limit' means a concurrent racer took the last slot after the
+ // pre-check above; the just-retained files are then unreferenced (a rare, minor
+ // leak of the user's own uploads) — acceptable vs. a partial/expiring commit.
+ // Inserted WITHOUT nextRunAt regardless of `enabled`: the engine claims by
+ // nextRunAt, so the row is inert until armed below. That is what makes the
+ // barrier re-check durable — every failure mode leaves a row that cannot fire,
+ // rather than one that fires for an account already being erased. The reconciler's
+ // unarmed sweep later arms anything left this way, so an inert row is a delay, not
+ // a permanent state.
+ const created = await deps.methods.createScheduleWithSlot(
+ {
+ ...parsed.data,
+ id,
+ user: user.id as never,
+ tenantId: user.tenantId,
+ clientRequestDigest: digest,
+ },
+ limits.maxPerUser,
+ );
+ if (created === 'limit') {
+ // A CONCURRENT first attempt with this key can be what filled the last slot;
+ // resolve to it rather than refusing the retry for being at capacity.
+ const raced = await deps.methods.getScheduleByClientRequestId(
+ user.id,
+ parsed.data.clientRequestId,
+ );
+ if (raced != null) {
+ await respondToReplay(raced);
+ return;
+ }
+ res.status(400).json({
+ error: `Schedule limit reached (${limits.maxPerUser}). Delete a schedule to add another.`,
+ });
+ return;
+ }
+ if (created.id !== id) {
+ // The allocator hit the idempotency index: a concurrent attempt with this key
+ // committed first. That row is established — hand it to the replay path.
+ await respondToReplay(created);
+ return;
+ }
+ // FRESH insert (this attempt owns the row) from here down; compensation is safe.
+ //
+ // POST-INSERT barrier re-check. The admission check at the top of this handler
+ // shrinks the window to roughly one request, but cannot close it: account deletion
+ // can raise the barrier after we passed that check and before this insert landed,
+ // and its one-shot disable scan would not have seen a row that did not exist yet.
+ // Re-checking AFTER the write is what makes the barrier authoritative.
+ if (await deps.isUserDeleting(user.id)) {
+ // Best-effort tidy-up. Hard-deletes only the exact unarmed revision this
+ // attempt inserted (a concurrent replay can have armed it, and the engine can
+ // have claimed the armed row); anything past that is soft-deleted into the
+ // ordinary drain-then-erase teardown. Its failure is reported but no longer
+ // load-bearing for BILLING: even once the reconciler's sweep arms a residual
+ // row, the fire path refuses it at the account-deletion barrier
+ // (isOwnerDeleting). The residual is a retained row, not a billed generation.
+ if (!(await compensateLateCreate(deps, id, user.id, created.configRevision ?? 0))) {
+ res.status(500).json({ error: 'Failed to roll back schedule creation' });
+ return;
+ }
+ res.status(410).json({ error: 'This account is being deleted' });
+ return;
+ }
+ // ARM last, through the SAME unarmed-guarded CAS the replay path and the
+ // reconciler's sweep use — one arming write per schedule, ever. Fenced on the
+ // revision stamped at insert: a PATCH racing this arm bumps configRevision, and
+ // this POST must not overwrite the nextRunAt that PATCH derived from newer
+ // config. Never updateScheduleById here: its claim-token rotation would fence an
+ // occurrence the engine may have claimed off a row a concurrent replay armed
+ // first, and its revision bump would break a concurrent PATCH's CAS.
+ if (nextRunAt) {
+ let armedNow: boolean;
+ try {
+ armedNow = await deps.methods.armSchedule(id, nextRunAt, created.configRevision ?? 0);
+ } catch (armError) {
+ // Roll back ONLY the exact unarmed, unedited revision this attempt inserted.
+ // An ambiguously-committed arm, a concurrent replay that armed the row and
+ // already answered 201, or a concurrent PATCH that edited it (even while
+ // leaving it unarmed) all leave a row that must survive.
+ logger.error(`[schedules] arming failed for ${id}; rolling back the create`, armError);
+ const rolledBack = await deps.methods
+ .deleteUnarmedSchedule(id, user.id, created.configRevision ?? 0)
+ .catch(() => null);
+ if (rolledBack === 'kept') {
+ const current = await deps.methods.getScheduleById(id, user.id);
+ if (current != null) {
+ res.status(201).json(toWireSchedule(current));
+ return;
+ }
+ }
+ // A retry is always safe: the same key collides on the idempotency index and
+ // resolves to whatever this attempt left behind (or a clean re-insert).
+ res.status(500).json({ error: 'Failed to create schedule. Please retry.' });
+ return;
+ }
+ // Whether THIS call armed or a concurrent actor got there first (a replay's
+ // arm, a PATCH whose own arming governs), the response reports the row as it
+ // now stands; only a row the deletion cascade claimed answers 410.
+ const current = await deps.methods.getScheduleById(id, user.id);
+ if (current == null) {
+ res.status(410).json({ error: 'Schedule no longer exists' });
+ return;
+ }
+ if (!armedNow && current.nextRunAt == null && current.enabled) {
+ // Unarmed with an unmoved revision would have matched the CAS; reaching here
+ // means the revision moved to an edit that keeps it enabled but unarmed —
+ // report it as-is, the reconciler's sweep recovers the arm.
+ logger.warn(`[schedules] create for ${id} left unarmed after a concurrent edit`);
+ }
+ logger.info(`[schedules] created ${id} for user ${user.id}`);
+ res.status(201).json(toWireSchedule(current));
+ return;
+ }
+ logger.info(`[schedules] created ${id} for user ${user.id}`);
+ res.status(201).json(toWireSchedule(created));
+ }
+
+ async function updateSchedule(req: ServerRequest, res: Response): Promise<void> {
+ const parsed = updateSchedulePayloadSchema.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({ error: 'Invalid schedule payload', issues: parsed.error.issues });
+ return;
+ }
+ // The fence input is not a schedule field: strip it before the emptiness check
+ // and the update spread below, or it would count as an edit and be written to
+ // the row.
+ const { expectedConfigRevision, ...editedFields } = parsed.data;
+ // A field-less PATCH is not a harmless no-op: updateScheduleById rotates the claim
+ // token and bumps the config revision on every write, so an empty update would
+ // fence a legitimate in-flight occurrence — its terminal bookkeeping revision-
+ // fences to a no-op, and a fire in the POST-to-controller window is refused at
+ // the admission boundary without ever running. Refuse before touching fencing.
+ if (Object.keys(editedFields).length === 0) {
+ res.status(400).json({ error: 'Schedule update must include at least one field' });
+ return;
+ }
+ const { id } = req.params as { id: string };
+ const user = requestUser(req);
+ if (await rejectIfUserDeleting(deps, user.id, res)) {
+ return;
+ }
+ const existing = await deps.methods.getScheduleById(id, user.id);
+ if (existing == null) {
+ res.status(404).json({ error: 'Schedule not found' });
+ return;
+ }
+ // Client-side revision fence: the dialog rebuilds compound fields (cadence)
+ // from the snapshot it opened with, so an edit from another tab is invisible
+ // to the fresh-read fence below — the payload is internally consistent with a
+ // row that no longer exists. Refuse before any side effect (file holds) when
+ // the client says which revision it edited.
+ if (expectedConfigRevision != null && existing.configRevision !== expectedConfigRevision) {
+ res.status(409).json({ error: 'Schedule was modified concurrently. Please retry.' });
+ return;
+ }
+ const limits = await deps.getLimits(user);
+ // When the owner's config disables schedules, block edits that keep the
+ // schedule enabled; still allow turning one OFF.
+ if (!limits.enabled && (parsed.data.enabled ?? existing.enabled)) {
+ res.status(403).json({ error: 'Scheduled chats are disabled' });
+ return;
+ }
+ if (!(await validatePayload(req, res, parsed.data, limits))) {
+ return;
+ }
+ const cadence = parsed.data.cadence ?? existing.cadence;
+ const timezone = parsed.data.timezone ?? existing.timezone;
+ const enabled = parsed.data.enabled ?? existing.enabled;
+ // Re-validate the EFFECTIVE (possibly stored) cadence against the current
+ // floor whenever this edit leaves the schedule enabled — otherwise a bare
+ // {enabled:true} could re-enable an existing schedule that now runs too often.
+ if (enabled && cadenceIntervalMinutes(cadence) < limits.minIntervalMinutes) {
+ res.status(400).json({
+ error: `Schedule interval must be at least ${limits.minIntervalMinutes} minutes`,
+ });
+ return;
+ }
+ // A supplied agent_id is validated in validatePayload; when an edit omits it
+ // but leaves the schedule enabled (e.g. toggling `enabled` back on after an
+ // agent_deleted/permission_revoked auto-disable), re-validate the STORED
+ // agent too. Otherwise re-enabling clears disabledReason for a target the
+ // next fire would immediately reject and disable again.
+ if (
+ enabled &&
+ parsed.data.agent_id == null &&
+ !(await deps.canViewAgent(existing.agent_id, req))
+ ) {
+ res.status(400).json({ error: 'Agent not found or not accessible' });
+ return;
+ }
+ const cadenceChanged =
+ parsed.data.cadence != null || parsed.data.timezone != null || parsed.data.enabled != null;
+ const reEnabled = parsed.data.enabled === true && existing.enabled === false;
+ // RECOVERY: an enabled schedule with no nextRunAt is inert — claimDueSchedule sorts
+ // on nextRunAt and can never select it. Creation arms in a second write, so a crash
+ // or a failed arm leaves exactly this state; re-arm on ANY edit rather than only a
+ // cadence one, or a name/prompt edit would silently leave it dead.
+ const needsArming = existing.nextRunAt == null;
+ const update: Partial<ISchedule> = { ...editedFields } as Partial<ISchedule>;
+ if (enabled && (cadenceChanged || needsArming)) {
+ const nextRunAt = computeNextRunAt({ cadence, timezone, scheduleId: existing.id });
+ if (nextRunAt == null) {
+ res.status(400).json({ error: 'Schedule has no computable next run' });
+ return;
+ }
+ update.nextRunAt = nextRunAt;
+ }
+ if (reEnabled) {
+ update.failureCount = 0;
+ update.balanceSkipCount = 0;
+ }
+ const unset = reEnabled ? { disabledReason: 1 as const } : undefined;
+ // Retain the new attachments BEFORE committing the edit, so a retention failure
+ // leaves the ENTIRE schedule unchanged rather than persisting prompt/cadence/
+ // agent/enabled changes while only reverting file_ids. A file whose TTL was
+ // cleared before the edit failed simply persists unreferenced (the user's own
+ // upload) — a minor leak, not a partial config change future runs would use.
+ if (parsed.data.file_ids?.length && !(await retainFiles(parsed.data.file_ids, user.id))) {
+ res.status(500).json({ error: 'Failed to retain schedule attachments' });
+ return;
+ }
+ // Re-enabling with STORED attachments: the bounded upload hold only renews while
+ // the schedule fires, so a schedule that sat disabled past the hold can have lost
+ // its uploads. Validate the effective list and renew its hold now, mirroring the
+ // stored-agent recheck above — otherwise the re-enable succeeds and the next run
+ // silently fires without the missing files instead of telling the user to
+ // replace them.
+ if (reEnabled && parsed.data.file_ids == null && existing.file_ids?.length) {
+ // Deduped before the count comparison: the ownership query returns one doc
+ // per unique id, so a legacy row holding the same id twice would otherwise
+ // read as a missing file. New payloads are deduped at the schema.
+ const storedIds = Array.from(new Set(existing.file_ids));
+ const stillOwned = await deps.filterOwnedFileIds(storedIds, user.id);
+ if (stillOwned.length !== storedIds.length) {
+ res.status(400).json({
+ error:
+ 'One or more attached files are no longer available. Replace the attachments before re-enabling.',
+ });
+ return;
+ }
+ if (!(await retainFiles(storedIds, user.id))) {
+ res.status(500).json({ error: 'Failed to retain schedule attachments' });
+ return;
+ }
+ }
+ // FENCED on the revision this edit was computed from. `nextRunAt` above is derived
+ // from (cadence, timezone) resolved against the row read at the top of this handler,
+ // so two overlapping edits — one changing cadence, one changing timezone — would
+ // each persist a nextRunAt consistent with neither final state, and it would stay
+ // wrong until the next edit (advanceSchedule recomputes from the row, so a fire
+ // propagates the bad occurrence rather than repairing it). configRevision is already
+ // the row's edit generation and is $inc'd inside the same atomic update, so the
+ // loser simply retries against fresh state.
+ const schedule = await deps.methods.updateScheduleById(existing.id, user.id, update, unset, {
+ expectedConfigRevision: existing.configRevision,
+ });
+ if (schedule == null) {
+ // Either the row is gone, or a concurrent edit moved the revision. Distinguish
+ // them so the owner sees a retryable conflict rather than a phantom 404.
+ const stillThere = await deps.methods.getScheduleById(existing.id, user.id);
+ if (stillThere != null) {
+ res.status(409).json({ error: 'Schedule was modified concurrently. Please retry.' });
+ return;
+ }
+ res.status(404).json({ error: 'Schedule not found' });
+ return;
+ }
+ res.json(toWireSchedule(schedule));
+ }
+
+ async function deleteSchedule(req: ServerRequest, res: Response): Promise<void> {
+ const { id } = req.params as { id: string };
+ // Quiesce-then-erase: disable + mark deleting (stops new claims, hides it),
+ // abort in-flight loopback jobs, and erase once drained — so a live run's
+ // evidence is never destroyed out from under it.
+ const result = await deps.deleteSchedule(id, requestUser(req).id);
+ if (result === 'not_found') {
+ res.status(404).json({ error: 'Schedule not found' });
+ return;
+ }
+ // HONEST failure: at least one active run could not be confirmed stopped, so its
+ // generation may still be producing and billing. The schedule is already hidden
+ // and fenced (no new claims), and the delete is idempotent — a retry re-runs the
+ // drain. Reporting success here would claim the run was stopped.
+ if (result === 'unconfirmed') {
+ res.set('Retry-After', '30');
+ res.status(503).json({
+ error: 'Could not confirm the active run was stopped. Please retry shortly.',
+ });
+ return;
+ }
+ // 202 for `draining`: the aborts were delivered but a generation has not yet
+ // recorded its terminal outcome. The schedule is hidden and erasure follows the
+ // settlement (erase-on-settle), in any topology.
+ res.status(result === 'draining' ? 202 : 200).json({ id });
+ }
+
+ async function runScheduleNow(req: ServerRequest, res: Response): Promise<void> {
+ const { id } = req.params as { id: string };
+ if (await rejectIfUserDeleting(deps, requestUser(req).id, res)) {
+ return;
+ }
+ const schedule = await deps.methods.getScheduleById(id, requestUser(req).id);
+ if (schedule == null) {
+ res.status(404).json({ error: 'Schedule not found' });
+ return;
+ }
+ const limits = await deps.getLimits(requestUser(req));
+ const result = await deps.fireNow(schedule, limits);
+ if (result == null) {
+ res.status(409).json({ error: 'A run for this schedule is already in progress' });
+ return;
+ }
+ if (!result.fired) {
+ // A limiter refusal is the caller's own quota, not a conflicting schedule state,
+ // so answer 429 rather than burying it in the generic 409.
+ res.status(result.skipped === 'rate_limited' ? 429 : 409).json({
+ error:
+ result.skipped === 'rate_limited'
+ ? 'Too many messages. Try running this schedule again shortly.'
+ : (result.error ?? `Run skipped (${result.skipped ?? 'unknown'})`),
+ skipped: result.skipped,
+ });
+ return;
+ }
+ res.json({ scheduleId: schedule.id, conversationId: result.conversationId, status: 'started' });
+ }
+
+ return {
+ listSchedules,
+ getSchedule,
+ createSchedule,
+ updateSchedule,
+ deleteSchedule,
+ runScheduleNow,
+ };
+}
+
+export type { ScheduleUserContext };
diff --git a/packages/api/src/schedules/index.ts b/packages/api/src/schedules/index.ts
new file mode 100644
index 0000000000..4abce8246d
--- /dev/null
+++ b/packages/api/src/schedules/index.ts
@@ -0,0 +1,8 @@
+export * from './access';
+export * from './cadence';
+export * from './engine';
+export * from './erasure';
+export * from './fire';
+export * from './handlers';
+export * from './trigger';
+export * from './types';
diff --git a/packages/api/src/schedules/service.spec.ts b/packages/api/src/schedules/service.spec.ts
new file mode 100644
index 0000000000..efc00f97b7
--- /dev/null
+++ b/packages/api/src/schedules/service.spec.ts
@@ -0,0 +1,1628 @@
+import { logger } from '@librechat/data-schemas';
+import type { SchedulesServiceDeps } from './service';
+import { isShutdownInProgress } from '../app/shutdown';
+import { createSchedulesService } from './service';
+
+/** Swappable per test: null keeps the no-job-store harness the drain tests rely on. */
+let mockJobStore: { getJob: jest.Mock; deleteJob?: jest.Mock } | null = null;
+
+jest.mock('../agents/checkpointer', () => ({
+ deleteAgentCheckpoint: jest.fn(async () => undefined),
+ // Non-empty by default so the scoped prune has something to delete in tests.
+ captureAgentCheckpointGeneration: jest.fn(async (threadId: string) => ({
+ threadId,
+ checkpointIds: ['ck-1'],
+ })),
+}));
+const checkpointerModule = jest.requireMock('../agents/checkpointer') as {
+ deleteAgentCheckpoint: jest.Mock;
+ captureAgentCheckpointGeneration: jest.Mock;
+};
+
+jest.mock('../stream/GenerationJobManager', () => ({
+ GenerationJobManager: {
+ // No configured job store by default: abortScheduledJob returns false, so the drain
+ // loop is driven purely by getActiveRunsForUser (the run rows).
+ getJobStore: () => mockJobStore,
+ abortJob: jest.fn(),
+ updateMetadata: jest.fn(async () => undefined),
+ isRedis: false,
+ },
+}));
+
+type ActiveRun = {
+ scheduleId: string;
+ scheduledFor: Date;
+ conversationId?: string;
+ status?: string;
+};
+
+let recordRunOutcome: jest.Mock;
+
+function makeService(
+ getActiveRunsForUser: jest.Mock<Promise<ActiveRun[]>, [string]>,
+ getAppConfig?: SchedulesServiceDeps['getAppConfig'],
+ enqueueAgentTrigger: SchedulesServiceDeps['enqueueAgentTrigger'] = jest.fn(async () => undefined),
+): ReturnType<typeof createSchedulesService> {
+ recordRunOutcome = jest.fn(async () => undefined);
+ const methods = {
+ suspendUserSchedulesForDeletion: jest.fn(async () => undefined),
+ restoreUserSchedulesFromDeletion: jest.fn(async () => undefined),
+ getActiveRunsForUser,
+ countActiveRuns: jest.fn(async () => 0),
+ requestRunAbort: jest.fn(async () => true),
+ getScheduleRunAbortState: jest.fn(async () => null),
+ markRunAbortPersisted: jest.fn(async () => undefined),
+ recordRunOutcome,
+ };
+ const deps = {
+ methods,
+ getAppConfig: getAppConfig ?? jest.fn(async () => ({})),
+ findUserById: jest.fn(async () => null),
+ findBalance: jest.fn(async () => null),
+ upsertBalance: jest.fn(async () => null),
+ initializeNullBalance: jest.fn(async () => null),
+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
+ isUserDeleting: jest.fn(async () => false),
+ enqueueAgentTrigger,
+ getTriggerDelivery: jest.fn(async () => null),
+ } as unknown as SchedulesServiceDeps;
+ // Short bounded waits so fail-closed drain paths resolve in test time.
+ return createSchedulesService(deps, {
+ drainTimeoutMs: 400,
+ drainPollMs: 25,
+ stopBarrierTimeoutMs: 400,
+ stopBarrierPollMs: 25,
+ });
+}
+
+const run = (): ActiveRun => ({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+});
+
+describe('shutdown wiring', () => {
+ it('carries the coordinator signal on the BASE deps so Run Now is gated too', async () => {
+ const service = makeService(jest.fn(async (_userId: string) => []));
+ // fireScheduleNow dispatches with engineDeps directly (not the engine's
+ // per-pass wrapper), so the shutdown gate must live on the base deps or a
+ // manual Run Now POSTs into a closing listener.
+ expect(service.engineDeps.isShuttingDown).toBe(isShutdownInProgress);
+ });
+});
+
+describe('manual Run Now lease cleanup', () => {
+ it('releases the old holder when an owner edit fences cleanup after a preflight throw', async () => {
+ const service = makeService(jest.fn(async (_userId: string) => []));
+ const leased = {
+ id: 's1',
+ user: 'user-1',
+ tenantId: 't1',
+ name: 'Digest',
+ prompt: 'Summarize',
+ agent_id: 'agent-1',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ claimToken: 'manual-token',
+ leaseBy: 'manual-holder',
+ runCount: 0,
+ failureCount: 0,
+ balanceSkipCount: 0,
+ } as never;
+ const methods = service.engineDeps.methods as unknown as {
+ acquireManualRunLease: jest.Mock;
+ releaseLease: jest.Mock;
+ releaseLeaseByHolder: jest.Mock;
+ };
+ methods.acquireManualRunLease = jest.fn(async () => leased);
+ // Simulates an edit rotating claimToken while deliberately preserving leaseBy.
+ methods.releaseLease = jest.fn(async () => false);
+ methods.releaseLeaseByHolder = jest.fn(async () => undefined);
+ service.engineDeps.getUserContext = jest.fn(async () => {
+ throw new Error('user lookup failed');
+ });
+
+ await expect(
+ service.fireScheduleNow(leased, {
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+ }),
+ ).rejects.toThrow('user lookup failed');
+
+ expect(methods.releaseLease).toHaveBeenCalledWith('s1', 'manual-token');
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'manual-holder');
+ });
+});
+
+describe('balance initialization', () => {
+ const balanceConfig = {
+ interfaceConfig: {},
+ balance: { enabled: true, startBalance: 20000 },
+ } as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;
+
+ function serviceWithBalance(
+ existing: Record<string, unknown> | null,
+ overrides: {
+ initializeNullBalance?: jest.Mock;
+ findBalance?: jest.Mock;
+ } = {},
+ ) {
+ const upsertBalance = jest.fn(async () => ({ tokenCredits: 20000 }));
+ const initializeNullBalance =
+ overrides.initializeNullBalance ?? jest.fn(async () => ({ tokenCredits: 20000 }));
+ const findBalance = overrides.findBalance ?? jest.fn(async () => existing);
+ const service = createSchedulesService({
+ methods: {} as unknown as SchedulesServiceDeps['methods'],
+ getAppConfig: (async () => balanceConfig) as SchedulesServiceDeps['getAppConfig'],
+ findUserById: jest.fn(async () => null),
+ findBalance,
+ upsertBalance,
+ initializeNullBalance,
+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
+ isUserDeleting: jest.fn(async () => false),
+ enqueueAgentTrigger: jest.fn(async () => undefined),
+ getTriggerDelivery: jest.fn(async () => null),
+ } as unknown as SchedulesServiceDeps);
+ return { service, upsertBalance, initializeNullBalance, findBalance };
+ }
+
+ const updateFrom = (spy: jest.Mock) =>
+ (
+ spy.mock.calls[0] as unknown as [
+ string,
+ { set: Record<string, unknown>; setOnInsert: Record<string, unknown> },
+ ]
+ )[1];
+
+ const casUpdateFrom = (spy: jest.Mock) =>
+ (
+ spy.mock.calls[0] as unknown as [
+ string,
+ { tokenCredits: number; sync: Record<string, unknown> },
+ ]
+ )[1];
+
+ /**
+ * The balance READ and this write are separate statements. A concurrent charge that
+ * creates the record in between would be overwritten by a blind `$set`, handing back
+ * credits the user had already spent.
+ */
+ it('initializes the starting credit via setOnInsert, never $set', async () => {
+ const { service, upsertBalance } = serviceWithBalance(null);
+
+ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
+
+ expect(upsertBalance).toHaveBeenCalledTimes(1);
+ const update = updateFrom(upsertBalance);
+ expect(update.setOnInsert).toMatchObject({ tokenCredits: 20000 });
+ expect(update.set).not.toHaveProperty('tokenCredits');
+ });
+
+ /**
+ * The record was observed with a null credit, but the read and the write are separate
+ * statements: a concurrent initializer/charge could have set and spent it in between.
+ * The credit must go through a `{ tokenCredits: null }` CAS, never a blind `$set` that
+ * would restore spent credits.
+ */
+ it('initializes an EXISTING null credit through the CAS, never a blind $set', async () => {
+ const { service, upsertBalance, initializeNullBalance } = serviceWithBalance({
+ autoRefillEnabled: false,
+ });
+
+ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
+
+ expect(initializeNullBalance).toHaveBeenCalledTimes(1);
+ const cas = casUpdateFrom(initializeNullBalance);
+ expect(cas.tokenCredits).toBe(20000);
+ // The credit write goes ONLY through the CAS, never the unconditional upsert path.
+ expect(upsertBalance).not.toHaveBeenCalled();
+ });
+
+ /**
+ * When the CAS misses — a concurrent initializer/charge won — the preflight must re-read
+ * the winner's balance rather than restore the starting credit.
+ */
+ it('re-reads the winner on a CAS miss and never restores spent credits', async () => {
+ const initializeNullBalance = jest.fn(async () => null);
+ // First read observes a null credit; the winner's re-read returns the spent balance.
+ const findBalance = jest
+ .fn()
+ .mockResolvedValueOnce({ autoRefillEnabled: false, tokenCredits: null })
+ .mockResolvedValueOnce({ tokenCredits: 5 });
+ const { service } = serviceWithBalance(null, { initializeNullBalance, findBalance });
+
+ const outOfBalance = await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
+
+ expect(initializeNullBalance).toHaveBeenCalledTimes(1);
+ // Re-read after the miss (the two isOutOfBalance reads), and the small winner balance
+ // wins: > 0 credits means the user is not pre-skipped for balance.
+ expect(findBalance).toHaveBeenCalledTimes(2);
+ expect(outOfBalance).toBe(false);
+ });
+
+ /**
+ * A stale record whose credit is already set but whose refill config drifted still syncs
+ * that config, and the sync must not widen into a credit write.
+ */
+ it('syncs refill config on a credited record without writing tokenCredits', async () => {
+ const refillConfig = {
+ interfaceConfig: {},
+ balance: {
+ enabled: true,
+ startBalance: 20000,
+ autoRefillEnabled: true,
+ refillIntervalValue: 30,
+ refillIntervalUnit: 'days',
+ refillAmount: 10000,
+ },
+ } as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;
+ const upsertBalance = jest.fn(async () => ({ tokenCredits: 100 }));
+ const initializeNullBalance = jest.fn(async () => null);
+ const service = createSchedulesService({
+ methods: {} as unknown as SchedulesServiceDeps['methods'],
+ getAppConfig: (async () => refillConfig) as SchedulesServiceDeps['getAppConfig'],
+ findUserById: jest.fn(async () => null),
+ findBalance: jest.fn(async () => ({
+ tokenCredits: 100,
+ autoRefillEnabled: false,
+ lastRefill: new Date('2026-01-01T00:00:00.000Z'),
+ })),
+ upsertBalance,
+ initializeNullBalance,
+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
+ isUserDeleting: jest.fn(async () => false),
+ enqueueAgentTrigger: jest.fn(async () => undefined),
+ getTriggerDelivery: jest.fn(async () => null),
+ } as unknown as SchedulesServiceDeps);
+
+ await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
+
+ // Credit is already set, so the CAS path is never taken.
+ expect(initializeNullBalance).not.toHaveBeenCalled();
+ expect(upsertBalance).toHaveBeenCalledTimes(1);
+ const update = updateFrom(upsertBalance);
+ expect(update.set).toMatchObject({ autoRefillEnabled: true });
+ expect(update.set).not.toHaveProperty('tokenCredits');
+ expect(update.setOnInsert).toEqual({});
+ });
+});
+
+describe('deleteScheduleForOwner', () => {
+ beforeEach(() => {
+ // Module-level mock: without this it accumulates calls across tests and the
+ // prune assertions below read a previous test's invocation.
+ checkpointerModule.deleteAgentCheckpoint.mockClear();
+ });
+
+ /**
+ * markScheduleDeleting runs FIRST and is one-shot: it matches only a not-yet-deleting
+ * row, so a retry answers 404. Anything between it and the aborts that can throw
+ * therefore strands the schedule — hidden and fenced, but with its paused job still
+ * resumable and its checkpoint unpruned, until the 25-hour abandonment sweep.
+ */
+ it('still aborts a paused run when the checkpointer config cannot be resolved', async () => {
+ const pausedRun = {
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'requires_action',
+ };
+ const service = makeService(
+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]),
+ // The checkpointer lookup resolves the owner's config; a transient failure here
+ // must not cost the abort.
+ jest.fn(async () => {
+ throw new Error('config plane down');
+ }) as unknown as SchedulesServiceDeps['getAppConfig'],
+ );
+ const methods = service.engineDeps.methods as unknown as {
+ markScheduleDeleting: jest.Mock;
+ getActiveRunsForSchedule: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ };
+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
+ methods.getActiveRunsForSchedule = jest.fn(async () => [pausedRun]);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+ // The owner must RESOLVE, or the checkpointer lookup short-circuits before it ever
+ // reads the config and the throw under test never happens.
+ (service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(
+ async () => ({ id: 'user-1', tenantId: 't1', role: 'USER' }),
+ );
+
+ const abortJob = jest.fn(async () => ({ success: true }));
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'requires_action',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = abortJob;
+
+ // A paused (not running) job is positive evidence: settled synchronously, erased.
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
+ expect(abortJob).toHaveBeenCalled();
+ });
+
+ /** Shared double set for the drain-discipline tests below. */
+ function makeDeleteHarness(run: Partial<ActiveRun> & { scheduleId: string }) {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ markScheduleDeleting: jest.Mock;
+ getActiveRunsForSchedule: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ recordRunOutcome: jest.Mock;
+ getScheduleById: jest.Mock;
+ };
+ methods.markScheduleDeleting = jest.fn(async () => ({ id: run.scheduleId, user: 'user-1' }));
+ methods.getActiveRunsForSchedule = jest.fn(async () => [run]);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+ methods.recordRunOutcome = jest.fn(async () => undefined);
+ methods.getScheduleById = jest.fn(async () => ({ user: 'user-1' }));
+ return { service, methods };
+ }
+
+ it('settles a pause hand-off after the exact provider drain is confirmed', async () => {
+ const { service, methods } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ });
+ // The job can report requires_action before the run row records the pause. The
+ // delete path must abort through #14925's provider-drain barrier before settling,
+ // so no trailing controller write can race the outcome.
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'requires_action',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({
+ success: true,
+ signalDelivered: true,
+ jobData: { status: 'requires_action' },
+ }));
+
+ await service.deleteScheduleForOwner('s1', 'user-1');
+
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'interrupted', conversationId: 'c1' }),
+ );
+ });
+
+ it('does not report success when the abort of a live run is not delivered', async () => {
+ const { service, methods } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ });
+ // A RUNNING identity-matched job: not settleable, must be aborted...
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ deleteJob: jest.fn(async () => true),
+ } as unknown as typeof mockJobStore;
+ // ...and the abort delivery FAILS (job store write rejected).
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => {
+ throw new Error('store unreachable');
+ });
+
+ // The generation may still be producing and billing; claiming success would say
+ // it was stopped. The schedule stays hidden and fenced; the delete is idempotent.
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
+ expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();
+ });
+
+ it('treats an unreadable job store as unknown, not absent', async () => {
+ const { service, methods } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ });
+ mockJobStore = {
+ getJob: jest.fn(async () => {
+ throw new Error('redis gone');
+ }),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => {
+ throw new Error('redis gone');
+ });
+
+ // With the store unreadable NOTHING is proven: the row must not be settled as
+ // an orphan (its generation may be live) and the delete must not read as done.
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('settles a provably job-less run synchronously and erases without a reconciler', async () => {
+ const { service, methods } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ });
+ // Confirmed absence: the lookup SUCCEEDS and returns null (a crashed fire, or a
+ // stale row from a previous topology). The clustered entrypoint has no reconciler,
+ // so deferring this row to one retained the deleted schedule indefinitely there.
+ mockJobStore = { getJob: jest.fn(async () => null) } as unknown as typeof mockJobStore;
+
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
+ expect(methods.recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
+ );
+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');
+ });
+
+ /**
+ * The durable checkpoint is keyed by conversationId ALONE, so pruning it when a
+ * REPLACEMENT turn owns the conversation strips the resume state of a live generation
+ * that has nothing to do with this schedule — the same hazard the interactive abort
+ * route refuses with a 409.
+ */
+ it('does not prune the checkpoint when a replacement owns the conversation', async () => {
+ const { service } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'requires_action',
+ });
+ // A job IS present, but it carries a DIFFERENT occurrence's identity: this
+ // conversation now belongs to someone else's turn.
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 99,
+ scheduleId: 'a-different-schedule',
+ scheduledFor: new Date('2030-01-01T00:00:00.000Z').toISOString(),
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: true }));
+
+ await service.deleteScheduleForOwner('s1', 'user-1');
+
+ expect(checkpointerModule.deleteAgentCheckpoint).not.toHaveBeenCalled();
+ });
+
+ it('prunes the checkpoint of a paused run it does own', async () => {
+ const { service } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'requires_action',
+ });
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'requires_action',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: true }));
+
+ await service.deleteScheduleForOwner('s1', 'user-1');
+
+ // SCOPED to the checkpoint ids captured before the terminal transition, so a
+ // replacement's later checkpoints can never be swept up by this prune.
+ expect(checkpointerModule.deleteAgentCheckpoint).toHaveBeenCalledWith('c1', undefined, {
+ threadId: 'c1',
+ checkpointIds: ['ck-1'],
+ });
+ });
+
+ it('reports draining when a live run was aborted but has not yet settled', async () => {
+ const { service, methods } = makeDeleteHarness({
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ });
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ deleteJob: jest.fn(async () => true),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: true }));
+ // The run row is still active, so the erase declines; settlement (and the
+ // erase-on-settle it triggers) belongs to the aborted generation's outcome write.
+ methods.eraseScheduleIfDrained = jest.fn(async () => false);
+ // Public reads intentionally hide deleting rows. That absence must not turn a
+ // declined erasure into a false `deleted` response.
+ methods.getScheduleById.mockResolvedValue(null);
+
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('draining');
+ });
+
+ afterEach(() => {
+ mockJobStore = null;
+ jest.restoreAllMocks();
+ });
+});
+
+describe('erase-on-settle', () => {
+ /**
+ * Whichever process records a run's terminal outcome also attempts the deferred
+ * erase of a deleting schedule. This is what makes a delete's `draining` state
+ * converge in EVERY topology — the clustered entrypoint runs no reconciler, so
+ * without it the hidden schedule (and its prompt, which has no TTL) survived its
+ * last run indefinitely there.
+ */
+ it('attempts the deferred erase after recording a terminal outcome', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ getScheduleById: jest.Mock;
+ recordRunOutcome: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ };
+ methods.getScheduleById = jest.fn(async () => null);
+ methods.recordRunOutcome = jest.fn(async () => undefined);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+
+ await expect(
+ service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'success',
+ }),
+ ).resolves.toBe(true);
+ expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');
+ });
+
+ it('does not erase on a pause, which is not a settlement', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ getScheduleById: jest.Mock;
+ recordRunOutcome: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ };
+ methods.getScheduleById = jest.fn(async () => null);
+ methods.recordRunOutcome = jest.fn(async () => undefined);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+
+ await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'requires_action',
+ });
+ expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();
+ });
+});
+
+describe('interactive Stop persistence barrier', () => {
+ function outcomeService() {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ getScheduleById: jest.Mock;
+ recordRunOutcome: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ getScheduleRunAbortState: jest.Mock;
+ requestRunAbort: jest.Mock;
+ markRunAbortPersisted: jest.Mock;
+ };
+ methods.getScheduleById = jest.fn(async () => null);
+ methods.recordRunOutcome = jest.fn(async () => undefined);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+ return { service, methods };
+ }
+
+ it('beginScheduledStop stamps a serialized stop; acknowledge marks persistence', async () => {
+ const { service, methods } = outcomeService();
+ methods.requestRunAbort = jest.fn(async () => 'in_progress' as const);
+ methods.markRunAbortPersisted = jest.fn(async () => undefined);
+
+ const stamp = await service.beginScheduledStop({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ });
+ expect(stamp).toBe('in_progress');
+ expect(methods.requestRunAbort).toHaveBeenCalledWith('s1', expect.any(Date), 'stop');
+
+ await service.acknowledgeScheduledStopPersistence({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ });
+ expect(methods.markRunAbortPersisted).toHaveBeenCalledWith('s1', expect.any(Date));
+ });
+
+ it('owner terminal settlement waits until the Stop persistence is acknowledged', async () => {
+ const { service, methods } = outcomeService();
+ const fresh = new Date();
+ let reads = 0;
+ // Unresolved fresh stop for the first two reads, then acknowledged.
+ methods.getScheduleRunAbortState = jest.fn(async () => {
+ reads += 1;
+ return {
+ status: 'started',
+ abortSource: 'stop',
+ abortRequestedAt: fresh,
+ ...(reads > 2 ? { abortPersistedAt: new Date() } : {}),
+ };
+ });
+
+ await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'success',
+ });
+
+ // Polled until the acknowledgement landed, and only THEN recorded the outcome.
+ expect(reads).toBeGreaterThanOrEqual(3);
+ expect(methods.recordRunOutcome).toHaveBeenCalled();
+ expect(methods.getScheduleRunAbortState.mock.invocationCallOrder[0]).toBeLessThan(
+ methods.recordRunOutcome.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('does not wait on a non-stop abort source', async () => {
+ const { service, methods } = outcomeService();
+ methods.getScheduleRunAbortState = jest.fn(async () => ({
+ status: 'started',
+ abortSource: 'deletion',
+ abortRequestedAt: new Date(),
+ }));
+
+ await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'interrupted',
+ });
+
+ expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
+ expect(methods.recordRunOutcome).toHaveBeenCalled();
+ });
+
+ it('does not wait on a STALE stop stamp (route presumed dead)', async () => {
+ const { service, methods } = outcomeService();
+ // Older than ABORT_OWNER_PRESUMED_ALIVE_MS: the owner presumes the route dead.
+ const stale = new Date(Date.now() - 31 * 60_000);
+ methods.getScheduleRunAbortState = jest.fn(async () => ({
+ status: 'started',
+ abortSource: 'stop',
+ abortRequestedAt: stale,
+ }));
+
+ await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'error',
+ });
+
+ expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
+ expect(methods.recordRunOutcome).toHaveBeenCalled();
+ });
+
+ /**
+ * The poll budget expiring proves nothing about the Stop route's writes (slow checkpoint
+ * cleanup looks identical), so the barrier must DEFER rather than terminalize the run and
+ * release its capacity mid-persistence.
+ */
+ it('defers settlement when a fresh Stop never acknowledges within the poll budget', async () => {
+ const { service, methods } = outcomeService();
+ methods.getScheduleRunAbortState = jest.fn(async () => ({
+ status: 'started',
+ abortSource: 'stop',
+ abortRequestedAt: new Date(),
+ }));
+
+ const settled = await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'interrupted',
+ });
+
+ // Reported NOT settled so a durable-retry caller re-drives it, and the run is left
+ // active rather than terminalized while persistence may still be in flight.
+ expect(settled).toBe(false);
+ expect(methods.recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('does not gate a pause (requires_action) on the Stop barrier', async () => {
+ const { service, methods } = outcomeService();
+ methods.getScheduleRunAbortState = jest.fn(async () => null);
+
+ await service.recordScheduleOutcome({
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ status: 'requires_action',
+ });
+
+ // A pause is not a settlement, so the barrier is never consulted.
+ expect(methods.getScheduleRunAbortState).not.toHaveBeenCalled();
+ });
+});
+
+describe('attachment hold renewal', () => {
+ it('renews the bounded upload hold at fire preflight, best-effort', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ extendFilesTTL: jest.Mock;
+ getFiles: jest.Mock;
+ };
+ // The hold bridges upload -> first consumption; a renewal failure must not fail
+ // the fire (the file resolves now; at worst the hold lapses later).
+ methods.extendFilesTTL = jest.fn(async () => {
+ throw new Error('mongo down');
+ });
+ methods.getFiles = jest.fn(async () => [
+ { file_id: 'f1', filepath: '/f1', filename: 'a.png', type: 'image/png', source: 'local' },
+ ]);
+
+ const files = await service.engineDeps.resolveFiles(['f1'], { id: 'user-1', tenantId: 't1' });
+ expect(methods.extendFilesTTL).toHaveBeenCalledWith(
+ ['f1'],
+ expect.objectContaining({ renewMs: expect.any(Number), maxLifetimeMs: expect.any(Number) }),
+ { user: 'user-1', tenantId: 't1' },
+ );
+ expect(files).toHaveLength(1);
+ });
+});
+
+describe('isScheduleLive policy recheck', () => {
+ const liveRow = { id: 's1', user: 'u1', enabled: true } as never;
+
+ it('refuses a resume while the operator kill switch is up', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as { getScheduleById: jest.Mock };
+ methods.getScheduleById = jest.fn(async () => liveRow);
+ process.env.SCHEDULES_DISABLED = 'true';
+ try {
+ // The row-level checks alone pass; only the policy recheck sees the switch.
+ await expect(service.isScheduleLive('s1')).resolves.toBe(true);
+ // A pause can sit for hours; an approval must not start a billed continuation
+ // the operator believes is stopped.
+ await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
+ } finally {
+ delete process.env.SCHEDULES_DISABLED;
+ }
+ });
+
+ it('refuses a resume when the owner lost SCHEDULES:USE', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ getScheduleById: jest.Mock;
+ getRoleByName: jest.Mock;
+ };
+ methods.getScheduleById = jest.fn(async () => liveRow);
+ methods.getRoleByName = jest.fn(async () => ({ permissions: { SCHEDULES: { USE: false } } }));
+ (service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(
+ async () => ({ id: 'u1', tenantId: 't1', role: 'USER' }),
+ );
+
+ await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
+ });
+});
+
+describe('quiesceUserSchedules drain wait', () => {
+ afterEach(() => {
+ mockJobStore = null;
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+ });
+
+ it('polls the active runs until they drain, then returns', async () => {
+ jest.useFakeTimers();
+ const active = [run()];
+ const getActive = jest
+ .fn<Promise<ActiveRun[]>, [string]>()
+ .mockResolvedValueOnce(active) // initial collection for abort
+ .mockResolvedValueOnce(active) // first poll: still settling
+ .mockResolvedValue([]); // subsequent polls: drained
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ // Each poll waits one interval; advance twice so the loop observes the drain.
+ await jest.advanceTimersByTimeAsync(250);
+ await jest.advanceTimersByTimeAsync(250);
+ // The rows DRAINED, so the runs are genuinely settled even though this harness has
+ // no job store and the aborts could not be confirmed delivered. The drain is the
+ // authority; an undelivered abort whose run then settled must not defer deletion
+ // forever.
+ await expect(pending).resolves.toBe(true);
+
+ // Initial read + at least one poll that observed a non-empty set + the empty one.
+ expect(getActive.mock.calls.length).toBeGreaterThanOrEqual(3);
+ });
+
+ it('stops waiting at the bounded deadline when runs never drain', async () => {
+ jest.useFakeTimers();
+ const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([run()]); // never drains
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ // Advance past the full bounded timeout; the loop must give up, not hang, and must
+ // report the drain as UNCONFIRMED so deletion defers rather than destroying.
+ await jest.advanceTimersByTimeAsync(10_000);
+ await expect(pending).resolves.toBe(false);
+
+ // It polled repeatedly (bounded by the deadline) and surfaced the un-drained runs.
+ expect(getActive.mock.calls.length).toBeGreaterThan(1);
+ expect(warn).toHaveBeenCalled();
+ });
+
+ /**
+ * A `started` row whose job is already terminal is the preserve-for-reconcile case:
+ * the inline outcome write exhausted its retries, so the retained job is the ONLY
+ * evidence the run finished. Account deletion deletes that job (nothing will ever
+ * reconcile it once the rows are hard-deleted), so it has to project the job's
+ * terminal state onto the row first — otherwise the row stays active, the bounded
+ * drain can never confirm, and deletion defers until the 30-minute orphan cutoff.
+ */
+ it('does not settle a pause hand-off still in flight (started row, paused job)', async () => {
+ jest.useFakeTimers();
+ const started = { ...run(), status: 'started' };
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]);
+ const service = makeService(getActive);
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'requires_action',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
+ })),
+ } as unknown as typeof mockJobStore;
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ // The pause hand-off's writes are still in flight; confirming the drain here
+ // let the destructive cascade run before they landed. Deferral is bounded: the
+ // controller (or the paused-window reconciler) flips the row to requires_action
+ // and the pending-deletion sweep retries.
+ await expect(pending).resolves.toBe(false);
+
+ expect(recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('settles a started run from its retained terminal job before dropping the evidence', async () => {
+ jest.useFakeTimers();
+ const started = { ...run(), status: 'started' };
+ const getActive = jest
+ .fn<Promise<ActiveRun[]>, [string]>()
+ .mockResolvedValueOnce([started])
+ .mockResolvedValue([]);
+ const service = makeService(getActive);
+ // deleteJob MUST be present: without it the store call throws, the delete never
+ // happens, and this test would pass while proving nothing about the ordering.
+ const deleteJob = jest.fn(async () => undefined);
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'complete',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
+ })),
+ deleteJob,
+ } as unknown as typeof mockJobStore;
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(500);
+ await expect(pending).resolves.toBe(true);
+
+ expect(recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'success' }),
+ );
+ // The retained job is the ONLY evidence this run finished, so it may not be deleted
+ // until the outcome is durably recorded.
+ expect(deleteJob).toHaveBeenCalled();
+ expect(recordRunOutcome.mock.invocationCallOrder[0]).toBeLessThan(
+ deleteJob.mock.invocationCallOrder[0],
+ );
+ });
+
+ /**
+ * `aborted` is the abort REQUEST landing, not the generation finishing: abortJob wins
+ * its status CAS before the owner unwinds, and both owner paths settle LAST, after
+ * saveMessage. Settling on that evidence confirms the drain while the partial write is
+ * still in flight — account deletion then destroys the user's data and the owner
+ * writes a message back for the account that no longer exists.
+ */
+ it('does not settle a run whose abort is still in flight', async () => {
+ jest.useFakeTimers();
+ const aborting = {
+ ...run(),
+ status: 'started',
+ abortRequestedAt: new Date(Date.now() - 1000),
+ };
+ const service = makeService(
+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([aborting]),
+ );
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'aborted',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
+ })),
+ deleteJob: jest.fn(async () => undefined),
+ } as unknown as typeof mockJobStore;
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ // Unconfirmed, so the caller defers deletion (503 + Retry-After) instead of
+ // destroying data the owner is still writing.
+ await expect(pending).resolves.toBe(false);
+ expect(recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ /**
+ * The other direction: an abort whose owner never came back is presumed dead past the
+ * grace, or the account could never be deleted at all.
+ */
+ it('settles a run whose abort request has outlived the owner grace', async () => {
+ jest.useFakeTimers();
+ const abandoned = {
+ ...run(),
+ status: 'started',
+ abortRequestedAt: new Date(Date.now() - 45 * 60 * 1000),
+ };
+ const getActive = jest
+ .fn<Promise<ActiveRun[]>, [string]>()
+ .mockResolvedValueOnce([abandoned])
+ .mockResolvedValue([]);
+ const service = makeService(getActive);
+ mockJobStore = {
+ getJob: jest.fn(async () => null),
+ deleteJob: jest.fn(async () => undefined),
+ } as unknown as typeof mockJobStore;
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(500);
+ await expect(pending).resolves.toBe(true);
+ expect(recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
+ );
+ });
+
+ it('keeps the retained job when settling it fails', async () => {
+ jest.useFakeTimers();
+ const started = { ...run(), status: 'started' };
+ const service = makeService(
+ jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]),
+ );
+ const deleteJob = jest.fn(async () => undefined);
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'complete',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
+ })),
+ deleteJob,
+ } as unknown as typeof mockJobStore;
+ recordRunOutcome.mockRejectedValue(new Error('mongo down'));
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(11_000);
+ // The row never settles, so the drain cannot confirm and deletion defers.
+ await expect(pending).resolves.toBe(false);
+
+ // Evidence intact for the next pass.
+ expect(deleteJob).not.toHaveBeenCalled();
+ });
+
+ it('does not wait when the user has no active runs', async () => {
+ jest.useFakeTimers();
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
+ const service = makeService(getActive);
+
+ // Nothing to abort and nothing to drain, so the quiesce is trivially CONFIRMED and
+ // the deletion cascade may proceed to its destructive steps.
+ await expect(service.quiesceUserSchedules('user-1', 'attempt-1')).resolves.toBe(true);
+ // Only the initial collection read; the drain loop is skipped for an empty set.
+ expect(getActive).toHaveBeenCalledTimes(1);
+ });
+
+ /** A `requires_action` row whose HITL approval was resumed through the chat UI. */
+ const pausedRun = (): ActiveRun => ({ ...run(), status: 'requires_action' });
+
+ it('does not terminalize a paused row whose resumed generation is still running', async () => {
+ jest.useFakeTimers();
+ // Backward-compatibility case from a pre-capacity resume (or rolling deploy):
+ // the job is running while the durable row still reads paused.
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ createdAt: 1,
+ })),
+ };
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ // Terminalizing here would drop the row out of the active set, so the drain would
+ // report nothing to wait for and the destructive cascade could delete messages the
+ // resumed generation is still able to persist.
+ await expect(pending).resolves.toBe(false);
+ expect(recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('still terminalizes a genuinely paused row so deletion is not blocked forever', async () => {
+ jest.useFakeTimers();
+ // Paused for approval, no live generation behind it.
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'requires_action',
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ createdAt: 1,
+ })),
+ };
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ await pending;
+ // An approval that will never be consumed must not keep the account undeletable.
+ expect(recordRunOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
+ );
+ });
+
+ it('does not terminalize a paused row when the job lookup FAILS', async () => {
+ jest.useFakeTimers();
+ mockJobStore = {
+ getJob: jest.fn(async () => {
+ throw new Error('redis unavailable');
+ }),
+ };
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ // A thrown lookup is evidence of NOTHING. Reading it as "genuinely paused" would
+ // terminalize a row whose resumed generation may still be running, and the drain
+ // would then permit the destructive cascade.
+ await expect(pending).resolves.toBe(false);
+ expect(recordRunOutcome).not.toHaveBeenCalled();
+ });
+
+ it('terminalizes a paused row when a replacement turn took over the conversation', async () => {
+ jest.useFakeTimers();
+ // A replacement user turn reuses the conversationId but strips the scheduled
+ // identity, so this running job is NOT this occurrence's generation.
+ mockJobStore = {
+ getJob: jest.fn(async () => ({ status: 'running', createdAt: 1 })),
+ };
+ const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
+ const service = makeService(getActive);
+
+ const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
+ await jest.advanceTimersByTimeAsync(10_000);
+ await pending;
+ expect(recordRunOutcome).toHaveBeenCalled();
+ });
+});
+
+describe('global kill switch', () => {
+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
+
+ afterEach(() => {
+ delete process.env.SCHEDULES_DISABLED;
+ });
+
+ it('is off by default', async () => {
+ const service = makeService(noRuns());
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
+ });
+
+ it('trips on the SCHEDULES_DISABLED env lever without reading config', async () => {
+ process.env.SCHEDULES_DISABLED = 'true';
+ // Throwing getAppConfig proves the env lever works even when the config plane is
+ // unhealthy — the case where a config-dependent kill switch would fail.
+ const getAppConfig = jest.fn(async () => {
+ throw new Error('config plane down');
+ }) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
+ expect(getAppConfig).not.toHaveBeenCalled();
+ });
+
+ it('trips on `interface.schedules: false` read from the BASE config only', async () => {
+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
+ options?.baseOnly === true
+ ? { interfaceConfig: { schedules: false } }
+ : // A principal-merged view that re-enables must NOT be consulted: the global
+ // stop is base-only so no role/user/tenant override can widen past it.
+ { interfaceConfig: { schedules: true } },
+ ) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
+ expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });
+ });
+
+ it('does not trip when only a principal-merged config disables it', async () => {
+ // Per-principal availability is NOT the global stop; the engine keeps claiming so
+ // other principals still fire, and the fire path skips this owner's occurrences.
+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
+ options?.baseOnly === true
+ ? { interfaceConfig: { schedules: true } }
+ : { interfaceConfig: { schedules: false } },
+ ) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
+ });
+
+ it('trips on the object form `{ use: false }` exactly like the boolean stop', async () => {
+ // Both stop shapes must FREEZE occurrences (engine stops claiming, nothing
+ // advances). A shape-blind gate left the engine claiming while getLimits refused
+ // fires, so the disabled path ADVANCED each occurrence — a short maintenance stop
+ // silently dropped everything it covered instead of leaving it due.
+ const getAppConfig = jest.fn(async () => ({
+ interfaceConfig: { schedules: { use: false, maxPerUser: 5 } },
+ })) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
+ });
+
+ it('does not trip on the object form while `use` stays enabled', async () => {
+ const getAppConfig = jest.fn(async () => ({
+ interfaceConfig: { schedules: { use: true, maxPerUser: 5 } },
+ })) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
+ });
+});
+
+describe('scheduled resume capacity', () => {
+ function makeResumeService(occupancy: { takenSlots: number[]; unslotted: number }) {
+ const methods = {
+ getScheduleById: jest.fn(async () => ({
+ id: 's1',
+ user: 'user-1',
+ enabled: true,
+ configRevision: 3,
+ })),
+ getRoleByName: jest.fn(async () => ({ permissions: { SCHEDULES: { USE: true } } })),
+ getCapacityOccupancy: jest.fn(async () => occupancy),
+ acquireResumeLease: jest.fn(async () => ({
+ id: 's1',
+ claimToken: 'resume-token',
+ leaseBy: 'resume:resume-token',
+ })),
+ consumeResumeLease: jest.fn(async () => true),
+ releaseLeaseByHolder: jest.fn(async () => undefined),
+ markRunResumeClaimed: jest.fn(async (_id, _scheduledFor, capacitySlot) => ({
+ capacitySlot,
+ })),
+ releaseRunResumeClaim: jest.fn(async () => true),
+ };
+ const service = createSchedulesService({
+ methods: methods as unknown as SchedulesServiceDeps['methods'],
+ getAppConfig: jest.fn(async () => ({
+ interfaceConfig: {
+ schedules: {
+ use: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 1,
+ },
+ },
+ })),
+ findUserById: jest.fn(async () => ({ _id: 'user-1', role: 'USER' })),
+ findBalance: jest.fn(async () => null),
+ upsertBalance: jest.fn(async () => null),
+ initializeNullBalance: jest.fn(async () => null),
+ resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
+ isUserDeleting: jest.fn(async () => false),
+ enqueueAgentTrigger: jest.fn(async () => undefined),
+ getTriggerDelivery: jest.fn(async () => null),
+ } as unknown as SchedulesServiceDeps);
+ return { service, methods };
+ }
+
+ it('promotes a pause through the global slot allocator and releases by exact slot', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ const scheduledFor = '2026-08-17T12:00:00.000Z';
+
+ await expect(service.claimScheduleResume('s1', scheduledFor)).resolves.toEqual({
+ capacitySlot: 0,
+ claimToken: 'resume-token',
+ leaseBy: 'resume:resume-token',
+ });
+ expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', undefined, true, 60_000);
+ expect(methods.markRunResumeClaimed).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);
+ await expect(
+ service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {
+ expectedConfigRevision: 3,
+ automatic: true,
+ }),
+ ).resolves.toBe(true);
+ expect(methods.consumeResumeLease).toHaveBeenCalledWith(
+ 's1',
+ 'resume-token',
+ 'resume:resume-token',
+ true,
+ 3,
+ );
+ await expect(service.releaseScheduleResumeClaim('s1', scheduledFor, 0)).resolves.toBe(true);
+ expect(methods.releaseRunResumeClaim).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);
+ });
+
+ it('leaves the paused row untouched when deployment capacity is full', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [0], unslotted: 0 });
+
+ await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({
+ conflict: 'capacity',
+ });
+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');
+ });
+
+ it('normalizes an exhausted slot-collision retry to public capacity conflict', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ methods.markRunResumeClaimed.mockResolvedValue({ conflict: 'slot-taken' } as never);
+
+ await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({
+ conflict: 'capacity',
+ });
+ });
+
+ it('refuses a resume whose schedule revision changed before the capacity handoff', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+
+ await expect(
+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
+ expectedConfigRevision: 2,
+ automatic: true,
+ }),
+ ).resolves.toEqual({ conflict: 'inactive' });
+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
+ });
+
+ it('refuses an edit that wins during async policy checks at the final schedule CAS', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ methods.acquireResumeLease.mockResolvedValue(null as never);
+
+ await expect(
+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
+ expectedConfigRevision: 3,
+ automatic: true,
+ }),
+ ).resolves.toEqual({ conflict: 'inactive' });
+ expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', 3, true, 60_000);
+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
+ });
+
+ it('releases the old holder when the post-approval schedule handoff loses', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ methods.consumeResumeLease.mockResolvedValue(false);
+
+ await expect(
+ service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {
+ expectedConfigRevision: 3,
+ automatic: true,
+ }),
+ ).resolves.toBe(false);
+ expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');
+ });
+
+ it('refuses a disabled automatic resume but preserves explicit Run Now semantics', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ methods.getScheduleById.mockResolvedValue({
+ id: 's1',
+ user: 'user-1',
+ enabled: false,
+ configRevision: 3,
+ });
+
+ await expect(
+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
+ expectedConfigRevision: 3,
+ automatic: true,
+ }),
+ ).resolves.toEqual({ conflict: 'inactive' });
+ await expect(
+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
+ expectedConfigRevision: 3,
+ automatic: false,
+ }),
+ ).resolves.toEqual({
+ capacitySlot: 0,
+ claimToken: 'resume-token',
+ leaseBy: 'resume:resume-token',
+ });
+ });
+
+ it('refuses a resume when the owner lost schedule access before the capacity handoff', async () => {
+ const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
+ methods.getRoleByName.mockResolvedValue({ permissions: { SCHEDULES: { USE: false } } });
+
+ await expect(
+ service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
+ expectedConfigRevision: 3,
+ automatic: true,
+ }),
+ ).resolves.toEqual({ conflict: 'inactive' });
+ expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
+ });
+});
+
+describe('deployment-wide limits', () => {
+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
+
+ it('resolves a principal-less getLimits from the BASE config only', async () => {
+ const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
+ options?.baseOnly === true
+ ? { interfaceConfig: { schedules: { use: true, fireConcurrency: 1 } } }
+ : // The principal/tenant-merged view. A bare getAppConfig() resolves THIS,
+ // including whatever tenant the ALS context happens to carry.
+ { interfaceConfig: { schedules: { use: true, fireConcurrency: 5 } } },
+ ) as unknown as SchedulesServiceDeps['getAppConfig'];
+ const service = makeService(noRuns(), getAppConfig);
+ const limits = await service.getLimits();
+ // NO principal must mean the DEPLOYMENT's config. Both callers of this form run
+ // inside a tenant context (fireSchedule clamps the global capacity allocator from
+ // within runInTenantContext(owner); the engine tick budgets from within
+ // runAsSystem), so resolving the merged view would let a tenant override widen the
+ // very global cap it is clamped against.
+ expect(limits.fireConcurrency).toBe(1);
+ expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });
+ });
+});
+
+describe('durable trigger wiring', () => {
+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
+
+ it('uses the shared trigger service instead of owning a second loopback transport', async () => {
+ const enqueueAgentTrigger = jest.fn(async () => ({ status: 'queued' }));
+ const service = makeService(noRuns(), undefined, enqueueAgentTrigger);
+ const envelope = { version: 1, mode: 'fire' } as never;
+
+ await service.engineDeps.enqueueTrigger(envelope, { orderingKey: 'schedule-1' });
+
+ expect(enqueueAgentTrigger).toHaveBeenCalledWith(envelope, {
+ orderingKey: 'schedule-1',
+ });
+ });
+});
+
+describe('admission revision fence', () => {
+ const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
+
+ function serviceWithSchedule(schedule: { configRevision?: number; enabled?: boolean } | null) {
+ const service = makeService(noRuns());
+ (
+ service.engineDeps.methods as unknown as {
+ getScheduleById: jest.Mock;
+ }
+ ).getScheduleById = jest.fn(async () => schedule);
+ return service;
+ }
+
+ it('admits when the claimed revision still matches', async () => {
+ const service = serviceWithSchedule({ configRevision: 3 });
+ expect(await service.isScheduleLive('sched-1', 3)).toBe(true);
+ });
+
+ it('REFUSES when an owner edit moved the revision on after the claim', async () => {
+ // The fire was claimed under revision 3; the owner edited since (now 4). Persisting
+ // would write the OLD prompt/agent into the edited schedule's history.
+ const service = serviceWithSchedule({ configRevision: 4 });
+ expect(await service.isScheduleLive('sched-1', 3)).toBe(false);
+ });
+
+ /**
+ * A policy auto-disable flips `enabled` WITHOUT touching configRevision (an older
+ * paused occurrence can resume, fail and cross the threshold while a newer occurrence
+ * is already in the claim-to-controller window), so the revision fence cannot see it.
+ */
+ it('refuses an AUTOMATIC fire once the schedule was disabled', async () => {
+ const service = serviceWithSchedule({ configRevision: 3, enabled: false });
+ expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(false);
+ });
+
+ it('still admits Run Now on a disabled schedule', async () => {
+ // An explicit user action, matching fireScheduleNow's own relaxation.
+ const service = serviceWithSchedule({ configRevision: 3, enabled: false });
+ expect(await service.isScheduleLive('sched-1', 3, { automatic: false })).toBe(true);
+ expect(await service.isScheduleLive('sched-1', 3)).toBe(true);
+ });
+
+ it('admits an automatic fire while the schedule is still enabled', async () => {
+ const service = serviceWithSchedule({ configRevision: 3, enabled: true });
+ expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(true);
+ });
+
+ it('refuses a schedule that is gone regardless of revision', async () => {
+ const service = serviceWithSchedule(null);
+ expect(await service.isScheduleLive('sched-1', 3)).toBe(false);
+ });
+
+ it('stays permissive when either side has no revision (pre-existing rows)', async () => {
+ expect(await serviceWithSchedule({}).isScheduleLive('sched-1', 3)).toBe(true);
+ expect(await serviceWithSchedule({ configRevision: 4 }).isScheduleLive('sched-1')).toBe(true);
+ });
+});
+
+describe('abort stamp is load-bearing (withheld abort on stamp failure)', () => {
+ it('withholds the abort and reports unconfirmed when the stamp cannot be made durable', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ markScheduleDeleting: jest.Mock;
+ getActiveRunsForSchedule: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ requestRunAbort: jest.Mock;
+ };
+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
+ methods.getActiveRunsForSchedule = jest.fn(async () => [
+ {
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ },
+ ]);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+ // The stamp is what makes concurrent drains and the reconciler defer to the
+ // owner's settle; signalling an abort without it re-opens the drain-mid-write
+ // window the stamp exists to close.
+ methods.requestRunAbort = jest.fn(async () => {
+ throw new Error('mongo down');
+ });
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: true }));
+
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
+ expect(manager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('reports an exact generation that remains active as not stopped', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({
+ success: false,
+ failureReason: 'job_still_active',
+ }));
+
+ const delivered = await service.engineDeps.abortScheduledJob(
+ 'c1',
+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
+ { preserve: true },
+ );
+ expect(delivered).toBe(false);
+ });
+});
+
+describe('deleteScheduleForOwner waits for the settle acknowledgement', () => {
+ it('converges when an unconfirmed abort settles during the bounded drain', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const methods = service.engineDeps.methods as unknown as {
+ markScheduleDeleting: jest.Mock;
+ getActiveRunsForSchedule: jest.Mock;
+ eraseScheduleIfDrained: jest.Mock;
+ };
+ methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
+ // Active at the abort pass, drained by the first drain poll: the owner settled.
+ methods.getActiveRunsForSchedule = jest
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ scheduleId: 's1',
+ scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
+ conversationId: 'c1',
+ status: 'started',
+ },
+ ])
+ .mockResolvedValue([]);
+ methods.eraseScheduleIfDrained = jest.fn(async () => true);
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'running',
+ createdAt: 1,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ // Delivery cannot be locally proven (peer-owned generation): honest false...
+ manager.abortJob = jest.fn(async () => ({ success: true, signalDelivered: false }));
+
+ // ...but the run row leaving the active set is the durable acknowledgement, so
+ // the delete converges instead of answering a spurious 503.
+ await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
+ });
+});
+
+describe('provider-drained schedule aborts', () => {
+ it('waits on the exact terminal generation instead of using the removed re-signal path', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'aborted',
+ createdAt: 7,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: false }));
+
+ const delivered = await service.engineDeps.abortScheduledJob(
+ 'c1',
+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
+ { preserve: true },
+ );
+ expect(manager.abortJob).toHaveBeenCalledWith('c1', {
+ expectedCreatedAt: 7,
+ awaitProviderDrain: true,
+ });
+ expect(delivered).toBe(true);
+ });
+
+ it('deletes terminal evidence only after the exact provider drain is confirmed', async () => {
+ const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
+ const deleteJob = jest.fn(async () => true);
+ mockJobStore = {
+ getJob: jest.fn(async () => ({
+ status: 'aborted',
+ createdAt: 7,
+ scheduleId: 's1',
+ scheduledFor: '2026-01-01T00:00:00.000Z',
+ })),
+ deleteJob,
+ } as unknown as typeof mockJobStore;
+ const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
+ manager.abortJob = jest.fn(async () => ({ success: false }));
+
+ const delivered = await service.engineDeps.abortScheduledJob(
+ 'c1',
+ { scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
+ { preserve: false },
+ );
+ expect(delivered).toBe(true);
+ expect(manager.abortJob).toHaveBeenCalledWith('c1', {
+ expectedCreatedAt: 7,
+ awaitProviderDrain: true,
+ });
+ expect(deleteJob).toHaveBeenCalledWith('c1', 7);
+ });
+});
diff --git a/packages/api/src/schedules/service.ts b/packages/api/src/schedules/service.ts
new file mode 100644
index 0000000000..393a724228
--- /dev/null
+++ b/packages/api/src/schedules/service.ts
@@ -0,0 +1,1610 @@
+import { logger, runAsSystem, tenantStorage, isRuntimeDisabled } from '@librechat/data-schemas';
+import { getRefillEligibilityDate, Permissions, PermissionTypes } from 'librechat-data-provider';
+import type { ScheduleMethods, AppConfig, IBalance } from '@librechat/data-schemas';
+import type { TCheckpointerConfig } from 'librechat-data-provider';
+import type { Types } from 'mongoose';
+import type {
+ ScheduleEngineDeps,
+ ScheduleDeleteResult,
+ ScheduleLimits,
+ ScheduleUserContext,
+ FireableSchedule,
+ FireResult,
+ JobIdentity,
+} from './types';
+import type { SerializableJobData } from '../stream/interfaces/IJobStore';
+import type { BalanceUpdateFields } from '../types/balance';
+import type { GetAppConfigOptions } from '../app/service';
+import {
+ DEFAULT_SCHEDULE_LIMITS,
+ SCHEDULE_FILE_HOLD,
+ hasResumeHandoffInFlight,
+ hasAbortInFlight,
+} from './types';
+import { deleteAgentCheckpoint, captureAgentCheckpointGeneration } from '../agents/checkpointer';
+import { fireSchedule, BALANCE_SKIP_DISABLE_THRESHOLD } from './fire';
+import { GenerationJobManager } from '../stream/GenerationJobManager';
+import { buildBalanceUpdateFields } from '../middleware/balance';
+import { getAppConfigOptionsFromUser } from '../app/service';
+import { isShutdownInProgress } from '../app/shutdown';
+import { startScheduleErasureSweep } from './erasure';
+import { getBalanceConfig } from '../app/config';
+import { startScheduleEngine } from './engine';
+import { withCapacitySlot } from './capacity';
+import { isEnabled } from '../utils/common';
+
+/** Recordable terminal/paused run outcome, as accepted by `recordRunOutcome`. */
+type ScheduleRunOutcomeStatus = Parameters<ScheduleMethods['recordRunOutcome']>[0]['status'];
+
+/** How a TERMINAL job status projects onto its run row. Mirrors the reconciler's
+ * mapping, so a run settled from a retained job reads the same either way. */
+const TERMINAL_JOB_OUTCOMES: Record<string, ScheduleRunOutcomeStatus | undefined> = {
+ complete: 'success',
+ error: 'error',
+ aborted: 'interrupted',
+};
+
+/** Reason text recorded when account-deletion quiesce settles a run itself. */
+const QUIESCE_SETTLE_ERRORS: Partial<Record<ScheduleRunOutcomeStatus, string>> = {
+ interrupted: 'Account deleted while awaiting approval',
+ error: 'Run ended in error',
+};
+
+/** Short schedule-document fence spanning capacity admission -> approval CAS. */
+const SCHEDULE_RESUME_LEASE_MS = 60_000;
+
+/**
+ * Whether this process may arm the scheduler at all.
+ *
+ * v1 is single-process by design, but the standard entrypoint runs
+ * `initializeScheduleEngine` in EVERY replica and nothing stops an operator from scaling
+ * it. That is only safe when replicas can see each other's generations: a shared stream
+ * store (Redis) gives every replica the same job view, so reconciliation and
+ * deletion-time aborts reach the run's real owner. With the process-local store a peer
+ * sees the globally visible `started` row but no job, and after the orphan cutoff it
+ * marks a still-running generation interrupted and frees its capacity slot.
+ *
+ * A single replica with the in-memory store is perfectly safe, but a process cannot
+ * observe its own replica count, so that case needs an explicit operator assertion.
+ */
+function isTopologySafeToArm(): boolean {
+ return GenerationJobManager.isRedis || isEnabled(process.env.SCHEDULES_SINGLE_PROCESS);
+}
+
+/** Whether a persisted job still carries a given scheduled occurrence's identity. */
+function jobMatchesIdentity(
+ job: Pick<SerializableJobData, 'scheduleId' | 'scheduledFor'>,
+ identity: JobIdentity,
+): boolean {
+ if (job.scheduleId !== identity.scheduleId || job.scheduledFor == null) {
+ return false;
+ }
+ return new Date(job.scheduledFor).getTime() === new Date(identity.scheduledFor).getTime();
+}
+
+export interface RecordScheduleOutcomeInput {
+ scheduleId?: string;
+ scheduledFor?: string | Date;
+ /** Exact generation whose terminal evidence is being persisted. */
+ streamId?: string;
+ jobCreatedAt?: number;
+ status: ScheduleRunOutcomeStatus;
+ conversationId?: string;
+ /** Erase the row's reserved conversationId (pre-start abort: no conversation exists). */
+ clearConversationId?: boolean;
+ error?: string;
+}
+
+export type ScheduleResumeClaimResult =
+ | { capacitySlot: number; claimToken: string; leaseBy: string }
+ | { conflict: 'capacity' | 'overlap' | 'not-paused' | 'inactive' };
+
+/**
+ * Api-side dependencies the schedules service needs injected: model methods,
+ * config/balance access, and the owner-scoped agent access check. Everything
+ * else (job store and tenant context) lives in `@librechat/api` and is imported
+ * directly.
+ */
+export interface SchedulesServiceDeps {
+ methods: ScheduleMethods & {
+ getRoleByName: (
+ role?: string,
+ ) => Promise<{ permissions?: Record<string, Record<string, boolean | undefined>> } | null>;
+ getFiles: (
+ filter: unknown,
+ sort: unknown,
+ select: unknown,
+ ) => Promise<Array<{
+ file_id: string;
+ filepath: string;
+ filename: string;
+ type: string;
+ height?: number;
+ width?: number;
+ source: string;
+ }> | null>;
+ /** Owner-scoped bounded TTL hold (`db.extendFilesTTL`-shaped). */
+ extendFilesTTL: (
+ fileIds: string[],
+ hold: { renewMs: number; maxLifetimeMs: number },
+ owner: { user: string; tenantId?: string | null },
+ ) => Promise<number>;
+ };
+ getAppConfig: (options?: GetAppConfigOptions) => Promise<AppConfig | undefined>;
+ findUserById: (
+ userId: string | Types.ObjectId,
+ ) => Promise<{ _id: Types.ObjectId; tenantId?: string; role?: string } | null>;
+ findBalance: (userId: string) => Promise<IBalance | null>;
+ /**
+ * Upserts a balance record. `setOnInsert` carries fields that must ONLY apply to a
+ * document this call creates — chiefly the starting credit — so a record created by a
+ * concurrent charge is never overwritten with a fresh balance.
+ */
+ upsertBalance: (
+ userId: string,
+ update: { set: Partial<BalanceUpdateFields>; setOnInsert: Partial<BalanceUpdateFields> },
+ ) => Promise<IBalance | null>;
+ /**
+ * Compare-and-set initialization for an EXISTING balance whose `tokenCredits` is still
+ * null. Writes the starting credit (and any refill-config sync) only while
+ * `{ user, tokenCredits: null }` still matches — never an upsert — so a concurrent
+ * initializer/charge that already set credits between the read and this write is not
+ * clobbered. Returns the winning document, or `null` when the CAS did not match (the
+ * caller re-reads the winner rather than restoring credits).
+ */
+ initializeNullBalance: (
+ userId: string,
+ update: { tokenCredits: number; sync: Partial<BalanceUpdateFields> },
+ ) => Promise<IBalance | null>;
+ resolveAgentFireAccess: (
+ agentId: string,
+ user: ScheduleUserContext,
+ ) => Promise<'ok' | 'missing' | 'forbidden'>;
+ /** Whether this user's account deletion has begun. Fail-closed (unknown == true). */
+ isUserDeleting: (userId: string) => Promise<boolean>;
+ /** Shared durable trigger admission from the merged agent-trigger service. */
+ enqueueAgentTrigger: ScheduleEngineDeps['enqueueTrigger'];
+ /** Reads a durable trigger delivery by key from the merged agent-trigger service, for
+ * reconciliation's live-vs-dead delivery check. */
+ getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'];
+}
+
+export interface SchedulesService {
+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;
+ engineDeps: ScheduleEngineDeps;
+ fireScheduleNow: (
+ schedule: FireableSchedule,
+ limits: ScheduleLimits,
+ ) => Promise<FireResult | null>;
+ recordScheduleOutcome: (input: RecordScheduleOutcomeInput) => Promise<boolean>;
+ /**
+ * Stamps a scheduled run's interactive Stop BEFORE the abort is signalled, so the owner
+ * settlement barrier, reconciliation, and schedule/account deletion hold off settling or
+ * erasing until {@link acknowledgeScheduledStopPersistence}. Serialized: `'in_progress'`
+ * means a fresh Stop already owns the stamp and the caller must not signal a second abort;
+ * `false` means there is no active run to stop.
+ */
+ beginScheduledStop: (input: {
+ scheduleId: string;
+ scheduledFor: string | Date;
+ }) => Promise<boolean | 'in_progress'>;
+ /**
+ * Releases the Stop settlement barrier once the route's partial-message/checkpoint writes
+ * have landed. Call ONLY after persistence succeeds — a failed persistence must leave the
+ * barrier unresolved so the run stays preserved and recovers via the stale-owner timeout.
+ */
+ acknowledgeScheduledStopPersistence: (input: {
+ scheduleId: string;
+ scheduledFor: string | Date;
+ /** Optional terminal outcome to re-drive once the barrier clears, so a settlement the
+ * owner deferred past its poll budget still converges where no reconciler is armed. */
+ settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };
+ }) => Promise<void>;
+ /** Re-enters a paused occurrence into the DB-enforced global/same-schedule
+ * capacity set before its approval job is allowed to resume. */
+ claimScheduleResume: (
+ scheduleId: string,
+ scheduledFor: string | Date,
+ options?: { expectedConfigRevision?: number; automatic?: boolean },
+ ) => Promise<ScheduleResumeClaimResult>;
+ /** Guarded rollback when the approval CAS did not consume the paused action. */
+ releaseScheduleResumeClaim: (
+ scheduleId: string,
+ scheduledFor: string | Date,
+ capacitySlot: number,
+ ) => Promise<boolean>;
+ /** Atomically validates the live schedule generation after the approval CAS and
+ * releases its short-lived document fence. This is the resume linearization point. */
+ finalizeScheduleResumeClaim: (
+ scheduleId: string,
+ claimToken: string,
+ leaseBy: string,
+ options?: { expectedConfigRevision?: number; automatic?: boolean },
+ ) => Promise<boolean>;
+ /** Releases only the schedule-document fence after an unconsumed/ambiguous approval CAS. */
+ releaseScheduleResumeFence: (scheduleId: string, leaseBy: string) => Promise<void>;
+ /**
+ * Whether a schedule is still live (exists and not soft-deleted). The loopback
+ * chat controller calls this right after creating the generation job to re-fence
+ * a fire against a delete/quiesce that landed in the claim -> POST window (when
+ * the reservation row exists but the job did not yet, so the deletion's abort
+ * missed it) — aborting before any messages are persisted.
+ */
+ isScheduleLive: (
+ scheduleId: string,
+ expectedConfigRevision?: number,
+ options?: { automatic?: boolean; policy?: boolean },
+ ) => Promise<boolean>;
+ /** Soft-deletes an owner's schedule: stop claims, abort active runs, drain, erase. */
+ deleteScheduleForOwner: (scheduleId: string, userId: string) => Promise<ScheduleDeleteResult>;
+ /**
+ * Quiesces all of a user's schedules ahead of account deletion (reversible suspension +
+ * abort + drain). `token` identifies this deletion attempt and is what a later
+ * {@link restoreUserSchedulesFromDeletion} restores against. Returns whether the drain was
+ * CONFIRMED: false means at least one run could not be confirmed settled, and the caller
+ * must NOT proceed to destructive deletion — the durable barrier keeps refusing new work
+ * while a later pass finishes the cascade.
+ */
+ quiesceUserSchedules: (userId: string, token: string) => Promise<boolean>;
+ /**
+ * Reverses a quiesce whose account deletion was cancelled, re-enabling and re-arming only
+ * the rows this exact attempt suspended. Safe to call even if quiesce never suspended a
+ * row (no matching token → no-op).
+ */
+ restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise<void>;
+ /** Arms the scheduler for THIS process. v1 is single-process only; the clustered
+ * entrypoint does not start it. Returns undefined when index creation failed, or when
+ * the topology cannot be shown safe (process-local job store with no single-process
+ * assertion) — see isTopologySafeToArm. */
+ initializeScheduleEngine: () => Promise<ReturnType<typeof startScheduleEngine> | undefined>;
+ /** Starts erasure-ONLY maintenance for an entrypoint that never arms the engine (the
+ * clustered worker). Idempotent per process and a no-op once the full engine is armed.
+ * Arms nothing else — no claims, firing, cadence advancement, or absence-based
+ * reconciliation — and refuses to infer owner death from a process-local missing job
+ * (isTopologySafeToArm gates that). See startScheduleErasureSweep. */
+ initializeScheduleErasureSweep: () => void;
+}
+
+/** Test-only overrides for the service's bounded waits (drains, barriers). */
+export interface ScheduleServiceTimings {
+ drainTimeoutMs?: number;
+ drainPollMs?: number;
+ stopBarrierTimeoutMs?: number;
+ stopBarrierPollMs?: number;
+}
+
+/**
+ * Builds the scheduler service around api-side dependencies. Each call owns its
+ * own engine singleton and job-store-shared flag, so state never leaks between
+ * instances.
+ */
+export function createSchedulesService(
+ deps: SchedulesServiceDeps,
+ timings?: ScheduleServiceTimings,
+): SchedulesService {
+ const { methods } = deps;
+
+ // Fail LOUDLY at construction, not per-fire. The JS adapter (api/server/services/
+ // Schedules) is not typechecked against SchedulesServiceDeps, so a missing dep would
+ // otherwise surface only as a `deps.X is not a function` deep inside a live fire —
+ // which is exactly how the deletion-barrier probe shipped unwired twice.
+ const REQUIRED_DEPS: Array<keyof SchedulesServiceDeps> = [
+ 'methods',
+ 'getAppConfig',
+ 'findUserById',
+ 'findBalance',
+ 'upsertBalance',
+ 'initializeNullBalance',
+ 'resolveAgentFireAccess',
+ 'isUserDeleting',
+ 'enqueueAgentTrigger',
+ 'getTriggerDelivery',
+ ];
+ for (const key of REQUIRED_DEPS) {
+ if (deps[key] == null) {
+ throw new Error(`createSchedulesService: missing required dependency "${key}"`);
+ }
+ }
+
+ /**
+ * Resolves schedule limits, honoring per-principal (role/user) config overrides
+ * when a user is supplied (routes pass req.user, the fire path passes the owner).
+ */
+ async function getLimits(user?: ScheduleUserContext): Promise<ScheduleLimits> {
+ // The BASE `interface.schedules: false` is a global stop and must win over any
+ // principal override. Without this a tenant/role/user override resolving to
+ // enabled would let the sidebar and CRUD handlers admit schedules that
+ // isGloballyDisabled() correctly refuses to ever fire.
+ // isRuntimeDisabled, NOT `=== false`: the base stop has TWO shapes (`false` and
+ // `{ use: false }`), and deepMerge turns a base `{ use: false, ... }` plus a principal
+ // override of `true` into `{ use: true, ... }`. A literal-false check missed that, so
+ // getLimits reported the feature enabled and Run Now dispatched straight through
+ // fireSchedule — bypassing the operator's object-form emergency stop. Same predicate
+ // the engine gate (isGloballyDisabled) already uses.
+ if (
+ user != null &&
+ isRuntimeDisabled((await deps.getAppConfig({ baseOnly: true }))?.interfaceConfig?.schedules)
+ ) {
+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
+ }
+ // NO principal means the DEPLOYMENT's config, so it must be read base-only. A bare
+ // getAppConfig() still resolves principals and picks up the tenant from the ALS
+ // context, and both callers of this form run inside one: fireSchedule clamps the
+ // global capacity allocator from within runInTenantContext(owner), and the engine
+ // tick budgets from within runAsSystem. Either would otherwise resolve a TENANT
+ // override as if it were the deployment-wide value, which is exactly what the
+ // global cap exists to prevent an override from widening.
+ const appConfig = user
+ ? await deps.getAppConfig(getAppConfigOptionsFromUser(user))
+ : await deps.getAppConfig({ baseOnly: true });
+ // The env kill switch is a GLOBAL stop and must be visible everywhere limits are
+ // consulted (write handlers, fire path), not only at the engine tick.
+ if (isEnabled(process.env.SCHEDULES_DISABLED)) {
+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
+ }
+ const config = appConfig?.interfaceConfig?.schedules;
+ // EXPERIMENTAL, default-OFF (v1): scheduled chats are disabled unless an admin
+ // explicitly enables them. Absence, null, or `false` all resolve to disabled, so a
+ // deployment that never opts in never runs the scheduler. `true` uses the defaults;
+ // an object opts in unless it sets `use: false`.
+ if (config == null || config === false) {
+ return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
+ }
+ if (config === true) {
+ return DEFAULT_SCHEDULE_LIMITS;
+ }
+ return {
+ enabled: config.use !== false,
+ maxPerUser: config.maxPerUser ?? DEFAULT_SCHEDULE_LIMITS.maxPerUser,
+ minIntervalMinutes: config.minIntervalMinutes ?? DEFAULT_SCHEDULE_LIMITS.minIntervalMinutes,
+ autoDisableAfterFailures:
+ config.autoDisableAfterFailures ?? DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
+ fireConcurrency: config.fireConcurrency ?? DEFAULT_SCHEDULE_LIMITS.fireConcurrency,
+ };
+ }
+
+ const MANUAL_RUN_LEASE_MS = 5 * 60 * 1000;
+ // Bounded wait for aborted scheduled runs to settle during account-deletion quiesce,
+ // before the message/conversation cascade runs. Long enough to cover a generation that
+ // already returned from the model finishing its persistence; capped so account deletion
+ // never blocks indefinitely on an unreachable peer-worker run.
+ const QUIESCE_DRAIN_TIMEOUT_MS = timings?.drainTimeoutMs ?? 10 * 1000;
+ const QUIESCE_DRAIN_POLL_MS = timings?.drainPollMs ?? 250;
+ // The owner's terminal settlement waits (briefly, bounded) for an interactive Stop to
+ // acknowledge its partial-message/checkpoint persistence before releasing capacity or
+ // erasing evidence. Short: the Stop route's persistence is a couple of writes; if it
+ // does not land in this window the stale-owner timeout (ABORT_OWNER_PRESUMED_ALIVE_MS)
+ // takes over as the bounded recovery, so the settlement never blocks indefinitely.
+ const STOP_BARRIER_TIMEOUT_MS = timings?.stopBarrierTimeoutMs ?? 5 * 1000;
+ const STOP_BARRIER_POLL_MS = timings?.stopBarrierPollMs ?? 100;
+
+ /**
+ * Whether a refill would top up this zero-credit balance record right now,
+ * mirroring the chat balance check's auto-refill eligibility (record-based).
+ */
+ function isRefillEligible(record: IBalance | null | undefined): boolean {
+ if (record?.autoRefillEnabled !== true) {
+ return false;
+ }
+ if (!(typeof record.refillAmount === 'number' && record.refillAmount > 0)) {
+ return false;
+ }
+ const lastRefillDate = new Date(record.lastRefill ?? 0);
+ if (Number.isNaN(lastRefillDate.getTime())) {
+ return true;
+ }
+ // Mirror checkBalanceRecord's fallbacks exactly (interval 0 / 'days' when a
+ // partially-synced record is missing them) so we never pre-skip a record the
+ // interactive chat balance check would have refilled.
+ return (
+ new Date() >=
+ getRefillEligibilityDate(
+ lastRefillDate,
+ record.refillIntervalValue ?? 0,
+ record.refillIntervalUnit ?? 'days',
+ )
+ );
+ }
+
+ const engineDeps: ScheduleEngineDeps = {
+ methods,
+ getLimits,
+ // On the BASE deps, not only the engine's per-pass wrapper: fireScheduleNow
+ // (manual Run Now) dispatches with these deps directly, and its POST must be
+ // gated by the same coordinator signal as the engine tick's.
+ isShuttingDown: isShutdownInProgress,
+ getUserContext: async (userId) => {
+ const user = await deps.findUserById(userId);
+ if (user == null) {
+ return null;
+ }
+ return { id: user._id.toString(), tenantId: user.tenantId, role: user.role };
+ },
+ hasScheduleAccess: async (user) => {
+ const role = await methods.getRoleByName(user.role);
+ return role?.permissions?.[PermissionTypes.SCHEDULES]?.[Permissions.USE] === true;
+ },
+ isOutOfBalance: async (user) => {
+ const appConfig = await deps.getAppConfig(getAppConfigOptionsFromUser(user));
+ const balanceConfig = getBalanceConfig(appConfig);
+ if (balanceConfig?.enabled !== true) {
+ return false;
+ }
+ let record = await deps.findBalance(user.id);
+ // Initialize/sync the record exactly as the chat's balance middleware would,
+ // so a new user's startBalance is applied before we read it (avoids skipping
+ // a schedule that an interactive chat would have allowed).
+ if (balanceConfig.startBalance != null) {
+ const updateFields = buildBalanceUpdateFields(balanceConfig, record, user.id);
+ if (Object.keys(updateFields).length > 0) {
+ // The read above and every write below are separate statements, and the credit
+ // field is an INITIALIZATION value: a concurrent charge that set-and-spent the
+ // record in between must never be handed back its starting balance. Each case
+ // fences the credit write so it can only create, never restore, credits. The
+ // refill-config fields are a genuine sync and never carry a credit.
+ const { user: initUser, tokenCredits, ...syncFields } = updateFields;
+ if (record == null) {
+ // ABSENT record: upsert. The credit rides `$setOnInsert` so a document created
+ // by a concurrent charge keeps its charged balance; the refill-config sync is
+ // a legitimate `$set`.
+ record = await deps.upsertBalance(user.id, {
+ set: syncFields,
+ setOnInsert: {
+ ...(initUser != null ? { user: initUser } : {}),
+ ...(tokenCredits != null ? { tokenCredits } : {}),
+ },
+ });
+ } else if (tokenCredits != null) {
+ // EXISTING record with a null credit: a blind `$set` would restore credits a
+ // concurrent initializer/charge already set and spent between the read and
+ // here. Initialize under a `{ tokenCredits: null }` CAS; on a miss re-read the
+ // winner's balance instead of overwriting it.
+ const initialized = await deps.initializeNullBalance(user.id, {
+ tokenCredits,
+ sync: syncFields,
+ });
+ record = initialized ?? (await deps.findBalance(user.id)) ?? record;
+ } else if (Object.keys(syncFields).length > 0) {
+ // EXISTING record with a real credit: only refill-config sync remains, which
+ // never touches `tokenCredits` and is safe to `$set` directly.
+ record = await deps.upsertBalance(user.id, { set: syncFields, setOnInsert: {} });
+ }
+ }
+ }
+ const credits = record?.tokenCredits ?? 0;
+ if (credits > 0) {
+ return false;
+ }
+ // At/below zero: an auto-refill user is only spared a pre-skip when a refill
+ // would actually fire now (mirrors the chat balance check's eligibility). If
+ // they aren't eligible yet, or the refill settings are incomplete, pre-skip as
+ // a balance skip — otherwise the zero-credit fire reaches the chat, is rejected
+ // there, and records a generic error that walks the schedule toward
+ // too_many_failures instead of skipped_balance/insufficient_balance.
+ if (balanceConfig.autoRefillEnabled === true && isRefillEligible(record)) {
+ return false;
+ }
+ return true;
+ },
+ // Mirrors the loopback chat route's authorization (role AGENTS:USE + resource
+ // VIEW with the manage:agents bypass); shared with the create/update precheck
+ // so the two never diverge.
+ agentAccess: (agentId, user) => deps.resolveAgentFireAccess(agentId, user),
+ resolveFiles: async (fileIds, user) => {
+ // Renew the bounded upload hold at every fire preflight, BEST-EFFORT: the hold
+ // only has to bridge upload -> first consumption (a real send clears the TTL
+ // permanently), and a failed renewal must not fail the fire — at worst the hold
+ // lapses later and resolveFiles drops the reaped file (droppedFileIds records it).
+ await methods
+ .extendFilesTTL(fileIds, SCHEDULE_FILE_HOLD, { user: user.id, tenantId: user.tenantId })
+ .catch((err) => logger.warn('[schedules] attachment hold renewal failed:', err));
+ const files = await methods.getFiles(
+ { file_id: { $in: fileIds }, user: user.id },
+ null,
+ '-text',
+ );
+ return (files ?? []).map((file) => ({
+ file_id: file.file_id,
+ filepath: file.filepath,
+ filename: file.filename,
+ type: file.type,
+ height: file.height,
+ width: file.width,
+ source: file.source,
+ }));
+ },
+ enqueueTrigger: deps.enqueueAgentTrigger,
+ getTriggerDelivery: deps.getTriggerDelivery,
+ runInTenantContext: (user, fn) =>
+ tenantStorage.run({ tenantId: user.tenantId, userId: user.id }, fn),
+ getJobStatus: async (conversationId) => {
+ const job = await GenerationJobManager.getJobStore()?.getJob(conversationId);
+ if (job == null) {
+ return null;
+ }
+ return {
+ status: job.status,
+ createdAt: job.createdAt,
+ scheduleId: job.scheduleId,
+ scheduledFor: job.scheduledFor,
+ createdEventEmitted: job.createdEventEmitted === true,
+ preserveForScheduleReconcile: job.preserveForScheduleReconcile === true,
+ ...(job.scheduleOutcome != null && { scheduleOutcome: job.scheduleOutcome }),
+ ...(job.scheduleOutcomeError != null && {
+ scheduleOutcomeError: job.scheduleOutcomeError,
+ }),
+ };
+ },
+ abortScheduledJob: async (conversationId, identity, options) => {
+ const store = GenerationJobManager.getJobStore();
+ if (store == null) {
+ return false;
+ }
+ const job = await store.getJob(conversationId);
+ // A null/identity-mismatched job is NOT reachable from this replica: it may be
+ // a live generation on a peer worker's private in-memory store (unshared
+ // topology). Report false so the caller knows the abort was NOT delivered.
+ if (job == null || !jobMatchesIdentity(job, identity)) {
+ return false;
+ }
+ if (options?.preserve !== false) {
+ await GenerationJobManager.updateMetadata(
+ conversationId,
+ {
+ preserveForScheduleReconcile: true,
+ scheduleOutcome: 'interrupted',
+ scheduleOutcomeError: 'Schedule deleted',
+ },
+ job.createdAt,
+ );
+ }
+
+ // #14925's provider-drain contract is the authority here. It covers both
+ // locally owned and cross-replica generations and does not return until the
+ // exact provider segment can no longer persist user data. Calling abortJob on
+ // an already-terminal job is intentional: awaitProviderDrain still waits for
+ // trailing owner work even though no new abort transition is needed.
+ const aborted = await GenerationJobManager.abortJob(conversationId, {
+ expectedCreatedAt: job.createdAt,
+ awaitProviderDrain: true,
+ });
+ if (
+ aborted.failureReason === 'generation_replaced' ||
+ aborted.failureReason === 'job_still_active'
+ ) {
+ return false;
+ }
+ if (options?.preserve === false) {
+ await store.deleteJob(conversationId, job.createdAt);
+ }
+ return true;
+ },
+ clearReconciledJob: async (conversationId, identity) => {
+ const store = GenerationJobManager.getJobStore();
+ if (store == null) {
+ return;
+ }
+ const job = await store.getJob(conversationId);
+ // Only delete when the job still carries THIS run's identity, so a
+ // replacement generation occupying the same conversationId is never destroyed.
+ if (job == null || !jobMatchesIdentity(job, identity)) {
+ return;
+ }
+ // CAS, not read-then-delete: the identity check above is a READ, and a replacement
+ // generation can land between it and the delete. Passing the observed createdAt
+ // makes the delete conditional on the job still being that exact generation, so
+ // the store itself rejects the write if one did.
+ await store.deleteJob(conversationId, job.createdAt);
+ },
+ // Counted in system scope so the cap is GLOBAL — a per-owner (tenant-scoped)
+ // count would let multiple tenants collectively exceed fireConcurrency.
+ countActiveRunsGlobal: () => runAsSystem(() => methods.countActiveRuns()),
+ isOwnerDeleting: (userId) => deps.isUserDeleting(userId),
+ isGloballyDisabled: async () => {
+ // Env first: an incident lever that must work even if the DB/config plane is the
+ // thing failing (a kill switch that needs a healthy DB is the one that fails when
+ // you need it).
+ if (isEnabled(process.env.SCHEDULES_DISABLED)) {
+ return true;
+ }
+ // BASE config only: DB principal overrides can narrow availability but must never
+ // widen past an operator's global stop, so `schedules: false` in librechat.yaml is
+ // genuinely non-overridable rather than emergent from the override filters.
+ // isRuntimeDisabled reads BOTH stop shapes (`false` and `{ use: false }`) — the
+ // same predicate the override merge preserves base stops with. A shape-blind
+ // check here made the object form disable getLimits but not this gate, so the
+ // engine kept claiming and fireSchedule ADVANCED each occurrence: a short
+ // maintenance stop silently dropped every occurrence it covered instead of
+ // leaving them due.
+ const base = await deps.getAppConfig({ baseOnly: true });
+ return isRuntimeDisabled(base?.interfaceConfig?.schedules);
+ },
+ // Occupancy is read in SYSTEM scope so the cap is global across tenants (the
+ // owner's tenant context would only see its own runs); the claim itself stays in
+ // the caller's context so the inserted row keeps correct tenant ownership.
+ withGlobalCapacitySlot: (cap, claim) =>
+ withCapacitySlot(cap, () => runAsSystem(() => methods.getCapacityOccupancy()), claim),
+ };
+
+ let engine: ReturnType<typeof startScheduleEngine> | undefined;
+ let erasureSweep: ReturnType<typeof startScheduleErasureSweep> | undefined;
+
+ /**
+ * Fallback cleanup for a process whose engine refused to arm (unsafe topology, or
+ * index creation failed): DELETE stays open on this entrypoint, so soft-deleted rows
+ * still accrue — and with no reconciler, an owner-death case (account deletion begun
+ * elsewhere, or a failed one-shot erase) would retain the hidden prompt forever.
+ * The engine's own reconcile pass covers this when armed, so never run both.
+ */
+ function startErasureFallback(): void {
+ if (erasureSweep != null || engine != null) {
+ return;
+ }
+ erasureSweep = startScheduleErasureSweep({
+ methods,
+ getJobStatus: engineDeps.getJobStatus,
+ // Positive-evidence delivery convergence: a dead delivery settles its reservation
+ // and frees the global capacity slot even where no engine is armed.
+ getTriggerDelivery: engineDeps.getTriggerDelivery,
+ // Releases a run's retained terminal job once its outcome is durable, so preserved
+ // evidence (kept without `completedAt`, and therefore invisible to the store's
+ // finished-job sweep) cannot outlive the run it belonged to.
+ clearReconciledJob: engineDeps.clearReconciledJob,
+ // If topology itself prevented arming, this process's missing job says
+ // nothing about peer liveness. If only index creation failed, the topology
+ // proof still holds and the existing owner-death backstop remains valid.
+ canInferOwnerDeathFromMissingJob: isTopologySafeToArm(),
+ });
+ }
+
+ async function initializeScheduleEngine(): Promise<
+ ReturnType<typeof startScheduleEngine> | undefined
+ > {
+ if (engine != null) {
+ return engine;
+ }
+ // Always arm. The engine owns BOTH firing and reconciliation, and reconciliation must
+ // run even while firing is stopped: a previous process can leave `started` rows and
+ // preserved terminal jobs that would otherwise never settle until scheduling is
+ // re-enabled. Firing is gated separately — runTick refuses to claim while globally
+ // disabled, and getLimits reports disabled to the write handlers and the fire path —
+ // so "off" still means nothing fires, without stranding prior state.
+ // Deliberately not gated on the base config either: schedules can be enabled for a
+ // role/user/tenant while the base YAML stays default-off.
+ // Explicitly build the Schedule/ScheduleRun indexes first — the unique
+ // idempotency index and TTL retention index would otherwise never exist when
+ // MONGO_AUTO_INDEX is disabled (the production default). If this fails the
+ // unique {scheduleId, scheduledFor} guard may be absent, so leave the engine
+ // DISABLED rather than firing without duplicate protection — the app still
+ // runs; schedules simply don't fire until an operator resolves the index.
+ if (!isTopologySafeToArm()) {
+ logger.error(
+ "[schedules] scheduler NOT started: this process cannot see other replicas' generations. " +
+ 'The job store is process-local (Redis is off), but nothing here proves this is the only ' +
+ 'replica — and the standard server arms the scheduler in EVERY replica. A peer would ' +
+ 'reconcile runs whose jobs it cannot see, eventually marking a still-running generation ' +
+ 'interrupted and releasing its capacity, and deletions routed elsewhere could not abort ' +
+ 'the generation at all. Enable a shared stream store (USE_REDIS_STREAMS), or set ' +
+ 'SCHEDULES_SINGLE_PROCESS=true to assert this deployment runs exactly one replica. ' +
+ 'Schedule writes are refused (503) until then.',
+ );
+ startErasureFallback();
+ return undefined;
+ }
+ try {
+ await runAsSystem(() => methods.ensureScheduleIndexes());
+ } catch (err) {
+ logger.error(
+ '[schedules] index creation failed — scheduler NOT started (fires need the unique idempotency index):',
+ err,
+ );
+ startErasureFallback();
+ return undefined;
+ }
+ engine = startScheduleEngine(engineDeps);
+ if (erasureSweep != null) {
+ // A later successful arm supersedes the fallback: the engine's reconcile pass
+ // owns erasure from here.
+ erasureSweep.stop();
+ erasureSweep = undefined;
+ }
+ return engine;
+ }
+
+ /**
+ * The clustered entrypoint's ONLY schedule maintenance. Exposes the same erasure sweep
+ * the standard entrypoint falls back to, so a soft-deleted row whose delete/terminal
+ * erase-on-settle attempts missed still drains instead of retaining the owner's prompt
+ * forever. It shares startErasureFallback's idempotent startup guard, the sweep's own
+ * shutdown registration, and the topology-fenced owner-death policy — and arms nothing
+ * else, so running it in every clustered replica changes nothing about v1 scheduling.
+ */
+ function initializeScheduleErasureSweep(): void {
+ startErasureFallback();
+ }
+
+ /**
+ * Manual run-now fire. Acquires the schedule lease to serialize concurrent
+ * run-now clicks (and to block against a background engine claim), then fires
+ * in manual mode so the next automatic occurrence is left untouched. Returns
+ * null when the lease is already held (a run is in progress).
+ */
+ async function fireScheduleNow(
+ schedule: FireableSchedule,
+ limits: ScheduleLimits,
+ ): Promise<FireResult | null> {
+ // The global stop means STOP: a manual run dispatches the same billed generation as
+ // an automatic one, so gating only the engine tick would leave Run Now wide open.
+ if (await engineDeps.isGloballyDisabled()) {
+ return { fired: false, skipped: 'disabled' as const };
+ }
+ const leased = await methods.acquireManualRunLease(
+ schedule.id,
+ schedule.user,
+ MANUAL_RUN_LEASE_MS,
+ );
+ if (leased == null) {
+ return null;
+ }
+ const claimToken = leased.claimToken;
+ try {
+ // Fire the FRESH leased row (post-image with the new claim token), not the
+ // snapshot the route read before the lease — an edit that committed in the
+ // window in between is reflected, so a stale prompt/agent is never dispatched.
+ return await fireSchedule(engineDeps, leased, limits, new Date(), { manual: true });
+ } catch (err) {
+ const released =
+ claimToken != null
+ ? await methods.releaseLease(schedule.id, claimToken).catch(() => false)
+ : false;
+ if (!released && leased.leaseBy != null) {
+ // An owner edit during the failed preflight rotates the token while preserving
+ // this unique holder. Clear only that holder; a takeover has a different value
+ // and is therefore never stripped by this exception safety net.
+ await methods.releaseLeaseByHolder(schedule.id, leased.leaseBy).catch(() => undefined);
+ }
+ throw err;
+ }
+ }
+
+ const OUTCOME_RETRY_ATTEMPTS = 3;
+
+ /**
+ * Completion hook: called from the agents controller finalize paths when the
+ * request carried a scheduleId. The caller deletes the job (`completeJob`) right
+ * after, destroying the only evidence the reconciler could use — so a transient
+ * Mongo failure here is RETRIED (bounded) before giving up, and the failure is
+ * surfaced to the caller (returns false) so it can keep the job when it matters.
+ */
+ /**
+ * OWNER-SIDE Stop barrier. An interactive Stop stamps `abortSource: 'stop'` before it
+ * signals the abort, and only acknowledges (`abortPersistedAt`) once its partial-message
+ * and checkpoint writes have landed. A terminal settlement here in between would release
+ * the run's capacity — and let a concurrent schedule/account deletion erase the data —
+ * while that write is still in flight. Wait (bounded) for the acknowledgement. A resolved
+ * marker, a non-stop source, or a stamp gone stale (its route presumed dead) proceeds
+ * immediately, so the stale-owner timeout remains the bounded recovery path.
+ */
+ async function waitForStopPersistence(scheduleId: string, scheduledFor: Date): Promise<boolean> {
+ const deadline = Date.now() + STOP_BARRIER_TIMEOUT_MS;
+ for (;;) {
+ const state = await methods.getScheduleRunAbortState(scheduleId, scheduledFor);
+ if (
+ state == null ||
+ state.abortSource !== 'stop' ||
+ state.abortPersistedAt != null ||
+ !hasAbortInFlight(state, Date.now())
+ ) {
+ // Cleared to settle: no Stop owns the run, it acknowledged, or its owner is past
+ // the stale cutoff and is presumed dead (the bounded recovery path).
+ return true;
+ }
+ if (Date.now() >= deadline) {
+ // Still an UNACKNOWLEDGED, FRESH Stop. The poll budget expiring proves nothing about
+ // the route's writes — slow checkpoint cleanup looks exactly like this — so treating
+ // the barrier as satisfied would terminalize the run and release its capacity (and
+ // its deletion/erasure barriers) while beforePublish may still be writing. DEFER
+ // instead: the acknowledgement settles it, or the stale-owner cutoff authorizes a
+ // later attempt.
+ return false;
+ }
+ await new Promise((resolve) => setTimeout(resolve, STOP_BARRIER_POLL_MS));
+ }
+ }
+
+ async function recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt,
+ status,
+ conversationId,
+ clearConversationId,
+ error,
+ }: RecordScheduleOutcomeInput): Promise<boolean> {
+ if (!scheduleId || !scheduledFor) {
+ return true;
+ }
+ const terminal = status !== 'requires_action';
+ if (terminal) {
+ // Honor an in-flight interactive Stop's persistence before terminalizing. A deferral
+ // is NOT a failure to record — the run is deliberately left active/preserved — but it
+ // must report "not settled" so callers with durable retry (the approval-expiry host
+ // action, reconciliation) re-drive it rather than assuming the outcome landed.
+ if (!(await waitForStopPersistence(scheduleId, new Date(scheduledFor)))) {
+ logger.info(
+ `[schedules] deferring terminal settlement for ${scheduleId}: interactive Stop persistence is still unacknowledged`,
+ );
+ return false;
+ }
+ }
+ if (terminal && streamId && jobCreatedAt != null) {
+ try {
+ await GenerationJobManager.updateMetadata(
+ streamId,
+ {
+ preserveForScheduleReconcile: true,
+ scheduleOutcome:
+ status === 'success' ||
+ status === 'error' ||
+ status === 'interrupted' ||
+ status === 'skipped_balance'
+ ? status
+ : 'error',
+ ...(error ? { scheduleOutcomeError: error } : {}),
+ },
+ jobCreatedAt,
+ );
+ } catch (err) {
+ logger.error('[schedules] failed to retain terminal outcome evidence:', err);
+ return false;
+ }
+ }
+ for (let attempt = 1; attempt <= OUTCOME_RETRY_ATTEMPTS; attempt++) {
+ try {
+ // Resolve the owner's limits so auto-disable uses the same per-principal
+ // threshold as the fire path (not the global default).
+ const schedule = await methods.getScheduleById(scheduleId);
+ const owner = schedule ? await engineDeps.getUserContext(schedule.user) : null;
+ const limits = await getLimits(owner ?? undefined);
+ await methods.recordRunOutcome({
+ scheduleId,
+ scheduledFor: new Date(scheduledFor),
+ status,
+ clearConversationId,
+ conversationId,
+ error,
+ autoDisableAfterFailures: limits.autoDisableAfterFailures,
+ balanceSkipDisableThreshold: BALANCE_SKIP_DISABLE_THRESHOLD,
+ });
+ // ERASE-ON-SETTLE: whichever process records a run's terminal outcome also
+ // attempts the deferred erase of a deleting schedule. This is what makes a
+ // delete's `draining` state converge in EVERY topology — the clustered
+ // entrypoint runs no reconciler, so without this the hidden schedule (and its
+ // prompt, which has no TTL) survived its last run indefinitely there. A cheap
+ // guarded no-op for live schedules (the erase filters on `deleting: true`).
+ if (status !== 'requires_action') {
+ await methods.eraseScheduleIfDrained(scheduleId).catch((err) => {
+ logger.warn(`[schedules] erase-on-settle failed for ${scheduleId}:`, err);
+ });
+ }
+ if (terminal && streamId && jobCreatedAt != null) {
+ await GenerationJobManager.updateMetadata(
+ streamId,
+ { preserveForScheduleReconcile: false },
+ jobCreatedAt,
+ ).catch((err) => {
+ logger.warn('[schedules] failed to release terminal outcome evidence:', err);
+ });
+
+ // A paused occurrence can be aborted while its provider is already drained.
+ // In that case the manager's normal drain-time cleanup has already passed;
+ // reap the exact terminal job after the Mongo outcome becomes durable.
+ const store = GenerationJobManager.getJobStore();
+ const job = await store?.getJob(streamId).catch(() => null);
+ if (
+ job?.createdAt === jobCreatedAt &&
+ job.providerDrained !== false &&
+ job.terminalPersistencePending !== true &&
+ job.status !== 'running' &&
+ job.status !== 'requires_action' &&
+ job.preserveForScheduleReconcile !== true
+ ) {
+ await store?.deleteJob(streamId, jobCreatedAt).catch((err) => {
+ logger.warn('[schedules] failed to clear settled generation evidence:', err);
+ });
+ }
+ }
+ return true;
+ } catch (err) {
+ logger.error(
+ `[schedules] failed to record run outcome (attempt ${attempt}/${OUTCOME_RETRY_ATTEMPTS}):`,
+ err,
+ );
+ }
+ }
+ return false;
+ }
+
+ async function beginScheduledStop({
+ scheduleId,
+ scheduledFor,
+ }: {
+ scheduleId: string;
+ scheduledFor: string | Date;
+ }): Promise<boolean | 'in_progress'> {
+ if (!scheduleId || !scheduledFor) {
+ return false;
+ }
+ return methods.requestRunAbort(scheduleId, new Date(scheduledFor), 'stop');
+ }
+
+ async function acknowledgeScheduledStopPersistence({
+ scheduleId,
+ scheduledFor,
+ settle,
+ }: {
+ scheduleId: string;
+ scheduledFor: string | Date;
+ /** Terminal outcome to (re-)drive once the barrier clears. Supplied by the Stop route
+ * so a settlement its owner already DEFERRED past the poll budget still converges. */
+ settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };
+ }): Promise<void> {
+ if (!scheduleId || !scheduledFor) {
+ return;
+ }
+ await methods.markRunAbortPersisted(scheduleId, new Date(scheduledFor));
+ if (settle == null) {
+ return;
+ }
+ // The owner calls recordScheduleOutcome ONCE. If its Stop barrier deferred (slow
+ // beforePublish), nothing would re-drive it where no schedule reconciler is armed —
+ // the run would stay `started`, its job preserved, and its global capacity slot held.
+ // Now that the barrier is acknowledged, settle from here; recordRunOutcome is
+ // match-guarded and idempotent, so a run the owner already settled is a no-op.
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ status: settle.status,
+ conversationId: settle.conversationId,
+ error: settle.error,
+ }).catch((err) => logger.warn('[schedules] post-acknowledgement settlement failed:', err));
+ }
+
+ async function isScheduleLive(
+ scheduleId: string,
+ expectedConfigRevision?: number,
+ options?: { automatic?: boolean; policy?: boolean },
+ ): Promise<boolean> {
+ if (!scheduleId) {
+ return false;
+ }
+ const schedule = await methods.getScheduleById(scheduleId);
+ if (schedule == null) {
+ return false;
+ }
+ // An AUTOMATIC fire must still be wanted. A policy auto-disable (too many failures,
+ // insufficient balance) flips `enabled` WITHOUT touching configRevision, so the
+ // revision fence below cannot see it — an occurrence already in the claim-to-
+ // controller window would otherwise start a billed generation for a schedule that
+ // has just been switched off. Run Now is an explicit user action and stays allowed
+ // on a disabled schedule, matching fireScheduleNow.
+ if (options?.automatic === true && schedule.enabled === false) {
+ return false;
+ }
+ // REVISION FENCE at the admission boundary. Existence alone is not enough: an owner
+ // edit landing between the claim and this point means the dispatched prompt/agent
+ // came from a config the owner has since replaced, and nothing downstream would
+ // catch it because the run persists under the NEW schedule. Refuse before any
+ // message is written. Absent on either side disables the fence, so pre-existing
+ // schedules and older fires keep working.
+ if (
+ expectedConfigRevision != null &&
+ typeof schedule.configRevision === 'number' &&
+ schedule.configRevision !== expectedConfigRevision
+ ) {
+ return false;
+ }
+ // LIVE dispatch policy, re-applied exactly as the fire path applies it. A pause
+ // can sit unanswered for hours, and an operator's global kill switch, a narrowed
+ // `interface.schedules` availability, or a revoked SCHEDULES:USE permission
+ // landing in that window touches neither the row nor its revision — none of the
+ // checks above can see it, and approving the pause would start a fresh billed
+ // continuation the operator believes is stopped. Applies to manual runs too: an
+ // emergency stop must stop those approvals as well.
+ if (options?.policy === true) {
+ if (await engineDeps.isGloballyDisabled()) {
+ return false;
+ }
+ const owner = await engineDeps.getUserContext(schedule.user);
+ if (owner == null) {
+ return false;
+ }
+ const limits = await engineDeps.runInTenantContext(owner, () => getLimits(owner));
+ if (!limits.enabled) {
+ return false;
+ }
+ if (!(await engineDeps.hasScheduleAccess(owner))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ async function claimScheduleResume(
+ scheduleId: string,
+ scheduledFor: string | Date,
+ options?: { expectedConfigRevision?: number; automatic?: boolean },
+ ): Promise<ScheduleResumeClaimResult> {
+ const schedule = await methods.getScheduleById(scheduleId);
+ if (schedule == null) {
+ return { conflict: 'inactive' };
+ }
+ if (options?.automatic !== false && schedule.enabled === false) {
+ return { conflict: 'inactive' };
+ }
+ if (
+ options?.expectedConfigRevision != null &&
+ typeof schedule.configRevision === 'number' &&
+ schedule.configRevision !== options.expectedConfigRevision
+ ) {
+ return { conflict: 'inactive' };
+ }
+ const owner = await engineDeps.getUserContext(schedule.user);
+ if (owner == null) {
+ return { conflict: 'inactive' };
+ }
+ return engineDeps.runInTenantContext(owner, async () => {
+ const [ownerLimits, deploymentLimits, globallyDisabled, hasAccess] = await Promise.all([
+ getLimits(owner),
+ getLimits(),
+ engineDeps.isGloballyDisabled(),
+ engineDeps.hasScheduleAccess(owner),
+ ]);
+ if (!ownerLimits.enabled || globallyDisabled || !hasAccess) {
+ return { conflict: 'inactive' };
+ }
+ // FINAL schedule-side admission fence. Everything above is asynchronous and an
+ // owner edit/disable can land while it runs. Claim the schedule document under
+ // the expected config generation now, carry this lease through the approval CAS,
+ // and atomically consume it before provider execution begins.
+ const resumeLease = await methods.acquireResumeLease(
+ scheduleId,
+ options?.expectedConfigRevision,
+ options?.automatic !== false,
+ SCHEDULE_RESUME_LEASE_MS,
+ );
+ if (resumeLease?.claimToken == null || resumeLease.leaseBy == null) {
+ return { conflict: 'inactive' };
+ }
+ let retainResumeLease = false;
+ try {
+ const allocation = await engineDeps.withGlobalCapacitySlot(
+ Math.min(ownerLimits.fireConcurrency, deploymentLimits.fireConcurrency),
+ async (capacitySlot) => {
+ const attempt = await methods.markRunResumeClaimed(
+ scheduleId,
+ new Date(scheduledFor),
+ capacitySlot,
+ );
+ return 'conflict' in attempt && attempt.conflict === 'slot-taken'
+ ? 'slot-taken'
+ : { claimed: attempt };
+ },
+ );
+ if (allocation === 'capacity') {
+ return { conflict: 'capacity' };
+ }
+ const claimed = allocation.claimed;
+ if ('conflict' in claimed) {
+ // withCapacitySlot consumes this internal collision sentinel by retrying the
+ // next free slot. Normalize defensively at the public boundary as well so a
+ // custom/test allocator can never leak an implementation-only conflict.
+ return {
+ conflict: claimed.conflict === 'slot-taken' ? 'capacity' : claimed.conflict,
+ };
+ }
+ retainResumeLease = true;
+ return {
+ capacitySlot: claimed.capacitySlot,
+ claimToken: resumeLease.claimToken,
+ leaseBy: resumeLease.leaseBy,
+ };
+ } finally {
+ if (!retainResumeLease) {
+ await methods.releaseLeaseByHolder(scheduleId, resumeLease.leaseBy);
+ }
+ }
+ });
+ }
+
+ function releaseScheduleResumeClaim(
+ scheduleId: string,
+ scheduledFor: string | Date,
+ capacitySlot: number,
+ ): Promise<boolean> {
+ return methods.releaseRunResumeClaim(scheduleId, new Date(scheduledFor), capacitySlot);
+ }
+
+ async function finalizeScheduleResumeClaim(
+ scheduleId: string,
+ claimToken: string,
+ leaseBy: string,
+ options?: { expectedConfigRevision?: number; automatic?: boolean },
+ ): Promise<boolean> {
+ const consumed = await methods.consumeResumeLease(
+ scheduleId,
+ claimToken,
+ leaseBy,
+ options?.automatic !== false,
+ options?.expectedConfigRevision,
+ );
+ if (!consumed) {
+ // An owner edit rotates claimToken but deliberately leaves the old holder's
+ // lease in place. Clear only that holder so the edited schedule is immediately
+ // usable; a takeover changed leaseBy and is therefore untouched.
+ await methods.releaseLeaseByHolder(scheduleId, leaseBy);
+ }
+ return consumed;
+ }
+
+ function releaseScheduleResumeFence(scheduleId: string, leaseBy: string): Promise<void> {
+ return methods.releaseLeaseByHolder(scheduleId, leaseBy);
+ }
+
+ /** Aborts an active run's loopback job (identity-guarded). Returns whether the
+ * abort was delivered (false when the job wasn't reachable — e.g. a peer worker's
+ * private store, or a transient error). */
+ async function abortActiveRun(
+ run: { scheduleId: string; scheduledFor: Date; conversationId?: string },
+ preserve: boolean,
+ options?: { stampRenewal?: boolean; settleAfterAbort?: boolean },
+ ): Promise<boolean> {
+ if (!run.conversationId) {
+ return false;
+ }
+ // Record the abort REQUEST before signalling. This keeps the run holding its global
+ // capacity slot until its generation owner writes a terminal outcome (settlement),
+ // so an abort that has been asked for but not yet honored cannot free capacity for a
+ // new run while the old generation is still alive. The stamp is LOAD-BEARING: it is
+ // what makes a concurrent drain (and the reconciler) defer to the owner's settle,
+ // so if it cannot be made durable the abort must NOT be signalled — report
+ // undelivered and let the caller's unconfirmed/retry path re-drive both.
+ //
+ // `stampRenewal: false` skips the stamp entirely: callers pass it for CLEANUP
+ // aborts of a job they just observed terminal/absent (nothing live will be
+ // signalled). Renewing there would re-arm the 30-minute owner-death fence on
+ // every retry against a dead owner, so the run could never age into the
+ // reconciler's recovery.
+ if (options?.stampRenewal !== false) {
+ try {
+ await methods.requestRunAbort(run.scheduleId, run.scheduledFor, 'deletion');
+ } catch (err) {
+ logger.warn('[schedules] failed to record abort request; abort withheld:', err);
+ return false;
+ }
+ }
+ const stopped = await engineDeps
+ .abortScheduledJob(
+ run.conversationId,
+ { scheduleId: run.scheduleId, scheduledFor: run.scheduledFor },
+ { preserve },
+ )
+ .catch((err) => {
+ logger.warn('[schedules] failed to abort run job on quiesce:', err);
+ return false;
+ });
+ if (!stopped || options?.settleAfterAbort === false) {
+ return stopped;
+ }
+
+ // awaitProviderDrain in abortScheduledJob is the positive persistence fence:
+ // the exact provider owner has unwound, so this row can now leave the active
+ // set without racing a late message write.
+ const recorded = await recordScheduleOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: 'interrupted',
+ conversationId: run.conversationId,
+ error: 'Schedule deleted while the run was active',
+ });
+ if (!recorded) {
+ return false;
+ }
+ await engineDeps
+ .clearReconciledJob(run.conversationId, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ })
+ .catch((err) => logger.warn('[schedules] failed to clear deleted run evidence:', err));
+ return true;
+ }
+
+ /** Resolve the owner's durable-checkpointer config (in their tenant context) so a
+ * paused-run checkpoint can be pruned on delete, mirroring the interactive abort. */
+ async function resolveOwnerCheckpointer(
+ ownerId: string | Types.ObjectId,
+ ): Promise<TCheckpointerConfig | undefined> {
+ const owner = await engineDeps.getUserContext(ownerId);
+ if (owner == null) {
+ return undefined;
+ }
+ const appConfig = await engineDeps.runInTenantContext(owner, () =>
+ deps.getAppConfig(getAppConfigOptionsFromUser(owner)),
+ );
+ return appConfig?.endpoints?.agents?.checkpointer;
+ }
+
+ /**
+ * Soft-deletes a schedule for its owner: disables + marks it `deleting` (so the
+ * engine can no longer claim it and it disappears from the owner's list), rotates
+ * the claim token to fence any in-flight worker, then DRAINS with the same evidence
+ * discipline as account-deletion quiesce — a run whose job is provably absent or
+ * settled is recorded and erased here, synchronously; a live generation is aborted
+ * and settles through its own outcome write (which erases on settle, so no
+ * reconciler is required in any topology). The result is honest: `unconfirmed`
+ * means at least one run could not be shown stopped, and the caller must not
+ * claim it was.
+ */
+ async function deleteScheduleForOwner(
+ scheduleId: string,
+ userId: string,
+ ): Promise<ScheduleDeleteResult> {
+ const schedule = await methods.markScheduleDeleting(scheduleId, userId);
+ if (schedule == null) {
+ return 'not_found';
+ }
+ const active = await methods.getActiveRunsForSchedule(scheduleId);
+ // Resolve the checkpointer config once (only when a paused run needs pruning) in
+ // the owner's tenant context, matching the interactive abort endpoint's prune.
+ const hasPausedRun = active.some(
+ (run) => run.status === 'requires_action' && run.conversationId != null,
+ );
+ // BEST-EFFORT: the prune this feeds is itself best-effort, and a lookup failure
+ // must not cost the aborts below.
+ const checkpointer = hasPausedRun
+ ? await resolveOwnerCheckpointer(schedule.user).catch((err) => {
+ logger.warn(`[schedules] checkpointer lookup failed for delete ${scheduleId}:`, err);
+ return undefined;
+ })
+ : undefined;
+ let unconfirmed = 0;
+ for (const run of active) {
+ // UNKNOWN is not ABSENT — the same distinction the quiesce path draws. A lookup
+ // that succeeded and returned null is positive evidence no generation holds this
+ // conversation; a lookup that THREW is evidence of nothing.
+ const live = run.conversationId
+ ? await engineDeps.getJobStatus(run.conversationId).then(
+ (job) => ({ known: true, job }),
+ () => ({ known: false, job: null }),
+ )
+ : { known: true, job: null };
+ const isThisGeneration =
+ live.job != null &&
+ jobMatchesIdentity(live.job, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ });
+ // Capture the paused run's checkpoint ids BEFORE any terminal transition below:
+ // the prune afterwards is scoped to exactly this set, so checkpoints a
+ // replacement turn writes after this point can never be swept up by it.
+ const checkpointGeneration =
+ run.status === 'requires_action' && run.conversationId
+ ? await captureAgentCheckpointGeneration(run.conversationId, checkpointer)
+ : undefined;
+ // Same abort-in-flight deferral as quiesce: post-abort job state (status `aborted`,
+ // or absence once the abort deleted the job) appears before the owner has persisted
+ // and settled, so it is not evidence that the generation is done.
+ //
+ // A paused job whose run row is still `started` is a pause HAND-OFF in flight:
+ // the job reports requires_action the instant the run interrupts, while the
+ // controller's pause branch is still flushing this segment's writes and records
+ // the pause on the row only after them. Settling on the job state alone let the
+ // cascade complete before those writes landed, which then recreated the paused
+ // response for the deleted schedule. The controller's record (or the paused-
+ // window reconciler) flips the row to requires_action, so the deferral is
+ // bounded — a later pass settles it.
+ const pauseHandoffInFlight =
+ isThisGeneration &&
+ live.job?.status === 'requires_action' &&
+ (run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));
+ const settleable =
+ live.known &&
+ !hasAbortInFlight(run, Date.now()) &&
+ !pauseHandoffInFlight &&
+ !(isThisGeneration && live.job?.status === 'running');
+ if (settleable) {
+ // Positive evidence nothing is generating: settle the row HERE so the erase
+ // below can proceed without any reconciler — the clustered entrypoint has
+ // none, and deferring these rows to it retained the deleted schedule (and its
+ // prompt) indefinitely in that topology. Settle BEFORE aborting: a retained
+ // job is the only evidence of a finished run whose outcome write failed.
+ const retainedOutcome = isThisGeneration
+ ? TERMINAL_JOB_OUTCOMES[live.job!.status]
+ : undefined;
+ const settledStatus = retainedOutcome ?? 'interrupted';
+ const settled = await methods
+ .recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: settledStatus,
+ conversationId: run.conversationId,
+ ...(settledStatus === 'interrupted' ? { error: 'Schedule deleted' } : {}),
+ autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
+ })
+ .then(
+ () => true,
+ (err) => {
+ logger.warn(`[schedules] failed to settle run on delete ${scheduleId}:`, err);
+ return false;
+ },
+ );
+ if (settled) {
+ // Cleanup abort of a job just observed terminal/absent: nothing live gets
+ // signalled, so the stamp is NOT renewed — re-arming the owner-death fence
+ // on every delete retry would keep a dead owner's row from ever aging into
+ // the reconciler's recovery.
+ await abortActiveRun(run, false, {
+ stampRenewal: false,
+ settleAfterAbort: false,
+ });
+ } else {
+ unconfirmed += 1;
+ }
+ } else {
+ // A live generation (or an unknown one): abort it, preserving the job so its
+ // outcome survives, and require the DELIVERY to be confirmed. An abort that
+ // was not delivered leaves a generation that keeps producing and billing —
+ // reporting this delete as a success would claim otherwise.
+ const aborted = await abortActiveRun(run, true);
+ if (!aborted) {
+ unconfirmed += 1;
+ }
+ }
+ // HITL: prune the durable checkpoints of a run aborted while paused so a new turn
+ // in this conversation can't rehydrate the stale interrupt before the Mongo TTL
+ // reclaims it (thread_id === conversationId). Idempotent / no-op otherwise.
+ //
+ // Two guards against a REPLACEMENT owning the conversation: ownership is re-read
+ // fresh (the settle/abort awaits above are a window a replacement can claim the
+ // conversationId in), and the deletion is SCOPED to the checkpoint ids captured
+ // before the terminal transition — so even a replacement that slips in between
+ // this read and the delete only ever loses checkpoints that predate the abort,
+ // never its own. An unreadable store proves nothing, so it is left alone.
+ if (
+ run.status === 'requires_action' &&
+ run.conversationId &&
+ checkpointGeneration != null &&
+ checkpointGeneration.checkpointIds.length > 0
+ ) {
+ const fresh = await engineDeps.getJobStatus(run.conversationId).then(
+ (job) => ({ known: true, job }),
+ () => ({ known: false, job: null }),
+ );
+ const freshIsThisGeneration =
+ fresh.job != null &&
+ jobMatchesIdentity(fresh.job, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ });
+ const ownsConversation = fresh.known && (fresh.job == null || freshIsThisGeneration);
+ if (ownsConversation) {
+ await deleteAgentCheckpoint(run.conversationId, checkpointer, checkpointGeneration).catch(
+ () => undefined,
+ );
+ }
+ }
+ }
+ if (unconfirmed > 0) {
+ // The run row leaving the active set is the durable acknowledgement that its
+ // generation stopped and persisted (the owner settles LAST) — wait for it,
+ // bounded, before answering. Without this, every cross-replica delete whose
+ // abort delivery cannot be locally proven would answer 503 even though the
+ // owner settles moments later.
+ const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;
+ let remaining = unconfirmed;
+ while (Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));
+ remaining = (await methods.getActiveRunsForSchedule(scheduleId)).length;
+ if (remaining === 0) {
+ break;
+ }
+ }
+ if (remaining > 0) {
+ return 'unconfirmed';
+ }
+ }
+ const erased = await methods.eraseScheduleIfDrained(scheduleId).catch((err) => {
+ logger.warn(`[schedules] erase failed for ${scheduleId}:`, err);
+ return false;
+ });
+ if (erased) {
+ return 'deleted';
+ }
+ // eraseScheduleIfDrained already returns true when the row was concurrently
+ // erased or is absent. A false result therefore means the deleting row is still
+ // protected by a live lease/run, or the erase could not be confirmed.
+ return 'draining';
+ }
+
+ /**
+ * Quiesces every schedule of a user ahead of account deletion: REVERSIBLY suspends them
+ * under the attempt `token` (non-claimable, so no new occurrence fires while the cascade
+ * runs) and aborts the loopback jobs of any in-flight runs, so a scheduled generation
+ * cannot keep persisting messages after the account's messages/conversations are deleted.
+ * The suspension is reversible so a cancelled deletion can restore it (see
+ * restoreUserSchedulesFromDeletion) rather than stranding a live user with erased rows.
+ */
+ async function quiesceUserSchedules(userId: string, token: string): Promise<boolean> {
+ await methods.suspendUserSchedulesForDeletion(userId, token);
+ const active = await methods.getActiveRunsForUser(userId);
+ const unconfirmed: string[] = [];
+ for (const run of active) {
+ // Current resumes promote the row back to `started` with a capacity slot, but a
+ // rolling deploy or crash-era row may still be `requires_action` while its job is
+ // already `running`. So the row status alone cannot prove a genuine pause. Read
+ // the live job BEFORE aborting, which settles or deletes it and would erase this
+ // evidence.
+ // UNKNOWN is not ABSENT. A lookup that THREW is evidence of nothing, while one
+ // that succeeded and returned null is positive evidence that no generation holds
+ // this conversation. Collapsing the two (a bare `.catch(() => null)`) would let a
+ // transient store failure read as "genuinely paused" and terminalize a row whose
+ // resumed generation is still running — the exact hazard this check exists for.
+ const live = run.conversationId
+ ? await engineDeps.getJobStatus(run.conversationId).then(
+ (job) => ({ known: true, job }),
+ () => ({ known: false, job: null }),
+ )
+ : { known: true, job: null };
+ const isThisGeneration =
+ live.job != null &&
+ jobMatchesIdentity(live.job, {
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ });
+ // Settle only on POSITIVE evidence that nothing is generating: an identity-matched
+ // job that is not running, an identity MISMATCH (a replacement turn owns the
+ // conversation, so this occurrence's generation is already gone), or a confirmed
+ // absence. Anything unknown falls through and the drain waits for it.
+ //
+ // An abort IN FLIGHT is not such evidence, and this is the subtle case: `aborted`
+ // and post-abort ABSENCE both appear the instant abortJob wins its CAS, while the
+ // owner has yet to persist. Deferring to the owner's settle (see
+ // ABORT_SETTLE_GRACE_MS) is what stops a drain from being confirmed mid-write.
+ const abortInFlight = hasAbortInFlight(run, Date.now());
+ // Same pause hand-off deferral as the schedule delete path: a paused job whose
+ // row is still `started` has the controller's pause-branch writes in flight,
+ // and settling on the job state alone confirms the drain before they land.
+ const pauseHandoffInFlight =
+ isThisGeneration &&
+ live.job?.status === 'requires_action' &&
+ (run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));
+ const settleable =
+ live.known &&
+ !abortInFlight &&
+ !pauseHandoffInFlight &&
+ !(isThisGeneration && live.job?.status === 'running');
+ // Aborts here never preserve for reconcile: account deletion hard-deletes these
+ // run rows, so no reconcile pass would ever finalize or clear a retained job.
+ const retainedOutcome = isThisGeneration
+ ? TERMINAL_JOB_OUTCOMES[live.job!.status]
+ : undefined;
+ // `settleable` ALREADY means positive evidence that nothing is generating: an
+ // identity-matched job that is not running, an identity MISMATCH (a replacement
+ // turn owns the conversation), or a confirmed absence. Any extra condition on top
+ // of it strands exactly the cases it was computed to cover — a `started` row whose
+ // process died before creating its job reads as a confirmed absence, and the old
+ // clause left it active through the whole drain, so account deletion answered 503
+ // on every attempt until the 30-minute orphan sweep.
+ if (settleable) {
+ // SETTLE BEFORE ABORTING. The abort deletes the retained job, and for a run
+ // whose inline outcome write exhausted its retries that job is the ONLY evidence
+ // it finished — reading its status into a local is not the same as durably
+ // recording it. If this write fails, the abort below has not yet run, so the
+ // evidence survives for the next pass; the drain simply does not confirm and
+ // deletion defers, which is the safe direction.
+ // Either the retained job's own outcome, or — for a genuinely PAUSED run whose
+ // approval will never be consumed for a deleted account — `interrupted`. Without
+ // this a single paused run blocked the account's deletion permanently.
+ const settledStatus = retainedOutcome ?? 'interrupted';
+ const settledError = QUIESCE_SETTLE_ERRORS[settledStatus];
+ const settled = await methods
+ .recordRunOutcome({
+ scheduleId: run.scheduleId,
+ scheduledFor: run.scheduledFor,
+ status: settledStatus,
+ conversationId: run.conversationId,
+ ...(settledError ? { error: settledError } : {}),
+ autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
+ })
+ .then(() => true)
+ .catch((err) => {
+ logger.warn('[schedules] failed to settle run on quiesce:', err);
+ return false;
+ });
+ // Only now is the job disposable. On a failed settle, leave it: the row is still
+ // active, so the drain reports unconfirmed and a later pass retries with its
+ // evidence intact. Cleanup abort of an observed-settled job — no stamp renewal,
+ // or every quiesce retry would re-arm the owner-death fence and a dead owner's
+ // row could never age into recovery.
+ if (settled) {
+ await abortActiveRun(run, false, {
+ stampRenewal: false,
+ settleAfterAbort: false,
+ });
+ }
+ continue;
+ }
+ // Not settleable here (a live generation, or an unknown job): abort it and let the
+ // bounded drain below wait for its own terminal outcome to land.
+ const aborted = await abortActiveRun(run, false);
+ if (!aborted && run.conversationId) {
+ unconfirmed.push(run.conversationId);
+ }
+ }
+ // WAIT (bounded) for the aborted generations to actually settle before the
+ // account-deletion cascade deletes messages/conversations: a run that already
+ // returned from the model can observe the abort but still persist its messages,
+ // which would otherwise resurrect data for the deleted account after the cascade
+ // ran. Poll the run rows (they leave the active set once their outcome is
+ // recorded) until drained or the deadline.
+ const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;
+ let remaining = active.length;
+ while (remaining > 0 && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));
+ remaining = (await methods.getActiveRunsForUser(userId)).length;
+ }
+ // Surface anything that did not drain / could not be confirmed so the deletion
+ // cascade defers rather than destroying while a generation may still persist.
+ // Re-evaluate rather than trusting the initial abort-delivery result: an abort can
+ // report false because the job was briefly unreachable, yet that generation then
+ // finishes and records a terminal outcome during the drain poll. The run is genuinely
+ // settled at that point, so keeping its id in `unconfirmed` would defer account
+ // deletion forever. The DRAIN is the authority; delivery is only a hint.
+ const confirmed = remaining === 0;
+ if (confirmed && unconfirmed.length > 0) {
+ logger.info(
+ `[schedules] ${unconfirmed.length} abort(s) were not confirmed delivered but their ` +
+ 'runs settled during the drain; treating the quiesce as complete.',
+ );
+ }
+ if (!confirmed) {
+ logger.warn(
+ `[schedules] account-deletion quiesce did not confirm ${Math.max(remaining, unconfirmed.length)} ` +
+ `in-flight scheduled run(s) settled${unconfirmed.length ? ` [${unconfirmed.join(', ')}]` : ''} ` +
+ '— a peer worker generation may still persist data. Guaranteed quiescing requires a ' +
+ 'shared stream store (USE_REDIS_STREAMS).',
+ );
+ }
+ return confirmed;
+ }
+
+ /**
+ * Reverses a quiesce whose account deletion was cancelled (a controller failure that
+ * released the user-deletion fence). Re-enables and re-arms only the rows this exact
+ * attempt suspended; a schedule the owner independently deleted, or one a newer attempt
+ * re-suspended, is left as-is.
+ */
+ async function restoreUserSchedulesFromDeletion(userId: string, token: string): Promise<void> {
+ await methods.restoreUserSchedulesFromDeletion(userId, token);
+ }
+
+ return {
+ getLimits,
+ engineDeps,
+ fireScheduleNow,
+ recordScheduleOutcome,
+ beginScheduledStop,
+ acknowledgeScheduledStopPersistence,
+ claimScheduleResume,
+ releaseScheduleResumeClaim,
+ finalizeScheduleResumeClaim,
+ releaseScheduleResumeFence,
+ isScheduleLive,
+ deleteScheduleForOwner,
+ quiesceUserSchedules,
+ restoreUserSchedulesFromDeletion,
+ initializeScheduleEngine,
+ initializeScheduleErasureSweep,
+ };
+}
diff --git a/packages/api/src/schedules/trigger.spec.ts b/packages/api/src/schedules/trigger.spec.ts
new file mode 100644
index 0000000000..249d05ca32
--- /dev/null
+++ b/packages/api/src/schedules/trigger.spec.ts
@@ -0,0 +1,84 @@
+import {
+ captureScheduleFireContext,
+ exemptFromConcurrencyLimiter,
+ exemptFromUserLimiter,
+ isScheduleFireRequest,
+ readScheduleFireContext,
+} from './trigger';
+
+function request(manual = false) {
+ return {
+ _isAgentTrigger: true,
+ body: {
+ scheduleId: 'spoofed',
+ agentTrigger: {
+ version: 1,
+ deliveryId: 'delivery-1',
+ event: {
+ id: 'occurrence-1',
+ type: 'schedule.occurrence',
+ occurredAt: 1_725_000_000_000,
+ source: { id: 'schedule-1', type: 'schedule' },
+ },
+ metadata: { manual, configRevision: 7 },
+ },
+ },
+ };
+}
+
+describe('schedule trigger context', () => {
+ it('projects a verified generic trigger into schedule lifecycle fields', () => {
+ const req = request();
+
+ expect(captureScheduleFireContext(req)).toEqual({
+ scheduleId: 'schedule-1',
+ scheduledFor: new Date(1_725_000_000_000).toISOString(),
+ manual: false,
+ configRevision: 7,
+ });
+ expect(req.body).toMatchObject({
+ scheduleId: 'schedule-1',
+ scheduledFor: new Date(1_725_000_000_000).toISOString(),
+ scheduleConfigRevision: 7,
+ });
+ expect(isScheduleFireRequest(req)).toBe(true);
+ expect(exemptFromUserLimiter(req)).toBe(true);
+ expect(exemptFromConcurrencyLimiter(req)).toBe(true);
+ });
+
+ it('keeps manual runs on the interactive user and concurrency limiters', () => {
+ const req = request(true);
+ captureScheduleFireContext(req);
+
+ expect(exemptFromUserLimiter(req)).toBe(false);
+ expect(exemptFromConcurrencyLimiter(req)).toBe(false);
+ });
+
+ it('rejects copied metadata without a verified agent-trigger identity', () => {
+ const req = request();
+ req._isAgentTrigger = false;
+
+ expect(readScheduleFireContext(req)).toBeUndefined();
+ expect(captureScheduleFireContext(req)).toBeUndefined();
+ expect(req.body.scheduleId).toBe('spoofed');
+ expect(isScheduleFireRequest(req)).toBe(false);
+ });
+
+ it.each([
+ { event: { type: 'schedule.occurrence', occurredAt: 1, source: { id: 's', type: 'webhook' } } },
+ {
+ event: { type: 'schedule.occurrence', occurredAt: -1, source: { id: 's', type: 'schedule' } },
+ },
+ {
+ event: { type: 'schedule.occurrence', occurredAt: 1, source: { id: 's', type: 'schedule' } },
+ metadata: { manual: 'yes' },
+ },
+ ])('fails closed for malformed schedule trigger metadata', (agentTrigger) => {
+ expect(
+ readScheduleFireContext({
+ _isAgentTrigger: true,
+ body: { agentTrigger: { version: 1, ...agentTrigger } },
+ }),
+ ).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/schedules/trigger.ts b/packages/api/src/schedules/trigger.ts
new file mode 100644
index 0000000000..16f7b020d8
--- /dev/null
+++ b/packages/api/src/schedules/trigger.ts
@@ -0,0 +1,113 @@
+interface ScheduleTriggerRequest {
+ _isAgentTrigger?: boolean;
+ _isScheduledFire?: boolean;
+ _isManualScheduledFire?: boolean;
+ body?: Record<string, unknown>;
+}
+
+interface TriggerRecord {
+ [key: string]: unknown;
+}
+
+export interface ScheduleFireContext {
+ scheduleId: string;
+ scheduledFor: string;
+ manual: boolean;
+ configRevision?: number;
+}
+
+function record(value: unknown): TriggerRecord | undefined {
+ return value != null && typeof value === 'object' && !Array.isArray(value)
+ ? (value as TriggerRecord)
+ : undefined;
+}
+
+/**
+ * Reads the schedule identity carried by the trusted trigger host. The generic
+ * trigger token is the trust boundary; ordinary chat requests cannot opt into
+ * schedule bookkeeping or limiter exemptions by copying these body fields.
+ */
+export function readScheduleFireContext(
+ req?: ScheduleTriggerRequest,
+): ScheduleFireContext | undefined {
+ if (req?._isAgentTrigger !== true) {
+ return undefined;
+ }
+ const trigger = record(req.body?.agentTrigger);
+ const event = record(trigger?.event);
+ const source = record(event?.source);
+ const metadata = record(trigger?.metadata);
+ if (
+ trigger?.version !== 1 ||
+ event?.type !== 'schedule.occurrence' ||
+ source?.type !== 'schedule' ||
+ typeof source.id !== 'string' ||
+ source.id.trim().length === 0 ||
+ typeof event.occurredAt !== 'number' ||
+ !Number.isSafeInteger(event.occurredAt) ||
+ event.occurredAt < 0
+ ) {
+ return undefined;
+ }
+ if (metadata?.manual != null && typeof metadata.manual !== 'boolean') {
+ return undefined;
+ }
+ if (
+ metadata?.configRevision != null &&
+ (typeof metadata.configRevision !== 'number' ||
+ !Number.isSafeInteger(metadata.configRevision) ||
+ metadata.configRevision < 0)
+ ) {
+ return undefined;
+ }
+ return {
+ scheduleId: source.id,
+ scheduledFor: new Date(event.occurredAt).toISOString(),
+ manual: metadata?.manual === true,
+ ...(typeof metadata?.configRevision === 'number' && {
+ configRevision: metadata.configRevision,
+ }),
+ };
+}
+
+/**
+ * Captures the verified trigger classification once, then projects the minimum
+ * schedule fields expected by the existing generation lifecycle hooks.
+ */
+export function captureScheduleFireContext(
+ req: ScheduleTriggerRequest,
+): ScheduleFireContext | undefined {
+ const context = readScheduleFireContext(req);
+ req._isScheduledFire = context != null;
+ req._isManualScheduledFire = context?.manual === true;
+ if (context != null && req.body != null) {
+ req.body.scheduleId = context.scheduleId;
+ req.body.scheduledFor = context.scheduledFor;
+ if (context.configRevision == null) {
+ delete req.body.scheduleConfigRevision;
+ } else {
+ req.body.scheduleConfigRevision = context.configRevision;
+ }
+ }
+ return context;
+}
+
+export function isScheduleFireRequest(req?: ScheduleTriggerRequest): boolean {
+ return typeof req?._isScheduledFire === 'boolean'
+ ? req._isScheduledFire
+ : readScheduleFireContext(req) != null;
+}
+
+/** Automatic occurrences are bounded by schedule cadence and global capacity. */
+export function exemptFromUserLimiter(req?: ScheduleTriggerRequest): boolean {
+ if (typeof req?._isScheduledFire === 'boolean') {
+ return req._isScheduledFire && req._isManualScheduledFire !== true;
+ }
+ const context = readScheduleFireContext(req);
+ return context != null && !context.manual;
+}
+
+/** Acquire/release sites share this exact predicate to keep counters balanced. */
+export function exemptFromConcurrencyLimiter(req?: ScheduleTriggerRequest): boolean {
+ return exemptFromUserLimiter(req);
+}
diff --git a/packages/api/src/schedules/types.ts b/packages/api/src/schedules/types.ts
new file mode 100644
index 0000000000..52c5febb42
--- /dev/null
+++ b/packages/api/src/schedules/types.ts
@@ -0,0 +1,282 @@
+import type {
+ ScheduleMethods,
+ ISchedule,
+ AgentTriggerDeliveryStatus,
+ AgentTriggerDeliveryFailure,
+} from '@librechat/data-schemas';
+import type { Types } from 'mongoose';
+import type { AgentTriggerEnqueueOptions, AgentTriggerEnvelope } from '../agents/triggers';
+import type { SlotClaimResult } from './capacity';
+
+export interface ScheduleLimits {
+ /** Feature-level switch: when false the engine claims/fires nothing. */
+ enabled: boolean;
+ maxPerUser: number;
+ minIntervalMinutes: number;
+ autoDisableAfterFailures: number;
+ fireConcurrency: number;
+}
+
+export const DEFAULT_SCHEDULE_LIMITS: ScheduleLimits = {
+ enabled: true,
+ maxPerUser: 10,
+ minIntervalMinutes: 60,
+ autoDisableAfterFailures: 5,
+ fireConcurrency: 5,
+};
+
+export interface ScheduleUserContext {
+ id: string;
+ tenantId?: string;
+ role?: string;
+}
+
+/**
+ * Outcome of an owner-initiated schedule delete.
+ * - `deleted`: drained and erased.
+ * - `draining`: every active run's abort was DELIVERED; erasure follows once the
+ * generation records its terminal outcome (erase-on-settle), in any topology.
+ * - `unconfirmed`: at least one run could not be confirmed stopped (job store
+ * unreachable, or the abort was not delivered). The schedule stays hidden and
+ * fenced, but its generation may still be producing — callers must not report
+ * the run as stopped.
+ */
+export type ScheduleDeleteResult = 'not_found' | 'deleted' | 'draining' | 'unconfirmed';
+
+/**
+ * How long an abort's OWNER is presumed alive and obliged to settle the run itself.
+ *
+ * The rule this enforces: **job state is never persistence acknowledgement.** A job
+ * reads `aborted` — or vanishes, or carries `completedAt` — the moment `abortJob` wins
+ * its status CAS, which is BEFORE the owner unwinds and writes anything. Both owner
+ * paths deliberately settle LAST, after `saveMessage`, so the ONLY acknowledgement that
+ * all persistence-producing work finished is the owner's own terminal outcome write
+ * (i.e. the run row leaving the active set). Reading post-abort job state as "nothing is
+ * generating" confirms a drain mid-write: account deletion destroys the user's data and
+ * the owner then writes a message back for the deleted account.
+ *
+ * So while an abort is in flight, nothing but the owner settles — deletion drains
+ * defer (503 + Retry-After) and the reconciler leaves the row alone. Past the window
+ * the owner is presumed dead, and the reconciler's orphan handling takes over.
+ *
+ * Deliberately the reconciler's ORPHAN cutoff rather than a shorter local grace: that
+ * presumption already exists and is already the thing that would eventually clear the
+ * row. One rule, not two.
+ */
+export const ABORT_OWNER_PRESUMED_ALIVE_MS: number = 30 * 60_000;
+
+/**
+ * Whether an abort was requested for this run recently enough that its owner is still
+ * expected to persist and settle. `abortRequestedAt` is stamped BEFORE any abort is
+ * signalled (see `abortActiveRun` and the interactive abort route), which is what makes
+ * it usable as evidence that post-abort job state is not yet a settled generation.
+ */
+/** How long a resume-claim stamp fences quiesce settling before it is presumed
+ * crashed. A resume's own generation shows `running` (already unsettleable);
+ * this bound only covers the re-pause hand-off window plus crash recovery. */
+export const RESUME_HANDOFF_STALE_MS: number = 10 * 60_000;
+
+/** Whether a paused run's RESUME hand-off is still in flight: its approval was
+ * consumed and the continuation's re-pause writes may still be landing, so the
+ * paused job state is not settleable evidence yet. */
+export function hasResumeHandoffInFlight(
+ run: { resumeClaimedAt?: Date | null },
+ now: number,
+): boolean {
+ return (
+ run.resumeClaimedAt != null && now - run.resumeClaimedAt.getTime() < RESUME_HANDOFF_STALE_MS
+ );
+}
+
+/**
+ * The outcome the generation owner intended for a RETAINED `complete` job, or `success`
+ * when it left none (a fire from before the stamp, or a store that never carried it).
+ * An unrecognized value is treated as `success` rather than forwarded: it crossed a
+ * serialization boundary, and `recordRunOutcome` would reject a status outside its
+ * union — losing a rare refinement beats failing the recovery write outright.
+ */
+export function retainedOutcome(
+ jobState: JobState | null,
+ fallback: 'success' | 'error',
+): {
+ status: 'success' | 'error' | 'interrupted' | 'skipped_balance';
+ error?: string;
+} {
+ const stamped = jobState?.scheduleOutcome;
+ if (stamped === 'skipped_balance') {
+ return { status: 'skipped_balance' };
+ }
+ if (stamped === 'interrupted') {
+ return { status: 'interrupted', error: jobState?.scheduleOutcomeError };
+ }
+ if (stamped === 'error' || fallback === 'error') {
+ return { status: 'error', error: jobState?.scheduleOutcomeError ?? 'Run ended in error' };
+ }
+ return { status: fallback };
+}
+
+export function hasAbortInFlight(run: { abortRequestedAt?: Date }, now: number): boolean {
+ if (run.abortRequestedAt == null) {
+ return false;
+ }
+ return now - new Date(run.abortRequestedAt).getTime() < ABORT_OWNER_PRESUMED_ALIVE_MS;
+}
+
+/**
+ * Renewable upload hold for schedule attachments (extendFilesTTL-shaped), replacing
+ * the earlier permanent TTL removal, which leaked the upload forever when the schedule
+ * was deleted before its first run, its file_ids were replaced, or creation failed.
+ * Touched at create/edit and at every fire preflight; the FIRST fire that actually
+ * sends the file clears its TTL permanently through the ordinary consumption path, so
+ * the hold only has to bridge upload -> first consumption. `renewMs` covers the longest
+ * cadence gap (weekly) twice over; `maxLifetimeMs` bounds a schedule that never manages
+ * to consume (auto-disable stops its renewals long before this ceiling).
+ */
+export const SCHEDULE_FILE_HOLD: { renewMs: number; maxLifetimeMs: number } = {
+ renewMs: 14 * 24 * 60 * 60 * 1000,
+ maxLifetimeMs: 90 * 24 * 60 * 60 * 1000,
+};
+
+export interface ScheduleFileRef {
+ file_id: string;
+ filepath?: string;
+ filename?: string;
+ type?: string;
+ height?: number;
+ width?: number;
+ source?: string;
+}
+
+export interface ScheduleEngineDeps {
+ methods: ScheduleMethods;
+ /** Resolves interface.schedules limits, per-principal when a user is given. */
+ getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;
+ /** Loads the owning user (id + tenant) or null when deleted/disabled. */
+ getUserContext: (userId: string | Types.ObjectId) => Promise<ScheduleUserContext | null>;
+ /** Whether the balance feature gates this user and they are out of credits. */
+ isOutOfBalance: (user: ScheduleUserContext) => Promise<boolean>;
+ /** Live agent access for the owner: 'missing' (deleted) vs 'forbidden' (ACL revoked). */
+ agentAccess: (
+ agentId: string,
+ user: ScheduleUserContext,
+ ) => Promise<'ok' | 'missing' | 'forbidden'>;
+ /** Whether the owning user's current role still grants SCHEDULES access. */
+ hasScheduleAccess: (user: ScheduleUserContext) => Promise<boolean>;
+ /** Re-resolves stored file_ids to attachment payloads; missing files are simply absent. */
+ resolveFiles: (fileIds: string[], user: ScheduleUserContext) => Promise<ScheduleFileRef[]>;
+ /** Persists a trusted trigger delivery before this occurrence releases its claim. */
+ enqueueTrigger: (
+ envelope: AgentTriggerEnvelope,
+ options?: AgentTriggerEnqueueOptions,
+ ) => Promise<unknown>;
+ /**
+ * Reads the durable trigger delivery for a reservation's `deliveryKey`, so
+ * reconciliation can tell a still-live admission (`staging`/`pending`/`leased`, which a
+ * `Retry-After` can defer up to 24h) or a dead-letter (`dead`, with its `lastError`)
+ * apart from a genuinely orphaned jobless run. Null when no delivery record exists.
+ */
+ getTriggerDelivery: (deliveryKey: string) => Promise<{
+ status: AgentTriggerDeliveryStatus;
+ lastError?: AgentTriggerDeliveryFailure;
+ } | null>;
+ /** Runs fn inside the owner's tenant ALS context. */
+ runInTenantContext: <T>(user: ScheduleUserContext, fn: () => Promise<T>) => Promise<T>;
+ /**
+ * Job-store state for a run's conversation, or null when the job is gone. Carries
+ * the job's scheduled identity so reconciliation can verify the job at this
+ * conversationId is THIS occurrence's generation (a replacement user turn reuses
+ * the conversationId but strips scheduleId/scheduledFor) before trusting the status.
+ */
+ getJobStatus: (conversationId: string) => Promise<JobState | null>;
+ /** True once graceful shutdown began; fires step aside instead of dispatching. */
+ isShuttingDown?: () => boolean;
+ /**
+ * Aborts the loopback generation for a scheduled occurrence, identity-guarded so
+ * it never signals/clobbers a replacement turn that reused the conversationId.
+ * Used by deletion quiescing to stop an in-flight run before its evidence is
+ * erased. `preserve` keeps the terminal job for the reconciler (per-schedule
+ * delete, whose run row survives to drive cleanup); account deletion passes false
+ * since it hard-deletes the run rows, so a preserved job would leak in the store.
+ */
+ abortScheduledJob: (
+ conversationId: string,
+ identity: JobIdentity,
+ options?: { preserve?: boolean },
+ ) => Promise<boolean>;
+ /**
+ * Deletes a retained terminal job after the reconciler has finalized its run.
+ * Identity-guarded: only deletes when the job still carries this run's
+ * scheduleId/scheduledFor, so a replacement generation occupying the same
+ * conversationId is never destroyed. Gives `preserveForReconcile` jobs (kept
+ * without `completedAt` so the finished-job sweep can't reap them early) a
+ * definitive cleanup path.
+ */
+ clearReconciledJob: (conversationId: string, identity: JobIdentity) => Promise<void>;
+ /** Global in-flight scheduled-run count (system tenant scope) for the fire cap. */
+ countActiveRunsGlobal: () => Promise<number>;
+ /**
+ * The GLOBAL kill switch, deliberately distinct from per-principal availability.
+ * True when scheduling is stopped for the whole deployment: the SCHEDULES_DISABLED
+ * env lever (works even when the config plane is unhealthy), or `interface.schedules:
+ * false` in the BASE config — read base-only so no role/user/tenant override can
+ * re-enable it. Checked once per engine tick, so the uncached read is negligible.
+ */
+ isGloballyDisabled: () => Promise<boolean>;
+ /** Whether the run owner's account deletion has begun. Fail-closed (unknown == true). */
+ isOwnerDeleting: (userId: string) => Promise<boolean>;
+ /**
+ * Runs `claim` against the lowest free GLOBAL capacity slot, retrying the next slot
+ * when the unique partial index rejects a collision. Enforces fireConcurrency in the
+ * database instead of via a read-then-compare count, so concurrent admissions of
+ * different schedules cannot both pass a cap-1 check. Occupancy is read in system
+ * scope so the cap stays global across tenants.
+ */
+ withGlobalCapacitySlot: <T>(
+ cap: number,
+ claim: (slot: number) => Promise<SlotClaimResult<T>>,
+ ) => Promise<{ claimed: T } | 'capacity'>;
+}
+
+/** The immutable scheduled identity of a generation job, for reconcile/abort fencing. */
+export interface JobIdentity {
+ scheduleId: string;
+ scheduledFor: string | Date;
+}
+
+/** Job-store state plus the job's scheduled identity (absent on a replacement turn). */
+export interface JobState {
+ status: string;
+ createdAt?: number;
+ scheduleId?: string;
+ scheduledFor?: string;
+ /** Whether the generation ever emitted its created event — false means a
+ * pre-start abort whose conversation never came to exist. */
+ createdEventEmitted?: boolean;
+ /** The outcome the generation owner intended to record, stamped on a retained
+ * terminal job. A generic `complete` cannot distinguish a clean run from a balance
+ * refusal or a swallowed provider failure, so reconciliation prefers this when the
+ * owner left it. See SerializableJobData.scheduleOutcome. */
+ scheduleOutcome?: string;
+ scheduleOutcomeError?: string;
+ preserveForScheduleReconcile?: boolean;
+}
+
+export interface FireResult {
+ fired: boolean;
+ conversationId?: string;
+ skipped?:
+ | 'overlap'
+ | 'balance'
+ | 'capacity'
+ | 'duplicate'
+ | 'superseded'
+ | 'agent_deleted'
+ | 'user_missing'
+ | 'user_deleting'
+ | 'permission_revoked'
+ | 'rate_limited'
+ | 'disabled';
+ error?: string;
+}
+
+export type FireableSchedule = ISchedule;
diff --git a/packages/api/src/stream/ApprovalLifecycle.ts b/packages/api/src/stream/ApprovalLifecycle.ts
index 29797a09cf..ac6961af51 100644
--- a/packages/api/src/stream/ApprovalLifecycle.ts
+++ b/packages/api/src/stream/ApprovalLifecycle.ts
@@ -1,6 +1,11 @@
import { logger } from '@librechat/data-schemas';
import type { Agents } from 'librechat-data-provider';
-import type { IJobStoreV2, JobMetadataPatch, SteerQueueItem } from '~/stream/interfaces/IJobStore';
+import type {
+ IJobStoreV2,
+ JobMetadataPatch,
+ SerializableJobData,
+ SteerQueueItem,
+} from '~/stream/interfaces/IJobStore';
import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';
import {
isPendingActionExpired,
@@ -377,15 +382,17 @@ export class ApprovalLifecycle {
}
/**
- * Expires the observed approval and returns the winning job identity. Callers
- * that need to notify runtime-local subscribers can use the identity to avoid
- * delivering the predecessor's terminal event to a replacement generation.
+ * Expires the observed approval and returns the winning pre-transition job.
+ * Callers that need to notify runtime-local subscribers or run host lifecycle
+ * hooks get both the exact generation identity and its trusted metadata without
+ * a racy post-transition read (the terminal TTL may already remove the hash).
*/
async expireWithIdentity(
streamId: string,
expectedActionId?: string,
expectedCreatedAt?: number,
- ): Promise<number | null> {
+ options?: { markHostActionPending?: boolean },
+ ): Promise<SerializableJobData | null> {
const job = await this.waitForPausePersistence(streamId, expectedCreatedAt);
if (
!job ||
@@ -394,14 +401,25 @@ export class ApprovalLifecycle {
) {
return null;
}
+ // Built-in in-memory stores may mutate the returned job object in place during
+ // transitionStatus. Snapshot it before the CAS so the host hook receives the
+ // observed generation metadata even when terminal TTL removes or rewrites it.
+ const expiredJob = { ...job };
const createdAt = job.createdAt;
const ok = await this.store.transitionStatus(streamId, {
from: 'requires_action',
to: 'aborted',
clear: ['pendingAction', 'pendingActionId'],
// completedAt lets the stores' terminal-cleanup reclaim the job; without
- // it an expired approval lingers in the in-memory map indefinitely.
- patch: { error: 'Approval expired before a decision was made', completedAt: Date.now() },
+ // it an expired approval lingers in the in-memory map indefinitely. When a host
+ // action is owed, `terminalHostActionPending` retains the job (and keeps it
+ // enumerable) until the adapter acknowledges — set ATOMICALLY here so a crash right
+ // after this CAS still leaves the durable retry evidence.
+ patch: {
+ error: 'Approval expired before a decision was made',
+ completedAt: Date.now(),
+ ...(options?.markHostActionPending === true && { terminalHostActionPending: true }),
+ },
expectActionId: expectedActionId,
expectCreatedAt: createdAt,
});
@@ -409,6 +427,6 @@ export class ApprovalLifecycle {
this.callbacks.onExpired?.(streamId, createdAt);
logger.debug(`[ApprovalLifecycle] expired pending review: ${streamId}`);
}
- return ok ? createdAt : null;
+ return ok ? expiredJob : null;
}
}
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index b4a859d896..4b40b6c205 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -449,6 +449,13 @@ export interface GenerationJobManagerOptions {
cleanupOnComplete?: boolean;
}
+/** Host-owned lifecycle seam for durable work layered on agent generations.
+ * Delivery is at-least-once: implementations must be idempotent by generation. */
+export type ApprovalExpiredHandler = (
+ streamId: string,
+ job: SerializableJobData,
+) => void | Promise<void>;
+
export interface CreateGenerationJobOptions {
startupTelemetry?: AgentStartupTelemetry;
initialMetadata?: Partial<t.GenerationJobMetadata>;
@@ -695,6 +702,10 @@ class GenerationJobManagerClass {
/** Whether to cleanup event transport immediately on job completion */
private _cleanupOnComplete = true;
+ /** Optional host hook; the generic stream runtime does not know what external
+ * durable work (scheduled chats, webhooks, etc.) a generation represents. */
+ private approvalExpiredHandler: ApprovalExpiredHandler | undefined;
+
constructor(options?: GenerationJobManagerOptions) {
const jobStore =
options?.jobStore ?? new InMemoryJobStore({ ttlAfterComplete: 0, maxJobs: 1000 });
@@ -823,6 +834,12 @@ class GenerationJobManagerClass {
return this._isRedis;
}
+ /** Installs the application-owned approval-expiry hook without coupling the
+ * stream package to any particular trigger/scheduler implementation. */
+ setApprovalExpiredHandler(handler?: ApprovalExpiredHandler): void {
+ this.approvalExpiredHandler = handler;
+ }
+
private get storeLabel(): GenerationJobStore {
return this._isRedis ? 'redis' : 'memory';
}
@@ -2499,6 +2516,13 @@ class GenerationJobManagerClass {
agent_id: jobData.agent_id,
// Surface whether the turn was temporary so a resume keeps it non-persisted.
isTemporary: jobData.isTemporary,
+ scheduleId: jobData.scheduleId,
+ scheduledFor: jobData.scheduledFor,
+ scheduleConfigRevision: jobData.scheduleConfigRevision,
+ scheduleManual: jobData.scheduleManual,
+ scheduleOutcome: jobData.scheduleOutcome,
+ scheduleOutcomeError: jobData.scheduleOutcomeError,
+ preserveForScheduleReconcile: jobData.preserveForScheduleReconcile,
// Surface deferred tools discovered before the pause so the resume route can
// replay them into createRun (the rebuilt graph passes `messages: []`).
discoveredTools: jobData.discoveredTools,
@@ -3503,6 +3527,7 @@ class GenerationJobManagerClass {
if (
!retainForTerminalReplay &&
terminalJob?.providerDrained !== false &&
+ terminalJob?.preserveForScheduleReconcile !== true &&
(terminalJob?.createdAt !== createdAt || terminalJob.terminalPersistencePending !== true)
) {
// A same-stream replacement created after the claim makes this a safe
@@ -6359,6 +6384,7 @@ class GenerationJobManagerClass {
job?.createdAt === expectedCreatedAt &&
job.providerExecutionId === providerExecutionId &&
job.providerDrained === true &&
+ job.preserveForScheduleReconcile !== true &&
job.terminalPersistencePending !== true &&
(job.status === 'complete' || job.status === 'aborted')
) {
@@ -7004,19 +7030,42 @@ class GenerationJobManagerClass {
);
}
}
- const expiredCreatedAt = await this._approvals.expireWithIdentity(
+ const expiredJob = await this._approvals.expireWithIdentity(
streamId,
actionId,
expectedCreatedAt ?? observedJob?.createdAt,
+ // Only retain a durable host-action marker when a host adapter is installed to
+ // consume it — a store with no handler owes no action and accumulates nothing.
+ { markHostActionPending: this.approvalExpiredHandler != null },
);
- if (expiredCreatedAt == null) {
+ if (expiredJob == null) {
return false;
}
- await this.notifyApprovalExpiredRuntime(streamId, expiredCreatedAt, observedRuntime);
+ await this.runApprovalExpiredHandler(streamId, expiredJob);
+ await this.notifyApprovalExpiredRuntime(streamId, expiredJob.createdAt, observedRuntime);
return true;
}
+ private async runApprovalExpiredHandler(
+ streamId: string,
+ job: SerializableJobData,
+ ): Promise<void> {
+ try {
+ await this.approvalExpiredHandler?.(streamId, job);
+ // Success: the durable host action is settled. Clear its pending marker, fenced to
+ // this exact generation so a replacement at the same streamId keeps its own state.
+ // A no-op handler (or none) clears harmlessly, so non-scheduled jobs never linger.
+ await this.jobStore.clearTerminalHostAction?.(streamId, job.createdAt);
+ } catch (err) {
+ // Expiry itself already won its exact CAS. Keep terminal notification moving; the
+ // `terminalHostActionPending` marker stays set so a later cleanup pass (this or
+ // another replica, across restarts) re-enumerates the job and retries this
+ // idempotent hook until it acknowledges.
+ logger.error(`[GenerationJobManager] Approval-expiry host hook failed: ${streamId}`, err);
+ }
+ }
+
private async notifyApprovalExpiredRuntime(
streamId: string,
createdAt: number,
@@ -7058,10 +7107,37 @@ class GenerationJobManagerClass {
private async expireStaleApprovals(): Promise<void> {
let changed = false;
- for (const streamId of this.runtimeState.keys()) {
+ // Scan durable pauses as well as local runtimes. A process can restart while an
+ // approval waits; if expiry were limited to runtimeState, store cleanup would
+ // terminalize that ownerless job without crossing the host lifecycle hook.
+ const candidates = new Map<string, SerializableJobData>();
+ if (this.jobStore.getRequiresActionJobs) {
+ try {
+ for (const job of await this.jobStore.getRequiresActionJobs()) {
+ candidates.set(job.streamId, job);
+ }
+ } catch (err) {
+ logger.error('[GenerationJobManager] Failed to enumerate pending approvals', err);
+ }
+ }
+ // Also scan terminal jobs that still owe a host lifecycle hook. An expired approval is
+ // no longer in the requires_action index, so without this a failed host hook (e.g. the
+ // schedule outcome write) would never be retried on a replica or after a restart that
+ // has no local runtime — the exact clustered-entrypoint gap, which runs no reconciler.
+ if (this.jobStore.getTerminalHostActionJobs) {
+ try {
+ for (const job of await this.jobStore.getTerminalHostActionJobs()) {
+ candidates.set(job.streamId, job);
+ }
+ } catch (err) {
+ logger.error('[GenerationJobManager] Failed to enumerate pending host actions', err);
+ }
+ }
+ const streamIds = new Set([...this.runtimeState.keys(), ...candidates.keys()]);
+ for (const streamId of streamIds) {
let job: SerializableJobData | null;
try {
- job = await this.jobStore.getJob(streamId);
+ job = candidates.get(streamId) ?? (await this.jobStore.getJob(streamId));
} catch (err) {
logger.error(
`[GenerationJobManager] Failed to read job during approval expiry sweep: ${streamId}`,
@@ -7078,6 +7154,13 @@ class GenerationJobManagerClass {
// The `errorEvent` flag (set by emitError) keeps this idempotent vs the win path.
const runtime = this.runtimeState.get(streamId);
if (job?.status === 'aborted' && job.error === APPROVAL_EXPIRED_ERROR) {
+ // Retry the durable host hook ONLY while its marker is unacknowledged, so a
+ // successful ack prevents duplicate work. The terminal SSE relay below is
+ // separately idempotent (emitError's errorEvent flag) and always runs so a
+ // loser-replica subscriber still gets a terminal event.
+ if (job.terminalHostActionPending === true) {
+ await this.runApprovalExpiredHandler(streamId, job);
+ }
await this.notifyApprovalExpiredRuntime(streamId, job.createdAt, runtime);
changed = this.releaseJobOwnership(streamId, job.createdAt) || changed;
continue;
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
index 8506f13f5a..ff3be25270 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
@@ -333,6 +333,13 @@ describe('RedisJobStore', () => {
model: 'test-model',
agent_id: 'agent-1',
isTemporary: false,
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleConfigRevision: 4,
+ scheduleManual: false,
+ scheduleOutcome: 'interrupted',
+ scheduleOutcomeError: 'Schedule deleted',
+ preserveForScheduleReconcile: true,
promptTokens: 0,
discoveredTools: [],
preemptCapable: true,
@@ -388,6 +395,13 @@ describe('RedisJobStore', () => {
model: 'test-model',
agent_id: 'agent-1',
isTemporary: false,
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleConfigRevision: 4,
+ scheduleManual: false,
+ scheduleOutcome: 'interrupted',
+ scheduleOutcomeError: 'Schedule deleted',
+ preserveForScheduleReconcile: true,
promptTokens: 0,
discoveredTools: [],
resolvedAskUserQuestions: [
@@ -411,6 +425,13 @@ describe('RedisJobStore', () => {
responseMessageId: 'response-1',
agent_id: 'agent-1',
isTemporary: '0',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ scheduleConfigRevision: '4',
+ scheduleManual: '0',
+ scheduleOutcome: 'interrupted',
+ scheduleOutcomeError: 'Schedule deleted',
+ preserveForScheduleReconcile: '1',
promptTokens: '0',
discoveredTools: '[]',
resolvedAskUserQuestions: JSON.stringify([
@@ -644,6 +665,7 @@ describe('RedisJobStore', () => {
const evalResult = createDeferred<number>();
const runningMembership = createDeferred<number>();
const requiresActionRemoval = createDeferred<number>();
+ const terminalHostActionRemoval = createDeferred<number>();
const userMembership = createDeferred<number>();
const userExpiry = createDeferred<number>();
const started: string[] = [];
@@ -667,9 +689,13 @@ describe('RedisJobStore', () => {
started.push('user');
return userMembership.promise;
}),
- srem: jest.fn(() => {
- started.push('requires_action');
- return requiresActionRemoval.promise;
+ srem: jest.fn((key: string) => {
+ if (key === 'stream:requires_action') {
+ started.push('requires_action');
+ return requiresActionRemoval.promise;
+ }
+ started.push('terminal_host_action');
+ return terminalHostActionRemoval.promise;
}),
hgetall: jest.fn(() => jobHashFromCreationCall(evalJobCreation.mock.calls[0])),
expire,
@@ -684,16 +710,23 @@ describe('RedisJobStore', () => {
expect(started).toEqual(['job']);
evalResult.resolve(1);
- await waitFor(() => started.length === 4);
+ await waitFor(() => started.length === 5);
- expect(started).toEqual(['job', 'running', 'requires_action', 'user']);
+ expect(started).toEqual(['job', 'running', 'requires_action', 'terminal_host_action', 'user']);
expect(settled).toBe(false);
expect(expire).not.toHaveBeenCalled();
userMembership.resolve(1);
await waitFor(() => expire.mock.calls.length === 1);
- expect(started).toEqual(['job', 'running', 'requires_action', 'user', 'user_expiry']);
+ expect(started).toEqual([
+ 'job',
+ 'running',
+ 'requires_action',
+ 'terminal_host_action',
+ 'user',
+ 'user_expiry',
+ ]);
expect(expire).toHaveBeenCalledWith('stream:user:{user-1}:jobs', 60);
expect(settled).toBe(false);
@@ -705,6 +738,10 @@ describe('RedisJobStore', () => {
await Promise.resolve();
expect(settled).toBe(false);
+ terminalHostActionRemoval.resolve(1);
+ await Promise.resolve();
+ expect(settled).toBe(false);
+
requiresActionRemoval.resolve(1);
await expect(creating).resolves.toMatchObject({
streamId: 'stream-1',
diff --git a/packages/api/src/stream/__tests__/pendingAction.spec.ts b/packages/api/src/stream/__tests__/pendingAction.spec.ts
index 551e68f60f..cdca10c1e1 100644
--- a/packages/api/src/stream/__tests__/pendingAction.spec.ts
+++ b/packages/api/src/stream/__tests__/pendingAction.spec.ts
@@ -782,6 +782,56 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
});
describe('expireApproval notification', () => {
+ test('delivers the exact expired generation to the host lifecycle hook', async () => {
+ const streamId = 'stream-expire-host-hook';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.updateMetadata(streamId, {
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ });
+ await manager.approvals.pause(streamId, buildAction(streamId));
+ const onApprovalExpired = jest.fn(async () => undefined);
+ manager.setApprovalExpiredHandler(onApprovalExpired);
+
+ expect(await manager.expireApproval(streamId)).toBe(true);
+
+ expect(onApprovalExpired).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({
+ createdAt: job.createdAt,
+ status: 'requires_action',
+ scheduleId: 'schedule-1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ }),
+ );
+ });
+
+ test('expires a durable paused job even when no process-local runtime survived', async () => {
+ const streamId = 'stream-expire-ownerless-host-hook';
+ const job = await jobStore.createJob(streamId, 'user-1', streamId, undefined, {
+ scheduleId: 'schedule-ownerless',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ });
+ const lifecycle = new ApprovalLifecycle(jobStore);
+ await lifecycle.pause(streamId, buildAction(streamId, { expiresAt: Date.now() - 1 }));
+ const onApprovalExpired = jest.fn(async () => undefined);
+ manager.setApprovalExpiredHandler(onApprovalExpired);
+
+ await (
+ manager as unknown as { expireStaleApprovals: () => Promise<void> }
+ ).expireStaleApprovals();
+
+ expect(onApprovalExpired).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({
+ createdAt: job.createdAt,
+ scheduleId: 'schedule-ownerless',
+ status: 'requires_action',
+ }),
+ );
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({ status: 'aborted' });
+ });
+
test('does not expire a replacement paused on the same action id', async () => {
const streamId = 'stream-expire-epoch-fence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
@@ -928,6 +978,154 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
});
});
+ describe('durable approval-expiry host action', () => {
+ const sweep = (m: GenerationJobManagerClass) =>
+ (m as unknown as { expireStaleApprovals: () => Promise<void> }).expireStaleApprovals();
+
+ async function pauseScheduled(streamId: string) {
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.updateMetadata(streamId, {
+ scheduleId: 's1',
+ scheduledFor: '2026-08-17T12:00:00.000Z',
+ });
+ await manager.approvals.pause(streamId, buildAction(streamId));
+ return job;
+ }
+
+ test('retains the marker on hook failure and retries it on a later cleanup pass', async () => {
+ const streamId = 'stream-host-retry';
+ await pauseScheduled(streamId);
+ const handler = jest
+ .fn<Promise<void>, [string, { scheduleId?: string }]>()
+ .mockRejectedValueOnce(new Error('mongo down'))
+ .mockResolvedValue(undefined);
+ manager.setApprovalExpiredHandler(handler);
+
+ // First expiry wins its CAS, but the host hook FAILS: the durable marker is retained.
+ expect(await manager.expireApproval(streamId)).toBe(true);
+ expect(handler).toHaveBeenCalledTimes(1);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(1);
+
+ // A later cleanup pass re-enumerates the aborted job and retries; it now succeeds
+ // and clears the marker.
+ await sweep(manager);
+ expect(handler).toHaveBeenCalledTimes(2);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
+ });
+
+ test('a fresh manager (restart / other replica) retries the pending host action', async () => {
+ const streamId = 'stream-host-restart';
+ const job = await pauseScheduled(streamId);
+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
+ expect(await manager.expireApproval(streamId)).toBe(true);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
+
+ // Another replica / a restarted process: a fresh manager over the SAME durable store,
+ // with NO local runtime for this stream.
+ const other = new GenerationJobManagerClass();
+ other.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ other.initialize();
+ const succeeding = jest.fn().mockResolvedValue(undefined);
+ other.setApprovalExpiredHandler(succeeding);
+
+ await sweep(other);
+
+ expect(succeeding).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({ createdAt: job.createdAt, scheduleId: 's1' }),
+ );
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
+ await other.destroy();
+ });
+
+ test('a successful acknowledgement prevents duplicate work on later sweeps', async () => {
+ const streamId = 'stream-host-ack';
+ await pauseScheduled(streamId);
+ const handler = jest.fn().mockResolvedValue(undefined);
+ manager.setApprovalExpiredHandler(handler);
+
+ expect(await manager.expireApproval(streamId)).toBe(true);
+ expect(handler).toHaveBeenCalledTimes(1);
+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
+
+ await sweep(manager);
+ expect(handler).toHaveBeenCalledTimes(1); // no duplicate invocation
+ });
+
+ test('clearTerminalHostAction is identity-fenced against a replacement generation', async () => {
+ const streamId = 'stream-host-fence';
+ const job = await pauseScheduled(streamId);
+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
+ await manager.expireApproval(streamId);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
+
+ // A clear fenced to a DIFFERENT generation must not clear this one's action.
+ await jobStore.clearTerminalHostAction(streamId, job.createdAt + 1);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
+
+ // The exact generation clears it.
+ await jobStore.clearTerminalHostAction(streamId, job.createdAt);
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
+ });
+
+ test('terminalizes and delivers the terminal state even when the first hook fails', async () => {
+ const streamId = 'stream-host-notify';
+ await pauseScheduled(streamId);
+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
+
+ // The CAS + terminal notification proceed regardless of the host hook outcome.
+ expect(await manager.expireApproval(streamId)).toBe(true);
+ expect(await manager.getJobStatus(streamId)).toBe('aborted');
+ });
+
+ test('retention is refreshed on each retry so a long host outage keeps the evidence', async () => {
+ const streamId = 'stream-host-refresh';
+ await pauseScheduled(streamId);
+ manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('mongo down')));
+ expect(await manager.expireApproval(streamId)).toBe(true);
+
+ const before = (await jobStore.getJob(streamId))?.terminalHostActionRefreshedAt;
+
+ // A later cleanup pass re-enumerates it for retry; the hook still fails.
+ await sweep(manager);
+
+ const after = await jobStore.getJob(streamId);
+ // Still pending, and its retention basis moved forward — so the bounded retention
+ // window measures from the LAST retry, not from when the approval expired.
+ expect(after?.terminalHostActionPending).toBe(true);
+ expect(after?.terminalHostActionRefreshedAt).toEqual(expect.any(Number));
+ if (before != null) {
+ expect(after!.terminalHostActionRefreshedAt!).toBeGreaterThanOrEqual(before);
+ }
+ });
+
+ test('a non-scheduled job with a no-op host hook accumulates no marker', async () => {
+ const streamId = 'stream-host-nonsched';
+ await manager.createJob(streamId, 'user-1'); // no schedule metadata
+ await manager.approvals.pause(streamId, buildAction(streamId));
+ // The schedule adapter is installed but owns no action for a non-scheduled job.
+ manager.setApprovalExpiredHandler(
+ jest.fn(async (_streamId: string, job: { scheduleId?: string }) => {
+ if (!job.scheduleId) {
+ return;
+ }
+ }),
+ );
+
+ expect(await manager.expireApproval(streamId)).toBe(true);
+ // Marked atomically then cleared on the no-op success — nothing lingers.
+ expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
+ expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
+ });
+ });
+
describe('facade integration', () => {
test('requires_action drops the running count but keeps the user-active set', async () => {
const streamId = 'stream-counts';
@@ -997,9 +1195,9 @@ describe('InMemoryJobStore — approval expiry cleanup', () => {
expect((await store.getJob('epoch-guard'))?.status).toBe('error');
});
- test('cleanup() finalizes and reclaims a past-expiry pending-approval job', async () => {
+ test('cleanup() finalizes a past-expiry approval, retaining it for host-hook retry', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
- await store.createJob('s1', 'u1');
+ const job = await store.createJob('s1', 'u1');
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
@@ -1011,7 +1209,17 @@ describe('InMemoryJobStore — approval expiry cleanup', () => {
patch: { pendingAction: action, pendingActionId: action.actionId },
});
- // A past-expiry approval must be finalized + reclaimed, not left resident.
+ // A past-expiry approval is finalized (aborted) but RETAINED with a pending host-action
+ // marker so the manager relay can still run its lifecycle hook (the store cannot know
+ // whether one is owed); it is enumerable for that retry.
+ await store.cleanup();
+ const aborted = await store.getJob('s1');
+ expect(aborted?.status).toBe('aborted');
+ expect(aborted?.terminalHostActionPending).toBe(true);
+ expect(await store.getTerminalHostActionJobs()).toHaveLength(1);
+
+ // Once the host action acknowledges, the marker clears and the next cleanup reclaims it.
+ await store.clearTerminalHostAction('s1', job.createdAt);
await store.cleanup();
expect(await store.getJob('s1')).toBeNull();
});
diff --git a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts b/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
index d6982ba4f8..fca8fff107 100644
--- a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
+++ b/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
@@ -34,6 +34,35 @@ describe('InMemoryJobStore - stale running-job failsafe', () => {
await store.destroy();
});
+ it('retains a terminal schedule job until reconciliation releases it', async () => {
+ const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+
+ const job = await store.createJob('scheduled-run', 'u1', 'scheduled-run', undefined, {
+ scheduleId: 'schedule-1',
+ preserveForScheduleReconcile: true,
+ });
+ await store.updateJob(
+ job.streamId,
+ {
+ status: 'complete',
+ completedAt: Date.now(),
+ scheduleOutcome: 'success',
+ },
+ job.createdAt,
+ );
+
+ expect(await store.cleanup()).toBe(0);
+ expect(await store.hasJob(job.streamId)).toBe(true);
+
+ await store.updateJob(job.streamId, { preserveForScheduleReconcile: false }, job.createdAt);
+
+ expect(await store.cleanup()).toBe(1);
+ expect(await store.hasJob(job.streamId)).toBe(false);
+
+ await store.destroy();
+ });
+
it('does not reap a running job with recent activity even if created long ago', async () => {
const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
const store = new InMemoryJobStore({ staleJobTimeout: 1000 });
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index 9b9653d00c..2ae4ad2d18 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -46,6 +46,10 @@ const STEER_RECEIPT_MAX_PER_STREAM: number = 100;
const CREATE_REPLACEMENT_RECEIPT_MAX = 32;
const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
const TERMINAL_PERSISTENCE_RETENTION_MS = 5 * 60 * 1000;
+/** Bounded window a terminal job owing a host lifecycle hook is retained for retry before
+ * it is reaped even if unacknowledged (a permanently-failing hook cannot leak forever).
+ * Mirrors the Redis store's 24h host-action TTL. */
+const HOST_ACTION_RETENTION_MS = 24 * 60 * 60 * 1000;
function assertCreateIdempotencyArguments(
claimKey?: string,
@@ -1026,6 +1030,40 @@ export class InMemoryJobStore implements IJobStoreV2 {
return running;
}
+ async getRequiresActionJobs(): Promise<SerializableJobData[]> {
+ const paused: SerializableJobData[] = [];
+ for (const job of this.jobs.values()) {
+ if (job.status === 'requires_action') {
+ paused.push(job);
+ }
+ }
+ return paused;
+ }
+
+ async getTerminalHostActionJobs(): Promise<SerializableJobData[]> {
+ const pending: SerializableJobData[] = [];
+ const now = Date.now();
+ for (const job of this.jobs.values()) {
+ if (job.terminalHostActionPending === true) {
+ // Enumerating IS the retry attempt: refresh retention so evidence outlives a host
+ // dependency that is unreachable for longer than the retention window.
+ job.terminalHostActionRefreshedAt = now;
+ pending.push(job);
+ }
+ }
+ return pending;
+ }
+
+ async clearTerminalHostAction(streamId: string, expectedCreatedAt?: number): Promise<void> {
+ const job = this.jobs.get(streamId);
+ // Identity-fenced: a replacement generation at this streamId (a newer createdAt) must
+ // not have a predecessor's marker cleared on its behalf.
+ if (job == null || (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)) {
+ return;
+ }
+ delete job.terminalHostActionPending;
+ }
+
async cleanup(): Promise<number> {
const now = Date.now();
const toDelete: Array<{ streamId: string; createdAt: number }> = [];
@@ -1089,7 +1127,19 @@ export class InMemoryJobStore implements IJobStoreV2 {
}
const isFinished = ['complete', 'error', 'aborted'].includes(job.status);
- if (isFinished && job.completedAt) {
+ // A terminal job that still owes a host lifecycle hook is retained (like a
+ // reconcile-preserved job) so a later cleanup pass can enumerate and retry it —
+ // but only within a bounded window, so a permanently-failing hook cannot leak.
+ const hostActionHeld =
+ job.terminalHostActionPending === true &&
+ now - (job.terminalHostActionRefreshedAt ?? job.completedAt ?? 0) <=
+ HOST_ACTION_RETENTION_MS;
+ if (
+ isFinished &&
+ job.completedAt &&
+ job.preserveForScheduleReconcile !== true &&
+ !hostActionHeld
+ ) {
// A pending final event is a persistence barrier, not an ordinary
// completed row. Give its owner/recovery path a bounded window even
// when normal terminal retention is configured to zero.
@@ -1117,11 +1167,13 @@ export class InMemoryJobStore implements IJobStoreV2 {
job.status = 'aborted';
job.completedAt = now;
job.error = 'Approval expired before a decision was made';
+ // Store-won expiry (the manager's own sweep did not win the CAS): mark the host
+ // action pending so the manager relay still runs its lifecycle hook, and RETAIN the
+ // job (even under zero terminal retention) so a sweep can enumerate it — cleared and
+ // reaped once the hook acknowledges.
+ job.terminalHostActionPending = true;
delete job.pendingAction;
delete job.pendingActionId;
- if (this.ttlAfterComplete === 0) {
- toDelete.push({ streamId, createdAt: job.createdAt });
- }
} else if (this.staleJobTimeout > 0 && job.status === 'running') {
// Failsafe: reap jobs stuck in "running" with no generation activity for
// longer than the stale timeout. These are crashed/hung generations that
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index b218d9d4ac..ea46b4a4e1 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -1540,6 +1540,9 @@ const KEYS = {
runningJobs: 'stream:running',
/** Jobs paused for human review (global set - single slot) */
requiresActionJobs: 'stream:requires_action',
+ /** Terminal jobs that still owe a durable host lifecycle hook (global set). Retains
+ * the aborted approval-expiry job for cross-replica / post-restart hook retry. */
+ terminalHostActionJobs: 'stream:terminal_host_action',
/** User's active jobs set, tenant-qualified when tenantId is available */
userJobs: (userId: string, tenantId?: string) =>
tenantId ? `stream:user:{${tenantId}:${userId}}:jobs` : `stream:user:{${userId}}:jobs`,
@@ -2246,6 +2249,11 @@ export class RedisJobStore implements IJobStoreV2 {
statusKey === KEYS.requiresActionJobs
? this.redis.sadd(KEYS.requiresActionJobs, streamId)
: this.redis.srem(KEYS.requiresActionJobs, streamId),
+ // Terminal host-action membership follows the durable hash field, not status, so
+ // an aborted approval-expiry job stays enumerable for hook retry until acked.
+ job?.terminalHostActionPending === true
+ ? this.redis.sadd(KEYS.terminalHostActionJobs, streamId)
+ : this.redis.srem(KEYS.terminalHostActionJobs, streamId),
];
for (const userJobsKey of observedUserKeys) {
if (userJobsKey !== activeUserKey) {
@@ -2277,6 +2285,11 @@ export class RedisJobStore implements IJobStoreV2 {
} else {
pipeline.srem(KEYS.requiresActionJobs, streamId);
}
+ if (job?.terminalHostActionPending === true) {
+ pipeline.sadd(KEYS.terminalHostActionJobs, streamId);
+ } else {
+ pipeline.srem(KEYS.terminalHostActionJobs, streamId);
+ }
for (const userJobsKey of observedUserKeys) {
if (userJobsKey !== activeUserKey) {
pipeline.srem(userJobsKey, streamId);
@@ -2438,6 +2451,12 @@ export class RedisJobStore implements IJobStoreV2 {
if (terminal && patch?.terminalPersistencePending === true) {
ttl = Math.max(ttl, TERMINAL_PERSISTENCE_RETENTION_TTL_S);
}
+ if (terminal && patch?.terminalHostActionPending === true) {
+ // A terminal job owing a host hook must outlive the normal completed TTL so cleanup
+ // can still enumerate and retry it across restarts; the pause backstop (24h) bounds
+ // the retry window. Cleared to the completed TTL on acknowledgement.
+ ttl = Math.max(ttl, this.ttl.requiresAction);
+ }
if (to === 'requires_action') {
// A paused job must outlive its approval window, even when that window is
// longer than the running TTL — otherwise Redis evicts it before a
@@ -2681,6 +2700,70 @@ export class RedisJobStore implements IJobStoreV2 {
return jobs;
}
+ async getRequiresActionJobs(): Promise<SerializableJobData[]> {
+ const streamIds = await this.redis.smembers(KEYS.requiresActionJobs);
+ if (streamIds.length === 0) {
+ return [];
+ }
+ const jobs = await Promise.all(streamIds.map((streamId) => this.getJob(streamId)));
+ return jobs.filter(
+ (job): job is SerializableJobData => job != null && job.status === 'requires_action',
+ );
+ }
+
+ async getTerminalHostActionJobs(): Promise<SerializableJobData[]> {
+ const streamIds = await this.redis.smembers(KEYS.terminalHostActionJobs);
+ if (streamIds.length === 0) {
+ return [];
+ }
+ const jobs = await Promise.all(streamIds.map((streamId) => this.getJob(streamId)));
+ // The durable hash field is the source of truth; a stale set entry (job reaped, or the
+ // marker already cleared) is filtered out and self-heals via reconcileJobMembership.
+ const stale: string[] = [];
+ const pending: SerializableJobData[] = [];
+ for (let i = 0; i < streamIds.length; i++) {
+ const job = jobs[i];
+ if (job != null && job.terminalHostActionPending === true) {
+ pending.push(job);
+ } else {
+ stale.push(streamIds[i]);
+ }
+ }
+ if (stale.length > 0) {
+ await this.redis.srem(KEYS.terminalHostActionJobs, ...stale).catch(() => undefined);
+ }
+ // Enumerating IS the retry attempt: extend each pending job's TTL so unacknowledged
+ // host-action evidence outlives a host dependency (e.g. Mongo) that stays unreachable
+ // longer than the retention window. A deployment that stops sweeping lets it age out.
+ if (pending.length > 0 && this.ttl.requiresAction > 0) {
+ await Promise.all(
+ pending.map((job) =>
+ this.redis.expire(KEYS.job(job.streamId), this.ttl.requiresAction).catch(() => undefined),
+ ),
+ );
+ }
+ return pending;
+ }
+
+ async clearTerminalHostAction(streamId: string, expectedCreatedAt?: number): Promise<void> {
+ // Identity-fenced: only clear when the hash still holds this exact generation, so a
+ // replacement at the same streamId is never cleared through its predecessor. The HDEL
+ // and the completed-TTL reset happen atomically; membership is then reconciled to SREM.
+ const cleared = (await this.redis.eval(
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'redis.call("HDEL", KEYS[1], "terminalHostActionPending") ' +
+ 'if tonumber(ARGV[2]) > 0 then redis.call("EXPIRE", KEYS[1], ARGV[2]) end ' +
+ 'return 1',
+ 1,
+ KEYS.job(streamId),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ String(this.ttl.completed),
+ )) as number;
+ if (cleared === 1) {
+ await this.redis.srem(KEYS.terminalHostActionJobs, streamId).catch(() => undefined);
+ }
+ }
+
async cleanup(): Promise<number> {
const now = Date.now();
const streamIds = await this.redis.smembers(KEYS.runningJobs);
@@ -2848,6 +2931,10 @@ export class RedisJobStore implements IJobStoreV2 {
patch: {
error: 'Approval expired before a decision was made',
completedAt: Date.now(),
+ // Store-won expiry: mark the host action pending (and extend retention via
+ // the TTL rule above) so the manager relay still runs its lifecycle hook on
+ // a replica that owns the runtime; cleared once that hook acknowledges.
+ terminalHostActionPending: true,
},
// Scope the CAS to the action we observed as stale: if the user resolved it
// and the run re-paused on a fresh action between the read and here, the
@@ -4477,6 +4564,26 @@ export class RedisJobStore implements IJobStoreV2 {
promptTokens: data.promptTokens ? parseInt(data.promptTokens, 10) : undefined,
agent_id: data.agent_id || undefined,
isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,
+ scheduleId: data.scheduleId || undefined,
+ scheduledFor: data.scheduledFor || undefined,
+ scheduleConfigRevision: data.scheduleConfigRevision
+ ? parseInt(data.scheduleConfigRevision, 10)
+ : undefined,
+ scheduleManual: data.scheduleManual != null ? data.scheduleManual === '1' : undefined,
+ scheduleOutcome:
+ data.scheduleOutcome === 'success' ||
+ data.scheduleOutcome === 'error' ||
+ data.scheduleOutcome === 'interrupted' ||
+ data.scheduleOutcome === 'skipped_balance'
+ ? data.scheduleOutcome
+ : undefined,
+ scheduleOutcomeError: data.scheduleOutcomeError || undefined,
+ preserveForScheduleReconcile:
+ data.preserveForScheduleReconcile != null
+ ? data.preserveForScheduleReconcile === '1'
+ : undefined,
+ terminalHostActionPending:
+ data.terminalHostActionPending != null ? data.terminalHostActionPending === '1' : undefined,
// Deferred tools discovered before a HITL pause; replayed into createRun on resume.
discoveredTools: data.discoveredTools ? JSON.parse(data.discoveredTools) : undefined,
activityPhaseSnapshot: data.activityPhaseSnapshot
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 10143c62cd..50cb847ebc 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -123,6 +123,31 @@ export interface SerializableJobData {
/** Whether sync has been sent to a client */
syncSent: boolean;
+ /** Trusted schedule identity copied atomically into the generation job. */
+ scheduleId?: string;
+ scheduledFor?: string;
+ scheduleConfigRevision?: number;
+ scheduleManual?: boolean;
+ /** Terminal outcome evidence retained when the schedule row could not be updated. */
+ scheduleOutcome?: 'success' | 'error' | 'interrupted' | 'skipped_balance';
+ scheduleOutcomeError?: string;
+ preserveForScheduleReconcile?: boolean;
+ /**
+ * A terminal transition (currently approval expiry) still owes a durable host
+ * lifecycle hook. Set atomically with that transition and cleared only once the host
+ * adapter acknowledges success, so the job is retained (not reaped) and enumerable by
+ * cleanup across restarts and replicas until the hook completes. Generic: a host with
+ * no action clears it immediately on its no-op success, so nothing accumulates.
+ */
+ terminalHostActionPending?: boolean;
+ /**
+ * Last time a cleanup pass enumerated this pending host action for retry. Retention is
+ * measured from this rather than `completedAt`, so evidence survives as long as some
+ * replica is still actively retrying the hook (e.g. Mongo unreachable for days), while a
+ * deployment that stops retrying entirely still lets it age out instead of leaking.
+ */
+ terminalHostActionRefreshedAt?: number;
+
/** Serialized final event for replay */
finalEvent?: string;
@@ -277,6 +302,13 @@ export type JobMetadataPatch = Partial<
| 'model'
| 'agent_id'
| 'isTemporary'
+ | 'scheduleId'
+ | 'scheduledFor'
+ | 'scheduleConfigRevision'
+ | 'scheduleManual'
+ | 'scheduleOutcome'
+ | 'scheduleOutcomeError'
+ | 'preserveForScheduleReconcile'
| 'promptTokens'
| 'discoveredTools'
| 'activityPhaseSnapshot'
@@ -661,6 +693,18 @@ export interface IJobStore {
deleteJob(streamId: string, expectedCreatedAt?: number): Promise<boolean>;
hasJob(streamId: string): Promise<boolean>;
getRunningJobs(): Promise<SerializableJobData[]>;
+ /** Optional durable paused-job enumeration. Built-in stores implement it so
+ * the manager can own approval expiry even after the original runtime died. */
+ getRequiresActionJobs?(): Promise<SerializableJobData[]>;
+ /** Optional durable enumeration of terminal jobs that still owe a host lifecycle
+ * hook (see `terminalHostActionPending`). Built-in stores implement it so cleanup can
+ * retry the host adapter after a restart / on another replica, even though the job is
+ * no longer in the requires_action index. */
+ getTerminalHostActionJobs?(): Promise<SerializableJobData[]>;
+ /** Clears the pending-host-action marker once the adapter acknowledges success.
+ * Identity-fenced on `expectedCreatedAt` so a replacement generation at the same
+ * streamId is never cleared through its predecessor. */
+ clearTerminalHostAction?(streamId: string, expectedCreatedAt?: number): Promise<void>;
cleanup(): Promise<number>;
recordActivity?(streamId: string, expectedCreatedAt?: number): void;
getJobCount(): Promise<number>;
@@ -902,6 +946,9 @@ export interface IJobStoreV2 extends IJobStore {
/** Get all running jobs (for cleanup) */
getRunningJobs(): Promise<SerializableJobData[]>;
+ /** Get durable paused jobs so approval expiry is not process-runtime-dependent. */
+ getRequiresActionJobs?(): Promise<SerializableJobData[]>;
+
/** Cleanup expired jobs */
cleanup(): Promise<number>;
diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts
index 1a4b2e3376..b334412267 100644
--- a/packages/api/src/stream/metadata.ts
+++ b/packages/api/src/stream/metadata.ts
@@ -30,6 +30,27 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): J
if (metadata.isTemporary !== undefined) {
patch.isTemporary = metadata.isTemporary;
}
+ if (metadata.scheduleId) {
+ patch.scheduleId = metadata.scheduleId;
+ }
+ if (metadata.scheduledFor) {
+ patch.scheduledFor = metadata.scheduledFor;
+ }
+ if (metadata.scheduleConfigRevision !== undefined) {
+ patch.scheduleConfigRevision = metadata.scheduleConfigRevision;
+ }
+ if (metadata.scheduleManual !== undefined) {
+ patch.scheduleManual = metadata.scheduleManual;
+ }
+ if (metadata.scheduleOutcome !== undefined) {
+ patch.scheduleOutcome = metadata.scheduleOutcome;
+ }
+ if (metadata.scheduleOutcomeError !== undefined) {
+ patch.scheduleOutcomeError = metadata.scheduleOutcomeError;
+ }
+ if (metadata.preserveForScheduleReconcile !== undefined) {
+ patch.preserveForScheduleReconcile = metadata.preserveForScheduleReconcile;
+ }
if (metadata.promptTokens !== undefined) {
patch.promptTokens = metadata.promptTokens;
}
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index 145df044b8..b6727ebfbf 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -31,6 +31,20 @@ export interface GenerationJobMetadata {
agent_id?: string;
/** Whether the originating turn was a temporary chat; a HITL resume keeps it so. */
isTemporary?: boolean;
+ /** Trusted scheduled-occurrence identity. These fields are accepted only from a
+ * verified agent-trigger request and let pause/resume/reconciliation keep the
+ * occurrence attached to the exact generation that owns it. */
+ scheduleId?: string;
+ scheduledFor?: string;
+ scheduleConfigRevision?: number;
+ scheduleManual?: boolean;
+ /** Intended terminal classification retained when Mongo outcome persistence
+ * fails. The scheduler reconciler consumes this evidence before clearing the job. */
+ scheduleOutcome?: 'success' | 'error' | 'interrupted' | 'skipped_balance';
+ scheduleOutcomeError?: string;
+ /** Prevent normal terminal cleanup until schedule reconciliation has consumed
+ * the retained outcome evidence. */
+ preserveForScheduleReconcile?: boolean;
/**
* Deferred-tool names discovered (via `tool_search`) before a HITL pause. A resume
* replays these into `createRun` because the rebuilt graph uses `messages: []`, so
diff --git a/packages/client/src/components/ControlCombobox.tsx b/packages/client/src/components/ControlCombobox.tsx
index 0233501cf6..05fc1a0bde 100644
--- a/packages/client/src/components/ControlCombobox.tsx
+++ b/packages/client/src/components/ControlCombobox.tsx
@@ -5,6 +5,7 @@ import { Search, ChevronDown } from 'lucide-react';
import { SelectRenderer } from '@ariakit/react-components/select/select-renderer';
import type { OptionWithIcon } from '~/common';
import { usePopoverZIndex } from './OriginalDialog';
+import { fieldControl } from './Field';
import './AnimatePopover.css';
import { JSX } from 'react/jsx-runtime';
import { cn } from '~/utils';
@@ -14,7 +15,10 @@ interface ControlComboboxProps {
displayValue?: string;
items: OptionWithIcon[];
setValue: (value: string) => void;
+ onBlur?: React.FocusEventHandler<HTMLButtonElement>;
ariaLabel: string;
+ ariaInvalid?: boolean;
+ ariaDescribedBy?: string;
searchPlaceholder?: string;
selectPlaceholder?: string;
isCollapsed: boolean;
@@ -29,6 +33,8 @@ interface ControlComboboxProps {
placement?: Ariakit.SelectStoreProps['placement'];
popoverClassName?: string;
matchTriggerWidth?: boolean;
+ /** `field` matches the `Input` primitive so this can sit in a form row. */
+ variant?: 'default' | 'field';
gutter?: number;
/**
* Radix dialogs trap focus, so a portaled popover rendered outside the dialog
@@ -46,7 +52,10 @@ function ControlCombobox({
displayValue,
items,
setValue,
+ onBlur,
ariaLabel,
+ ariaInvalid,
+ ariaDescribedBy,
searchPlaceholder,
selectPlaceholder,
containerClassName,
@@ -61,6 +70,7 @@ function ControlCombobox({
placement,
popoverClassName,
matchTriggerWidth = true,
+ variant = 'default',
gutter = 4,
portal = true,
}: ControlComboboxProps): JSX.Element {
@@ -136,7 +146,13 @@ function ControlCombobox({
);
return (
- <div className={cn('flex w-full items-center justify-center px-1', containerClassName)}>
+ <div
+ className={cn(
+ 'flex w-full items-center justify-center px-1',
+ variant === 'field' && 'px-0',
+ containerClassName,
+ )}
+ >
<Ariakit.SelectLabel store={select} className="sr-only">
{ariaLabel}
</Ariakit.SelectLabel>
@@ -145,11 +161,15 @@ function ControlCombobox({
store={select}
id={selectId}
disabled={disabled}
+ onBlur={onBlur}
+ aria-invalid={ariaInvalid || undefined}
+ aria-describedby={ariaDescribedBy}
className={cn(
'flex items-center justify-center gap-2 rounded-full bg-surface-secondary',
'text-text-primary hover:bg-surface-tertiary',
'border border-border-light',
isCollapsed ? 'h-9 w-9' : 'h-9 w-full rounded-xl px-3 py-2 text-sm',
+ variant === 'field' && cn(fieldControl, 'justify-start hover:bg-surface-hover'),
className,
)}
>
diff --git a/packages/client/src/components/Dropdown.tsx b/packages/client/src/components/Dropdown.tsx
index c18ddb9bd3..c4ba9a1ae9 100644
--- a/packages/client/src/components/Dropdown.tsx
+++ b/packages/client/src/components/Dropdown.tsx
@@ -4,6 +4,7 @@ import { matchSorter } from 'match-sorter';
import * as Select from '@ariakit/react/select';
import * as Combobox from '@ariakit/react/combobox';
import type { Option } from '~/common';
+import { fieldControl } from './Field';
import { cn } from '~/utils/';
import './Dropdown.css';
@@ -25,6 +26,8 @@ interface DropdownProps {
ariaLabel?: string;
'aria-labelledby'?: string;
portal?: boolean;
+ /** `field` matches the `Input` primitive so this can sit in a form row. */
+ variant?: 'default' | 'field';
/** Renders the popover into this element instead of document.body */
portalElement?: ((element: HTMLElement) => HTMLElement | null) | HTMLElement | null;
disabled?: boolean;
@@ -57,6 +60,7 @@ const Dropdown: React.FC<DropdownProps> = ({
ariaLabel,
'aria-labelledby': ariaLabelledBy,
portal = true,
+ variant = 'default',
portalElement,
disabled = false,
searchable = false,
@@ -140,7 +144,7 @@ const Dropdown: React.FC<DropdownProps> = ({
);
return (
- <div className={cn('relative', className)}>
+ <div className={cn('relative', variant === 'field' && 'w-full', className)}>
<Select.Select
store={selectProps}
disabled={disabled}
@@ -149,6 +153,7 @@ const Dropdown: React.FC<DropdownProps> = ({
'disabled:cursor-not-allowed disabled:opacity-50 disabled:hover:bg-transparent disabled:hover:text-text-primary',
/** Horizontal padding would squeeze the icon, which flex-shrinks to fit */
iconOnly ? 'size-10 justify-center px-0' : 'w-fit gap-2 px-3',
+ variant === 'field' && fieldControl,
triggerClassName,
)}
data-testid={testId}
diff --git a/packages/client/src/components/Field.ts b/packages/client/src/components/Field.ts
new file mode 100644
index 0000000000..5c09d95a6b
--- /dev/null
+++ b/packages/client/src/components/Field.ts
@@ -0,0 +1,11 @@
+/**
+ * The shared appearance of a form control — border, radius, type scale and focus
+ * treatment. Owned here so `Input`, `Textarea`, and the select/combobox triggers
+ * that have to sit beside them in a form cannot drift apart as the theme evolves.
+ * Callers compose a variant rather than restating these classes locally.
+ */
+export const fieldBase: string =
+ 'lc-field flex w-full rounded-lg border border-border-light px-3 py-2 text-sm text-text-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50';
+
+/** A single-line control sized to sit in a form row, matching `Input`. */
+export const fieldControl: string = `${fieldBase} h-10 bg-transparent`;
diff --git a/packages/client/src/components/Input.tsx b/packages/client/src/components/Input.tsx
index a8eae5c54f..0db5998670 100644
--- a/packages/client/src/components/Input.tsx
+++ b/packages/client/src/components/Input.tsx
@@ -1,4 +1,5 @@
import * as React from 'react';
+import { fieldControl } from './Field';
import { cn } from '~/utils';
import './Field.css';
@@ -8,10 +9,7 @@ const Input: React.ForwardRefExoticComponent<InputProps & React.RefAttributes<HT
React.forwardRef<HTMLInputElement, InputProps>(({ className, ...props }, ref) => {
return (
<input
- className={cn(
- 'lc-field flex h-10 w-full rounded-lg border border-border-light bg-transparent px-3 py-2 text-sm text-text-primary ring-offset-surface-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50',
- className ?? '',
- )}
+ className={cn(fieldControl, 'ring-offset-surface-primary', className ?? '')}
ref={ref}
{...props}
/>
diff --git a/packages/client/src/components/Textarea.tsx b/packages/client/src/components/Textarea.tsx
index 8c9a2ee1cc..19259faa18 100644
--- a/packages/client/src/components/Textarea.tsx
+++ b/packages/client/src/components/Textarea.tsx
@@ -1,6 +1,7 @@
/* eslint-disable */
import * as React from 'react';
import TextareaAutosize from 'react-textarea-autosize';
+import { fieldBase } from './Field';
import { cn } from '~/utils';
import './Field.css';
@@ -11,10 +12,7 @@ const Textarea: React.ForwardRefExoticComponent<
> = React.forwardRef<HTMLTextAreaElement, TextareaProps>(({ className = '', ...props }, ref) => {
return (
<textarea
- className={cn(
- 'lc-field flex min-h-20 w-full resize-none rounded-lg border border-border-light bg-surface-secondary px-3 py-2 text-sm text-text-primary placeholder:text-text-secondary focus-visible:border-border-medium focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary disabled:cursor-not-allowed disabled:opacity-50',
- className,
- )}
+ className={cn(fieldBase, 'min-h-20 resize-none bg-surface-secondary', className)}
ref={ref}
{...props}
/>
diff --git a/packages/client/src/components/index.ts b/packages/client/src/components/index.ts
index 8181ae945b..b03669d4ad 100644
--- a/packages/client/src/components/index.ts
+++ b/packages/client/src/components/index.ts
@@ -9,6 +9,7 @@ export * from './Checkbox';
export * from './Dialog';
export * from './DropdownMenu';
export * from './HoverCard';
+export * from './Field';
export * from './Input';
export * from './InputNumber';
export * from './SecretInput';
diff --git a/packages/client/src/theme/semanticTokens.spec.ts b/packages/client/src/theme/semanticTokens.spec.ts
index 66c763a82f..0b9cc25a19 100644
--- a/packages/client/src/theme/semanticTokens.spec.ts
+++ b/packages/client/src/theme/semanticTokens.spec.ts
@@ -112,12 +112,23 @@ describe('light brand text', () => {
describe('shared field and dropdown interaction styles', () => {
it('keeps pointer focus stable and keyboard focus visible on text fields', () => {
- ['Input.tsx', 'Textarea.tsx'].forEach((component) => {
+ /** The focus treatment lives in the shared field module, so guard it there and
+ * assert the primitives still compose it rather than restating the classes. */
+ const field = readFileSync(join(__dirname, '..', 'components', 'Field.ts'), 'utf8');
+
+ expect(field).toMatch(/focus-visible:border-border-medium/);
+ expect(field).toMatch(/focus-visible:ring-2/);
+ expect(field).toMatch(/focus-visible:ring-text-primary/);
+
+ const composers: Array<[string, RegExp]> = [
+ ['Input.tsx', /\bfieldControl\b/],
+ ['Textarea.tsx', /\bfieldBase\b/],
+ ['Dropdown.tsx', /\bfieldControl\b/],
+ ['ControlCombobox.tsx', /\bfieldControl\b/],
+ ];
+ composers.forEach(([component, token]) => {
const source = readFileSync(join(__dirname, '..', 'components', component), 'utf8');
-
- expect(source).toMatch(/focus-visible:border-border-medium/);
- expect(source).toMatch(/focus-visible:ring-2/);
- expect(source).toMatch(/focus-visible:ring-text-primary/);
+ expect(source).toMatch(token);
});
const secretInput = readFileSync(
diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.ts
index e20e9729a6..80acb4990a 100644
--- a/packages/data-provider/src/api-endpoints.ts
+++ b/packages/data-provider/src/api-endpoints.ts
@@ -392,6 +392,11 @@ export const getCategories = () => `${BASE_URL}/api/categories`;
export const getAllPromptGroups = () => `${prompts()}/all`;
+/* Scheduled chats */
+export const schedules = () => `${BASE_URL}/api/schedules`;
+export const schedule = (id: string) => `${schedules()}/${encodeURIComponent(id)}`;
+export const runSchedule = (id: string) => `${schedule(id)}/run`;
+
/* Skills */
export const skills = () => `${BASE_URL}/api/skills`;
export const importSkill = () => `${skills()}/import`;
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index bc97dcd1a1..9384f8ff1c 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1563,6 +1563,19 @@ export const interfaceSchema = z
}),
])
.optional(),
+ schedules: z
+ .union([
+ z.boolean(),
+ z.object({
+ use: z.boolean().optional(),
+ create: z.boolean().optional(),
+ maxPerUser: z.number().int().min(0).optional(),
+ minIntervalMinutes: z.number().int().min(1).optional(),
+ autoDisableAfterFailures: z.number().int().min(1).optional(),
+ fireConcurrency: z.number().int().min(1).optional(),
+ }),
+ ])
+ .optional(),
})
.default({
modelSelect: true,
@@ -1625,6 +1638,11 @@ export const interfaceSchema = z
public: true,
snapshotFiles: true,
},
+ // `schedules` is deliberately ABSENT from this default. It is experimental and
+ // default-off in v1, and zod applies this whole object when `interface` is omitted
+ // from librechat.yaml — including it would silently enable the feature (and permit
+ // billable scheduled runs) on every deployment that never opted in. The PERMISSION
+ // defaults live in updateInterfacePermissions, which is a separate concern.
});
export type TInterfaceConfig = z.infer<typeof interfaceSchema>;
diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts
index 3b94c80b6b..89383f070b 100644
--- a/packages/data-provider/src/data-service.ts
+++ b/packages/data-provider/src/data-service.ts
@@ -6,6 +6,7 @@ import * as permissions from './accessPermissions';
import * as endpoints from './api-endpoints';
import { uploadEventStream } from './upload';
import * as mcp from './types/mcpServers';
+import * as sch from './types/schedules';
import * as a from './types/assistants';
import * as m from './types/mutations';
import * as ag from './types/agents';
@@ -1078,6 +1079,30 @@ export function listSkills(params?: sk.TSkillListRequest): Promise<sk.TSkillList
return request.get(endpoints.listSkillsWithFilters(params ?? {}));
}
+export function getSchedules(): Promise<sch.TSchedulesResponse> {
+ return request.get(endpoints.schedules());
+}
+
+export function getSchedule(id: string): Promise<sch.TSchedule> {
+ return request.get(endpoints.schedule(id));
+}
+
+export function createSchedule(payload: sch.TCreateSchedule): Promise<sch.TSchedule> {
+ return request.post(endpoints.schedules(), payload);
+}
+
+export function updateSchedule(id: string, payload: sch.TUpdateSchedule): Promise<sch.TSchedule> {
+ return request.patch(endpoints.schedule(id), payload);
+}
+
+export function deleteSchedule(id: string): Promise<{ id: string }> {
+ return request.delete(endpoints.schedule(id));
+}
+
+export function runScheduleNow(id: string): Promise<sch.TScheduleRunNowResponse> {
+ return request.post(endpoints.runSchedule(id), {});
+}
+
export function getSkill(id: string): Promise<sk.TSkill> {
return request.get(endpoints.getSkill(id));
}
diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts
index 8c98967cf3..dbe8b51764 100644
--- a/packages/data-provider/src/index.ts
+++ b/packages/data-provider/src/index.ts
@@ -28,6 +28,7 @@ export * from './types/files';
export * from './types/mcpServers';
export * from './types/mutations';
export * from './types/queries';
+export * from './types/schedules';
export * from './types/skills';
export * from './types/runs';
export * from './types/web';
diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.ts
index 7a485996ef..8f7b8fa2e3 100644
--- a/packages/data-provider/src/keys.ts
+++ b/packages/data-provider/src/keys.ts
@@ -90,6 +90,9 @@ export enum QueryKeys {
skillStates = 'skillStates',
/* General user favorites */
favorites = 'favorites',
+ /* Scheduled chats */
+ schedules = 'schedules',
+ schedule = 'schedule',
}
// Dynamic query keys that require parameters
@@ -138,4 +141,8 @@ export enum MutationKeys {
updateSkillNodeContent = 'updateSkillNodeContent',
convoPin = 'convoPin',
archiveAllConversations = 'archiveAllConversations',
+ createSchedule = 'createSchedule',
+ updateSchedule = 'updateSchedule',
+ deleteSchedule = 'deleteSchedule',
+ runSchedule = 'runSchedule',
}
diff --git a/packages/data-provider/src/permissions.ts b/packages/data-provider/src/permissions.ts
index 08f2c7953d..deeedd57f4 100644
--- a/packages/data-provider/src/permissions.ts
+++ b/packages/data-provider/src/permissions.ts
@@ -68,6 +68,10 @@ export enum PermissionTypes {
* Type for Shared Link Permissions
*/
SHARED_LINKS = 'SHARED_LINKS',
+ /**
+ * Type for Scheduled Chats Permissions
+ */
+ SCHEDULES = 'SCHEDULES',
}
/**
@@ -92,11 +96,21 @@ export const PERMISSION_TYPE_INTERFACE_FIELDS: Record<PermissionTypes, string> =
[PermissionTypes.REMOTE_AGENTS]: 'remoteAgents',
[PermissionTypes.SKILLS]: 'skills',
[PermissionTypes.SHARED_LINKS]: 'sharedLinks',
+ [PermissionTypes.SCHEDULES]: 'schedules',
};
/** Set of interface config field names that correspond to role permissions. */
export const INTERFACE_PERMISSION_FIELDS = new Set(Object.values(PERMISSION_TYPE_INTERFACE_FIELDS));
+/**
+ * Interface fields that seed a permission (use/create) BUT also carry runtime
+ * config that must survive DB/tenant/user overrides. For these, override
+ * sanitizers strip only the permission sub-keys (use/create) from the object
+ * form and PRESERVE the boolean form — for `schedules`, `interface.schedules: false`
+ * is the runtime feature disable that `getLimits` reads, not a permission toggle.
+ */
+export const RUNTIME_CONFIG_INTERFACE_FIELDS = new Set<string>(['schedules']);
+
/**
* YAML sub-keys within composite interface permission fields that map to permission bits.
* When an interface permission field is an object, only these sub-keys are stripped from
@@ -245,6 +259,12 @@ export const skillPermissionsSchema = z.object({
});
export type TSkillPermissions = z.infer<typeof skillPermissionsSchema>;
+export const schedulesPermissionsSchema = z.object({
+ [Permissions.USE]: z.boolean().default(true),
+ [Permissions.CREATE]: z.boolean().default(true),
+});
+export type TSchedulesPermissions = z.infer<typeof schedulesPermissionsSchema>;
+
export const sharedLinksPermissionsSchema = z.object({
[Permissions.CREATE]: z.boolean().default(true),
[Permissions.SHARE]: z.boolean().default(true),
@@ -270,4 +290,5 @@ export const permissionsSchema = z.object({
[PermissionTypes.REMOTE_AGENTS]: remoteAgentsPermissionsSchema,
[PermissionTypes.SKILLS]: skillPermissionsSchema,
[PermissionTypes.SHARED_LINKS]: sharedLinksPermissionsSchema,
+ [PermissionTypes.SCHEDULES]: schedulesPermissionsSchema,
});
diff --git a/packages/data-provider/src/roles.spec.ts b/packages/data-provider/src/roles.spec.ts
index 3cc1740e47..f3b1068d0c 100644
--- a/packages/data-provider/src/roles.spec.ts
+++ b/packages/data-provider/src/roles.spec.ts
@@ -90,7 +90,8 @@ describe('roleDefaults', () => {
permType === PermissionTypes.PROMPTS ||
permType === PermissionTypes.AGENTS ||
permType === PermissionTypes.SKILLS ||
- permType === PermissionTypes.SHARED_LINKS;
+ permType === PermissionTypes.SHARED_LINKS ||
+ permType === PermissionTypes.SCHEDULES;
expect({
permType,
diff --git a/packages/data-provider/src/roles.ts b/packages/data-provider/src/roles.ts
index 42ce28ce1f..a4eda4eb7f 100644
--- a/packages/data-provider/src/roles.ts
+++ b/packages/data-provider/src/roles.ts
@@ -13,6 +13,7 @@ import {
fileSearchPermissionsSchema,
multiConvoPermissionsSchema,
mcpServersPermissionsSchema,
+ schedulesPermissionsSchema,
sharedLinksPermissionsSchema,
peoplePickerPermissionsSchema,
remoteAgentsPermissionsSchema,
@@ -117,6 +118,10 @@ const defaultRolesSchema = z.object({
[Permissions.SHARE]: z.boolean().default(true),
[Permissions.SHARE_PUBLIC]: z.boolean().default(true),
}),
+ [PermissionTypes.SCHEDULES]: schedulesPermissionsSchema.extend({
+ [Permissions.USE]: z.boolean().default(true),
+ [Permissions.CREATE]: z.boolean().default(true),
+ }),
}),
}),
[SystemRoles.USER]: roleSchema.extend({
@@ -211,6 +216,10 @@ export const roleDefaults = defaultRolesSchema.parse({
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
},
},
[SystemRoles.USER]: {
@@ -268,6 +277,10 @@ export const roleDefaults = defaultRolesSchema.parse({
[Permissions.SHARE]: true,
[Permissions.SHARE_PUBLIC]: true,
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: true,
+ [Permissions.CREATE]: true,
+ },
},
},
});
diff --git a/packages/data-provider/src/types/schedules.ts b/packages/data-provider/src/types/schedules.ts
new file mode 100644
index 0000000000..9e50b6ac83
--- /dev/null
+++ b/packages/data-provider/src/types/schedules.ts
@@ -0,0 +1,127 @@
+import { z } from 'zod';
+
+export const scheduleFrequencies = ['hourly', 'daily', 'weekdays', 'weekly'] as const;
+export type ScheduleFrequency = (typeof scheduleFrequencies)[number];
+
+export const scheduleTargets = ['new'] as const;
+export type ScheduleTarget = (typeof scheduleTargets)[number];
+
+export type ScheduleDisabledReason =
+ | 'too_many_failures'
+ | 'agent_deleted'
+ | 'invalid_schedule'
+ | 'permission_revoked'
+ | 'insufficient_balance';
+
+export type ScheduleRunStatus =
+ | 'started'
+ | 'requires_action'
+ | 'success'
+ | 'error'
+ | 'interrupted'
+ | 'skipped_overlap'
+ | 'skipped_balance';
+
+export const scheduleCadenceSchema = z.object({
+ frequency: z.enum(scheduleFrequencies),
+ hour: z.number().int().min(0).max(23),
+ minute: z.number().int().min(0).max(59),
+ daysOfWeek: z
+ .array(z.number().int().min(0).max(6))
+ .min(1)
+ .max(7)
+ .transform((days) => Array.from(new Set(days)))
+ .optional(),
+});
+export type TScheduleCadence = z.infer<typeof scheduleCadenceSchema>;
+
+export const createSchedulePayloadSchema = z.object({
+ name: z.string().trim().min(1).max(256),
+ prompt: z.string().trim().min(1).max(32000),
+ agent_id: z.string().trim().min(1),
+ cadence: scheduleCadenceSchema,
+ timezone: z.string().min(1),
+ target: z.enum(scheduleTargets).default('new'),
+ file_ids: z
+ .array(z.string())
+ .max(10)
+ .transform((ids) => Array.from(new Set(ids)))
+ .optional(),
+ enabled: z.boolean().default(true),
+ /**
+ * Client-generated key making creation idempotent across retries. Creation commits
+ * the row and arms it in two writes, so a failure between them leaves the client
+ * unable to tell whether anything persisted; retrying blind can produce two recurring
+ * schedules. A retry carrying the same key resolves to the original row instead.
+ * REQUIRED: an optional key preserves the keyless duplicate path for any client
+ * that omits it, which is exactly the failure the key exists to close.
+ */
+ clientRequestId: z.string().trim().min(1).max(128),
+});
+export type TCreateSchedule = z.infer<typeof createSchedulePayloadSchema>;
+
+/** Idempotency is a property of the CREATE attempt, not of the schedule's config. */
+export const updateSchedulePayloadSchema = createSchedulePayloadSchema
+ .omit({ clientRequestId: true })
+ .partial()
+ .extend({
+ /**
+ * The configRevision the client's edit was computed from (captured when the
+ * dialog opened). The server fences the update on it, so a concurrent edit
+ * from another tab answers 409 instead of being silently overwritten by a
+ * payload rebuilt from a stale snapshot (cadence is sent whole, so the
+ * server-side fresh-read fence alone cannot detect this).
+ */
+ expectedConfigRevision: z.number().int().min(0).optional(),
+ });
+export type TUpdateSchedule = z.infer<typeof updateSchedulePayloadSchema>;
+
+export type TScheduleLastRun = {
+ conversationId?: string;
+ status: ScheduleRunStatus;
+ error?: string;
+ firedAt: string;
+};
+
+export type TSchedule = {
+ id: string;
+ user: string;
+ name: string;
+ prompt: string;
+ agent_id: string;
+ cadence: TScheduleCadence;
+ timezone: string;
+ target: ScheduleTarget;
+ file_ids?: string[];
+ enabled: boolean;
+ disabledReason?: ScheduleDisabledReason;
+ nextRunAt?: string;
+ lastRun?: TScheduleLastRun;
+ runCount: number;
+ failureCount: number;
+ configRevision?: number;
+ createdAt: string;
+ updatedAt: string;
+};
+
+export type TScheduleRun = {
+ scheduleId: string;
+ scheduledFor: string;
+ firedAt?: string;
+ conversationId?: string;
+ status: ScheduleRunStatus;
+ error?: string;
+ droppedFileIds?: string[];
+ durationMs?: number;
+};
+
+export type TSchedulesResponse = {
+ schedules: TSchedule[];
+ limits: { maxPerUser: number };
+};
+
+export type TScheduleRunNowResponse = {
+ scheduleId: string;
+ conversationId: string;
+ status: 'started';
+};
diff --git a/packages/data-schemas/jest.config.mjs b/packages/data-schemas/jest.config.mjs
index 739c2eecf9..de247fbc11 100644
--- a/packages/data-schemas/jest.config.mjs
+++ b/packages/data-schemas/jest.config.mjs
@@ -19,6 +19,9 @@ export default {
// lines: 57,
// },
// },
+ // Download the in-memory MongoDB binary once, before workers fork: on a cold
+ // cache the parallel downloads race their final rename and fail whole suites.
+ globalSetup: '<rootDir>/jest.globalSetup.mjs',
maxWorkers: '50%',
restoreMocks: true,
testTimeout: 15000,
diff --git a/packages/data-schemas/jest.globalSetup.mjs b/packages/data-schemas/jest.globalSetup.mjs
new file mode 100644
index 0000000000..1c892c4129
--- /dev/null
+++ b/packages/data-schemas/jest.globalSetup.mjs
@@ -0,0 +1,27 @@
+import { MongoBinary } from 'mongodb-memory-server-core';
+
+/**
+ * Download the in-memory MongoDB binary ONCE, before jest forks its workers.
+ *
+ * Dozens of suites here spin up `MongoMemoryServer`, and on a cold binary cache
+ * every worker starts its own download of the same archive. They then race on the
+ * final `rename('<file>.tgz.downloading', '<file>.tgz')`, and whichever workers
+ * lose that race fail their whole suite with
+ * `ENOENT ... rename '.../mongodb-linux-...tgz.downloading'` — a beforeAll timeout
+ * plus an unrelated-looking error, most visibly on CI where the cache starts empty.
+ *
+ * Pre-warming serially makes the download happen exactly once; every worker then
+ * finds the extracted binary already present. Failures are swallowed: if the
+ * download cannot happen here it will be retried by the first suite that needs it,
+ * and a pre-warm problem must not fail the run on its own.
+ */
+export default async function globalSetup() {
+ try {
+ await MongoBinary.getPath({});
+ } catch (error) {
+ console.warn(
+ '[data-schemas jest] Could not pre-warm the mongodb-memory-server binary:',
+ error instanceof Error ? error.message : error,
+ );
+ }
+}
diff --git a/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts
index 9f73ef3bcb..33ea021d07 100644
--- a/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts
+++ b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts
@@ -20,6 +20,7 @@ import {
import { decrementTagCounts } from '~/methods/conversationTag';
import { tenantStorage } from '~/config/tenantContext';
import { supportsTransactions } from '~/utils/transactions';
+import { createScheduleMethods } from '~/methods/schedule';
import { createUserMethods } from '~/methods/user';
import { createFileMethods } from '~/methods/file';
import { createModels } from '~/models';
@@ -353,4 +354,80 @@ describeLive('Amazon DocumentDB live compatibility', () => {
expect(typeof disabled).toBe('boolean');
});
});
+
+ /**
+ * The scheduler's rewritten write shapes (classic operators, worker clock — the
+ * DocumentDB-portable replacements for its original pipeline/$$NOW CAS forms),
+ * exercised through the real methods against the live engine.
+ */
+ describe('scheduler write shapes', () => {
+ const scheduleMethods = () => createScheduleMethods(mongoose);
+ const scheduleData = (overrides: Record<string, unknown> = {}) => ({
+ id: `sched_compat_${runId}_${randomUUID().slice(0, 8)}`,
+ user: new mongoose.Types.ObjectId(),
+ name: `compat ${runId}`,
+ prompt: 'compat probe',
+ agent_id: 'agent_compat',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ nextRunAt: new Date(Date.now() - 60_000),
+ ...overrides,
+ });
+
+ afterAll(async () => {
+ await mongoose.models.Schedule.deleteMany({ name: `compat ${runId}` });
+ await mongoose.models.ScheduleRun.deleteMany({ scheduleId: { $regex: runId } });
+ });
+
+ it('claims a due schedule via the classic-operator lease CAS', async () => {
+ const methods = scheduleMethods();
+ const schedule = await methods.createSchedule(scheduleData() as never);
+ const claimed = await methods.claimDueSchedule({
+ instanceId: `compat-${runId}`,
+ leaseMs: 60_000,
+ });
+ expect(claimed?.leaseUntil).toBeInstanceOf(Date);
+ // Held lease: a second claim must not steal it.
+ const contender = await methods.claimDueSchedule({
+ instanceId: `compat2-${runId}`,
+ leaseMs: 60_000,
+ });
+ expect(contender?.id).not.toBe(schedule.id);
+ capabilities['scheduler lease CAS'] = 'supported';
+ });
+
+ it('stamps and resolves an abort request with guarded classic updates', async () => {
+ const methods = scheduleMethods();
+ const schedule = await methods.createSchedule(scheduleData() as never);
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+ await mongoose.models.ScheduleRun.create({
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor,
+ status: 'started',
+ firedAt: new Date(),
+ });
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);
+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);
+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(state?.abortSource).toBe('stop');
+ expect(state?.abortPersistedAt).toBeInstanceOf(Date);
+ capabilities['scheduler abort stamps'] = 'supported';
+ });
+
+ it('arms only an unarmed row through the shared arming CAS', async () => {
+ const methods = scheduleMethods();
+ const schedule = await methods.createSchedule(
+ scheduleData({ nextRunAt: undefined }) as never,
+ );
+ const armAt = new Date(Date.now() + HOUR);
+ expect(await methods.armSchedule(schedule.id, armAt, 0)).toBe(true);
+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 2 * HOUR), 0)).toBe(
+ false,
+ );
+ capabilities['scheduler arming CAS'] = 'supported';
+ });
+ });
});
diff --git a/packages/data-schemas/src/app/interface.ts b/packages/data-schemas/src/app/interface.ts
index 143e62bc5f..ec26054139 100644
--- a/packages/data-schemas/src/app/interface.ts
+++ b/packages/data-schemas/src/app/interface.ts
@@ -64,6 +64,7 @@ export async function loadDefaultInterface({
remoteAgents: interfaceConfig?.remoteAgents,
skills: interfaceConfig?.skills,
sharedLinks: interfaceConfig?.sharedLinks,
+ schedules: interfaceConfig?.schedules,
});
return loadedInterface;
diff --git a/packages/data-schemas/src/app/resolution.spec.ts b/packages/data-schemas/src/app/resolution.spec.ts
index de47138f39..41d0f7aec2 100644
--- a/packages/data-schemas/src/app/resolution.spec.ts
+++ b/packages/data-schemas/src/app/resolution.spec.ts
@@ -77,6 +77,121 @@ describe('mergeConfigOverrides', () => {
expect(iface.parameters).toBe(true);
});
+ it('folds a boolean schedules override onto inherited limits (does not collapse them)', () => {
+ const base = {
+ interfaceConfig: {
+ schedules: { use: true, maxPerUser: 50, minIntervalMinutes: 5 },
+ },
+ } as unknown as AppConfig;
+ // Enabling the feature for a role via the natural boolean toggle must PRESERVE
+ // the deployment's configured limits rather than reverting them to defaults.
+ const enabled = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: true } }, 10),
+ ]) as unknown as Record<string, unknown>;
+ const iface = enabled.interfaceConfig as Record<string, Record<string, unknown>>;
+ expect(iface.schedules).toEqual({ use: true, maxPerUser: 50, minIntervalMinutes: 5 });
+ // A boolean-false override disables while still preserving the limit values.
+ const disabled = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: false } }, 10),
+ ]) as unknown as Record<string, unknown>;
+ const dIface = disabled.interfaceConfig as Record<string, Record<string, unknown>>;
+ expect(dIface.schedules).toEqual({ use: false, maxPerUser: 50, minIntervalMinutes: 5 });
+ });
+
+ it('does not let a boolean override re-enable a globally-disabled feature', () => {
+ // Both values are booleans, so neither fold above applies and the plain fallback
+ // used to replace the base `false`. The service reads the BASE value and keeps
+ // refusing every write, so the client would render a panel whose create, edit and
+ // run actions all fail. A global stop may be narrowed, never widened.
+ const base = { interfaceConfig: { schedules: false } } as unknown as AppConfig;
+ const merged = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: true } }, 10),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(merged.interfaceConfig.schedules).toBe(false);
+
+ // An enabled base still honours a boolean override in both directions.
+ const enabledBase = { interfaceConfig: { schedules: true } } as unknown as AppConfig;
+ expect(
+ (
+ mergeConfigOverrides(enabledBase, [
+ fakeConfig({ interface: { schedules: false } }, 10),
+ ]) as unknown as Record<string, Record<string, unknown>>
+ ).interfaceConfig.schedules,
+ ).toBe(false);
+ });
+
+ /**
+ * Overrides are folded in priority order into an ACCUMULATED value, so a stop applied
+ * INSIDE the merge compares against whatever the previous override left — which let a
+ * low-priority `false` outrank a high-priority `true`. The base stop is the only
+ * non-negotiable one; overrides still order normally among themselves.
+ */
+ it('lets a higher-priority override win over a lower-priority one', () => {
+ const base = { interfaceConfig: { schedules: true } } as unknown as AppConfig;
+ const merged = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: false } }, 10),
+ fakeConfig({ interface: { schedules: true } }, 20),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(merged.interfaceConfig.schedules).toBe(true);
+
+ // And the reverse ordering still disables.
+ const disabled = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: true } }, 10),
+ fakeConfig({ interface: { schedules: false } }, 20),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(disabled.interfaceConfig.schedules).toBe(false);
+ });
+
+ it('keeps the base stop non-negotiable even for the highest-priority override', () => {
+ const base = { interfaceConfig: { schedules: false } } as unknown as AppConfig;
+ const merged = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: true } }, 10),
+ fakeConfig({ interface: { schedules: true } }, 99),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(merged.interfaceConfig.schedules).toBe(false);
+ });
+
+ it('applies the base stop through the object form too', () => {
+ const base = {
+ interfaceConfig: { schedules: { use: false, maxPerUser: 5 } },
+ } as unknown as AppConfig;
+ const merged = mergeConfigOverrides(base, [
+ fakeConfig({ interface: { schedules: { maxPerUser: 50 } } }, 10),
+ ]) as unknown as Record<string, Record<string, Record<string, unknown>>>;
+ // The limit tuning lands, but the stop survives it.
+ expect(merged.interfaceConfig.schedules).toMatchObject({ use: false, maxPerUser: 50 });
+ });
+
+ it('folds an object schedules override onto a boolean base, inheriting the enable state', () => {
+ // Enabled base, object override that only TUNES a limit (no `use`): the enable
+ // state must be inherited from the boolean base, not silently dropped.
+ const enabledBase = {
+ interfaceConfig: { schedules: true },
+ } as unknown as AppConfig;
+ const tuned = mergeConfigOverrides(enabledBase, [
+ fakeConfig({ interface: { schedules: { maxPerUser: 3 } } }, 10),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(tuned.interfaceConfig.schedules).toEqual({ use: true, maxPerUser: 3 });
+
+ // Disabled base, same object override: tuning a limit must NOT re-enable the
+ // globally-disabled feature.
+ const disabledBase = {
+ interfaceConfig: { schedules: false },
+ } as unknown as AppConfig;
+ const stillOff = mergeConfigOverrides(disabledBase, [
+ fakeConfig({ interface: { schedules: { maxPerUser: 3 } } }, 10),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(stillOff.interfaceConfig.schedules).toEqual({ use: false, maxPerUser: 3 });
+
+ // A config override cannot flip the enable state: `use` is a permission sub-key,
+ // stripped from interface overrides before merge (enable is permission-managed),
+ // so an override attempting `use: true` on a disabled base stays disabled.
+ const cannotReEnable = mergeConfigOverrides(disabledBase, [
+ fakeConfig({ interface: { schedules: { use: true, maxPerUser: 3 } } }, 10),
+ ]) as unknown as Record<string, Record<string, unknown>>;
+ expect(cannotReEnable.interfaceConfig.schedules).toEqual({ use: false, maxPerUser: 3 });
+ });
+
it('sorts by priority — higher priority wins', () => {
const configs = [
fakeConfig({ registration: { enabled: false } }, 100),
diff --git a/packages/data-schemas/src/app/resolution.ts b/packages/data-schemas/src/app/resolution.ts
index 1f79163df9..cdc77cf328 100644
--- a/packages/data-schemas/src/app/resolution.ts
+++ b/packages/data-schemas/src/app/resolution.ts
@@ -2,6 +2,7 @@ import {
BASE_PRINCIPAL_CONFIG_SECTIONS,
BASE_ONLY_CONFIG_SECTIONS,
INTERFACE_PERMISSION_FIELDS,
+ RUNTIME_CONFIG_INTERFACE_FIELDS,
PERMISSION_SUB_KEYS,
isProcessMCPServerConfig,
} from 'librechat-data-provider';
@@ -201,6 +202,30 @@ function deepMerge<T extends AnyObject>(target: T, source: AnyObject, depth = 0,
depth,
currentPath,
);
+ } else if (
+ typeof sourceVal === 'boolean' &&
+ targetVal != null &&
+ typeof targetVal === 'object' &&
+ !Array.isArray(targetVal) &&
+ RUNTIME_CONFIG_INTERFACE_FIELDS.has(key)
+ ) {
+ // A runtime-config interface field (e.g. schedules) toggled by a boolean
+ // override is a runtime enable/disable, not a replacement of its config. Fold
+ // the boolean into the `use` flag so inherited object-form limits (maxPerUser,
+ // minIntervalMinutes, ...) survive instead of collapsing to global defaults.
+ result[key] = { ...(targetVal as AnyObject), use: sourceVal };
+ } else if (
+ typeof targetVal === 'boolean' &&
+ sourceVal != null &&
+ typeof sourceVal === 'object' &&
+ !Array.isArray(sourceVal) &&
+ RUNTIME_CONFIG_INTERFACE_FIELDS.has(key)
+ ) {
+ // Symmetric case: an OBJECT override (e.g. tuning maxPerUser) on top of a
+ // BOOLEAN base must inherit the base's enable state unless it sets `use`
+ // explicitly — otherwise setting a limit on a globally-disabled feature
+ // (`schedules: false`) would silently re-enable it.
+ result[key] = { use: targetVal, ...(sourceVal as AnyObject) };
} else {
result[key] = sourceVal;
}
@@ -313,8 +338,12 @@ export function mergeConfigOverrides(baseConfig: AppConfig, configs: IConfig[]):
if (Object.keys(uiOnly).length > 0) {
filtered[field] = uiOnly;
}
+ } else if (RUNTIME_CONFIG_INTERFACE_FIELDS.has(field)) {
+ // Dual-purpose field: the boolean form is a runtime disable, not a
+ // permission toggle, so preserve it (e.g. schedules: false).
+ filtered[field] = fieldVal;
}
- // boolean permission fields (e.g. runCode: false) are fully stripped
+ // other boolean permission fields (e.g. runCode: false) are fully stripped
}
if (Object.keys(filtered).length > 0) {
remapped[mappedKey] = filtered;
@@ -327,5 +356,59 @@ export function mergeConfigOverrides(baseConfig: AppConfig, configs: IConfig[]):
}
}
- return merged;
+ return preserveRuntimeStops(baseConfig, merged);
+}
+
+/** Whether a runtime-config interface field reads as OFF in either of its two shapes:
+ * boolean `false`, or the object form with `use: false`. Exported so runtime gates
+ * (e.g. the schedule engine's global stop) apply the same semantics as the merge. */
+export function isRuntimeDisabled(value: unknown): boolean {
+ if (value === false) {
+ return true;
+ }
+ return (
+ value != null &&
+ typeof value === 'object' &&
+ !Array.isArray(value) &&
+ (value as AnyObject).use === false
+ );
+}
+
+/**
+ * Re-applies a BASE-level runtime stop after all overrides are merged.
+ *
+ * A deployment that turns a runtime feature off is not making a preference an override
+ * can outrank: the service reads the base value and keeps refusing writes, so an
+ * override that only flips the client's view produces a panel whose every action fails.
+ *
+ * Applied here rather than inside the merge deliberately. The merge folds overrides in
+ * priority order into an ACCUMULATED value, so a guard there compares against whatever
+ * the previous override left — which let a LOW-priority `false` block a HIGH-priority
+ * `true` and broke highest-priority-wins. Overrides still order normally among
+ * themselves; only the base stop is non-negotiable.
+ */
+function preserveRuntimeStops<T extends AppConfig>(baseConfig: AppConfig, merged: T): T {
+ const baseInterface = (baseConfig as unknown as AnyObject).interfaceConfig as
+ | AnyObject
+ | undefined;
+ const mergedInterface = (merged as unknown as AnyObject).interfaceConfig as AnyObject | undefined;
+ if (baseInterface == null || mergedInterface == null) {
+ return merged;
+ }
+ let patched: AnyObject | undefined;
+ for (const key of RUNTIME_CONFIG_INTERFACE_FIELDS) {
+ if (!isRuntimeDisabled(baseInterface[key]) || isRuntimeDisabled(mergedInterface[key])) {
+ continue;
+ }
+ const current = mergedInterface[key];
+ patched ??= { ...mergedInterface };
+ patched[key] =
+ current != null && typeof current === 'object' && !Array.isArray(current)
+ ? { ...(current as AnyObject), use: false }
+ : false;
+ }
+ if (patched == null) {
+ return merged;
+ }
+ return { ...(merged as unknown as AnyObject), interfaceConfig: patched } as unknown as T;
}
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index d9f16f327b..1370343de3 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -98,6 +98,7 @@ import {
type UpdateSkillResult,
type ValidationIssue,
} from './skill';
+import { createScheduleMethods, type ScheduleMethods } from './schedule';
import {
createAgentTriggerDeliveryMethods,
AgentTriggerDeliveryConflictError,
@@ -196,6 +197,7 @@ export type AllMethods = UserMethods &
SkillMethods &
SkillSyncMethods &
AgentTriggerDeliveryMethods &
+ ScheduleMethods &
AgentMethods &
ConfigMethods &
MCPAuthorityMethods &
@@ -334,6 +336,7 @@ export function createMethods(
...skillMethods,
...createSkillSyncMethods(mongoose),
...createAgentTriggerDeliveryMethods(mongoose),
+ ...createScheduleMethods(mongoose),
/* Tier 5 */
...agentMethods,
/* Config */
@@ -395,6 +398,7 @@ export type {
UpsertSkillSyncCredentialInput,
SkillSyncMethods,
AgentTriggerDeliveryMethods,
+ ScheduleMethods,
AgentMethods,
ConfigMethods,
MCPAuthorityMethods,
diff --git a/packages/data-schemas/src/methods/schedule.methods.spec.ts b/packages/data-schemas/src/methods/schedule.methods.spec.ts
new file mode 100644
index 0000000000..057ffcf2d2
--- /dev/null
+++ b/packages/data-schemas/src/methods/schedule.methods.spec.ts
@@ -0,0 +1,2759 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type { Model } from 'mongoose';
+import type {
+ ISchedule,
+ IScheduleRun,
+ IScheduleDocument,
+ IScheduleRunDocument,
+} from '~/types/schedule';
+import type { ScheduleMethods } from './schedule';
+import { createScheduleMethods } from './schedule';
+import { createModels } from '../models';
+
+jest.mock('~/config/winston', () => ({
+ error: jest.fn(),
+ warn: jest.fn(),
+ info: jest.fn(),
+ debug: jest.fn(),
+}));
+
+let mongoServer: MongoMemoryServer;
+let Schedule: Model<IScheduleDocument>;
+let ScheduleRun: Model<IScheduleRunDocument>;
+let methods: ScheduleMethods;
+
+/** Standing up a real in-memory MongoDB plus index builds routinely outruns the
+ * 15s default hook timeout when the suite runs alongside the rest of the package
+ * (and on a cold binary cache), which failed all 115 tests on an unrelated-looking
+ * beforeAll timeout. The work itself takes ~2s idle; this is headroom, not a
+ * slow expectation. */
+const DB_SETUP_TIMEOUT_MS = 60_000;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+ createModels(mongoose);
+ Schedule = mongoose.models.Schedule as Model<IScheduleDocument>;
+ ScheduleRun = mongoose.models.ScheduleRun as Model<IScheduleRunDocument>;
+ await Schedule.init();
+ await ScheduleRun.init();
+ methods = createScheduleMethods(mongoose);
+}, DB_SETUP_TIMEOUT_MS);
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+}, DB_SETUP_TIMEOUT_MS);
+
+beforeEach(async () => {
+ await Schedule.deleteMany({});
+ await ScheduleRun.deleteMany({});
+});
+
+let scheduleCounter = 0;
+
+function scheduleData(overrides: Partial<ISchedule> = {}): Partial<ISchedule> {
+ scheduleCounter += 1;
+ return {
+ id: `sched_${scheduleCounter}`,
+ user: new mongoose.Types.ObjectId(),
+ name: 'Morning digest',
+ prompt: 'Summarize my inbox',
+ agent_id: 'agent_test',
+ cadence: { frequency: 'daily', hour: 8, minute: 0 },
+ timezone: 'America/New_York',
+ target: 'new',
+ enabled: true,
+ nextRunAt: new Date(Date.now() - 60_000),
+ ...overrides,
+ };
+}
+
+function runData(
+ schedule: Pick<ISchedule, 'id' | 'user'>,
+ overrides: Partial<IScheduleRun> = {},
+): Partial<IScheduleRun> {
+ return {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: new Date('2026-07-20T12:00:00Z'),
+ status: 'started',
+ firedAt: new Date(),
+ ...overrides,
+ };
+}
+
+async function getSchedule(id: string): Promise<ISchedule> {
+ const schedule = await methods.getScheduleById(id);
+ if (schedule == null) {
+ throw new Error(`schedule ${id} not found`);
+ }
+ return schedule;
+}
+
+async function getRun(scheduleId: string, scheduledFor: Date): Promise<IScheduleRun> {
+ const run = await ScheduleRun.findOne({ scheduleId, scheduledFor }).lean<IScheduleRun>();
+ if (run == null) {
+ throw new Error(`run ${scheduleId} @ ${scheduledFor.toISOString()} not found`);
+ }
+ return run;
+}
+
+describe('claimDueSchedule', () => {
+ it('grants exactly one winner across 8 concurrent claims', async () => {
+ await methods.createSchedule(scheduleData());
+ const results = await Promise.all(
+ Array.from({ length: 8 }, (_, i) =>
+ methods.claimDueSchedule({ instanceId: `inst-${i}`, leaseMs: 60_000 }),
+ ),
+ );
+ const winners = results.filter((result) => result != null);
+ expect(winners).toHaveLength(1);
+ expect(winners[0]?.leaseBy).toMatch(/^inst-\d$/);
+ expect(winners[0]?.leaseUntil).toBeInstanceOf(Date);
+ });
+
+ it('blocks a second claim while the lease is live', async () => {
+ const created = await methods.createSchedule(scheduleData());
+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ expect(first?.id).toBe(created.id);
+ const second = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });
+ expect(second).toBeNull();
+ });
+
+ it('allows re-claim after the lease expires past the skew margin', async () => {
+ const created = await methods.createSchedule(scheduleData());
+ // Takeover requires expiry to be at least LEASE_SKEW_MARGIN_MS in the past
+ // (worker-clock comparisons tolerate inter-replica skew), so lease a NEGATIVE
+ // duration deep enough to clear the margin instead of sleeping it out.
+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: -31_000 });
+ expect(first?.id).toBe(created.id);
+ const second = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });
+ expect(second?.id).toBe(created.id);
+ expect(second?.leaseBy).toBe('inst-b');
+ });
+
+ it('does not take over a lease expired less than the skew margin ago', async () => {
+ await methods.createSchedule(scheduleData());
+ // Expired 1s ago by this clock — within the margin, so a skew-behind holder may
+ // still legitimately consider it live.
+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: -1_000 });
+ expect(first).not.toBeNull();
+ const contender = await methods.claimDueSchedule({ instanceId: 'inst-b', leaseMs: 60_000 });
+ expect(contender).toBeNull();
+ });
+
+ it('claims due schedules in nextRunAt order', async () => {
+ const now = Date.now();
+ const later = await methods.createSchedule(scheduleData({ nextRunAt: new Date(now - 60_000) }));
+ const earlier = await methods.createSchedule(
+ scheduleData({ nextRunAt: new Date(now - 120_000) }),
+ );
+ const first = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ const second = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ const third = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ expect(first?.id).toBe(earlier.id);
+ expect(second?.id).toBe(later.id);
+ expect(third).toBeNull();
+ });
+
+ it('never claims disabled, future, or never-scheduled schedules', async () => {
+ const now = Date.now();
+ await methods.createSchedule(
+ scheduleData({ enabled: false, nextRunAt: new Date(now - 60_000) }),
+ );
+ await methods.createSchedule(scheduleData({ nextRunAt: new Date(now + 3_600_000) }));
+ await methods.createSchedule(scheduleData({ nextRunAt: undefined }));
+ const claimed = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ expect(claimed).toBeNull();
+ });
+});
+
+describe('insertScheduleRun idempotency', () => {
+ it('returns null on a duplicate {scheduleId, scheduledFor} claim', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+ const first = await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ const second = await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ expect(first?.status).toBe('started');
+ expect(second).toBeNull();
+ expect(await ScheduleRun.countDocuments({ scheduleId: schedule.id })).toBe(1);
+ });
+
+ it('inserts distinct occurrences for different scheduledFor values', async () => {
+ // Terminal status so the single-active partial index (one `started` per
+ // schedule) doesn't interfere — this asserts occurrence-level distinctness.
+ const schedule = await methods.createSchedule(scheduleData());
+ const first = await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor: new Date('2026-07-20T12:00:00Z'), status: 'success' }),
+ );
+ const second = await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z'), status: 'success' }),
+ );
+ expect(first).not.toBeNull();
+ expect(second).not.toBeNull();
+ expect(await ScheduleRun.countDocuments({ scheduleId: schedule.id })).toBe(2);
+ });
+});
+
+describe('recordSkippedRun duplicate-occurrence guard', () => {
+ it('does not overwrite lastRun/counters when the occurrence already fired', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const when = new Date('2026-07-20T12:00:00Z');
+ // The occurrence already ran to success (e.g. a claim retry after the POST was
+ // accepted but the lease advance failed).
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: when, status: 'success' }));
+ // A stale balance preflight must NOT relabel that occurrence as a balance skip.
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: when,
+ status: 'skipped_balance',
+ },
+ 5,
+ );
+ const after = await getSchedule(schedule.id);
+ expect(after.lastRun).toBeUndefined();
+ expect(after.balanceSkipCount).toBe(0);
+ expect(await getRun(schedule.id, when)).toMatchObject({ status: 'success' });
+ });
+});
+
+describe('settledAt retention marker', () => {
+ it('stamps settledAt on terminal outcomes but never on a pause', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const when = new Date('2026-07-20T12:00:00Z');
+ await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));
+
+ // The retention TTL index expires on `settledAt`, so a PAUSE must not carry it:
+ // an approval window longer than the retention period would otherwise let Mongo
+ // reap a still-live requires_action row out from under its retained job.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: when,
+ status: 'requires_action',
+ autoDisableAfterFailures: 5,
+ });
+ expect((await getRun(schedule.id, when)).settledAt).toBeUndefined();
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: when,
+ status: 'success',
+ autoDisableAfterFailures: 5,
+ });
+ expect((await getRun(schedule.id, when)).settledAt).toBeInstanceOf(Date);
+ });
+
+ it('stamps settledAt on skip rows inserted terminal', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const when = new Date('2026-07-20T12:00:00Z');
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: when,
+ status: 'skipped_overlap',
+ },
+ 5,
+ );
+ expect((await getRun(schedule.id, when)).settledAt).toBeInstanceOf(Date);
+ });
+});
+
+describe('reserveStartedRun (single-active overlap guard)', () => {
+ it('reserves the slot, then rejects a concurrent occurrence as overlap', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const first = await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor: new Date('2026-07-20T12:00:00Z') }),
+ );
+ const second = await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z') }),
+ );
+ expect('run' in first).toBe(true);
+ expect(second).toEqual({ conflict: 'overlap' });
+ // Once the first terminalizes it leaves the active index, so the next occurrence reserves.
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor: new Date('2026-07-20T12:00:00Z') },
+ { $set: { status: 'success' } },
+ );
+ const third = await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor: new Date('2026-07-21T12:00:00Z') }),
+ );
+ expect('run' in third).toBe(true);
+ });
+
+ it('reports a re-fired same occurrence as duplicate', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const when = new Date('2026-07-20T12:00:00Z');
+ await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor: when },
+ { $set: { status: 'success' } },
+ );
+ const again = await methods.reserveStartedRun(runData(schedule, { scheduledFor: when }));
+ // The status travels with the conflict: this exact case (settled row, occurrence
+ // never advanced past) is the one the caller must ADVANCE past rather than treat as
+ // owned by a live worker.
+ expect(again).toEqual({ conflict: 'duplicate', existingStatus: 'success' });
+ });
+});
+
+describe('markRunResumeClaimed (paused resume capacity)', () => {
+ const when = new Date('2026-07-20T12:00:00Z');
+
+ it('atomically promotes a paused run and claims a global capacity slot', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { scheduledFor: when, status: 'requires_action' }));
+
+ await expect(methods.markRunResumeClaimed(schedule.id, when, 0)).resolves.toEqual({
+ capacitySlot: 0,
+ });
+ await expect(getRun(schedule.id, when)).resolves.toMatchObject({
+ status: 'started',
+ capacitySlot: 0,
+ resumeClaimedAt: expect.any(Date),
+ });
+ });
+
+ it('reports global-slot and same-schedule overlap conflicts without changing the pause', async () => {
+ const first = await methods.createSchedule(scheduleData());
+ const second = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(first, { capacitySlot: 0 }));
+ await ScheduleRun.create(runData(second, { scheduledFor: when, status: 'requires_action' }));
+
+ await expect(methods.markRunResumeClaimed(second.id, when, 0)).resolves.toEqual({
+ conflict: 'slot-taken',
+ });
+
+ const later = new Date('2026-07-21T12:00:00Z');
+ await ScheduleRun.create(runData(first, { scheduledFor: later, status: 'requires_action' }));
+ await expect(methods.markRunResumeClaimed(first.id, later, 1)).resolves.toEqual({
+ conflict: 'overlap',
+ });
+ await expect(getRun(second.id, when)).resolves.toMatchObject({ status: 'requires_action' });
+ await expect(getRun(first.id, later)).resolves.toMatchObject({ status: 'requires_action' });
+ });
+
+ it('rolls back only the exact resume reservation', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { scheduledFor: when, status: 'requires_action' }));
+ await methods.markRunResumeClaimed(schedule.id, when, 2);
+
+ await expect(methods.releaseRunResumeClaim(schedule.id, when, 1)).resolves.toBe(false);
+ await expect(methods.releaseRunResumeClaim(schedule.id, when, 2)).resolves.toBe(true);
+ await expect(getRun(schedule.id, when)).resolves.toMatchObject({ status: 'requires_action' });
+ expect((await getRun(schedule.id, when)).capacitySlot).toBeUndefined();
+ });
+});
+
+describe('createScheduleWithSlot (atomic per-user cap)', () => {
+ it('allows exactly maxPerUser and reports limit under concurrency', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const results = await Promise.all(
+ Array.from({ length: 8 }, () => methods.createScheduleWithSlot(scheduleData({ user }), 3)),
+ );
+ const created = results.filter((r) => r !== 'limit');
+ expect(created).toHaveLength(3);
+ expect(results.filter((r) => r === 'limit')).toHaveLength(5);
+ // Distinct slots [0,3) were assigned — the partial unique index is the arbiter.
+ const slots = new Set(
+ (created as ISchedule[]).map((s) => s.slot).filter((s) => typeof s === 'number'),
+ );
+ expect(slots.size).toBe(3);
+ expect(await methods.countSchedulesByUser(user)).toBe(3);
+ });
+
+ it('frees a slot on delete so a new create can take it', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const a = (await methods.createScheduleWithSlot(scheduleData({ user }), 2)) as ISchedule;
+ await methods.createScheduleWithSlot(scheduleData({ user }), 2);
+ expect(await methods.createScheduleWithSlot(scheduleData({ user }), 2)).toBe('limit');
+ await methods.deleteScheduleById(a.id, user);
+ const c = await methods.createScheduleWithSlot(scheduleData({ user }), 2);
+ expect(c).not.toBe('limit');
+ expect(await methods.countSchedulesByUser(user)).toBe(2);
+ });
+});
+
+describe('recordRunOutcome', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('success finalizes the run, increments runCount, and resets failure state', async () => {
+ const schedule = await methods.createSchedule(
+ scheduleData({
+ failureCount: 2,
+ balanceSkipCount: 3,
+ disabledReason: 'insufficient_balance',
+ }),
+ );
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-1',
+ durationMs: 1234,
+ autoDisableAfterFailures: 3,
+ });
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('success');
+ expect(run.conversationId).toBe('convo-1');
+ expect(run.durationMs).toBe(1234);
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(1);
+ expect(updated.failureCount).toBe(0);
+ expect(updated.balanceSkipCount).toBe(0);
+ expect(updated.disabledReason).toBeUndefined();
+ expect(updated.enabled).toBe(true);
+ expect(updated.lastRun?.status).toBe('success');
+ expect(updated.lastRun?.conversationId).toBe('convo-1');
+ expect(updated.lastRun?.firedAt).toBeInstanceOf(Date);
+ });
+
+ it('error increments failureCount without disabling below the threshold', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ error: 'provider exploded',
+ autoDisableAfterFailures: 3,
+ });
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('error');
+ expect(run.error).toBe('provider exploded');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.failureCount).toBe(1);
+ expect(updated.runCount).toBe(0);
+ expect(updated.enabled).toBe(true);
+ expect(updated.disabledReason).toBeUndefined();
+ expect(updated.lastRun?.status).toBe('error');
+ expect(updated.lastRun?.error).toBe('provider exploded');
+ });
+
+ it('error at the threshold disables the schedule and releases the lease', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 2 }));
+ const claimed = await methods.claimDueSchedule({ instanceId: 'inst-a', leaseMs: 60_000 });
+ expect(claimed?.id).toBe(schedule.id);
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ error: 'provider exploded',
+ autoDisableAfterFailures: 3,
+ });
+ const updated = await getSchedule(schedule.id);
+ expect(updated.failureCount).toBe(3);
+ expect(updated.enabled).toBe(false);
+ expect(updated.disabledReason).toBe('too_many_failures');
+ expect(updated.leaseUntil).toBeUndefined();
+ expect(updated.leaseBy).toBeUndefined();
+ });
+
+ it('requires_action surfaces lastRun for the card without touching counters', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 1 }));
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-paused',
+ autoDisableAfterFailures: 3,
+ });
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('requires_action');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(0);
+ expect(updated.failureCount).toBe(1);
+ // lastRun now reflects the pause so the card can show "Needs approval".
+ expect(updated.lastRun?.status).toBe('requires_action');
+ expect(updated.lastRun?.conversationId).toBe('convo-paused');
+ });
+
+ it('clearConversationId erases the reserved id so replays cannot restore a dead link', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor, conversationId: 'reserved-but-never-created' }),
+ );
+
+ // Pre-start abort: an id was reserved but no conversation ever came to exist.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'interrupted',
+ clearConversationId: true,
+ autoDisableAfterFailures: 3,
+ });
+
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.conversationId).toBeUndefined();
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun?.conversationId).toBeUndefined();
+
+ // The crash-retry replay reads the ROW; with the id erased it has nothing
+ // to project, so the dead link cannot be restored.
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor },
+ { $set: { bookkept: false } },
+ );
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'interrupted',
+ conversationId: run.conversationId,
+ autoDisableAfterFailures: 3,
+ });
+ const replayed = await getSchedule(schedule.id);
+ expect(replayed.lastRun?.conversationId).toBeUndefined();
+ });
+
+ it('re-affirmed pauses keep the ORIGINAL fire time and do not churn updatedAt', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const originalFiredAt = new Date('2026-07-20T12:00:02Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, firedAt: originalFiredAt }));
+ const pause = () =>
+ methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-paused',
+ autoDisableAfterFailures: 3,
+ });
+
+ await pause();
+ const afterFirst = await getSchedule(schedule.id);
+ // Reconciliation re-affirms a long-lived pause every pass; a fresh stamp per
+ // pass walked the card's timestamp forward and reordered updated-time listings
+ // for as long as the approval sat waiting.
+ await pause();
+ const afterSecond = await getSchedule(schedule.id);
+
+ expect(afterSecond.lastRun?.firedAt?.toISOString()).toBe(originalFiredAt.toISOString());
+ expect(afterSecond.updatedAt?.toISOString()).toBe(afterFirst.updatedAt?.toISOString());
+ });
+
+ it('does not write a pause card when no active run matches (spoof guard)', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ // No run row for this occurrence: the card is written only after a matching active
+ // run is confirmed, so a spoofed scheduleId can never stamp a pause onto a schedule.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-spoof',
+ autoDisableAfterFailures: 3,
+ });
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun).toBeUndefined();
+ });
+
+ it('re-affirms the pause card on a retried requires_action (already-paused row)', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+ // The row is already `requires_action`; a crash-replay retry must still write the
+ // card (keyed on existence, not modification) so the pause never goes invisible.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-2',
+ autoDisableAfterFailures: 3,
+ });
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('requires_action');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun?.status).toBe('requires_action');
+ expect(updated.lastRun?.conversationId).toBe('convo-2');
+ });
+
+ it('does not pin the card to Needs approval when a resume terminalizes first', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ // The user answers instantly: the resumed generation settles the run BEFORE the
+ // pause write lands. The pause must lose, or the card shows "Needs approval" for a
+ // job that already finished (the run row is terminal, so nothing corrects it).
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('success');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun?.status).toBe('success');
+ });
+
+ it('does not let a delayed older outcome overwrite a newer run on the card', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const older = new Date('2026-07-26T10:00:00.000Z');
+ const newer = new Date('2026-07-26T11:00:00.000Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: older }));
+ // Pausing the older occurrence is what frees the single-active-`started` slot and
+ // lets a later occurrence run at all — the interleaving under test.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'requires_action',
+ conversationId: 'convo-older',
+ autoDisableAfterFailures: 3,
+ });
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: newer }));
+
+ // The newer run finishes while the older one is still parked at its approval.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: newer,
+ status: 'success',
+ conversationId: 'convo-newer',
+ autoDisableAfterFailures: 3,
+ });
+ // The older one settles afterwards (resumed pause / reconciler replay).
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'error',
+ error: 'stale',
+ conversationId: 'convo-older',
+ autoDisableAfterFailures: 3,
+ });
+
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun?.status).toBe('success');
+ expect(updated.lastRun?.conversationId).toBe('convo-newer');
+ // The streak counters are order-fenced too, not just the card. This assertion used
+ // to expect 1 on the reasoning that counters are order-insensitive — but a streak
+ // is not: letting the stale failure land rebuilds one the newer success had
+ // cleared, and at `autoDisableAfterFailures: 1` that alone disables the schedule.
+ expect(updated.failureCount).toBe(0);
+ // The row still settles; only the schedule-level streak ignores the late outcome.
+ expect(await getRun(schedule.id, older)).toMatchObject({ status: 'error' });
+ });
+});
+
+describe('recordSkippedRun', () => {
+ it('skipped_balance increments the counter and disables at the threshold', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: new Date('2026-07-20T12:00:00Z'),
+ status: 'skipped_balance',
+ },
+ 2,
+ );
+ let updated = await getSchedule(schedule.id);
+ expect(updated.balanceSkipCount).toBe(1);
+ expect(updated.enabled).toBe(true);
+ const firstRun = await getRun(schedule.id, new Date('2026-07-20T12:00:00Z'));
+ expect(firstRun.status).toBe('skipped_balance');
+ expect(firstRun.firedAt).toBeInstanceOf(Date);
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: new Date('2026-07-21T12:00:00Z'),
+ status: 'skipped_balance',
+ },
+ 2,
+ );
+ updated = await getSchedule(schedule.id);
+ expect(updated.balanceSkipCount).toBe(2);
+ expect(updated.enabled).toBe(false);
+ expect(updated.disabledReason).toBe('insufficient_balance');
+ });
+
+ it('is retryable: a duplicate skipped_balance occurrence does not double-count the streak', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+ const data = {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor,
+ status: 'skipped_balance' as const,
+ };
+ // First attempt inserts the row + increments the streak.
+ await methods.recordSkippedRun(data, 3);
+ // Retry of the SAME occurrence (a crash before the counter landed on the first
+ // try): the row already exists, but the bookkeeping must still run — exactly once.
+ await methods.recordSkippedRun(data, 3);
+ const updated = await getSchedule(schedule.id);
+ expect(updated.balanceSkipCount).toBe(1);
+ expect(updated.lastRun?.status).toBe('skipped_balance');
+ });
+
+ it('skipped_overlap records the run without counter changes', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+ await methods.recordSkippedRun(
+ { scheduleId: schedule.id, user: schedule.user, scheduledFor, status: 'skipped_overlap' },
+ 2,
+ );
+ const run = await getRun(schedule.id, scheduledFor);
+ expect(run.status).toBe('skipped_overlap');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.balanceSkipCount).toBe(0);
+ expect(updated.enabled).toBe(true);
+ // the skip surfaces on the card via lastRun
+ expect(updated.lastRun?.status).toBe('skipped_overlap');
+ });
+
+ it('a subsequent success resets balanceSkipCount', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: new Date('2026-07-20T12:00:00Z'),
+ status: 'skipped_balance',
+ },
+ 3,
+ );
+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(1);
+ const scheduledFor = new Date('2026-07-21T12:00:00Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-2',
+ autoDisableAfterFailures: 3,
+ });
+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);
+ });
+});
+
+describe('countActiveRuns', () => {
+ it('counts started runs across all schedules for the fire cap', async () => {
+ expect(await methods.countActiveRuns()).toBe(0);
+ const a = await methods.createSchedule(scheduleData());
+ const b = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(a, { scheduledFor: new Date('2026-07-20T12:00:00Z') }));
+ await methods.insertScheduleRun(runData(b, { scheduledFor: new Date('2026-07-20T13:00:00Z') }));
+ expect(await methods.countActiveRuns()).toBe(2);
+ // terminal + skipped runs don't count as active
+ await methods.recordRunOutcome({
+ scheduleId: a.id,
+ scheduledFor: new Date('2026-07-20T12:00:00Z'),
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+ expect(await methods.countActiveRuns()).toBe(1);
+ });
+});
+
+describe('recordRunOutcome — reconciled completions and no-match guard', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('finalizes a resumed run from requires_action with full bookkeeping', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, status: 'requires_action' }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-resumed',
+ autoDisableAfterFailures: 3,
+ });
+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('success');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(1);
+ expect(updated.lastRun?.status).toBe('success');
+ expect(updated.lastRun?.conversationId).toBe('convo-resumed');
+ });
+
+ it('interrupted records lastRun without touching counters', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ runCount: 4, failureCount: 1 }));
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'interrupted',
+ autoDisableAfterFailures: 3,
+ });
+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('interrupted');
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(4);
+ expect(updated.failureCount).toBe(1);
+ expect(updated.lastRun?.status).toBe('interrupted');
+ });
+
+ it('does not touch schedule bookkeeping when no matching run row exists', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ runCount: 7 }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'spoofed',
+ autoDisableAfterFailures: 3,
+ });
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(7);
+ expect(updated.lastRun).toBeUndefined();
+ });
+});
+
+describe('recordRunOutcome idempotency + crash-retry (bookkeeping)', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('counts an occurrence at most once across repeated invocations', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ const outcome = {
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success' as const,
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ };
+ await methods.recordRunOutcome(outcome);
+ // A second call (e.g. reconciler racing the inline finish) must not re-count.
+ await methods.recordRunOutcome(outcome);
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(1);
+ expect(
+ (updated as { countedFor?: Date[] }).countedFor?.map((d) => new Date(d).toISOString()),
+ ).toContain(scheduledFor.toISOString());
+ expect((await getRun(schedule.id, scheduledFor)).bookkept).toBe(true);
+ });
+
+ it('does not double-count a crashed later occurrence when an earlier one interleaves', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const t1 = new Date('2026-07-20T10:00:00Z');
+ const t2 = new Date('2026-07-20T11:00:00Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: t1 }));
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: t2 }));
+
+ // Later occurrence B (t2) finishes + counts, then crashes before bookkept:true.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: t2,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor: t2 },
+ { $set: { bookkept: false } },
+ );
+
+ // Earlier occurrence A (t1) then counts — a single scalar guard would move to
+ // t1 here, so replaying B would look uncounted again.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: t1,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+
+ // Reconciler replays the still-unbookkept B: the per-occurrence guard blocks a re-count.
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor: t2,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(2); // A + B, each counted exactly once
+ });
+
+ it('finalizeBookkeeping recovers a terminalized-but-uncounted run (crash between writes)', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ // Simulate a crash after the run row was terminalized but before bookkeeping.
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor },
+ { $set: { status: 'success', bookkept: false } },
+ );
+ expect((await getSchedule(schedule.id)).runCount).toBe(0);
+
+ const unbookkept = await methods.getUnbookkeptRuns(new Date(Date.now() + 1000), 100);
+ expect(unbookkept.map((r) => r.scheduleId)).toContain(schedule.id);
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+ const updated = await getSchedule(schedule.id);
+ expect(updated.runCount).toBe(1);
+ expect((await getRun(schedule.id, scheduledFor)).bookkept).toBe(true);
+ // No longer surfaced as needing bookkeeping.
+ expect(await methods.getUnbookkeptRuns(new Date(Date.now() + 1000), 100)).toHaveLength(0);
+ });
+
+ it('rotates the unbookkept window past rows whose replay keeps failing', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const early = new Date('2026-07-20T10:00:00Z');
+ const late = new Date('2026-07-20T11:00:00Z');
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor: early, firedAt: early, status: 'success' }),
+ );
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor: late, firedAt: late, status: 'success' }),
+ );
+ await ScheduleRun.updateMany({ scheduleId: schedule.id }, { $set: { bookkept: false } });
+
+ const cutoff = new Date(Date.now() + 1000);
+ const first = await methods.getUnbookkeptRuns(cutoff, 1);
+ expect(first).toHaveLength(1);
+ expect(new Date(first[0].firedAt!).toISOString()).toBe(early.toISOString());
+
+ // The engine stamps every row it attempted (success or failure); the next
+ // bounded read must then surface the row BEHIND it, not the same one again.
+ await methods.markRunsReconciled(first);
+ const second = await methods.getUnbookkeptRuns(cutoff, 1);
+ expect(new Date(second[0].firedAt!).toISOString()).toBe(late.toISOString());
+ });
+
+ it('replayed bookkeeping projects the ORIGINAL firedAt, not the replay time', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const originalFiredAt = new Date('2026-07-20T12:00:03Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, firedAt: originalFiredAt }));
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor },
+ { $set: { status: 'success', bookkept: false } },
+ );
+
+ // The reconciler replays minutes (or several retries) later — lastRun must
+ // still show when the run actually fired, not when recovery caught up.
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+
+ const updated = await getSchedule(schedule.id);
+ expect(updated.lastRun?.firedAt?.toISOString()).toBe(originalFiredAt.toISOString());
+ });
+});
+
+describe('bookkeeping policy is crash-retryable / streak-correct', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('re-applies auto-disable on a bookkeeping retry after the count already landed', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ failureCount: 2 }));
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ // First terminal error: increments to threshold (3) and disables.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ autoDisableAfterFailures: 3,
+ });
+ expect((await getSchedule(schedule.id)).enabled).toBe(false);
+ // Simulate a crash where the count landed (countedFor set) but the disable
+ // was lost: re-enable and replay via finalizeBookkeeping. The count must NOT
+ // double (guard), but the disable policy must re-apply.
+ await Schedule.updateOne({ id: schedule.id }, { $set: { enabled: true } });
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ autoDisableAfterFailures: 3,
+ });
+ const updated = await getSchedule(schedule.id);
+ expect(updated.failureCount).toBe(3); // not double-counted
+ expect(updated.enabled).toBe(false); // disable re-applied
+ });
+
+ it('resets the balance-skip streak on an intervening error, then interrupt, then overlap', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 4 }));
+ const outcomes: Array<[Date, 'error' | 'interrupted']> = [
+ [new Date('2026-07-20T01:00:00Z'), 'error'],
+ [new Date('2026-07-20T02:00:00Z'), 'interrupted'],
+ ];
+ for (const [when, status] of outcomes) {
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: when }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: when,
+ status,
+ autoDisableAfterFailures: 10,
+ });
+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);
+ }
+ // Also an overlap skip breaks the streak.
+ await Schedule.updateOne({ id: schedule.id }, { $set: { balanceSkipCount: 3 } });
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor: new Date('2026-07-20T03:00:00Z'),
+ status: 'skipped_overlap',
+ },
+ 5,
+ );
+ expect((await getSchedule(schedule.id)).balanceSkipCount).toBe(0);
+ });
+});
+
+describe('getRunsForReconciliation fairness', () => {
+ it('always includes started runs even behind a backlog of paused rows', async () => {
+ const s = await methods.createSchedule(scheduleData());
+ // Older paused rows + a newer started row; started must still be returned.
+ for (let i = 0; i < 3; i++) {
+ await methods.insertScheduleRun(
+ runData(s, {
+ scheduledFor: new Date(`2026-07-1${i}T12:00:00Z`),
+ status: 'requires_action',
+ firedAt: new Date('2020-01-01T00:00:00Z'),
+ }),
+ );
+ }
+ await methods.insertScheduleRun(
+ runData(s, {
+ scheduledFor: new Date('2026-07-25T12:00:00Z'),
+ status: 'started',
+ firedAt: new Date('2020-06-01T00:00:00Z'),
+ }),
+ );
+ const runs = await methods.getRunsForReconciliation(new Date('2026-01-01T00:00:00Z'), 2);
+ expect(runs.some((r) => r.status === 'started')).toBe(true);
+ });
+});
+
+describe('deleteSchedulesByUser', () => {
+ it('removes the user’s schedules and their runs', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const a = await methods.createSchedule(scheduleData({ user: userId }));
+ await methods.createSchedule(scheduleData({ user: userId }));
+ const other = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(a, { scheduledFor: new Date('2026-07-20T12:00:00Z') }));
+ await methods.insertScheduleRun(
+ runData(other, { scheduledFor: new Date('2026-07-20T12:00:00Z') }),
+ );
+
+ await methods.deleteSchedulesByUser(userId);
+
+ expect(await methods.getSchedulesByUser(userId)).toHaveLength(0);
+ expect(await getSchedule(other.id)).not.toBeNull();
+ // The other user's run survives; the deleted user's runs are gone.
+ expect(await ScheduleRun.countDocuments({ scheduleId: a.id })).toBe(0);
+ expect(await ScheduleRun.countDocuments({ scheduleId: other.id })).toBe(1);
+ });
+});
+
+describe('reversible account-deletion suspension', () => {
+ async function rawSchedule(id: string): Promise<ISchedule | null> {
+ return Schedule.findOne({ id }).lean<ISchedule>();
+ }
+
+ it('suspends under a token, fencing firing while snapshotting the prior state', async () => {
+ const armedAt = new Date('2030-01-01T00:00:00Z');
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));
+ const before = await getSchedule(schedule.id);
+
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ const after = await getSchedule(schedule.id);
+ expect(after.enabled).toBe(false);
+ expect(after.nextRunAt).toBeUndefined();
+ expect(after.claimToken).not.toBe(before.claimToken);
+ // Reversible, NOT destructive: never marked for erasure.
+ expect(after.deleting).not.toBe(true);
+ expect(after.deletionSuspension?.token).toBe('attempt-1');
+ expect(after.deletionSuspension?.enabled).toBe(true);
+ expect(after.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());
+ });
+
+ it('restores enabled + next-run from the snapshot and clears the suspension', async () => {
+ const armedAt = new Date('2030-01-01T00:00:00Z');
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');
+
+ const restored = await getSchedule(schedule.id);
+ expect(restored.enabled).toBe(true);
+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());
+ expect(restored.deletionSuspension).toBeUndefined();
+ });
+
+ it('restores a previously DISABLED schedule as disabled with no next run', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ enabled: false }));
+ await Schedule.updateOne({ id: schedule.id }, { $unset: { nextRunAt: 1 } });
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');
+
+ const restored = await getSchedule(schedule.id);
+ expect(restored.enabled).toBe(false);
+ expect(restored.nextRunAt).toBeUndefined();
+ expect(restored.deletionSuspension).toBeUndefined();
+ });
+
+ it('restore is fenced to the exact attempt token', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ // A different attempt's restore must not touch this suspension.
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-2');
+
+ const after = await getSchedule(schedule.id);
+ expect(after.enabled).toBe(false);
+ expect(after.deletionSuspension?.token).toBe('attempt-1');
+ });
+
+ it('cannot resurrect a schedule the owner independently deleted mid-suspension', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+ // The owner deletes this one schedule while the account deletion is in flight.
+ await methods.markScheduleDeleting(schedule.id, schedule.user);
+
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');
+
+ const after = await rawSchedule(schedule.id);
+ expect(after?.deleting).toBe(true);
+ expect(after?.enabled).toBe(false); // NOT re-enabled by restore
+ });
+
+ it('is idempotent per token — a retry does not overwrite the snapshot', async () => {
+ const armedAt = new Date('2030-01-01T00:00:00Z');
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+ // A retry of the SAME attempt must not re-snapshot the now-disabled state.
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ const after = await getSchedule(schedule.id);
+ expect(after.deletionSuspension?.enabled).toBe(true);
+ expect(after.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());
+ });
+
+ /**
+ * A crash between suspend and restore leaves an ABANDONED suspension. A later attempt
+ * must adopt that attempt's snapshot rather than re-snapshotting the row's current
+ * (already-suspended) state — otherwise the true pre-suspension state is lost and the
+ * schedule stays disabled forever once the second attempt restores.
+ */
+ it('a second attempt adopts the abandoned snapshot instead of capturing suspended state', async () => {
+ const armedAt = new Date('2030-01-01T00:00:00Z');
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ // attempt-1's process died without restoring; attempt-2 now suspends the same rows.
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-2');
+
+ const suspended = await getSchedule(schedule.id);
+ expect(suspended.deletionSuspension?.token).toBe('attempt-2');
+ // The ORIGINAL pre-suspension state, not the suspended state attempt-2 observed.
+ expect(suspended.deletionSuspension?.enabled).toBe(true);
+ expect(suspended.deletionSuspension?.nextRunAt?.getTime()).toBe(armedAt.getTime());
+
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-2');
+ const restored = await getSchedule(schedule.id);
+ expect(restored.enabled).toBe(true);
+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());
+ expect(restored.deletionSuspension).toBeUndefined();
+ });
+
+ /**
+ * The restore's callers cannot retry it later: a cancelled deletion restores while the
+ * user-deletion fence is still armed and then releases that fence, after which nothing
+ * re-drives this. A single transient write failure would therefore leave a LIVE account
+ * holding silently disabled, next-run-less schedules.
+ */
+ it('retries a transient write failure instead of stranding the rows disabled', async () => {
+ const armedAt = new Date('2030-01-01T00:00:00Z');
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: armedAt }));
+ await methods.suspendUserSchedulesForDeletion(schedule.user, 'attempt-1');
+
+ const bulkWrite = jest.spyOn(Schedule, 'bulkWrite');
+ bulkWrite.mockImplementationOnce(() => {
+ throw new Error('transient primary stepdown');
+ });
+
+ await methods.restoreUserSchedulesFromDeletion(schedule.user, 'attempt-1');
+ bulkWrite.mockRestore();
+
+ const restored = await getSchedule(schedule.id);
+ expect(restored.enabled).toBe(true);
+ expect(restored.nextRunAt?.getTime()).toBe(armedAt.getTime());
+ expect(restored.deletionSuspension).toBeUndefined();
+ });
+
+ /** Retrying is only safe because each attempt re-reads the rows STILL carrying the
+ * token: a partially-applied unordered write converges on exactly the stragglers, and
+ * an already-restored row is simply no longer in the set. */
+ it('converges on the stragglers of a partially applied restore', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const first = await methods.createSchedule(scheduleData({ user: userId }));
+ const second = await methods.createSchedule(scheduleData({ user: userId }));
+ await methods.suspendUserSchedulesForDeletion(userId, 'attempt-1');
+
+ // Exactly what a partially-applied unordered bulkWrite leaves behind: one row already
+ // restored and out of the token set, one still suspended.
+ await methods.restoreUserSchedulesFromDeletion(userId, 'attempt-1');
+ await Schedule.updateOne(
+ { id: second.id },
+ {
+ $set: {
+ enabled: false,
+ deletionSuspension: { token: 'attempt-1', enabled: true },
+ },
+ $unset: { nextRunAt: 1 },
+ },
+ );
+
+ await methods.restoreUserSchedulesFromDeletion(userId, 'attempt-1');
+
+ for (const id of [first.id, second.id]) {
+ const restored = await getSchedule(id);
+ expect(restored.enabled).toBe(true);
+ expect(restored.deletionSuspension).toBeUndefined();
+ }
+ });
+
+ it('a successful deletion removes suspended rows and their snapshots', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const schedule = await methods.createSchedule(scheduleData({ user: userId }));
+ await methods.suspendUserSchedulesForDeletion(userId, 'attempt-1');
+
+ await methods.deleteSchedulesByUser(userId);
+
+ expect(await rawSchedule(schedule.id)).toBeNull();
+ });
+});
+
+describe('acquireManualRunLease / releaseLease', () => {
+ it('serializes concurrent run-now attempts and can be released without advancing', async () => {
+ const schedule = await methods.createSchedule(
+ scheduleData({ nextRunAt: new Date('2026-08-01T12:00:00Z') }),
+ );
+ const results = await Promise.all(
+ Array.from({ length: 6 }, () =>
+ methods.acquireManualRunLease(schedule.id, schedule.user, 60_000),
+ ),
+ );
+ expect(results.filter(Boolean)).toHaveLength(1);
+
+ await methods.releaseLease(schedule.id);
+ const afterRelease = await getSchedule(schedule.id);
+ expect(afterRelease.leaseUntil).toBeUndefined();
+ expect(afterRelease.leaseBy).toBeUndefined();
+ // release must NOT reschedule the next automatic occurrence
+ expect(afterRelease.nextRunAt?.toISOString()).toBe('2026-08-01T12:00:00.000Z');
+
+ const reacquired = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);
+ // Returns the fresh claim token (truthy) rather than a boolean.
+ expect(reacquired).toBeTruthy();
+ });
+
+ it('returns the FRESH row (reflecting an edit) with a new claim token', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ name: 'original' }));
+ // An edit commits after a caller read the schedule but before the lease is taken.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+ const leased = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);
+ // The lease returns the post-image, so a manual fire uses the edited snapshot.
+ expect(leased?.name).toBe('edited');
+ expect(leased?.claimToken).toBeTruthy();
+ });
+
+ it('blocks against a held engine lease and rejects a non-owner', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const claimed = await methods.claimDueSchedule({ instanceId: 'engine-1', leaseMs: 60_000 });
+ expect(claimed?.id).toBe(schedule.id);
+ expect(await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000)).toBeNull();
+ expect(
+ await methods.acquireManualRunLease(schedule.id, new mongoose.Types.ObjectId(), 60_000),
+ ).toBeNull();
+ });
+
+ it('uses a unique per-lease holder so a stale run-now cannot strip a replacement lease', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const first = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);
+ expect(first?.leaseBy).toMatch(/^manual:/);
+ // Release and re-acquire (a fresh run-now): the replacement lease has a DIFFERENT holder.
+ await methods.releaseLease(schedule.id);
+ const second = await methods.acquireManualRunLease(schedule.id, schedule.user, 60_000);
+ expect(second?.leaseBy).toMatch(/^manual:/);
+ expect(second?.leaseBy).not.toBe(first?.leaseBy);
+ // A stalled fire from the FIRST lease releasing by its (old) holder must NOT clear the
+ // second lease — the unique holder makes releaseLeaseByHolder no-op there.
+ await methods.releaseLeaseByHolder(schedule.id, first!.leaseBy!);
+ const afterStale = await getSchedule(schedule.id);
+ expect(afterStale.leaseBy).toBe(second?.leaseBy);
+ expect(afterStale.leaseUntil).toBeDefined();
+ // Releasing by the CURRENT holder does clear it.
+ await methods.releaseLeaseByHolder(schedule.id, second!.leaseBy!);
+ const cleared = await getSchedule(schedule.id);
+ expect(cleared.leaseBy).toBeUndefined();
+ });
+});
+
+describe('scheduled resume lease fencing', () => {
+ it('atomically consumes the same live config generation and releases its lease', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const fence = await methods.acquireResumeLease(schedule.id, 0, true, 60_000);
+
+ expect(fence?.claimToken).toBeTruthy();
+ expect(fence?.leaseBy).toMatch(/^resume:/);
+ await expect(
+ methods.consumeResumeLease(schedule.id, fence!.claimToken!, fence!.leaseBy!, true, 0),
+ ).resolves.toBe(true);
+ const consumed = await getSchedule(schedule.id);
+ expect(consumed.leaseUntil).toBeUndefined();
+ expect(consumed.leaseBy).toBeUndefined();
+ });
+
+ it('rejects an owner edit that wins before the final resume handoff', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ name: 'before' }));
+ const fence = await methods.acquireResumeLease(schedule.id, 0, true, 60_000);
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'after' });
+
+ await expect(
+ methods.consumeResumeLease(schedule.id, fence!.claimToken!, fence!.leaseBy!, true, 0),
+ ).resolves.toBe(false);
+ // Owner edits intentionally leave the in-flight holder fields intact. The stale
+ // resume can release only its unique holder without touching any newer takeover.
+ expect((await getSchedule(schedule.id)).leaseBy).toBe(fence!.leaseBy);
+ await methods.releaseLeaseByHolder(schedule.id, fence!.leaseBy!);
+ expect((await getSchedule(schedule.id)).leaseBy).toBeUndefined();
+ });
+});
+
+describe('claim-token fencing (stale worker cannot mutate an edited/deleted schedule)', () => {
+ it('an owner edit rotates the token so a stale disable/advance no-ops and revalidate fails', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 });
+ const staleToken = claim!.claimToken!;
+ expect(staleToken).toBeTruthy();
+ // The stale worker still holds a valid claim until the owner edits.
+ expect(await methods.revalidateClaim(schedule.id, staleToken)).toBe(true);
+
+ // Owner edits (re-enable / rename) — updateScheduleById rotates the claim token.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });
+
+ // The stale worker's fenced writes now match nothing.
+ await methods.disableSchedule(schedule.id, 'too_many_failures', staleToken);
+ await methods.advanceSchedule(
+ schedule.id,
+ new Date('2030-01-01T00:00:00Z'),
+ claim!.nextRunAt,
+ staleToken,
+ );
+ const after = await methods.getScheduleById(schedule.id);
+ expect(after!.enabled).toBe(true); // stale disable no-op'd
+ expect(after!.disabledReason).toBeUndefined();
+ expect(after!.nextRunAt?.getTime()).toBe(claim!.nextRunAt?.getTime()); // stale advance no-op'd
+ expect(await methods.revalidateClaim(schedule.id, staleToken)).toBe(false);
+ });
+
+ it('revalidateClaim fails once the lease expires (a re-claim could be imminent)', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: -1 }); // already expired
+ expect(await methods.revalidateClaim(schedule.id, claim!.claimToken!)).toBe(false);
+ });
+});
+
+describe('owner edit clears a dead paused card', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ /** Projects a `requires_action` card whose run captured revision 0, matching a real
+ * fire: the reservation stamps the config generation on the run row. */
+ async function pauseUnderRevisionZero(): Promise<ISchedule> {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-paused',
+ autoDisableAfterFailures: 3,
+ });
+ const paused = await getSchedule(schedule.id);
+ expect(paused.lastRun?.status).toBe('requires_action');
+ return schedule;
+ }
+
+ it('a rename drops the pause the stale approval can no longer clear', async () => {
+ const schedule = await pauseUnderRevisionZero();
+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {
+ name: 'renamed',
+ });
+ expect(edited?.name).toBe('renamed');
+ expect(edited?.configRevision).toBe(1);
+ expect(edited?.lastRun).toBeUndefined();
+ });
+
+ it('a disabling edit drops the pause (no future occurrence would repair it)', async () => {
+ const schedule = await pauseUnderRevisionZero();
+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, { enabled: false });
+ expect(edited?.enabled).toBe(false);
+ expect(edited?.lastRun).toBeUndefined();
+ });
+
+ it('preserves a terminal success card across an edit', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-ok',
+ autoDisableAfterFailures: 3,
+ });
+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {
+ name: 'renamed',
+ });
+ expect(edited?.lastRun?.status).toBe('success');
+ expect(edited?.lastRun?.conversationId).toBe('convo-ok');
+ });
+
+ /** A resume terminalizes the paused occurrence between the handler's read and the
+ * edit; the clear is fenced on the card STILL being the pause, so the settled result
+ * survives rather than being cleared on a stale pre-read. */
+ it('preserves a terminal outcome that races the edit ahead of it', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor, configRevision: 0 }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+ // The pause settles to interrupted before the edit lands.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'interrupted',
+ error: 'stopped',
+ autoDisableAfterFailures: 3,
+ });
+ const edited = await methods.updateScheduleById(schedule.id, schedule.user, {
+ name: 'renamed',
+ });
+ expect(edited?.lastRun?.status).toBe('interrupted');
+ });
+
+ it('a stale-revision pause cannot resurrect the cleared card', async () => {
+ const schedule = await pauseUnderRevisionZero();
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });
+ // The run row is still `requires_action` under revision 0; a reconciliation re-affirm
+ // of the pause must not re-stamp the card now that the schedule advanced to revision 1.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-paused',
+ autoDisableAfterFailures: 3,
+ });
+ const after = await getSchedule(schedule.id);
+ expect(after.lastRun).toBeUndefined();
+ });
+
+ it('a stale old-revision terminal outcome cannot overwrite the edited card or counters', async () => {
+ const schedule = await pauseUnderRevisionZero();
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });
+ // The dead occurrence (revision 0) finally settles; its bookkeeping is revision-fenced,
+ // so it neither re-projects the card nor advances counters on the revision-1 schedule.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'convo-paused',
+ autoDisableAfterFailures: 3,
+ });
+ const after = await getSchedule(schedule.id);
+ expect(after.lastRun).toBeUndefined();
+ expect(after.runCount).toBe(0);
+ });
+});
+
+describe('schedule deletion is retryable', () => {
+ /**
+ * The mark is the FIRST of several teardown steps (read active runs, abort their
+ * jobs, prune checkpoints, erase when drained). A one-shot mark made every retry
+ * answer 404 after any of those threw, stranding the schedule with its job and
+ * checkpoint alive until the background expiry windows.
+ */
+ it('re-marks an already-deleting schedule so the caller can re-drive teardown', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const first = await methods.markScheduleDeleting(schedule.id, schedule.user);
+ expect(first).not.toBeNull();
+
+ const retry = await methods.markScheduleDeleting(schedule.id, schedule.user);
+ expect(retry).not.toBeNull();
+ expect(retry!.deleting).toBe(true);
+ expect(retry!.enabled).toBe(false);
+ // The token rotates again, which only re-fences stale workers.
+ expect(retry!.claimToken).not.toBe(first!.claimToken);
+ });
+
+ it('still refuses a schedule owned by someone else', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const other = new mongoose.Types.ObjectId();
+ expect(await methods.markScheduleDeleting(schedule.id, other)).toBeNull();
+ });
+});
+
+describe('unarmed-schedule recovery', () => {
+ /**
+ * Creation arms in a second write, so a crash or a failed arm leaves a row that looks
+ * enabled to its owner and occupies a slot while claimDueSchedule — which sorts on
+ * nextRunAt — can never select it.
+ */
+ it('finds enabled schedules with no nextRunAt and arms them', async () => {
+ const schedule = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });
+ expect((await getSchedule(schedule.id)).nextRunAt).toBeUndefined();
+
+ const unarmed = await methods.getUnarmedSchedules(10);
+ expect(unarmed.map((s) => s.id)).toContain(schedule.id);
+
+ const when = new Date('2030-01-01T00:00:00.000Z');
+ await methods.armSchedule(schedule.id, when);
+ expect((await getSchedule(schedule.id)).nextRunAt?.getTime()).toBe(when.getTime());
+ // Now armed, so it drops out of the sweep.
+ expect((await methods.getUnarmedSchedules(10)).map((s) => s.id)).not.toContain(schedule.id);
+ });
+
+ it('never disturbs a schedule that armed itself in the meantime', async () => {
+ const schedule = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });
+ const real = new Date('2030-06-01T00:00:00.000Z');
+ await methods.armSchedule(schedule.id, real);
+
+ // A concurrent sweep pass computed its own value from a stale read.
+ await methods.armSchedule(schedule.id, new Date('2030-01-01T00:00:00.000Z'));
+
+ expect((await getSchedule(schedule.id)).nextRunAt?.getTime()).toBe(real.getTime());
+ });
+
+ it('excludes disabled and deleting schedules from the sweep', async () => {
+ const disabled = await methods.createSchedule({
+ ...scheduleData(),
+ nextRunAt: undefined,
+ enabled: false,
+ });
+ const deleting = await methods.createSchedule({ ...scheduleData(), nextRunAt: undefined });
+ await methods.markScheduleDeleting(deleting.id, deleting.user);
+
+ const ids = (await methods.getUnarmedSchedules(10)).map((s) => s.id);
+ expect(ids).not.toContain(disabled.id);
+ expect(ids).not.toContain(deleting.id);
+ });
+});
+
+describe('card projection fences (regressions in the ordered projection)', () => {
+ /**
+ * The terminal path passes the row's revision to projectLastRun; the pause path did
+ * not. An owner edit landing between fire and approval therefore got the OLD config's
+ * pause stamped on it, and because the later terminal write IS revision-fenced it
+ * could never replace that status — the card stuck on "Needs approval".
+ */
+ it('does not stamp a pause onto a schedule edited after the run started', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');
+ const runRevision = (await getSchedule(schedule.id)).configRevision;
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor, configRevision: runRevision }),
+ );
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+
+ // The row still pauses (the generation really is waiting) but the edited
+ // schedule's card is not stamped by the superseded config.
+ expect(await getRun(schedule.id, scheduledFor)).toMatchObject({ status: 'requires_action' });
+ expect((await getSchedule(schedule.id)).lastRun).toBeUndefined();
+ });
+
+ /**
+ * A skip wrote `lastRun` directly, dropping the `scheduledFor` ordering marker that
+ * projectLastRun keys off — so the NEXT projection saw an absent marker and let an
+ * older occurrence's result overwrite the newer skip.
+ */
+ /**
+ * The pause wins its row transition and THEN projects, so a resume that terminalizes
+ * in between leaves exactly this state: row still active, card already settled for the
+ * SAME occurrence. Occurrence ordering alone permits the pause (equal `scheduledFor`),
+ * which would pin the card to "Needs approval" for a run that had already finished —
+ * and nothing corrects it until the next occurrence runs.
+ *
+ * The interleaving needs two concurrent writers, but the state it produces does not:
+ * constructing it directly tests the guard on exactly the input it exists for.
+ */
+ it('refuses to walk an occurrence card back from a settled status to a pause', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+ // The resumed generation already settled this occurrence's card.
+ await mongoose.models.Schedule.updateOne(
+ { id: schedule.id },
+ {
+ $set: {
+ lastRun: {
+ status: 'success',
+ firedAt: new Date(),
+ scheduledFor,
+ conversationId: 'convo-1',
+ },
+ },
+ },
+ );
+
+ // The in-flight pause now lands for that same occurrence.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+
+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('success');
+ });
+
+ it('still projects a pause when the occurrence card is not yet settled', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-1',
+ autoDisableAfterFailures: 3,
+ });
+
+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('requires_action');
+ });
+
+ it('keeps the occurrence marker when recording a skip', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const older = new Date('2026-07-26T10:00:00.000Z');
+ const newer = new Date('2026-07-26T11:00:00.000Z');
+ await methods.insertScheduleRun(runData(schedule, { scheduledFor: older }));
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'requires_action',
+ conversationId: 'convo-older',
+ autoDisableAfterFailures: 3,
+ });
+
+ // The newer occurrence is skipped for balance while the older one sits paused.
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ scheduledFor: newer,
+ user: schedule.user,
+ status: 'skipped_balance',
+ firedAt: new Date(),
+ },
+ 5,
+ );
+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_balance');
+
+ // The older occurrence then resumes and finishes. It must NOT roll the card back.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'success',
+ conversationId: 'convo-older',
+ autoDisableAfterFailures: 3,
+ });
+
+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_balance');
+ });
+});
+
+describe('auto-disable carries the run config generation', () => {
+ /**
+ * The auto-disable policy re-reads the schedule and then writes, so the decision and
+ * the write are not atomic. Carrying the run's config generation on the write closes
+ * that window: an owner edit landing inside it rotates configRevision, and the stale
+ * decision no longer applies.
+ *
+ * NOTE the ordinary re-enable path is NOT this scenario — `updateSchedule` zeroes
+ * failureCount/balanceSkipCount when re-enabling, so a re-enabled schedule cannot be
+ * sitting at the threshold. The state below is reached only through the read→write
+ * window itself, which is why it is constructed directly.
+ */
+ it('does not disable when the run config generation is stale', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const runRevision = (await getSchedule(schedule.id)).configRevision;
+ // A plain rename — NOT a re-enable, so the counters are deliberately left alone.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'renamed' });
+ expect((await getSchedule(schedule.id)).configRevision).toBeGreaterThan(runRevision!);
+ // At threshold and still enabled: what a decision made just before that edit saw.
+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });
+
+ const stale = new Date('2026-07-26T11:00:00.000Z');
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor: stale, configRevision: runRevision }),
+ );
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: stale,
+ status: 'error',
+ error: 'stale decision',
+ autoDisableAfterFailures: 1,
+ });
+
+ expect((await getSchedule(schedule.id)).enabled).toBe(true);
+ });
+
+ /**
+ * The threshold read and the disable write are separate statements. A concurrent
+ * success resets the failure streak in between, so a revision-only fence would still
+ * let the stale decision disable a schedule whose streak had just been cleared.
+ */
+ it('does not disable once a concurrent success reset the failure streak', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const current = await getSchedule(schedule.id);
+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });
+
+ // The decision was made on failureCount >= 5; a success then zeroed it.
+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 0 } });
+ await methods.disableSchedule(
+ schedule.id,
+ 'too_many_failures',
+ undefined,
+ current.configRevision,
+ {
+ failureCount: { $gte: 5 },
+ },
+ );
+
+ expect((await getSchedule(schedule.id)).enabled).toBe(true);
+ });
+
+ it('still disables when the streak is intact', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const current = await getSchedule(schedule.id);
+ await mongoose.models.Schedule.updateOne({ id: schedule.id }, { $set: { failureCount: 5 } });
+
+ await methods.disableSchedule(
+ schedule.id,
+ 'too_many_failures',
+ undefined,
+ current.configRevision,
+ {
+ failureCount: { $gte: 5 },
+ },
+ );
+
+ expect(await getSchedule(schedule.id)).toMatchObject({
+ enabled: false,
+ disabledReason: 'too_many_failures',
+ });
+ });
+
+ it('still auto-disables when the run matches the current config generation', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');
+ const current = await getSchedule(schedule.id);
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor, configRevision: current.configRevision }),
+ );
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ error: 'boom',
+ autoDisableAfterFailures: 1,
+ });
+
+ expect(await getSchedule(schedule.id)).toMatchObject({
+ enabled: false,
+ disabledReason: 'too_many_failures',
+ });
+ });
+});
+
+describe('reserveStartedRun duplicate reporting', () => {
+ /**
+ * The caller has to tell "another worker is still running this" from "this occurrence
+ * already finished and was merely never advanced past" — they need opposite handling,
+ * and a bare `duplicate` cannot distinguish them.
+ */
+ it('reports the existing row status on a duplicate', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T10:00:00.000Z');
+ const base = {
+ scheduleId: schedule.id,
+ scheduledFor,
+ user: schedule.user,
+ firedAt: new Date(),
+ };
+ await methods.reserveStartedRun({ ...base, conversationId: 'first' });
+
+ const active = await methods.reserveStartedRun({ ...base, conversationId: 'second' });
+ expect(active).toMatchObject({ conflict: 'duplicate', existingStatus: 'started' });
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ autoDisableAfterFailures: 3,
+ });
+
+ const settled = await methods.reserveStartedRun({ ...base, conversationId: 'third' });
+ expect(settled).toMatchObject({ conflict: 'duplicate', existingStatus: 'success' });
+ });
+});
+
+describe('deleteScheduleRun conversation fence', () => {
+ it('deletes only the reservation the caller inserted', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T12:00:00.000Z');
+ await methods.reserveStartedRun({
+ scheduleId: schedule.id,
+ scheduledFor,
+ user: schedule.user,
+ conversationId: 'conv-mine',
+ firedAt: new Date(),
+ });
+
+ // Another fire's id must not match: this is what makes the rollback safe without
+ // consulting lease ownership, which the preflight can outlive.
+ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-theirs');
+ expect(await methods.getActiveRunsForSchedule(schedule.id)).toHaveLength(1);
+
+ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-mine');
+ expect(await methods.getActiveRunsForSchedule(schedule.id)).toHaveLength(0);
+ });
+
+ it('still honors the status fence so a settled run is never rolled back', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2026-07-26T13:00:00.000Z');
+ await methods.reserveStartedRun({
+ scheduleId: schedule.id,
+ scheduledFor,
+ user: schedule.user,
+ conversationId: 'conv-mine',
+ firedAt: new Date(),
+ });
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ autoDisableAfterFailures: 5,
+ });
+
+ await methods.deleteScheduleRun(schedule.id, scheduledFor, 'started', 'conv-mine');
+
+ const runs = await methods.getRunsForReconciliation(new Date(Date.now() + 1000), 10);
+ expect(runs).toHaveLength(0);
+ expect(await methods.getScheduleById(schedule.id)).toMatchObject({
+ lastRun: expect.objectContaining({ status: 'success' }),
+ });
+ });
+});
+
+describe('erasure leaves an idempotency tombstone', () => {
+ it('retains the create key so a late retry cannot resurrect deleted work', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const schedule = await methods.createSchedule(
+ scheduleData({
+ user,
+ deleting: true,
+ nextRunAt: undefined,
+ clientRequestId: 'late-retry-key',
+ clientRequestDigest: 'digest-1',
+ }),
+ );
+
+ await expect(methods.eraseScheduleIfDrained(schedule.id)).resolves.toBe(true);
+
+ // The content is gone, the identity is not: a create retry with this key must
+ // resolve to the tombstone (the handler answers 410 on deleting rows) instead
+ // of missing entirely and recreating the recurring work the owner deleted.
+ const tombstone = await methods.getScheduleByClientRequestId(user, 'late-retry-key');
+ expect(tombstone).not.toBeNull();
+ expect(tombstone?.erased).toBe(true);
+ expect(tombstone?.deleting).toBe(true);
+ expect(tombstone?.clientRequestDigest).toBe('digest-1');
+ // ALLOWLIST semantics: everything except the replay-detection identity is gone,
+ // derived from the live schema paths so fields added later cannot leak either.
+ const contentKeys = Object.keys(tombstone ?? {}).filter(
+ (key) =>
+ ![
+ '_id',
+ '__v',
+ 'id',
+ 'user',
+ 'tenantId',
+ 'clientRequestId',
+ 'clientRequestDigest',
+ 'deleting',
+ 'erased',
+ 'erasedAt',
+ 'enabled',
+ 'createdAt',
+ 'updatedAt',
+ ].includes(key),
+ );
+ expect(contentKeys).toEqual([]);
+ expect(tombstone?.prompt).toBeUndefined();
+ expect(tombstone?.name).toBeUndefined();
+ expect(tombstone?.cadence).toBeUndefined();
+ expect(tombstone?.timezone).toBeUndefined();
+
+ // Tombstones are inert to every sweep: re-erasing or re-deleting them forever
+ // would pin the bounded windows.
+ const sweepRows = await methods.getDeletingSchedules(50);
+ expect(sweepRows.some((row) => row.id === schedule.id)).toBe(false);
+ const retryIds = await methods.getDeletingScheduleIds(user, 50);
+ expect(retryIds).not.toContain(schedule.id);
+ });
+
+ it('hard-deletes rows that never carried a create key', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const schedule = await methods.createSchedule(
+ scheduleData({ user, deleting: true, nextRunAt: undefined }),
+ );
+
+ await expect(methods.eraseScheduleIfDrained(schedule.id)).resolves.toBe(true);
+ const gone = await mongoose.models.Schedule.findOne({ id: schedule.id }).lean();
+ expect(gone).toBeNull();
+ });
+});
+
+describe('deletion quiescing (soft-delete, drain, erase)', () => {
+ it('markScheduleDeleting hides + un-claims; erase waits for active runs to drain', async () => {
+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);
+ const sched = schedule as ISchedule;
+ const when = new Date('2026-07-20T12:00:00Z');
+ await methods.reserveStartedRun(runData(sched, { scheduledFor: when }));
+
+ const marked = await methods.markScheduleDeleting(sched.id, sched.user);
+ expect(marked!.deleting).toBe(true);
+ expect(marked!.enabled).toBe(false);
+ // Hidden from the owner and no longer claimable by the engine.
+ expect(await methods.getSchedulesByUser(sched.user)).toHaveLength(0);
+ expect(await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 })).toBeNull();
+
+ // Erase is blocked while a run is still active (evidence preserved). The raw
+ // doc still exists (getScheduleById hides `deleting` schedules by design).
+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(false);
+ expect(await Schedule.findOne({ id: sched.id }).lean()).not.toBeNull();
+
+ // Once the run terminalizes, the schedule and its runs erase.
+ await ScheduleRun.updateOne(
+ { scheduleId: sched.id, scheduledFor: when },
+ { $set: { status: 'interrupted' } },
+ );
+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(true);
+ expect(await Schedule.findOne({ id: sched.id }).lean()).toBeNull();
+ expect(await ScheduleRun.countDocuments({ scheduleId: sched.id })).toBe(0);
+ // Its slot is freed for reuse.
+ const replacement = await methods.createScheduleWithSlot(
+ scheduleData({ user: sched.user }),
+ 10,
+ );
+ expect(replacement).not.toBe('limit');
+ });
+
+ it('a deleting schedule is hidden, non-mutable, and non-claimable (no re-enable)', async () => {
+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);
+ const sched = schedule as ISchedule;
+ await methods.markScheduleDeleting(sched.id, sched.user);
+ // Hidden and unreadable through the owner-facing read.
+ expect(await methods.getScheduleById(sched.id)).toBeNull();
+ expect(await methods.getScheduleById(sched.id, sched.user)).toBeNull();
+ // A stale/concurrent re-enable PATCH is refused (returns null -> handler 404).
+ const reEnabled = await methods.updateScheduleById(sched.id, sched.user, {
+ enabled: true,
+ nextRunAt: new Date(Date.now() - 60_000),
+ });
+ expect(reEnabled).toBeNull();
+ // Even if a nextRunAt/enabled somehow lingered, the engine won't claim it.
+ expect(await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 })).toBeNull();
+ });
+
+ it('keeps a live lease and does not erase until it is released (worker mid-claim)', async () => {
+ const schedule = await methods.createScheduleWithSlot(scheduleData(), 10);
+ const sched = schedule as ISchedule;
+ // A worker has CLAIMED the schedule (live lease) but not yet inserted a run row.
+ const claim = await methods.claimDueSchedule({ instanceId: 'w1', leaseMs: 60_000 });
+ expect(claim?.id).toBe(sched.id);
+ const marked = await methods.markScheduleDeleting(sched.id, sched.user);
+ // The lease is PRESERVED so the worker can prove ownership on its rollback.
+ expect(marked?.leaseBy).toBe('w1');
+ // Erase is blocked while the lease is live, even with no active run row.
+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(false);
+ expect(await Schedule.findOne({ id: sched.id }).lean()).not.toBeNull();
+ // Once the worker releases its own lease (by holder), it drains and erases.
+ await methods.releaseLeaseByHolder(sched.id, 'w1');
+ expect(await methods.eraseScheduleIfDrained(sched.id)).toBe(true);
+ expect(await Schedule.findOne({ id: sched.id }).lean()).toBeNull();
+ });
+
+ it('frees the slot immediately on soft-delete so a new create can take it under the cap', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const a = (await methods.createScheduleWithSlot(scheduleData({ user }), 1)) as ISchedule;
+ expect(await methods.createScheduleWithSlot(scheduleData({ user }), 1)).toBe('limit');
+ await methods.markScheduleDeleting(a.id, user);
+ // The deleting schedule no longer occupies the cap, so a replacement fits.
+ const b = await methods.createScheduleWithSlot(scheduleData({ user }), 1);
+ expect(b).not.toBe('limit');
+ });
+});
+
+describe('config fence is DERIVED at the seam, not passed by callers', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('skips bookkeeping when the owner edited the schedule after the run started', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ // The run captured the config generation it started under.
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor, configRevision: schedule.configRevision ?? 0 }),
+ );
+ // Owner edits -> configRevision moves on.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+
+ // NOTE: no fence token is passed — recordRunOutcome derives it from the run row.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ error: 'boom',
+ autoDisableAfterFailures: 3,
+ });
+
+ // The run still terminalizes (evidence is preserved)...
+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('error');
+ // ...but it must NOT count a failure against the schedule the owner just edited.
+ const after = await getSchedule(schedule.id);
+ expect(after.failureCount).toBe(0);
+ expect(after.lastRun).toBeUndefined();
+ });
+
+ it('applies bookkeeping normally when the revision still matches', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(
+ runData(schedule, { scheduledFor, configRevision: schedule.configRevision ?? 0 }),
+ );
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'success',
+ conversationId: 'c1',
+ autoDisableAfterFailures: 3,
+ });
+ const after = await getSchedule(schedule.id);
+ expect(after.runCount).toBe(1);
+ expect(after.lastRun?.status).toBe('success');
+ });
+
+ it('fences the crash-retry path (finalizeBookkeeping) the same way', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.insertScheduleRun(
+ runData(schedule, {
+ scheduledFor,
+ status: 'error',
+ bookkept: false,
+ configRevision: schedule.configRevision ?? 0,
+ }),
+ );
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+
+ // The reconciler replays bookkeeping for an unbookkept run; it must honor the same
+ // fence as the inline path, or a crash would let it apply what inline refused.
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'error',
+ autoDisableAfterFailures: 3,
+ });
+ expect((await getSchedule(schedule.id)).failureCount).toBe(0);
+ });
+});
+
+describe('skip bookkeeping is fenced by the same derived seam', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('does not stamp the card when the owner edited after the occurrence was claimed', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ // Owner edits AFTER the worker claimed at the old revision.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+
+ await methods.recordSkippedRun({
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor,
+ status: 'skipped_overlap',
+ configRevision: schedule.configRevision ?? 0, // the CLAIMED revision, now stale
+ });
+
+ // The skip row still exists as evidence, but the schedule card is untouched.
+ expect((await getRun(schedule.id, scheduledFor)).status).toBe('skipped_overlap');
+ expect((await getSchedule(schedule.id)).lastRun).toBeUndefined();
+ });
+
+ it('stamps the card normally when the revision still matches', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.recordSkippedRun({
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor,
+ status: 'skipped_overlap',
+ configRevision: schedule.configRevision ?? 0,
+ });
+ expect((await getSchedule(schedule.id)).lastRun?.status).toBe('skipped_overlap');
+ });
+
+ it('does not walk the balance streak toward auto-disable under a stale revision', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+ await methods.recordSkippedRun(
+ {
+ scheduleId: schedule.id,
+ user: schedule.user,
+ scheduledFor,
+ status: 'skipped_balance',
+ configRevision: schedule.configRevision ?? 0,
+ },
+ 1, // threshold of 1 would disable immediately if the fence did not hold
+ );
+ const after = await getSchedule(schedule.id);
+ expect(after.balanceSkipCount).toBe(0);
+ expect(after.enabled).toBe(true);
+ });
+});
+
+describe('streak counters are order-fenced', () => {
+ const older = new Date('2026-07-20T08:00:00Z');
+ const newer = new Date('2026-07-20T09:00:00Z');
+
+ /**
+ * A `requires_action` run holds no capacity slot and does not block its schedule's
+ * later occurrences, so an older occurrence routinely settles AFTER a newer one — a
+ * resumed pause, or a reconciler replay. Unfenced, its late `error` rebuilds a streak
+ * the newer success had cleared.
+ */
+ it('does not let an older failure rebuild a streak a newer success cleared', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));
+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: newer,
+ status: 'success',
+ autoDisableAfterFailures: 5,
+ });
+ expect((await getSchedule(schedule.id)).failureCount).toBe(0);
+
+ // The older occurrence settles late, in error.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'error',
+ error: 'late failure',
+ autoDisableAfterFailures: 5,
+ });
+
+ const after = await getSchedule(schedule.id);
+ expect(after.failureCount).toBe(0);
+ });
+
+ /**
+ * `autoDisableAfterFailures` may be configured as low as 1, where a single stale
+ * failure is the difference between a healthy schedule and a disabled one.
+ */
+ it('does not auto-disable a healthy schedule on a stale failure at threshold 1', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));
+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: newer,
+ status: 'success',
+ autoDisableAfterFailures: 1,
+ });
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'error',
+ autoDisableAfterFailures: 1,
+ });
+
+ const after = await getSchedule(schedule.id);
+ expect(after.enabled).toBe(true);
+ expect(after.disabledReason).toBeUndefined();
+ });
+
+ it('still counts occurrences that settle in order', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ // The older row is `requires_action`: the single-active partial index admits only
+ // one `started` run per schedule, which is exactly why pauses can pile up.
+ await ScheduleRun.create(runData(schedule, { scheduledFor: older, status: 'requires_action' }));
+ await ScheduleRun.create(runData(schedule, { scheduledFor: newer }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: older,
+ status: 'error',
+ autoDisableAfterFailures: 5,
+ });
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor: newer,
+ status: 'error',
+ autoDisableAfterFailures: 5,
+ });
+
+ expect((await getSchedule(schedule.id)).failureCount).toBe(2);
+ });
+});
+
+describe('updateScheduleById revision fence', () => {
+ it('refuses an edit computed from a superseded revision', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const staleRevision = schedule.configRevision ?? 0;
+ // A concurrent edit lands first and moves the revision on.
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'winner' });
+
+ const loser = await methods.updateScheduleById(
+ schedule.id,
+ schedule.user,
+ { name: 'loser' },
+ undefined,
+ { expectedConfigRevision: staleRevision },
+ );
+
+ // Otherwise the loser would persist a nextRunAt derived from a (cadence, timezone)
+ // pair that no longer describes the row.
+ expect(loser).toBeNull();
+ expect((await getSchedule(schedule.id)).name).toBe('winner');
+ });
+
+ it('applies an edit whose revision is still current', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const updated = await methods.updateScheduleById(
+ schedule.id,
+ schedule.user,
+ { name: 'renamed' },
+ undefined,
+ { expectedConfigRevision: schedule.configRevision ?? 0 },
+ );
+ expect(updated?.name).toBe('renamed');
+ });
+});
+
+describe('pause replay is fenced against a concurrent resume claim', () => {
+ /**
+ * Every clustered replica runs the erasure sweep, so several can observe the same
+ * unprojected pause. The first projects it; the owner's approval then claims a fresh
+ * slot; a peer still holding the PRE-projection snapshot passes its in-memory
+ * hand-off check and replays the pause — unsetting `capacitySlot` and
+ * `resumeClaimedAt` under a continuation that is already running. The fence has to
+ * live in the write, not in the caller's snapshot.
+ */
+ it('refuses to replay a pause onto a row a resume has claimed', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2030-03-01T00:00:00Z');
+ await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor, conversationId: 'convo-race', capacitySlot: 0 }),
+ );
+ // Sweeper A wins: the row leaves `started` and frees its slot.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-race',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),
+ });
+ // The approval now claims a FRESH slot; this is the state a stale peer must not touch.
+ const claimed = await methods.markRunResumeClaimed(schedule.id, scheduledFor, 3);
+ expect(claimed).toEqual({ capacitySlot: 3 });
+
+ // Sweeper B replays from its pre-projection snapshot.
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-race',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),
+ });
+
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();
+ expect(run?.status).toBe('started');
+ expect(run?.capacitySlot).toBe(3);
+ expect(run?.resumeClaimedAt).toBeDefined();
+ });
+
+ /** The fence must not block the recovery it exists to enable: a genuinely stuck
+ * `started` row carries no resume claim, because markRunResumeClaimed only matches
+ * `requires_action` — the stuck row blocks its own approval until this replay runs. */
+ it('still recovers a stuck row that carries no resume claim', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2030-03-02T00:00:00Z');
+ await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor, conversationId: 'convo-stuck', capacitySlot: 1 }),
+ );
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-stuck',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),
+ });
+
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();
+ expect(run?.status).toBe('requires_action');
+ expect(run?.capacitySlot).toBeUndefined();
+ });
+
+ /**
+ * The fence is a staleness CUTOFF, not an existence check, and this is why: a worker
+ * that dies between markRunResumeClaimed and either resuming or rolling back leaves
+ * `resumeClaimedAt` set forever. Rejecting on existence alone would pin the row
+ * `started` for good — holding its global capacity slot, and with its approval
+ * permanently unresumable, since markRunResumeClaimed only matches `requires_action`.
+ * That is precisely the stuck state this replay exists to clear.
+ */
+ it('recovers a row whose resume claim was abandoned by a dead worker', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const scheduledFor = new Date('2030-03-03T00:00:00Z');
+ await methods.reserveStartedRun(
+ runData(schedule, { scheduledFor, conversationId: 'convo-abandoned', capacitySlot: 2 }),
+ );
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-abandoned',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),
+ });
+ await methods.markRunResumeClaimed(schedule.id, scheduledFor, 4);
+ // The claiming worker died here: the stamp is now well past the hand-off bound.
+ await ScheduleRun.updateOne(
+ { scheduleId: schedule.id, scheduledFor },
+ { $set: { resumeClaimedAt: new Date(Date.now() - 60 * 60_000) } },
+ { timestamps: false },
+ );
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'requires_action',
+ conversationId: 'convo-abandoned',
+ autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
+ resumeClaimStaleBefore: new Date(Date.now() - 10 * 60_000),
+ });
+
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id, scheduledFor }).lean();
+ expect(run?.status).toBe('requires_action');
+ expect(run?.capacitySlot).toBeUndefined();
+ expect(run?.resumeClaimedAt).toBeUndefined();
+ });
+});
+
+describe('reconciliation rotates the paused window', () => {
+ /**
+ * Paused runs accumulate without bound (no capacity slot, no occurrence block) and
+ * the approval TTL that ends them has no ceiling, so a full window of still-LIVE
+ * pauses can sit at the front of an oldest-first ordering forever. Ordering by when
+ * each row was last examined is what guarantees every row eventually gets a turn.
+ */
+ it('serves rows behind a full window once the leaders have been examined', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ const base = Date.parse('2026-07-01T00:00:00Z');
+ const olderThan = new Date(base + 10_000_000);
+ for (let i = 0; i < 100; i++) {
+ await ScheduleRun.create(
+ runData(schedule, {
+ scheduledFor: new Date(base + i * 60_000),
+ status: 'requires_action',
+ firedAt: new Date(base + i * 60_000),
+ }),
+ );
+ }
+ // Newer than the 100 leaders, but still inside the age filter — the point is that
+ // ORDERING keeps it out of the first window, not that it is too young to qualify.
+ const queuedAt = new Date(base + 120 * 60_000);
+ await ScheduleRun.create(
+ runData(schedule, {
+ scheduledFor: queuedAt,
+ status: 'requires_action',
+ firedAt: queuedAt,
+ }),
+ );
+
+ const first = await methods.getRunsForReconciliation(olderThan, 100);
+ // The newest row is behind a full batch, so this pass cannot reach it.
+ expect(first.some((run) => run.scheduledFor?.getTime() === queuedAt.getTime())).toBe(false);
+
+ // Examining the batch rotates it to the back.
+ await methods.markRunsReconciled(first);
+
+ const second = await methods.getRunsForReconciliation(olderThan, 100);
+ expect(second.some((run) => run.scheduledFor?.getTime() === queuedAt.getTime())).toBe(true);
+ });
+});
+
+describe('requestRunAbort (renewable, source-aware stamp)', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('renews freshness on a deletion retry without stealing an unresolved stop source', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);
+ const first = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(first?.abortRequestedAt).toBeInstanceOf(Date);
+ expect(first?.abortSource).toBe('stop');
+
+ // A racing deletion must not steal the source (the settlement barrier reads it to
+ // know the Stop route still has writes in flight) but MUST renew the freshness
+ // stamp — its abort is a new signal the fences have to cover.
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);
+ const second = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(second?.abortSource).toBe('stop');
+ expect(second?.abortRequestedAt!.getTime()).toBeGreaterThanOrEqual(
+ first!.abortRequestedAt!.getTime(),
+ );
+ });
+
+ it('lets a stop attempt take over a deletion-stamped run and re-open its persistence', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+
+ // Deletion stamps first; the user's Stop then wins the abort — the owner's
+ // barrier must now wait on the ROUTE's persistence, so the stop takes the source.
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);
+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(state?.abortSource).toBe('stop');
+ expect(state?.abortPersistedAt).toBeUndefined();
+ });
+
+ it('lets deletion claim the source once the stop attempt resolved', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+
+ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');
+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'deletion')).toBe(true);
+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(state?.abortSource).toBe('deletion');
+ });
+
+ it('re-opens the persistence marker on a fresh stop attempt', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+
+ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');
+ await methods.markRunAbortPersisted(schedule.id, scheduledFor);
+ // A SECOND stop attempt persists anew; the resolved marker of the first must not
+ // release the owner's barrier while the new attempt's writes are in flight.
+ await methods.requestRunAbort(schedule.id, scheduledFor, 'stop');
+ const state = await methods.getScheduleRunAbortState(schedule.id, scheduledFor);
+ expect(state?.abortPersistedAt).toBeUndefined();
+ });
+
+ it('reports false when no active run holds the occurrence', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { status: 'success' }));
+ expect(
+ await methods.requestRunAbort(schedule.id, new Date('2026-07-20T12:00:00Z'), 'stop'),
+ ).toBe(false);
+ });
+
+ it('serializes concurrent stop attempts: the second answers in_progress', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe(true);
+ // A second Stop while the first is unresolved must neither re-open the
+ // persistence marker nor be told to proceed — its lost-CAS resolution would
+ // release the settlement barrier while the winner is still persisting.
+ expect(await methods.requestRunAbort(schedule.id, scheduledFor, 'stop')).toBe('in_progress');
+ });
+
+ it('marks abort persistence for the settlement barrier', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule));
+ await methods.requestRunAbort(schedule.id, new Date('2026-07-20T12:00:00Z'), 'stop');
+ await methods.markRunAbortPersisted(schedule.id, new Date('2026-07-20T12:00:00Z'));
+ const state = await methods.getScheduleRunAbortState(
+ schedule.id,
+ new Date('2026-07-20T12:00:00Z'),
+ );
+ expect(state?.abortPersistedAt).toBeInstanceOf(Date);
+ });
+});
+
+describe('recordRunOutcome settles a mid-generation balance refusal as skipped_balance', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('walks the balance streak instead of the failure streak and frees capacity', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0, conversationId: 'c1' }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'skipped_balance',
+ conversationId: 'c1',
+ autoDisableAfterFailures: 5,
+ balanceSkipDisableThreshold: 5,
+ });
+
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();
+ expect(run?.status).toBe('skipped_balance');
+ expect(run?.capacitySlot).toBeUndefined();
+ expect(run?.bookkept).toBe(true);
+
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.balanceSkipCount).toBe(1);
+ expect(row?.failureCount).toBe(0);
+ expect(row?.enabled).toBe(true);
+ expect(row?.lastRun?.status).toBe('skipped_balance');
+ });
+
+ it('auto-disables with insufficient_balance at the threshold, not too_many_failures', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 4 }));
+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0 }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'skipped_balance',
+ autoDisableAfterFailures: 5,
+ balanceSkipDisableThreshold: 5,
+ });
+
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.enabled).toBe(false);
+ expect(row?.disabledReason).toBe('insufficient_balance');
+ });
+
+ it('is idempotent across the crash-retry replay (finalizeBookkeeping)', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(runData(schedule, { capacitySlot: 0 }));
+
+ await methods.recordRunOutcome({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'skipped_balance',
+ autoDisableAfterFailures: 5,
+ balanceSkipDisableThreshold: 5,
+ });
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor,
+ status: 'skipped_balance',
+ autoDisableAfterFailures: 5,
+ balanceSkipDisableThreshold: 5,
+ });
+
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.balanceSkipCount).toBe(1);
+ });
+
+ it('surfaces an unbookkept skipped_balance run to the reconciler', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+ await ScheduleRun.create(
+ runData(schedule, {
+ status: 'skipped_balance',
+ bookkept: false,
+ firedAt: new Date(Date.now() - 10 * 60_000),
+ }),
+ );
+ const rows = await methods.getUnbookkeptRuns(new Date(Date.now() - 60_000), 10);
+ expect(rows.map((run) => run.scheduleId)).toContain(schedule.id);
+ });
+});
+
+describe('skip bookkeeping is crash-recoverable', () => {
+ const scheduledFor = new Date('2026-07-20T12:00:00Z');
+
+ it('marks skip rows bookkept only after the schedule-side writes land', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 3 }));
+ await methods.recordSkippedRun({
+ scheduleId: schedule.id,
+ scheduledFor,
+ user: schedule.user,
+ status: 'skipped_overlap',
+ });
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();
+ expect(run?.bookkept).toBe(true);
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.lastRun?.status).toBe('skipped_overlap');
+ expect(row?.balanceSkipCount).toBe(0);
+ });
+
+ it('replays a half-applied overlap skip through the reconciler', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ balanceSkipCount: 3 }));
+ // Simulate the crash: the row exists but no schedule-side write landed.
+ await ScheduleRun.create(
+ runData(schedule, {
+ status: 'skipped_overlap',
+ bookkept: false,
+ firedAt: new Date(Date.now() - 10 * 60_000),
+ }),
+ );
+ const rows = await methods.getUnbookkeptRuns(new Date(Date.now() - 60_000), 10);
+ expect(rows.map((run) => run.scheduleId)).toContain(schedule.id);
+
+ await methods.finalizeBookkeeping({
+ scheduleId: schedule.id,
+ scheduledFor: new Date('2026-07-20T12:00:00Z'),
+ status: 'skipped_overlap',
+ autoDisableAfterFailures: 5,
+ });
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ // Without the replay, the missed reset left a stale balance streak that a later
+ // balance skip would push over the auto-disable threshold.
+ expect(row?.balanceSkipCount).toBe(0);
+ expect(row?.lastRun?.status).toBe('skipped_overlap');
+ const run = await ScheduleRun.findOne({ scheduleId: schedule.id }).lean();
+ expect(run?.bookkept).toBe(true);
+ });
+});
+
+describe('deleteUnarmedSchedule (guarded create rollback)', () => {
+ it('erases a drained unarmed row and its settled runs', async () => {
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: undefined }));
+ await ScheduleRun.create(runData(schedule, { status: 'success' }));
+
+ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('deleted');
+ expect(await Schedule.findOne({ id: schedule.id }).lean()).toBeNull();
+ expect(await ScheduleRun.findOne({ scheduleId: schedule.id }).lean()).toBeNull();
+ });
+
+ it('defers to the drain when a live manual run holds the row', async () => {
+ // An unarmed row is visible and Run Now-able: rollback must never hard-delete a
+ // live generation's evidence out from under it. The soft-claim hides the row and
+ // the ordinary erase-on-settle/sweep teardown erases it once the run settles.
+ const schedule = await methods.createSchedule(scheduleData({ nextRunAt: undefined }));
+ await ScheduleRun.create(runData(schedule, { status: 'started' }));
+
+ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('draining');
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.deleting).toBe(true);
+ expect(await ScheduleRun.findOne({ scheduleId: schedule.id }).lean()).not.toBeNull();
+ });
+
+ it('refuses to delete a row that has since been armed', async () => {
+ const schedule = await methods.createSchedule(scheduleData());
+
+ // An ambiguously-committed arm (or a concurrent replay's arm) landed: the row is a
+ // live schedule another response may have confirmed, so the rollback must not
+ // erase it.
+ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user)).toBe('kept');
+ expect(await Schedule.findOne({ id: schedule.id }).lean()).not.toBeNull();
+ });
+
+ it('refuses to delete an unarmed row a concurrent edit has moved past', async () => {
+ const schedule = await methods.createSchedule(
+ scheduleData({ nextRunAt: undefined, configRevision: 0 }),
+ );
+ // A PATCH that disables the schedule leaves it UNARMED but bumps the revision —
+ // the row now embodies the owner's edit, and the POST's rollback must not erase it.
+ await methods.updateScheduleById(schedule.id, schedule.user, { enabled: false });
+
+ expect(await methods.deleteUnarmedSchedule(schedule.id, schedule.user, 0)).toBe('kept');
+ expect(await Schedule.findOne({ id: schedule.id }).lean()).not.toBeNull();
+ });
+
+ it('arms through the shared CAS without rotating the claim token or revision', async () => {
+ const schedule = await methods.createSchedule(
+ scheduleData({ nextRunAt: undefined, claimToken: 'ct-live', configRevision: 0 }),
+ );
+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 60_000), 0)).toBe(true);
+ // A second armer (the original POST racing a replay) must collapse to a no-op.
+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 120_000), 0)).toBe(false);
+ const row = await Schedule.findOne({ id: schedule.id }).lean();
+ expect(row?.claimToken).toBe('ct-live');
+ expect(row?.configRevision).toBe(0);
+ });
+
+ it('refuses the revision-fenced arm after a concurrent edit', async () => {
+ const schedule = await methods.createSchedule(
+ scheduleData({ nextRunAt: undefined, configRevision: 0 }),
+ );
+ await methods.updateScheduleById(schedule.id, schedule.user, { name: 'edited' });
+ expect(await methods.armSchedule(schedule.id, new Date(Date.now() + 60_000), 0)).toBe(false);
+ });
+
+ it('reports a missing row distinctly', async () => {
+ expect(await methods.deleteUnarmedSchedule('sched_gone', new mongoose.Types.ObjectId())).toBe(
+ 'missing',
+ );
+ });
+});
+
+describe('erasure sweep rotation and idempotency-key lookup', () => {
+ it('rotates the deleting window so undrainable rows cannot starve later ones', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const first = scheduleData({ user, deleting: true, nextRunAt: undefined });
+ const second = scheduleData({ user, deleting: true, nextRunAt: undefined });
+ await Schedule.create(first);
+ await Schedule.create(second);
+
+ const window = await methods.getDeletingSchedules(1);
+ expect(window).toHaveLength(1);
+ await methods.markEraseAttempted(window.map((row) => row.id));
+
+ const rotated = await methods.getDeletingSchedules(1);
+ expect(rotated[0].id).not.toBe(window[0].id);
+ });
+
+ it('rotates the per-owner deletion-retry window the same way', async () => {
+ const user = new mongoose.Types.ObjectId();
+ await Schedule.create(scheduleData({ user, deleting: true, nextRunAt: undefined }));
+ await Schedule.create(scheduleData({ user, deleting: true, nextRunAt: undefined }));
+
+ const window = await methods.getDeletingScheduleIds(user, 1);
+ expect(window).toHaveLength(1);
+ await methods.markEraseAttempted(window);
+
+ // The list-path retry is bounded too; without least-recently-attempted
+ // ordering an owner with more stuck rows than the limit re-drives the same
+ // unconfirmable few on every list and never reaches the rest.
+ const rotated = await methods.getDeletingScheduleIds(user, 1);
+ expect(rotated[0]).not.toBe(window[0]);
+ });
+
+ it('resolves an idempotency key even while its row is draining', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const schedule = scheduleData({ user, deleting: true, clientRequestId: 'intent-9' });
+ await Schedule.create(schedule);
+
+ const found = await methods.getScheduleByClientRequestId(user, 'intent-9');
+ expect(found?.id).toBe(schedule.id);
+ });
+});
diff --git a/packages/data-schemas/src/methods/schedule.ts b/packages/data-schemas/src/methods/schedule.ts
new file mode 100644
index 0000000000..222a8c5275
--- /dev/null
+++ b/packages/data-schemas/src/methods/schedule.ts
@@ -0,0 +1,2058 @@
+import { randomUUID } from 'node:crypto';
+import type { ScheduleRunStatus, ScheduleDisabledReason } from 'librechat-data-provider';
+import type { Model, Types, AnyBulkWriteOperation } from 'mongoose';
+import type {
+ ISchedule,
+ IScheduleDocument,
+ IScheduleRun,
+ IScheduleRunDocument,
+} from '~/types/schedule';
+import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
+import { createIndexesWithRetry } from '~/utils/retry';
+
+const DUPLICATE_KEY = 11000;
+
+/**
+ * Tolerance for inter-replica wall-clock skew on the worker-clock lease
+ * comparisons (DocumentDB rules out the server-clock `$$NOW` CAS). Applied only
+ * where one worker judges ANOTHER worker's lease expired — taking over a due
+ * claim, or declaring a deleting schedule drained — so a clock-ahead worker
+ * cannot steal a lease its holder still considers live. Never applied where a
+ * worker checks its OWN lease (revalidateClaim), which is self-consistent.
+ * NTP-managed fleets sit orders of magnitude under this.
+ */
+const LEASE_SKEW_MARGIN_MS = 30_000;
+
+/**
+ * Age past which an unresolved interactive-stop attempt is presumed dead and its
+ * per-run serialization claim becomes takeable again. Mirrors the api layer's
+ * ABORT_OWNER_PRESUMED_ALIVE_MS — one presumption, two enforcement points.
+ */
+const ABORT_STAMP_STALE_MS = 30 * 60_000;
+
+/**
+ * Upper bound on the per-schedule `countedFor` idempotency set. Far larger than
+ * the number of a single schedule's occurrences that can be terminal (or awaiting
+ * reconciliation) at once, so an occurrence's marker is never evicted before any
+ * possible crash-replay, while the array stays bounded.
+ */
+const COUNTED_FOR_WINDOW = 64;
+
+/** Statuses that occupy a schedule's live capacity / block a concurrent occurrence. */
+const ACTIVE_RUN_STATUSES: ScheduleRunStatus[] = ['started', 'requires_action'];
+
+/** Card statuses that represent a SETTLED occurrence — everything except the pause. */
+const TERMINAL_CARD_STATUSES: ScheduleRunStatus[] = [
+ 'success',
+ 'error',
+ 'interrupted',
+ 'skipped_overlap',
+ 'skipped_balance',
+];
+
+type DuplicateKeyError = { code?: number; keyPattern?: Record<string, unknown> };
+
+/** A duplicate-key error whose conflict is the {scheduleId, scheduledFor} occurrence index. */
+function isOccurrenceDuplicate(error: unknown): boolean {
+ const err = error as DuplicateKeyError;
+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'scheduledFor' in err.keyPattern;
+}
+
+/** A duplicate-key error whose conflict is the single-active-run partial index
+ * ({scheduleId} where status:'started'). Matched EXACTLY on scheduleId so the
+ * global {capacitySlot} index below is never misread as a per-schedule overlap. */
+function isActiveRunConflict(error: unknown): boolean {
+ const err = error as DuplicateKeyError;
+ return (
+ err?.code === DUPLICATE_KEY &&
+ err.keyPattern != null &&
+ 'scheduleId' in err.keyPattern &&
+ !('scheduledFor' in err.keyPattern)
+ );
+}
+
+/** A duplicate-key error whose conflict is the GLOBAL {capacitySlot} cap index. */
+function isCapacitySlotConflict(error: unknown): boolean {
+ const err = error as DuplicateKeyError;
+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'capacitySlot' in err.keyPattern;
+}
+
+/** A duplicate-key error whose conflict is the per-user {user, slot} cap index. */
+function isSlotConflict(error: unknown): boolean {
+ const err = error as DuplicateKeyError;
+ return err?.code === DUPLICATE_KEY && err.keyPattern != null && 'slot' in err.keyPattern;
+}
+
+/** A duplicate-key error whose conflict is the per-user create-idempotency index. */
+function isClientRequestConflict(error: unknown): boolean {
+ const err = error as DuplicateKeyError;
+ return (
+ err?.code === DUPLICATE_KEY && err.keyPattern != null && 'clientRequestId' in err.keyPattern
+ );
+}
+
+export interface ClaimDueScheduleParams {
+ instanceId: string;
+ leaseMs: number;
+}
+
+export interface RecordRunOutcomeParams {
+ scheduleId: string;
+ scheduledFor: Date;
+ status: Extract<
+ ScheduleRunStatus,
+ 'success' | 'error' | 'requires_action' | 'interrupted' | 'skipped_balance' | 'skipped_overlap'
+ >;
+ conversationId?: string;
+ /** Erase the run row's RESERVED conversationId in the same terminal write: a
+ * pre-start abort reserved an id but never created the conversation, and any
+ * recovery replay that reads the row would otherwise project a dead link. */
+ clearConversationId?: boolean;
+ error?: string;
+ durationMs?: number;
+ autoDisableAfterFailures: number;
+ /** Consecutive-balance-skip auto-disable threshold; required to settle a run as
+ * `skipped_balance` (a mid-generation balance refusal), ignored otherwise. */
+ balanceSkipDisableThreshold?: number;
+ /**
+ * RECOVERY REPLAY of a pause, from an actor holding a possibly-stale row snapshot:
+ * apply the transition only while no FRESH resume claim owns the row — that is, the
+ * row carries no `resumeClaimedAt`, or one older than this cutoff. Without the fence,
+ * two clustered sweepers observing the same unprojected pause race: the first projects
+ * it, the owner's approval then claims a fresh slot (`markRunResumeClaimed` sets
+ * `started` + `resumeClaimedAt` in ONE write), and the second still passes its
+ * snapshot-based hand-off check and unsets the capacity slot and claim stamp out from
+ * under the running continuation.
+ *
+ * A CUTOFF rather than a mere existence check, because the stamp outlives its meaning:
+ * a worker that dies between claiming and resuming leaves it set forever, and rejecting
+ * on existence alone would strand that row `started` — holding its capacity slot, with
+ * its approval unresumable (`markRunResumeClaimed` only matches `requires_action`) —
+ * which is the very state this replay exists to recover. Pass the same staleness bound
+ * the caller's own in-flight check uses so the two agree.
+ *
+ * Never set by the generation owner: its own re-pause legitimately clears the stamp as
+ * the hand-off's completion signal.
+ */
+ resumeClaimStaleBefore?: Date;
+}
+
+/** Result of claiming/leasing a schedule: the snapshot plus the fencing token to carry. */
+export interface ScheduleClaim {
+ schedule: ISchedule;
+ claimToken: string;
+}
+
+/** Outcome of reserving the single-active-run slot for a fired occurrence. */
+export type StartedRunReservation =
+ | { run: IScheduleRun }
+ | {
+ conflict: 'duplicate' | 'overlap' | 'slot-taken';
+ /** For a `duplicate`, the status of the row that already holds this occurrence.
+ * A TERMINAL status means the occurrence is finished and merely never advanced
+ * past; an active one means another worker is still running it. */
+ existingStatus?: ScheduleRunStatus;
+ };
+
+/** Outcome of promoting a paused occurrence back into the capacity-consuming
+ * `started` set. Both conflicts are database-arbitrated by the same partial
+ * unique indexes used for a first fire. */
+export type ResumedRunReservation =
+ | { capacitySlot: number }
+ | { conflict: 'not-paused' | 'overlap' | 'slot-taken' };
+
+export type ScheduleMethods = {
+ ensureScheduleIndexes: () => Promise<void>;
+ createSchedule: (data: Partial<ISchedule>) => Promise<ISchedule>;
+ createScheduleWithSlot: (
+ data: Partial<ISchedule>,
+ maxPerUser: number,
+ ) => Promise<ISchedule | 'limit'>;
+ updateScheduleById: (
+ id: string,
+ userId: string | Types.ObjectId,
+ update: Partial<ISchedule>,
+ unset?: Record<string, 1>,
+ options?: { expectedConfigRevision?: number },
+ ) => Promise<ISchedule | null>;
+ deleteScheduleById: (id: string, userId: string | Types.ObjectId) => Promise<boolean>;
+ deleteUnarmedSchedule: (
+ id: string,
+ userId: string | Types.ObjectId,
+ expectedConfigRevision?: number,
+ ) => Promise<'deleted' | 'draining' | 'kept' | 'missing'>;
+ getScheduleById: (id: string, userId?: string | Types.ObjectId) => Promise<ISchedule | null>;
+ getSchedulesByUser: (userId: string | Types.ObjectId) => Promise<ISchedule[]>;
+ countSchedulesByUser: (userId: string | Types.ObjectId) => Promise<number>;
+ claimDueSchedule: (params: ClaimDueScheduleParams) => Promise<ISchedule | null>;
+ acquireManualRunLease: (
+ id: string,
+ userId: string | Types.ObjectId,
+ leaseMs: number,
+ ) => Promise<ISchedule | null>;
+ acquireResumeLease: (
+ id: string,
+ expectedConfigRevision: number | undefined,
+ requireEnabled: boolean,
+ leaseMs: number,
+ ) => Promise<ISchedule | null>;
+ consumeResumeLease: (
+ id: string,
+ expectedClaimToken: string,
+ expectedLeaseBy: string,
+ requireEnabled: boolean,
+ expectedConfigRevision?: number,
+ ) => Promise<boolean>;
+ releaseLease: (id: string, expectedClaimToken?: string) => Promise<boolean>;
+ releaseLeaseByHolder: (id: string, leaseBy: string) => Promise<void>;
+ revalidateClaim: (id: string, claimToken: string, requireEnabled?: boolean) => Promise<boolean>;
+ advanceSchedule: (
+ id: string,
+ nextRunAt: Date | null,
+ expectedNextRunAt?: Date | null,
+ expectedClaimToken?: string,
+ ) => Promise<boolean>;
+ disableSchedule: (
+ id: string,
+ reason: ScheduleDisabledReason,
+ expectedClaimToken?: string,
+ expectedConfigRevision?: number,
+ /** Extra filter pinning the counter the disable DECISION was made on, so a
+ * concurrent outcome that reset the streak invalidates this stale write. */
+ counterGuard?: Record<string, unknown>,
+ ) => Promise<void>;
+ insertScheduleRun: (data: Partial<IScheduleRun>) => Promise<IScheduleRun | null>;
+ reserveStartedRun: (data: Partial<IScheduleRun>) => Promise<StartedRunReservation>;
+ getCapacityOccupancy: () => Promise<{ takenSlots: number[]; unslotted: number }>;
+ requestRunAbort: (
+ scheduleId: string,
+ scheduledFor: Date,
+ source?: IScheduleRun['abortSource'],
+ ) => Promise<boolean | 'in_progress'>;
+ getScheduleRunAbortState: (
+ scheduleId: string,
+ scheduledFor: Date,
+ ) => Promise<Pick<
+ IScheduleRun,
+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'
+ > | null>;
+ markRunResumeClaimed: (
+ scheduleId: string,
+ scheduledFor: Date,
+ capacitySlot: number,
+ ) => Promise<ResumedRunReservation>;
+ releaseRunResumeClaim: (
+ scheduleId: string,
+ scheduledFor: Date,
+ capacitySlot: number,
+ ) => Promise<boolean>;
+ markRunAbortPersisted: (scheduleId: string, scheduledFor: Date) => Promise<void>;
+ setRunFireDetails: (
+ scheduleId: string,
+ scheduledFor: Date,
+ details: { conversationId: string; droppedFileIds?: string[] },
+ ) => Promise<void>;
+ countActiveRuns: () => Promise<number>;
+ deleteScheduleRun: (
+ scheduleId: string,
+ scheduledFor: Date,
+ expectedStatus?: ScheduleRunStatus,
+ expectedConversationId?: string,
+ ) => Promise<void>;
+ markScheduleDeleting: (id: string, userId: string | Types.ObjectId) => Promise<ISchedule | null>;
+ getActiveRunsForSchedule: (scheduleId: string) => Promise<IScheduleRun[]>;
+ getActiveRunsForUser: (userId: string | Types.ObjectId) => Promise<IScheduleRun[]>;
+ suspendUserSchedulesForDeletion: (
+ userId: string | Types.ObjectId,
+ token: string,
+ ) => Promise<void>;
+ restoreUserSchedulesFromDeletion: (
+ userId: string | Types.ObjectId,
+ token: string,
+ ) => Promise<void>;
+ getDeletingSchedules: (limit: number) => Promise<ISchedule[]>;
+ markEraseAttempted: (ids: string[]) => Promise<void>;
+ getScheduleByClientRequestId: (
+ userId: string | Types.ObjectId,
+ clientRequestId: string,
+ ) => Promise<ISchedule | null>;
+ getDeletingScheduleIds: (userId: string | Types.ObjectId, limit: number) => Promise<string[]>;
+ getUnarmedSchedules: (limit: number) => Promise<ISchedule[]>;
+ armSchedule: (id: string, nextRunAt: Date, expectedConfigRevision?: number) => Promise<boolean>;
+ eraseScheduleIfDrained: (id: string) => Promise<boolean>;
+ deleteSchedulesByUser: (userId: string | Types.ObjectId) => Promise<void>;
+ getUnbookkeptRuns: (olderThan: Date, limit: number) => Promise<IScheduleRun[]>;
+ finalizeBookkeeping: (params: RecordRunOutcomeParams) => Promise<void>;
+ recordRunOutcome: (params: RecordRunOutcomeParams) => Promise<void>;
+ recordSkippedRun: (
+ data: Partial<IScheduleRun> & {
+ scheduleId: string;
+ scheduledFor: Date;
+ status: Extract<ScheduleRunStatus, 'skipped_overlap' | 'skipped_balance'>;
+ },
+ balanceSkipDisableThreshold?: number,
+ ) => Promise<void>;
+ getRunsForReconciliation: (olderThan: Date, limit: number) => Promise<IScheduleRun[]>;
+ markRunsReconciled: (runs: Array<Pick<IScheduleRun, '_id'>>) => Promise<void>;
+};
+
+export function createScheduleMethods(mongoose: typeof import('mongoose')): ScheduleMethods {
+ const Schedule = () => mongoose.models.Schedule as Model<IScheduleDocument>;
+ const ScheduleRun = () => mongoose.models.ScheduleRun as Model<IScheduleRunDocument>;
+
+ /**
+ * Explicitly builds the Schedule/ScheduleRun indexes. Required because the
+ * standard production setting `MONGO_AUTO_INDEX=` (empty) disables Mongoose's
+ * automatic index creation — without this the unique idempotency index and the
+ * TTL retention index would never exist. Called once before the engine starts.
+ */
+ async function ensureScheduleIndexes(): Promise<void> {
+ await createIndexesWithRetry(Schedule());
+ await createIndexesWithRetry(ScheduleRun());
+ }
+
+ async function createSchedule(data: Partial<ISchedule>): Promise<ISchedule> {
+ const doc = await Schedule().create(data);
+ return doc.toObject();
+ }
+
+ /**
+ * Creates a schedule in the lowest free per-user slot, enforcing maxPerUser
+ * ATOMICALLY: the {user, slot} partial unique index is the sole arbiter, so
+ * concurrent creates that pick the same slot collide (duplicate key) and one
+ * retries the next free slot — no read-then-count window, no drift. Returns
+ * 'limit' when all slots in [0, maxPerUser) are held by the user's live schedules.
+ */
+ async function createScheduleWithSlot(
+ data: Partial<ISchedule>,
+ maxPerUser: number,
+ ): Promise<ISchedule | 'limit'> {
+ const userId = data.user;
+ // Bound the retries by maxPerUser+1: each collision advances to a distinct
+ // slot, so a caller can lose at most (slots occupied) races before it either
+ // wins a free slot or finds every slot taken.
+ for (let attempt = 0; attempt <= maxPerUser; attempt++) {
+ const used = await Schedule()
+ .find({ user: userId, deleting: { $ne: true } })
+ .select('slot')
+ .lean<Array<{ slot?: number }>>();
+ const taken = new Set(
+ used.map((s) => s.slot).filter((s): s is number => typeof s === 'number'),
+ );
+ if (taken.size >= maxPerUser) {
+ return 'limit';
+ }
+ let slot = 0;
+ while (taken.has(slot)) {
+ slot++;
+ }
+ if (slot >= maxPerUser) {
+ return 'limit';
+ }
+ try {
+ const doc = await Schedule().create({ ...data, slot, deleting: false });
+ return doc.toObject();
+ } catch (error) {
+ if (isSlotConflict(error)) {
+ continue;
+ }
+ // A RETRY of a create whose first attempt already committed. Hand back the
+ // original row so the caller re-arms and reports that one, instead of minting a
+ // second recurring schedule for a single user intent.
+ if (isClientRequestConflict(error) && data.clientRequestId) {
+ const existing = await Schedule()
+ .findOne({ user: userId, clientRequestId: data.clientRequestId })
+ .lean<ISchedule>();
+ if (existing != null) {
+ return existing;
+ }
+ }
+ throw error;
+ }
+ }
+ return 'limit';
+ }
+
+ /**
+ * Owner edit. Rotates `claimToken` on every update so an in-flight engine claim
+ * (which captured the prior token) is fenced off: its disable/advance/release
+ * writes and its pre-dispatch revalidation no-op, and it cannot fire an edited
+ * or re-enabled schedule. Delete removes the row, so those writes no-op too.
+ */
+ async function updateScheduleById(
+ id: string,
+ userId: string | Types.ObjectId,
+ update: Partial<ISchedule>,
+ unset?: Record<string, 1>,
+ /** Optional edit-generation fence, so two overlapping owner edits cannot each
+ * persist state derived from a row the other has already replaced. */
+ options?: { expectedConfigRevision?: number },
+ ): Promise<ISchedule | null> {
+ const filter = {
+ id,
+ user: userId,
+ deleting: { $ne: true },
+ ...(options?.expectedConfigRevision !== undefined
+ ? { configRevision: options.expectedConfigRevision }
+ : {}),
+ };
+ const mutation = {
+ $set: { ...update, claimToken: randomUUID() },
+ // The ONLY writer of configRevision: an owner edit moves the config
+ // generation forward atomically with the claim-token rotation, so a run
+ // that started under the old config can detect it and skip bookkeeping.
+ // Worker/policy writes (claim, lease, advance, disable, bookkeeping) never
+ // bump it, and deletion deliberately does not either — a draining run must
+ // still be able to record its outcome before erasure.
+ $inc: { configRevision: 1 },
+ };
+ // FIRST branch: if the card still shows a pause, drop that projection in the SAME
+ // atomic write as the revision bump. A `requires_action` card present at edit time
+ // was projected by a run that captured the pre-edit revision; its terminal outcome
+ // is revision-fenced (projectLastRun), so once this bump lands it can never replace
+ // the card — and a disabling edit means no future occurrence will either, leaving it
+ // stuck on "Needs approval" for a dead action. The clear is fenced on the projection
+ // STILL being that pause (not a stale handler pre-read), so a terminal outcome or a
+ // newer occurrence that raced in fails this branch and is preserved by the second.
+ // DocumentDB rules out a conditional pipeline `$unset`, so this is a classic-operator
+ // CAS branch rather than one update.
+ const cleared = await Schedule()
+ .findOneAndUpdate(
+ { ...filter, 'lastRun.status': 'requires_action' },
+ { ...mutation, $unset: { lastRun: 1, ...(unset ?? {}) } },
+ { new: true },
+ )
+ .lean<ISchedule>();
+ if (cleared != null) {
+ return cleared;
+ }
+ // SECOND branch: no dead pause to clear (terminal history survives untouched), or the
+ // row is gone / a concurrent edit moved the revision — a null here is the true
+ // gone/conflict signal the caller distinguishes.
+ return Schedule()
+ .findOneAndUpdate(filter, { ...mutation, ...(unset ? { $unset: unset } : {}) }, { new: true })
+ .lean<ISchedule>();
+ }
+
+ async function deleteScheduleById(id: string, userId: string | Types.ObjectId): Promise<boolean> {
+ const result = await Schedule().deleteOne({ id, user: userId });
+ if (result.deletedCount > 0) {
+ await ScheduleRun().deleteMany({ scheduleId: id });
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Rolls back a fresh create whose arming write failed — but ONLY while the row is
+ * still the unarmed, unedited revision this attempt inserted. An
+ * ambiguously-committed arm, or a concurrent create-retry that armed the row and
+ * already answered 201, leaves an ARMED row this rollback must not erase; a
+ * concurrent PATCH that edited the row while leaving it unarmed (e.g. disabling
+ * it) moved the revision, and deleting it would erase the owner's edit. 'kept'
+ * tells the caller the row survived in some later state.
+ *
+ * The claim is a SOFT delete, never a hard one: an unarmed row is visible in the
+ * owner's list and Run Now-able, so a live manual generation can already hold its
+ * lease and an active run row — hard-deleting would erase a billed generation's
+ * evidence out from under it. The erase runs only once drained (no active run, no
+ * live lease); otherwise 'draining' hands the hidden row to the ordinary
+ * erase-on-settle / sweep teardown.
+ */
+ async function deleteUnarmedSchedule(
+ id: string,
+ userId: string | Types.ObjectId,
+ expectedConfigRevision?: number,
+ ): Promise<'deleted' | 'draining' | 'kept' | 'missing'> {
+ const claimed = await Schedule()
+ .findOneAndUpdate(
+ {
+ id,
+ user: userId,
+ deleting: { $ne: true },
+ nextRunAt: { $exists: false },
+ ...(expectedConfigRevision != null ? { configRevision: expectedConfigRevision } : {}),
+ },
+ { $set: { enabled: false, deleting: true, claimToken: randomUUID() } },
+ { new: true },
+ )
+ .select('_id')
+ .lean();
+ if (claimed == null) {
+ const still = await Schedule()
+ .findOne({ id, user: userId })
+ .select('deleting')
+ .lean<Pick<ISchedule, 'deleting'>>();
+ if (still == null) {
+ return 'missing';
+ }
+ return still.deleting === true ? 'draining' : 'kept';
+ }
+ const erased = await eraseScheduleIfDrained(id);
+ return erased ? 'deleted' : 'draining';
+ }
+
+ async function getScheduleById(
+ id: string,
+ userId?: string | Types.ObjectId,
+ ): Promise<ISchedule | null> {
+ const filter: Record<string, unknown> = { id, deleting: { $ne: true } };
+ if (userId != null) {
+ filter.user = userId;
+ }
+ return Schedule().findOne(filter).lean<ISchedule>();
+ }
+
+ async function getSchedulesByUser(userId: string | Types.ObjectId): Promise<ISchedule[]> {
+ // Hide schedules pending erasure (soft-deleted, draining their active runs)
+ // so a deleted schedule disappears immediately for the owner.
+ return Schedule()
+ .find({ user: userId, deleting: { $ne: true } })
+ .sort({ updatedAt: -1 })
+ .select('-leaseUntil -leaseBy -claimToken')
+ .lean<ISchedule[]>();
+ }
+
+ async function countSchedulesByUser(userId: string | Types.ObjectId): Promise<number> {
+ return Schedule().countDocuments({ user: userId, deleting: { $ne: true } });
+ }
+
+ /**
+ * Atomically claims one due schedule by taking a lease. The per-document CAS
+ * is the sole multi-instance dispatch arbiter: exactly one caller wins each
+ * due schedule regardless of replica count, with or without Redis. Stamps a
+ * fresh `claimToken` the winner carries through every subsequent write.
+ *
+ * Due-ness and lease expiry compare against THIS worker's clock. DocumentDB
+ * supports neither pipeline-form updates nor `$$NOW` (see
+ * misc/documentdb/documentdb-compat.md), so the server-clock CAS is not
+ * expressible portably; the CAS itself still arbitrates every race, and the
+ * exposure is bounded by inter-replica clock skew (NTP-order) against a
+ * 5-minute lease and per-minute cadence granularity. `{ $lte }` matches
+ * neither missing nor null `nextRunAt`, so unarmed rows stay unclaimable;
+ * a missing `leaseUntil` is claimable via the `$or`.
+ */
+ async function claimDueSchedule(params: ClaimDueScheduleParams): Promise<ISchedule | null> {
+ const claimToken = randomUUID();
+ const now = new Date();
+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);
+ return Schedule()
+ .findOneAndUpdate(
+ {
+ enabled: true,
+ deleting: { $ne: true },
+ nextRunAt: { $lte: now },
+ $or: [
+ { leaseUntil: { $exists: false } },
+ { leaseUntil: null },
+ { leaseUntil: { $lt: takeoverCutoff } },
+ ],
+ },
+ {
+ $set: {
+ leaseUntil: new Date(now.getTime() + params.leaseMs),
+ leaseBy: params.instanceId,
+ claimToken,
+ },
+ },
+ { new: true, sort: { nextRunAt: 1 } },
+ )
+ .lean<ISchedule>();
+ }
+
+ /**
+ * Takes the schedule's lease for a manual run-now, serializing concurrent
+ * `POST /:id/run` requests (and blocking against an engine claim) so a
+ * double-click can't start two runs. Owner-scoped. Returns the FRESH schedule row
+ * (post-image, with the new claim token) so the caller fires the current snapshot
+ * — an edit that committed after the route read the schedule but before this lease
+ * is reflected here, not the stale pre-edit prompt/agent. Null if already leased.
+ */
+ async function acquireManualRunLease(
+ id: string,
+ userId: string | Types.ObjectId,
+ leaseMs: number,
+ ): Promise<ISchedule | null> {
+ // Same worker-clock CAS shape as claimDueSchedule (DocumentDB rules out `$$NOW`
+ // and pipeline updates); the CAS still serializes concurrent run-now clicks, and
+ // clock skew only shifts WHEN an expired lease becomes re-acquirable.
+ const claimToken = randomUUID();
+ // A UNIQUE per-lease holder (not the constant 'manual'): the superseded-fire
+ // cleanup releases by holder (leaseBy), so a stale run-now that stalled past its
+ // lease must not match — and strip — the fresh lease a newer run-now acquired.
+ // The claimToken already fences the lease, so reuse it as the holder discriminator.
+ const leaseBy = `manual:${claimToken}`;
+ const now = new Date();
+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);
+ return Schedule()
+ .findOneAndUpdate(
+ {
+ id,
+ user: userId,
+ deleting: { $ne: true },
+ $or: [
+ { leaseUntil: { $exists: false } },
+ { leaseUntil: null },
+ { leaseUntil: { $lt: takeoverCutoff } },
+ ],
+ },
+ { $set: { leaseUntil: new Date(now.getTime() + leaseMs), leaseBy, claimToken } },
+ { new: true },
+ )
+ .lean<ISchedule>();
+ }
+
+ /**
+ * Takes a short-lived schedule-document fence for a paused run's resume. The
+ * approval and its ScheduleRun row live in different stores, so this claim is
+ * carried through the approval CAS and atomically consumed immediately after it.
+ * An owner edit rotates claimToken; a policy disable clears the lease/enabled bit.
+ * Either therefore makes consumeResumeLease fail before provider execution starts.
+ */
+ async function acquireResumeLease(
+ id: string,
+ expectedConfigRevision: number | undefined,
+ requireEnabled: boolean,
+ leaseMs: number,
+ ): Promise<ISchedule | null> {
+ const claimToken = randomUUID();
+ const leaseBy = `resume:${claimToken}`;
+ const now = new Date();
+ const takeoverCutoff = new Date(now.getTime() - LEASE_SKEW_MARGIN_MS);
+ return Schedule()
+ .findOneAndUpdate(
+ {
+ id,
+ deleting: { $ne: true },
+ ...(requireEnabled ? { enabled: true } : {}),
+ ...(expectedConfigRevision !== undefined
+ ? { configRevision: expectedConfigRevision }
+ : {}),
+ $or: [
+ { leaseUntil: { $exists: false } },
+ { leaseUntil: null },
+ { leaseUntil: { $lt: takeoverCutoff } },
+ ],
+ },
+ {
+ $set: {
+ leaseUntil: new Date(now.getTime() + leaseMs),
+ leaseBy,
+ claimToken,
+ },
+ },
+ { new: true },
+ )
+ .lean<ISchedule>();
+ }
+
+ /**
+ * Atomically linearizes a scheduled resume against owner edits, deletion, policy
+ * disable, lease expiry, and takeover, then releases the short-lived fence. A
+ * successful match means the approval was consumed under the same live schedule
+ * generation that acquired the fence; a miss must stop before provider execution.
+ */
+ async function consumeResumeLease(
+ id: string,
+ expectedClaimToken: string,
+ expectedLeaseBy: string,
+ requireEnabled: boolean,
+ expectedConfigRevision?: number,
+ ): Promise<boolean> {
+ const result = await Schedule().updateOne(
+ {
+ id,
+ claimToken: expectedClaimToken,
+ leaseBy: expectedLeaseBy,
+ deleting: { $ne: true },
+ leaseUntil: { $gt: new Date() },
+ ...(requireEnabled ? { enabled: true } : {}),
+ ...(expectedConfigRevision !== undefined ? { configRevision: expectedConfigRevision } : {}),
+ },
+ { $unset: { leaseUntil: 1, leaseBy: 1 } },
+ );
+ return (result.matchedCount ?? 0) > 0;
+ }
+
+ /**
+ * Releases a lease WITHOUT advancing nextRunAt (manual runs never reschedule).
+ * Fenced on the claim token when provided so a stale worker cannot strip a lease
+ * a different claimer now holds.
+ */
+ async function releaseLease(id: string, expectedClaimToken?: string): Promise<boolean> {
+ const filter: Record<string, unknown> = { id };
+ if (expectedClaimToken !== undefined) {
+ filter.claimToken = expectedClaimToken;
+ }
+ const result = await Schedule().updateOne(filter, { $unset: { leaseUntil: 1, leaseBy: 1 } });
+ return (result.matchedCount ?? 0) > 0;
+ }
+
+ /**
+ * Releases a lease fenced on the lease HOLDER (`leaseBy`) rather than the claim
+ * token. Used when a fire is superseded by an owner edit that rotated the token
+ * (so a token-fenced release would no-op): the worker still owns the lease, so it
+ * must clear it — otherwise the edited schedule (and Run now) is reported "already
+ * in progress" until the lease TTL, even though no run was dispatched. A takeover
+ * changed `leaseBy`, so this correctly no-ops and never strips the new holder's lease.
+ */
+ async function releaseLeaseByHolder(id: string, leaseBy: string): Promise<void> {
+ await Schedule().updateOne({ id, leaseBy }, { $unset: { leaseUntil: 1, leaseBy: 1 } });
+ }
+
+ /**
+ * Whether the caller still holds an authoritative claim on the schedule: it is
+ * not being deleted, its claim token is unchanged, and its lease has not expired
+ * (Mongo `$$NOW`). Called as the last check before the loopback POST so an owner
+ * delete/edit or a lease-expiry re-claim between claim and fire aborts the fire
+ * instead of dispatching a stale occurrence. `requireEnabled` additionally checks
+ * `enabled` (automatic fires must stop once disabled); a manual run-now passes
+ * false since the user explicitly triggered it, but delete/edit still fence it.
+ */
+ async function revalidateClaim(
+ id: string,
+ claimToken: string,
+ requireEnabled = true,
+ ): Promise<boolean> {
+ // Worker clock (DocumentDB-portable): `$gt` matches neither a missing nor a null
+ // leaseUntil, so a released lease reads as invalid — the fail-closed direction.
+ const row = await Schedule()
+ .findOne({
+ id,
+ claimToken,
+ deleting: { $ne: true },
+ ...(requireEnabled ? { enabled: true } : {}),
+ leaseUntil: { $gt: new Date() },
+ })
+ .select('_id')
+ .lean();
+ return row != null;
+ }
+
+ /**
+ * Advances past a fired (or skipped) occurrence and releases the lease. When
+ * `expectedNextRunAt` is given, the update is predicated on the schedule still
+ * sitting on the claimed occurrence; when `expectedClaimToken` is given it is
+ * additionally fenced on the claim, so a stale worker (whose lease expired and
+ * was re-claimed, or whose schedule the owner edited) cannot clobber a newer
+ * claimer's nextRunAt/lease. A predicate miss simply no-ops the stale write.
+ */
+ async function advanceSchedule(
+ id: string,
+ nextRunAt: Date | null,
+ expectedNextRunAt?: Date | null,
+ expectedClaimToken?: string,
+ ): Promise<boolean> {
+ const filter: Record<string, unknown> = { id };
+ if (expectedNextRunAt !== undefined) {
+ filter.nextRunAt = expectedNextRunAt;
+ }
+ if (expectedClaimToken !== undefined) {
+ filter.claimToken = expectedClaimToken;
+ }
+ const result = await Schedule().updateOne(filter, {
+ $set: { ...(nextRunAt ? { nextRunAt } : {}) },
+ $unset: { leaseUntil: 1, leaseBy: 1, ...(nextRunAt ? {} : { nextRunAt: 1 }) },
+ });
+ return (result.matchedCount ?? 0) > 0;
+ }
+
+ /**
+ * Disables a schedule. Preflight disables from the leased worker pass their
+ * `expectedClaimToken` so a stale worker cannot flip a schedule the owner just
+ * re-enabled/edited (rotating the token) back to disabled or clear a newer
+ * claimer's lease. POLICY disables (auto-disable) instead pass the config
+ * generation of the RUN that tripped the threshold: the counter update they follow
+ * is revision-fenced, so leaving the disable itself unfenced let a stale run (or a
+ * reconciler replay of one) disable a schedule the owner had since edited and
+ * re-enabled. Absent on either side disables the fence.
+ */
+ async function disableSchedule(
+ id: string,
+ reason: ScheduleDisabledReason,
+ expectedClaimToken?: string,
+ expectedConfigRevision?: number,
+ counterGuard?: Record<string, unknown>,
+ ): Promise<void> {
+ const filter: Record<string, unknown> = { id, ...counterGuard };
+ if (expectedClaimToken !== undefined) {
+ filter.claimToken = expectedClaimToken;
+ }
+ if (expectedConfigRevision !== undefined) {
+ filter.configRevision = expectedConfigRevision;
+ }
+ await Schedule().updateOne(filter, {
+ $set: { enabled: false, disabledReason: reason },
+ $unset: { leaseUntil: 1, leaseBy: 1 },
+ });
+ }
+
+ /**
+ * Inserts the run row BEFORE firing. The unique {scheduleId, scheduledFor}
+ * index makes this the durable idempotency claim: null means this occurrence
+ * was already fired (or is in flight) by another claimer or a prior life.
+ * Used for non-`started` rows (skips); started rows go through reserveStartedRun.
+ */
+ async function insertScheduleRun(data: Partial<IScheduleRun>): Promise<IScheduleRun | null> {
+ try {
+ const doc = await ScheduleRun().create(data);
+ return doc.toObject();
+ } catch (error) {
+ if ((error as { code?: number }).code === DUPLICATE_KEY) {
+ return null;
+ }
+ throw error;
+ }
+ }
+
+ /**
+ * Reserves the single active-run slot for a fired occurrence by inserting a
+ * `started` run. Two distinct duplicate-key outcomes: the {scheduleId,
+ * scheduledFor} index means this occurrence already fired ('duplicate'); the
+ * single-active partial index means another occurrence of the same schedule is
+ * already active ('overlap'). The DB enforces both atomically — no read-then-insert.
+ */
+ async function reserveStartedRun(data: Partial<IScheduleRun>): Promise<StartedRunReservation> {
+ try {
+ const doc = await ScheduleRun().create({ ...data, status: 'started' });
+ return { run: doc.toObject() };
+ } catch (error) {
+ if (isOccurrenceDuplicate(error)) {
+ // Report the EXISTING row's status. A duplicate means another fire owns this
+ // occurrence — but "owns" and "already finished" need opposite handling, and the
+ // caller cannot distinguish them without this.
+ const existing = await ScheduleRun()
+ .findOne({ scheduleId: data.scheduleId, scheduledFor: data.scheduledFor })
+ .select('status')
+ .lean<Pick<IScheduleRun, 'status'>>();
+ return { conflict: 'duplicate', existingStatus: existing?.status };
+ }
+ // Checked BEFORE overlap: the global cap index and the per-schedule active
+ // index are different failures and drive different caller behavior (retry the
+ // next slot vs skip the occurrence).
+ if (isCapacitySlotConflict(error)) {
+ return { conflict: 'slot-taken' };
+ }
+ if (isActiveRunConflict(error)) {
+ return { conflict: 'overlap' };
+ }
+ throw error;
+ }
+ }
+
+ /** Capacity-slot occupancy for the allocator: which slots are held by `started`
+ * runs, plus how many legacy rows hold no slot (they shrink the effective cap so
+ * the bound stays conservative during rollout rather than transiently overshooting). */
+ async function getCapacityOccupancy(): Promise<{ takenSlots: number[]; unslotted: number }> {
+ const rows = await ScheduleRun()
+ .find({ status: 'started' })
+ .select('capacitySlot')
+ .lean<Array<{ capacitySlot?: number }>>();
+ const takenSlots: number[] = [];
+ let unslotted = 0;
+ for (const row of rows) {
+ if (typeof row.capacitySlot === 'number') {
+ takenSlots.push(row.capacitySlot);
+ } else {
+ unslotted += 1;
+ }
+ }
+ return { takenSlots, unslotted };
+ }
+
+ /**
+ * Records that an abort was requested WITHOUT freeing the capacity slot: the run
+ * keeps counting against fireConcurrency until its generation owner confirms
+ * settlement by writing a terminal outcome.
+ *
+ * The stamp is RENEWED per attempt, not first-wins: the fences read
+ * `abortRequestedAt` for freshness, and a retry abort signalled against a live
+ * generation must re-arm them — a 31-minute-old stamp from a failed first attempt
+ * must not let a drain read the retry's post-abort job state as settled. Callers
+ * therefore only invoke this against a generation they observed LIVE (or unknown);
+ * stamping cleanup aborts of already-terminal jobs would re-fence a dead owner
+ * forever and block the reconciler's recovery.
+ *
+ * Source semantics: a 'stop' attempt takes the source and re-opens the persistence
+ * marker (the Stop route persists writes the owner's barrier must wait on, and each
+ * attempt resolves it anew); a 'deletion' request renews freshness but never steals
+ * the source from an UNRESOLVED stop attempt — the route may still be persisting,
+ * and the owner's barrier keys off `abortSource === 'stop'`.
+ * Classic operators only (DocumentDB).
+ */
+ async function requestRunAbort(
+ scheduleId: string,
+ scheduledFor: Date,
+ source?: IScheduleRun['abortSource'],
+ ): Promise<boolean | 'in_progress'> {
+ const now = new Date();
+ if (source === 'stop') {
+ // SERIALIZED per run: only one unresolved fresh stop attempt may exist. A second
+ // Stop stamping over a live one would re-open the persistence marker, and its
+ // own lost-CAS resolution would then release the settlement barrier while the
+ // FIRST attempt (the terminal-CAS winner) was still pruning and saving. The
+ // guarded update is the arbiter; a loser answers 'in_progress' and must neither
+ // signal the abort nor resolve anything. A stale unresolved attempt (its route
+ // presumed dead) is claimable again.
+ const staleCutoff = new Date(now.getTime() - ABORT_STAMP_STALE_MS);
+ const stamped = await ScheduleRun().updateOne(
+ {
+ scheduleId,
+ scheduledFor,
+ status: { $in: ACTIVE_RUN_STATUSES },
+ $nor: [
+ {
+ abortSource: 'stop',
+ abortPersistedAt: { $exists: false },
+ abortRequestedAt: { $gt: staleCutoff },
+ },
+ ],
+ },
+ { $set: { abortRequestedAt: now, abortSource: 'stop' }, $unset: { abortPersistedAt: 1 } },
+ );
+ if ((stamped.matchedCount ?? 0) > 0) {
+ return true;
+ }
+ const holder = await ScheduleRun()
+ .findOne({ scheduleId, scheduledFor, status: { $in: ACTIVE_RUN_STATUSES } })
+ .select('_id')
+ .lean();
+ return holder != null ? 'in_progress' : false;
+ }
+ const claimed = await ScheduleRun().updateOne(
+ {
+ scheduleId,
+ scheduledFor,
+ status: { $in: ACTIVE_RUN_STATUSES },
+ $or: [{ abortSource: { $ne: 'stop' } }, { abortPersistedAt: { $exists: true } }],
+ },
+ { $set: { abortRequestedAt: now, ...(source ? { abortSource: source } : {}) } },
+ );
+ if ((claimed.matchedCount ?? 0) > 0) {
+ return true;
+ }
+ // An unresolved stop attempt holds the source; renew freshness only.
+ const renewed = await ScheduleRun().updateOne(
+ { scheduleId, scheduledFor, status: { $in: ACTIVE_RUN_STATUSES } },
+ { $set: { abortRequestedAt: now } },
+ );
+ return (renewed.matchedCount ?? 0) > 0;
+ }
+
+ /** The abort-coordination view of a run row, for the generation owner's settlement
+ * barrier (see abortPersistedAt). */
+ async function getScheduleRunAbortState(
+ scheduleId: string,
+ scheduledFor: Date,
+ ): Promise<Pick<
+ IScheduleRun,
+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'
+ > | null> {
+ return ScheduleRun()
+ .findOne({ scheduleId, scheduledFor })
+ .select('status abortRequestedAt abortSource abortPersistedAt')
+ .lean<Pick<
+ IScheduleRun,
+ 'status' | 'abortRequestedAt' | 'abortSource' | 'abortPersistedAt'
+ > | null>();
+ }
+
+ /**
+ * Stamps a paused run as RESUME-CLAIMED: its approval was consumed and a
+ * continuation is running. While fresh, a re-paused job's state is a hand-off in
+ * flight (segment writes still landing), not settleable evidence. The pause
+ * record ($unset below) is the completion signal; a crash leaves the stamp to
+ * age out on the caller's staleness bound.
+ */
+ async function markRunResumeClaimed(
+ scheduleId: string,
+ scheduledFor: Date,
+ capacitySlot: number,
+ ): Promise<ResumedRunReservation> {
+ try {
+ const claimed = await ScheduleRun().findOneAndUpdate(
+ { scheduleId, scheduledFor, status: 'requires_action' },
+ {
+ $set: {
+ status: 'started',
+ capacitySlot,
+ resumeClaimedAt: new Date(),
+ },
+ },
+ { new: true, timestamps: false },
+ );
+ return claimed == null ? { conflict: 'not-paused' } : { capacitySlot };
+ } catch (error) {
+ if (isCapacitySlotConflict(error)) {
+ return { conflict: 'slot-taken' };
+ }
+ if (isActiveRunConflict(error)) {
+ return { conflict: 'overlap' };
+ }
+ throw error;
+ }
+ }
+
+ /** Roll back a resume reservation only while the exact slot this caller claimed
+ * still belongs to this occurrence. The approval job is checked separately by the
+ * controller before crossing this seam, so a committed/racing resume never has its
+ * capacity released by a losing request. */
+ async function releaseRunResumeClaim(
+ scheduleId: string,
+ scheduledFor: Date,
+ capacitySlot: number,
+ ): Promise<boolean> {
+ const released = await ScheduleRun().updateOne(
+ { scheduleId, scheduledFor, status: 'started', capacitySlot },
+ {
+ $set: { status: 'requires_action' },
+ $unset: { capacitySlot: 1, resumeClaimedAt: 1 },
+ },
+ { timestamps: false },
+ );
+ return (released.modifiedCount ?? 0) > 0;
+ }
+
+ /** Stamped by the interactive Stop route once ALL its writes (checkpoint prune,
+ * partial-response save) have landed; releases the generation owner's settlement
+ * barrier. Bookkeeping only: never touches `updatedAt`. */
+ async function markRunAbortPersisted(scheduleId: string, scheduledFor: Date): Promise<void> {
+ await ScheduleRun().updateOne(
+ { scheduleId, scheduledFor },
+ { $set: { abortPersistedAt: new Date() } },
+ { timestamps: false },
+ );
+ }
+
+ /** Count of in-flight scheduled runs (across all schedules) for the fire cap. */
+ async function countActiveRuns(): Promise<number> {
+ return ScheduleRun().countDocuments({ status: 'started' });
+ }
+
+ /**
+ * Writes an occurrence's result onto the schedule card, fenced on occurrence order
+ * so a delayed outcome can never overwrite a NEWER occurrence's projection. The
+ * fence is absent-tolerant: rows written before `lastRun.scheduledFor` existed
+ * project once and then start ordering normally.
+ */
+ async function projectLastRun(
+ scheduleId: string,
+ lastRun: NonNullable<ISchedule['lastRun']>,
+ scheduledFor: Date,
+ revisionFilter: Record<string, unknown> = {},
+ ): Promise<void> {
+ // A PAUSE is the only non-terminal projection, and it is the one that can still
+ // arrive late for its own occurrence: the row transition is won first, and a resume
+ // can terminalize and project between that CAS and this write. Occurrence ordering
+ // alone permits it (same `scheduledFor`), so refuse to walk this occurrence's card
+ // backwards from a settled result to "Needs approval" — which nothing would correct
+ // until the next occurrence ran.
+ const notAlreadySettled =
+ lastRun.status === 'requires_action'
+ ? {
+ $nor: [
+ {
+ $and: [
+ { 'lastRun.scheduledFor': scheduledFor },
+ { 'lastRun.status': { $in: TERMINAL_CARD_STATUSES } },
+ ],
+ },
+ ],
+ }
+ : {};
+ await Schedule().updateOne(
+ {
+ id: scheduleId,
+ $or: [
+ { 'lastRun.scheduledFor': { $exists: false } },
+ { 'lastRun.scheduledFor': { $lte: scheduledFor } },
+ ],
+ ...notAlreadySettled,
+ ...revisionFilter,
+ },
+ { $set: { lastRun: { ...lastRun, scheduledFor } } },
+ // Run-state bookkeeping, not a config edit: re-affirmed pauses would
+ // otherwise bump updatedAt (and reorder updated-time listings) every
+ // reconciliation pass for as long as an approval sits waiting.
+ { timestamps: false },
+ );
+ }
+
+ /**
+ * Applies the schedule-side bookkeeping (lastRun + counters + auto-disable) for
+ * a terminal occurrence. Idempotent via the per-occurrence `countedFor` guard:
+ * the $inc lands at most once per occurrence no matter how many times it is
+ * retried (inline finish, reconciler catch of an un-`bookkept` run, crash-replay),
+ * even when a later occurrence's counting interleaves with an earlier paused one.
+ */
+ async function applyTerminalBookkeeping(
+ params: RecordRunOutcomeParams & { firedAt: Date; expectConfigRevision?: number },
+ ): Promise<void> {
+ const lastRun = {
+ conversationId: params.conversationId,
+ status: params.status,
+ error: params.error,
+ firedAt: params.firedAt,
+ };
+ const isFailure = params.status === 'error';
+ const isSuccess = params.status === 'success';
+ // The COUNT is idempotent per occurrence: `countedFor` is a bounded set of
+ // recently-counted occurrence timestamps, so an interleaved earlier occurrence
+ // can't clear this one's marker (a single scalar could). Everything that must
+ // land atomically WITH the count goes in this one update so a crash can't leave
+ // it half-applied: the balance-skip streak resets on ANY non-balance outcome,
+ // and a success clears the failure streak inline (never a lost follow-up).
+ // CONFIG-REVISION FENCE: a run that started under an older owner config must not
+ // apply counters (or walk toward auto-disable) against a schedule the owner has
+ // since edited or re-enabled. Absent on either side disables the fence, so
+ // pre-existing rows/schedules keep today's behavior instead of wedging.
+ const revisionFilter =
+ params.expectConfigRevision != null ? { configRevision: params.expectConfigRevision } : {};
+ // ORDER FENCE, the same one the card projection has always had. `failureCount` and
+ // `balanceSkipCount` are CONSECUTIVE streaks, so they are order-sensitive in a way
+ // per-occurrence idempotency (`countedFor`) cannot express: a `requires_action` run
+ // does not block later occurrences, so an older occurrence — a resumed pause, a
+ // reconciler replay — routinely settles AFTER a newer one. Unfenced, its late `error`
+ // rebuilds a streak a newer success had already cleared, and since
+ // `autoDisableAfterFailures` may be configured as low as 1, a single stale failure
+ // can disable a healthy schedule.
+ //
+ // The fence covers the WHOLE counter update rather than the streak fields alone:
+ // splitting it would break the atomicity that keeps a crash from half-applying the
+ // count, and DocumentDB rules out expressing it as a pipeline update. The cost is
+ // that a late occurrence's success does not add to `runCount` — a display total
+ // under-counting a rare out-of-order run, which is the cheap side of this trade.
+ // Absent-tolerant, so schedules written before the watermark existed count once and
+ // then order normally.
+ const counterOrderFilter = {
+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],
+ };
+ await Schedule().updateOne(
+ {
+ id: params.scheduleId,
+ countedFor: { $ne: params.scheduledFor },
+ ...revisionFilter,
+ ...counterOrderFilter,
+ },
+ {
+ $set: {
+ balanceSkipCount: 0,
+ countersAsOf: params.scheduledFor,
+ ...(isSuccess ? { failureCount: 0 } : {}),
+ },
+ $push: { countedFor: { $each: [params.scheduledFor], $slice: -COUNTED_FOR_WINDOW } },
+ ...(isSuccess ? { $inc: { runCount: 1 } } : {}),
+ ...(isFailure ? { $inc: { failureCount: 1 } } : {}),
+ },
+ );
+ // The card projection is ORDER-fenced, unlike the counters above (idempotent and
+ // order-insensitive): a `requires_action` run doesn't block later occurrences, so a
+ // resumed pause — or a reconciler replay — routinely settles AFTER a newer run
+ // already finished, and must not roll the card back to its older result.
+ await projectLastRun(params.scheduleId, lastRun, params.scheduledFor, revisionFilter);
+ // A success clears a transient disable reason ONLY while the schedule is still
+ // enabled. An older run (e.g. a resumed pause) can succeed AFTER newer outcomes
+ // already auto-disabled the schedule — since `requires_action` runs don't block
+ // later occurrences — and must not wipe the reason that explains why it's off.
+ // Predicated on `enabled` separately so it can't leak into the count-guarded
+ // update above (which must run regardless of enabled state).
+ if (isSuccess) {
+ await Schedule().updateOne(
+ { id: params.scheduleId, enabled: true },
+ { $unset: { disabledReason: 1 } },
+ );
+ }
+ // Auto-disable is a POLICY re-evaluated on EVERY call (idempotent), NOT gated
+ // on the count guard — so if a crash landed the $inc but not the disable, the
+ // reconciler's replay still disables. Reads current state after the count.
+ if (isFailure) {
+ const schedule = await Schedule().findOne({ id: params.scheduleId }).lean<ISchedule>();
+ if (schedule?.enabled && schedule.failureCount >= params.autoDisableAfterFailures) {
+ // Carry the COUNT this decision was made on, not just the revision. The read
+ // above and this write are separate statements, and a concurrent success resets
+ // the failure streak to zero — a revision-only fence would still let this stale
+ // decision disable a schedule whose streak had just been cleared.
+ await disableSchedule(
+ params.scheduleId,
+ 'too_many_failures',
+ undefined,
+ params.expectConfigRevision,
+ { failureCount: { $gte: params.autoDisableAfterFailures } },
+ );
+ }
+ }
+ }
+
+ /**
+ * Balance-skip streak bookkeeping, shared by the pre-fire skip (recordSkippedRun)
+ * and a mid-generation balance refusal settling a `started` row (recordRunOutcome
+ * with `skipped_balance`): projects the card, walks the consecutive-balance-skip
+ * streak (per-occurrence idempotent via `countedFor`, order-fenced via
+ * `countersAsOf`), and applies the insufficient_balance auto-disable policy.
+ */
+ async function applyBalanceSkipBookkeeping(params: {
+ scheduleId: string;
+ scheduledFor: Date;
+ firedAt: Date;
+ conversationId?: string;
+ rowRevision?: number;
+ balanceSkipDisableThreshold?: number;
+ }): Promise<void> {
+ const revisionFilter = params.rowRevision != null ? { configRevision: params.rowRevision } : {};
+ await projectLastRun(
+ params.scheduleId,
+ {
+ ...(params.conversationId ? { conversationId: params.conversationId } : {}),
+ status: 'skipped_balance',
+ firedAt: params.firedAt,
+ },
+ params.scheduledFor,
+ revisionFilter,
+ );
+ if (params.balanceSkipDisableThreshold == null) {
+ return;
+ }
+ // Same order fence as the terminal counters: these are consecutive streaks, so an
+ // older occurrence settling late must not reset or extend one a newer outcome owns.
+ const orderFilter = {
+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],
+ };
+ // Per-occurrence guard: increment the streak at most once for this occurrence even
+ // across crash retries (same `countedFor` set the terminal counters use).
+ const schedule = await Schedule()
+ .findOneAndUpdate(
+ {
+ id: params.scheduleId,
+ countedFor: { $ne: params.scheduledFor },
+ ...revisionFilter,
+ ...orderFilter,
+ },
+ {
+ $inc: { balanceSkipCount: 1 },
+ $set: { countersAsOf: params.scheduledFor },
+ $push: { countedFor: { $each: [params.scheduledFor], $slice: -COUNTED_FOR_WINDOW } },
+ },
+ { new: true },
+ )
+ .lean<ISchedule>();
+ // Auto-disable is a POLICY re-evaluated on EVERY call (idempotent), NOT gated on
+ // the count guard — if a crash landed the $inc but not the disable, the replay's
+ // guarded update no-ops to null, so re-read and still disable at/over threshold.
+ const current =
+ schedule ?? (await Schedule().findOne({ id: params.scheduleId }).lean<ISchedule>());
+ if (current?.enabled && current.balanceSkipCount >= params.balanceSkipDisableThreshold) {
+ // Same read-then-write window as the failure policy: any non-balance outcome
+ // resets this streak, so the write must carry the count it decided on.
+ await disableSchedule(
+ params.scheduleId,
+ 'insufficient_balance',
+ undefined,
+ params.rowRevision,
+ { balanceSkipCount: { $gte: params.balanceSkipDisableThreshold } },
+ );
+ }
+ }
+
+ /**
+ * Overlap-skip bookkeeping, shared by the pre-fire skip (recordSkippedRun) and its
+ * crash-retry replay (finalizeBookkeeping): projects the skip onto the card and
+ * BREAKS the consecutive-balance-skip streak (an overlap is an intervening
+ * non-balance outcome). Both writes are idempotent and order-fenced, so replays
+ * are safe.
+ */
+ async function applyOverlapSkipBookkeeping(params: {
+ scheduleId: string;
+ scheduledFor: Date;
+ firedAt: Date;
+ rowRevision?: number;
+ }): Promise<void> {
+ const revisionFilter = params.rowRevision != null ? { configRevision: params.rowRevision } : {};
+ // Through the ORDERED projection: writing `lastRun` directly dropped the
+ // `scheduledFor` marker, after which the next projection read the marker as absent
+ // and let an older occurrence's outcome overwrite this newer skip.
+ await projectLastRun(
+ params.scheduleId,
+ { status: 'skipped_overlap', firedAt: params.firedAt },
+ params.scheduledFor,
+ revisionFilter,
+ );
+ // Same order fence as the terminal counters: these are consecutive streaks, so an
+ // older occurrence settling late must not reset or extend one a newer outcome owns.
+ const orderFilter = {
+ $or: [{ countersAsOf: { $exists: false } }, { countersAsOf: { $lte: params.scheduledFor } }],
+ };
+ await Schedule().updateOne(
+ { id: params.scheduleId, ...revisionFilter, ...orderFilter },
+ { $set: { balanceSkipCount: 0, countersAsOf: params.scheduledFor } },
+ );
+ }
+
+ /**
+ * Terminal (or pause) transition for a run + lastRun/failure bookkeeping.
+ * Matches a run row still in `started` OR `requires_action`. Crash-retryable:
+ * the run row is marked `bookkept:false` at terminalization and only flipped
+ * to `true` after bookkeeping lands, so a crash in between is re-applied by the
+ * reconciler (`getUnbookkeptRuns`), while `countedFor` keeps it idempotent.
+ * A `skipped_balance` settlement (a mid-generation balance refusal) takes the same
+ * guarded row transition but walks the balance-skip streak instead of the failure
+ * streak — the owner's credits, not the schedule, are at fault.
+ */
+ async function recordRunOutcome(params: RecordRunOutcomeParams): Promise<void> {
+ const firedAt = new Date();
+ if (params.status === 'requires_action') {
+ // PAUSE (HITL): win the ROW transition first, then project the card. A read-then-
+ // write guard let a concurrent resume terminalize the run between the two, after
+ // which the card was pinned to "Needs approval" forever — the row update no-op'd
+ // (already terminal) while the card write had landed. Matching `requires_action`
+ // too keeps a retried pause re-affirming the card, and keeps the transition
+ // crash-recoverable: the reconciler re-invokes this for a row in either state.
+ // The match is also the anti-spoof guard a bare scheduleId would bypass.
+ const paused = await ScheduleRun().findOneAndUpdate(
+ {
+ scheduleId: params.scheduleId,
+ scheduledFor: params.scheduledFor,
+ status: { $in: ['started', 'requires_action'] },
+ // See resumeClaimStaleBefore: fences a stale-snapshot recovery replay against a
+ // resume that claimed the row after the snapshot was taken, while still letting
+ // an ABANDONED claim (its worker died mid-hand-off) be recovered.
+ ...(params.resumeClaimStaleBefore != null
+ ? {
+ $or: [
+ { resumeClaimedAt: { $exists: false } },
+ { resumeClaimedAt: { $lt: params.resumeClaimStaleBefore } },
+ ],
+ }
+ : {}),
+ },
+ {
+ $set: {
+ status: 'requires_action',
+ ...(params.conversationId ? { conversationId: params.conversationId } : {}),
+ },
+ // Leaving `started` frees the global capacity slot; the resume claims a
+ // fresh one from the allocator rather than re-adopting a possibly-taken slot.
+ // The pause record is also the resume hand-off's COMPLETION signal: the
+ // re-pause branch persists its segment before recording, so clearing the
+ // claim stamp here re-opens the run to quiesce settling.
+ $unset: { capacitySlot: 1, resumeClaimedAt: 1 },
+ },
+ { new: false },
+ );
+ if (paused == null) {
+ return;
+ }
+ // Revision-fenced like the terminal path: an owner edit landing between the fire
+ // and the approval must not have the OLD config's pause stamped on it. Without
+ // this the later terminal write (which IS fenced) could never replace the stale
+ // status, so the card stuck on "Needs approval".
+ // The ROW's fire time, not the projection time: reconciliation re-affirms a
+ // long-lived pause every pass, and a fresh stamp per pass walked the card's
+ // timestamp forward for as long as the approval sat waiting.
+ await projectLastRun(
+ params.scheduleId,
+ {
+ conversationId: params.conversationId,
+ status: params.status,
+ firedAt: paused.firedAt ?? firedAt,
+ },
+ params.scheduledFor,
+ paused.configRevision != null ? { configRevision: paused.configRevision } : {},
+ );
+ return;
+ }
+ // TERMINAL: flip the run row (match-guarded), then apply bookkeeping. `bookkept` is
+ // set false at the flip and true only after bookkeeping lands, so a crash between is
+ // re-applied by the reconciler (getUnbookkeptRuns) while countedFor keeps counters idempotent.
+ const settled = await ScheduleRun()
+ .findOneAndUpdate(
+ {
+ scheduleId: params.scheduleId,
+ scheduledFor: params.scheduledFor,
+ status: { $in: ['started', 'requires_action'] },
+ },
+ {
+ $set: {
+ status: params.status,
+ bookkept: false,
+ settledAt: firedAt,
+ ...(params.conversationId && !params.clearConversationId
+ ? { conversationId: params.conversationId }
+ : {}),
+ ...(params.error ? { error: params.error } : {}),
+ ...(params.durationMs != null ? { durationMs: params.durationMs } : {}),
+ },
+ // SETTLEMENT: a terminal outcome is the generation owner confirming the run
+ // actually stopped, so this is the ONLY place the global capacity slot is
+ // released. An abort request alone does not free it (see requestRunAbort).
+ $unset: { capacitySlot: 1, ...(params.clearConversationId ? { conversationId: 1 } : {}) },
+ },
+ { new: false },
+ )
+ .lean<IScheduleRun>();
+ // No-match guard: never touch schedule bookkeeping without a matching run
+ // (protects against a spoofed scheduleId on a normal chat).
+ if (settled == null) {
+ return;
+ }
+ // SINGLE SEAM: the config fence is DERIVED here from the row being settled, not
+ // passed in by each caller. Callers only say "this occurrence reached status X" and
+ // structurally cannot forget a token — which is exactly how the reconcile and
+ // balance-skip paths previously shipped unfenced.
+ if (params.status === 'skipped_balance') {
+ await applyBalanceSkipBookkeeping({
+ scheduleId: params.scheduleId,
+ scheduledFor: params.scheduledFor,
+ firedAt: settled.firedAt ?? firedAt,
+ conversationId: params.conversationId,
+ rowRevision: settled.configRevision,
+ balanceSkipDisableThreshold: params.balanceSkipDisableThreshold,
+ });
+ } else {
+ await applyTerminalBookkeeping({
+ ...params,
+ firedAt: settled.firedAt ?? firedAt,
+ expectConfigRevision: settled.configRevision,
+ });
+ }
+ await ScheduleRun().updateOne(
+ { scheduleId: params.scheduleId, scheduledFor: params.scheduledFor },
+ { $set: { bookkept: true } },
+ );
+ }
+
+ async function recordSkippedRun(
+ data: Partial<IScheduleRun> & {
+ scheduleId: string;
+ scheduledFor: Date;
+ status: Extract<ScheduleRunStatus, 'skipped_overlap' | 'skipped_balance'>;
+ },
+ balanceSkipDisableThreshold?: number,
+ ): Promise<void> {
+ const firedAt = new Date();
+ // A duplicate {scheduleId, scheduledFor} row means a prior attempt inserted it.
+ // Proceed as a retry ONLY when that row is the SAME skip — otherwise this claim
+ // is a retry of an occurrence that actually started/terminalized (e.g. the POST
+ // was accepted but advanceSchedule failed before releasing the lease), and
+ // rewriting lastRun/counters would mislabel a real run as a skip (and could
+ // walk it toward auto-disable). The streak $inc is separately guarded per
+ // occurrence by `countedFor`, so a genuine same-skip retry can't double-count.
+ // `bookkept: false` until the schedule-side writes land: a crash in between is
+ // replayed by the reconciler (getUnbookkeptRuns), exactly like a terminal run —
+ // a duplicate claim of this occurrence advances past the terminal row without
+ // re-running bookkeeping, so nothing else would ever repair a half-applied skip.
+ const inserted = await insertScheduleRun({
+ ...data,
+ firedAt,
+ settledAt: firedAt,
+ bookkept: false,
+ });
+ let rowRevision = inserted?.configRevision;
+ let rowFiredAt = firedAt;
+ if (inserted == null) {
+ const existing = await ScheduleRun()
+ .findOne({ scheduleId: data.scheduleId, scheduledFor: data.scheduledFor })
+ .select('status configRevision firedAt')
+ .lean<Pick<IScheduleRun, 'status' | 'configRevision' | 'firedAt'>>();
+ if (existing == null || existing.status !== data.status) {
+ return;
+ }
+ rowRevision = existing.configRevision;
+ // A retry re-projects the ORIGINAL occurrence, not the retry moment.
+ rowFiredAt = existing.firedAt ?? firedAt;
+ }
+ // Same single seam as recordRunOutcome: the config fence is DERIVED from the row,
+ // never passed by the caller. A skip decided under an older owner config must not
+ // stamp the card (or walk the balance streak toward auto-disable) on a schedule the
+ // owner has since edited. Absent on either side disables the fence.
+ if (data.status === 'skipped_balance') {
+ await applyBalanceSkipBookkeeping({
+ scheduleId: data.scheduleId,
+ scheduledFor: data.scheduledFor,
+ firedAt: rowFiredAt,
+ rowRevision,
+ balanceSkipDisableThreshold,
+ });
+ } else {
+ await applyOverlapSkipBookkeeping({
+ scheduleId: data.scheduleId,
+ scheduledFor: data.scheduledFor,
+ firedAt: rowFiredAt,
+ rowRevision,
+ });
+ }
+ await ScheduleRun().updateOne(
+ { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor },
+ { $set: { bookkept: true } },
+ );
+ }
+
+ async function setRunFireDetails(
+ scheduleId: string,
+ scheduledFor: Date,
+ details: { conversationId: string; droppedFileIds?: string[] },
+ ): Promise<void> {
+ await ScheduleRun().updateOne(
+ { scheduleId, scheduledFor },
+ {
+ $set: {
+ conversationId: details.conversationId,
+ ...(details.droppedFileIds?.length ? { droppedFileIds: details.droppedFileIds } : {}),
+ },
+ },
+ );
+ }
+
+ /**
+ * Non-terminal runs old enough to need a job-store status check. Fetches
+ * `started` (capacity-consuming) and `requires_action` (paused) in separate
+ * budgeted, firedAt-ordered buckets so a backlog of long-lived paused rows
+ * can't starve orphaned `started` runs out of every sweep.
+ */
+ async function getRunsForReconciliation(olderThan: Date, limit: number): Promise<IScheduleRun[]> {
+ const [started, paused] = await Promise.all([
+ // `started` runs are bounded by the global fireConcurrency cap, so this window
+ // can never fill with rows that have nothing to do — oldest-first is right here.
+ ScheduleRun()
+ .find({ status: 'started', firedAt: { $lt: olderThan } })
+ .sort({ firedAt: 1 })
+ .limit(limit)
+ .lean<IScheduleRun[]>(),
+ // ROUND-ROBIN, not oldest-first. A paused run holds no capacity slot and does not
+ // block its schedule's later occurrences, so pauses accumulate without bound — a
+ // handful of hourly schedules can leave hundreds live at once, and the approval
+ // TTL that ends them is operator-configured with no ceiling. Under oldest-first
+ // the same full window of still-live pauses is re-fetched every tick while an
+ // ABANDONED row behind them is never reached, so its run stays active forever.
+ // Ordering by when each row was last examined (never-examined first) gives every
+ // row a turn, which makes that starvation impossible for any number of pauses.
+ ScheduleRun()
+ .find({ status: 'requires_action', firedAt: { $lt: olderThan } })
+ .sort({ reconciledAt: 1, firedAt: 1 })
+ .limit(limit)
+ .lean<IScheduleRun[]>(),
+ ]);
+ return [...started, ...paused];
+ }
+
+ /** Stamps rows as examined, so the paused window rotates instead of re-serving the
+ * same rows forever. Bookkeeping only: never touches `updatedAt`. */
+ async function markRunsReconciled(runs: Array<Pick<IScheduleRun, '_id'>>): Promise<void> {
+ const ids = runs.map((run) => run._id).filter((id) => id != null);
+ if (ids.length === 0) {
+ return;
+ }
+ await ScheduleRun().updateMany(
+ { _id: { $in: ids } },
+ { $set: { reconciledAt: new Date() } },
+ { timestamps: false },
+ );
+ }
+
+ /** Settled runs (terminal outcomes AND skips) whose schedule bookkeeping never
+ * landed — a crash between the row write and the card/counter writes.
+ * Least-recently-attempted first (the paused window's rotation): a row whose
+ * replay keeps failing must rotate to the back, or a batch of them re-fills
+ * this bounded window forever and every later row's counters never land. */
+ async function getUnbookkeptRuns(olderThan: Date, limit: number): Promise<IScheduleRun[]> {
+ return ScheduleRun()
+ .find({
+ status: {
+ $in: ['success', 'error', 'interrupted', 'skipped_balance', 'skipped_overlap'],
+ },
+ bookkept: false,
+ firedAt: { $lt: olderThan },
+ })
+ .sort({ reconciledAt: 1, firedAt: 1 })
+ .limit(limit)
+ .lean<IScheduleRun[]>();
+ }
+
+ /** Re-applies (idempotent) bookkeeping for a terminal run and marks it bookkept. */
+ async function finalizeBookkeeping(params: RecordRunOutcomeParams): Promise<void> {
+ // Same single seam as recordRunOutcome: derive the config fence from the row, so the
+ // crash-retry path cannot apply bookkeeping the inline path would have refused.
+ // The row's ORIGINAL firedAt is reused too: this replay runs minutes (or many
+ // retries) after the fact, and stamping the replay time onto lastRun would keep
+ // walking the card's timestamp forward every time the recovery path fires.
+ const run = await ScheduleRun()
+ .findOne({ scheduleId: params.scheduleId, scheduledFor: params.scheduledFor })
+ .select('configRevision firedAt')
+ .lean<Pick<IScheduleRun, 'configRevision' | 'firedAt'>>();
+ const firedAt = run?.firedAt ?? new Date();
+ if (params.status === 'skipped_balance') {
+ await applyBalanceSkipBookkeeping({
+ scheduleId: params.scheduleId,
+ scheduledFor: params.scheduledFor,
+ firedAt,
+ conversationId: params.conversationId,
+ rowRevision: run?.configRevision,
+ balanceSkipDisableThreshold: params.balanceSkipDisableThreshold,
+ });
+ } else if (params.status === 'skipped_overlap') {
+ await applyOverlapSkipBookkeeping({
+ scheduleId: params.scheduleId,
+ scheduledFor: params.scheduledFor,
+ firedAt,
+ rowRevision: run?.configRevision,
+ });
+ } else {
+ await applyTerminalBookkeeping({
+ ...params,
+ firedAt,
+ expectConfigRevision: run?.configRevision,
+ });
+ }
+ await ScheduleRun().updateOne(
+ { scheduleId: params.scheduleId, scheduledFor: params.scheduledFor },
+ { $set: { bookkept: true } },
+ );
+ }
+
+ /**
+ * Deletes a run row (used to roll back a capacity reservation). Status-fenced
+ * when `expectedStatus` is provided so a rollback cannot delete a row a
+ * concurrent process already advanced (e.g. to a terminal outcome). Fenced
+ * additionally on `expectedConversationId` so a fire only ever deletes the
+ * reservation IT inserted.
+ */
+ async function deleteScheduleRun(
+ scheduleId: string,
+ scheduledFor: Date,
+ expectedStatus?: ScheduleRunStatus,
+ expectedConversationId?: string,
+ ): Promise<void> {
+ const filter: Record<string, unknown> = { scheduleId, scheduledFor };
+ if (expectedStatus !== undefined) {
+ filter.status = expectedStatus;
+ }
+ if (expectedConversationId !== undefined) {
+ filter.conversationId = expectedConversationId;
+ }
+ await ScheduleRun().deleteOne(filter);
+ }
+
+ /**
+ * Soft-deletes a schedule for the owner: disables it (so the engine can no
+ * longer claim it), rotates the claim token (fencing any in-flight worker), and
+ * marks it `deleting` so it is hidden and awaits erasure once its active runs
+ * drain. Returns the updated row (for aborting its in-flight jobs) or null.
+ */
+ async function markScheduleDeleting(
+ id: string,
+ userId: string | Types.ObjectId,
+ ): Promise<ISchedule | null> {
+ // Keep leaseUntil/leaseBy so a fire that already leased this occurrence still
+ // serializes against a re-claim while it unwinds. Only clear nextRunAt
+ // (belt-and-suspenders atop enabled:false to stop new claims); the lease
+ // releases itself when the fire finishes, or via TTL.
+ //
+ // IDEMPOTENT: deliberately NOT gated on `deleting: { $ne: true }`. The mark is the
+ // first of several steps (read active runs, abort their jobs, prune checkpoints,
+ // erase when drained), and if any of those threw — or the process exited — a
+ // one-shot mark made every retry answer 404, stranding the schedule with its job
+ // and checkpoint alive. Re-marking an already-deleting schedule is harmless (the
+ // fields are already at these values; the token rotates again, which only re-fences
+ // stale workers), and it lets the caller re-drive the rest of the teardown.
+ return Schedule()
+ .findOneAndUpdate(
+ { id, user: userId },
+ {
+ $set: { enabled: false, deleting: true, claimToken: randomUUID() },
+ $unset: { nextRunAt: 1 },
+ },
+ { new: true },
+ )
+ .lean<ISchedule>();
+ }
+
+ /** In-flight (non-terminal) runs of a schedule — the jobs a delete must abort. */
+ async function getActiveRunsForSchedule(scheduleId: string): Promise<IScheduleRun[]> {
+ return ScheduleRun()
+ .find({ scheduleId, status: { $in: ACTIVE_RUN_STATUSES } })
+ .lean<IScheduleRun[]>();
+ }
+
+ /** In-flight runs across all of a user's schedules — for account-deletion quiescing. */
+ async function getActiveRunsForUser(userId: string | Types.ObjectId): Promise<IScheduleRun[]> {
+ return ScheduleRun()
+ .find({ user: userId, status: { $in: ACTIVE_RUN_STATUSES } })
+ .lean<IScheduleRun[]>();
+ }
+
+ /**
+ * Marks all of a user's schedules non-claimable ahead of account deletion, so
+ * the engine cannot fire a new occurrence while the cascade runs. Rotates each
+ * claim token to fence any in-flight worker.
+ */
+ /**
+ * REVERSIBLE quiesce ahead of account deletion. Snapshots each schedule's pre-suspension
+ * enabled/next-run state under this attempt's `token`, then fences firing (disable, clear
+ * `nextRunAt`, rotate `claimToken`). It deliberately does NOT set `deleting`: a suspended
+ * row must not be erased, because a deletion the controller later cancels has to restore
+ * it. On a successful deletion the hard-delete cascade removes the row (and this snapshot)
+ * anyway. Snapshotting needs each row's current values, which a classic (DocumentDB-safe)
+ * update cannot copy, so it reads then `bulkWrite`s — fenced per row so a row already
+ * suspended by this token, soft-deleted, or edited out from under the read is left alone.
+ */
+ async function suspendUserSchedulesForDeletion(
+ userId: string | Types.ObjectId,
+ token: string,
+ ): Promise<void> {
+ const schedules = await Schedule()
+ .find({
+ user: userId,
+ deleting: { $ne: true },
+ 'deletionSuspension.token': { $ne: token },
+ })
+ .select('id enabled nextRunAt deletionSuspension')
+ .lean<Array<Pick<ISchedule, 'id' | 'enabled' | 'nextRunAt' | 'deletionSuspension'>>>();
+ if (schedules.length === 0) {
+ return;
+ }
+ const ops = schedules.map((schedule) => {
+ // A row already suspended by an ABANDONED earlier attempt (its process died before
+ // restoring) still holds the only record of the true pre-suspension state. Carry that
+ // snapshot forward and merely take ownership with this attempt's token — re-reading
+ // the row's CURRENT values would snapshot the suspended state itself (disabled, no
+ // next run) and permanently strand the schedule when this attempt later restores.
+ const existing = schedule.deletionSuspension;
+ const snapshotEnabled =
+ existing != null ? existing.enabled === true : schedule.enabled === true;
+ const snapshotNextRunAt = existing != null ? existing.nextRunAt : schedule.nextRunAt;
+ return {
+ updateOne: {
+ filter: {
+ id: schedule.id,
+ user: userId,
+ deleting: { $ne: true },
+ 'deletionSuspension.token': { $ne: token },
+ },
+ update: {
+ $set: {
+ enabled: false,
+ claimToken: randomUUID(),
+ deletionSuspension: {
+ token,
+ enabled: snapshotEnabled,
+ ...(snapshotNextRunAt != null ? { nextRunAt: snapshotNextRunAt } : {}),
+ },
+ },
+ $unset: { nextRunAt: 1 as const },
+ },
+ },
+ };
+ });
+ await tenantSafeBulkWrite(Schedule(), ops as AnyBulkWriteOperation[], { ordered: false });
+ }
+
+ const RESTORE_ATTEMPTS = 3;
+
+ /**
+ * Reverses {@link suspendUserSchedulesForDeletion} when a deletion attempt is cancelled.
+ * Restores enabled/next-run from the snapshot and clears the suspension — but ONLY for
+ * rows still carrying this exact attempt's token and not independently soft-deleted, so a
+ * schedule the owner deleted (or a newer attempt re-suspended) is never resurrected.
+ *
+ * RETRIED, because its callers cannot retry it later. A cancelled deletion restores while
+ * the user-deletion fence is still armed and then releases that fence; once released,
+ * nothing re-drives this, and a single transient failure would strand the owner's live
+ * account with silently disabled, next-run-less schedules. Retrying is safe: every attempt
+ * re-reads only the rows STILL carrying this token, so a partially-applied unordered write
+ * converges on exactly the stragglers, and a fully-applied one finds nothing and returns.
+ * Throws the last failure so callers still surface it.
+ */
+ async function restoreUserSchedulesFromDeletion(
+ userId: string | Types.ObjectId,
+ token: string,
+ ): Promise<void> {
+ let lastError: unknown;
+ for (let attempt = 1; attempt <= RESTORE_ATTEMPTS; attempt++) {
+ try {
+ await restoreSuspendedBatch(userId, token);
+ return;
+ } catch (error) {
+ lastError = error;
+ if (attempt < RESTORE_ATTEMPTS) {
+ await new Promise((resolve) => setTimeout(resolve, 100 * attempt));
+ }
+ }
+ }
+ throw lastError;
+ }
+
+ async function restoreSuspendedBatch(
+ userId: string | Types.ObjectId,
+ token: string,
+ ): Promise<void> {
+ const suspended = await Schedule()
+ .find({ user: userId, 'deletionSuspension.token': token, deleting: { $ne: true } })
+ .select('id deletionSuspension')
+ .lean<Array<Pick<ISchedule, 'id' | 'deletionSuspension'>>>();
+ if (suspended.length === 0) {
+ return;
+ }
+ const ops = suspended.map((schedule) => {
+ const snapshot = schedule.deletionSuspension;
+ const restoreNextRunAt = snapshot?.nextRunAt;
+ return {
+ updateOne: {
+ filter: {
+ id: schedule.id,
+ user: userId,
+ 'deletionSuspension.token': token,
+ deleting: { $ne: true },
+ },
+ update: {
+ $set: {
+ enabled: snapshot?.enabled === true,
+ claimToken: randomUUID(),
+ ...(restoreNextRunAt != null ? { nextRunAt: restoreNextRunAt } : {}),
+ },
+ $unset: {
+ deletionSuspension: 1 as const,
+ ...(restoreNextRunAt == null ? { nextRunAt: 1 as const } : {}),
+ },
+ },
+ },
+ };
+ });
+ await tenantSafeBulkWrite(Schedule(), ops as AnyBulkWriteOperation[], { ordered: false });
+ }
+
+ /**
+ * Enabled schedules with no `nextRunAt`. Creation arms in a second write, so a crash
+ * or a failed arm leaves this state — and `claimDueSchedule` sorts on `nextRunAt`, so
+ * the row is permanently inert while still occupying the owner's slot.
+ */
+ async function getUnarmedSchedules(limit: number): Promise<ISchedule[]> {
+ return Schedule()
+ .find({ enabled: true, deleting: { $ne: true }, nextRunAt: { $exists: false } })
+ .limit(limit)
+ .lean<ISchedule[]>();
+ }
+
+ /**
+ * Arms an unarmed schedule — the ONE arming CAS every creation path shares (the
+ * original POST, its idempotent replay, and the reconciler's unarmed sweep).
+ * Conditional on still being unarmed, so concurrent armers collapse to a single
+ * write: it never rotates the claim token (which would fence an occurrence the
+ * engine already claimed off the freshly-armed row) and never bumps the config
+ * revision (which would break a concurrent PATCH's CAS). `expectedConfigRevision`
+ * additionally fences the original POST's arm against a PATCH that landed after
+ * the insert — the edit's own arming governs then. Returns whether THIS call armed.
+ */
+ async function armSchedule(
+ id: string,
+ nextRunAt: Date,
+ expectedConfigRevision?: number,
+ ): Promise<boolean> {
+ const result = await Schedule().updateOne(
+ {
+ id,
+ enabled: true,
+ deleting: { $ne: true },
+ nextRunAt: { $exists: false },
+ ...(expectedConfigRevision != null ? { configRevision: expectedConfigRevision } : {}),
+ },
+ { $set: { nextRunAt } },
+ );
+ return (result.modifiedCount ?? 0) > 0;
+ }
+
+ /** Soft-deleted schedules awaiting erasure (drained of active runs). ROUND-ROBIN
+ * on the last sweep attempt (never-attempted first): an unsorted window re-served
+ * the same first rows every pass, so a batch of undrainable rows (live runs or
+ * leases) starved every row behind it out of the sweep indefinitely. */
+ async function getDeletingSchedules(limit: number): Promise<ISchedule[]> {
+ return Schedule()
+ .find({ deleting: true, erased: { $ne: true } })
+ .sort({ eraseAttemptedAt: 1 })
+ .limit(limit)
+ .lean<ISchedule[]>();
+ }
+
+ /** Stamps deleting rows as attempted so the erasure window rotates. Bookkeeping
+ * only: never touches `updatedAt`. */
+ async function markEraseAttempted(ids: string[]): Promise<void> {
+ if (ids.length === 0) {
+ return;
+ }
+ await Schedule().updateMany(
+ { id: { $in: ids }, deleting: true },
+ { $set: { eraseAttemptedAt: new Date() } },
+ { timestamps: false },
+ );
+ }
+
+ /** Resolves the schedule a prior create attempt with this idempotency key committed,
+ * including one already soft-deleted (a retry must see its key as claimed while the
+ * row drains, mirroring the unique index, which deliberately spans deleting rows). */
+ async function getScheduleByClientRequestId(
+ userId: string | Types.ObjectId,
+ clientRequestId: string,
+ ): Promise<ISchedule | null> {
+ return Schedule().findOne({ user: userId, clientRequestId }).lean<ISchedule>();
+ }
+
+ /** Ids of an owner's soft-deleted schedules, for a lazy erase retry on a read path
+ * in topologies that run no reconciler. Projected to ids: the caller only re-drives
+ * eraseScheduleIfDrained, and the rows carry prompt text this need not load. */
+ async function getDeletingScheduleIds(
+ userId: string | Types.ObjectId,
+ limit: number,
+ ): Promise<string[]> {
+ // Least-recently-attempted first (missing sorts before any date), for the same
+ // reason the erasure sweep rotates: an unsorted `.limit()` window pins the same
+ // stuck rows forever once the owner has more deleting rows than the limit, and
+ // the ones beyond it never get their deletion re-driven. Callers stamp
+ // markEraseAttempted after each attempt to rotate the window.
+ const rows = await Schedule()
+ .find({ user: userId, deleting: true, erased: { $ne: true } })
+ .sort({ eraseAttemptedAt: 1 })
+ .select('id')
+ .limit(limit)
+ .lean<Array<Pick<ISchedule, 'id'>>>();
+ return rows.map((row) => row.id);
+ }
+
+ /**
+ * Erases a soft-deleted schedule and its runs ONLY once it has fully drained, so a
+ * live loopback generation's evidence is never destroyed out from under it. Drained
+ * means BOTH: (a) no run is active, and (b) no LIVE lease is held. The lease check
+ * is essential — a worker can have CLAIMED the schedule but not yet inserted its
+ * `started` reservation (or be mid-rollback of one); erasing in that window would
+ * let the worker then insert a ghost row against a gone schedule that it can no
+ * longer prove it owns. Returns whether it erased.
+ */
+ /** The ONLY fields an erasure tombstone keeps: replay-detection identity plus the
+ * flags that keep it inert to sweeps and resolvable by the create replay path. */
+ const TOMBSTONE_IDENTITY_FIELDS = new Set([
+ '_id',
+ '__v',
+ 'id',
+ 'user',
+ 'tenantId',
+ 'clientRequestId',
+ 'clientRequestDigest',
+ 'deleting',
+ 'erased',
+ 'erasedAt',
+ 'enabled',
+ 'createdAt',
+ 'updatedAt',
+ ]);
+
+ async function eraseScheduleIfDrained(id: string): Promise<boolean> {
+ // A live lease (leaseUntil in the future) means a worker still holds the claim.
+ // Worker clock, DocumentDB-portable: `$gt` matches neither missing nor null, so a
+ // released lease reads as drained. Skew margin: a lease reads as live until
+ // MARGIN past its expiry, so a clock-ahead erasure worker cannot destroy a row a
+ // skew-behind holder still legitimately claims.
+ const leased = await Schedule()
+ .findOne({
+ id,
+ deleting: true,
+ leaseUntil: { $gt: new Date(Date.now() - LEASE_SKEW_MARGIN_MS) },
+ })
+ .select('_id')
+ .lean();
+ if (leased != null) {
+ return false;
+ }
+ const active = await ScheduleRun()
+ .findOne({ scheduleId: id, status: { $in: ACTIVE_RUN_STATUSES } })
+ .select('_id')
+ .lean();
+ if (active != null) {
+ return false;
+ }
+ await ScheduleRun().deleteMany({ scheduleId: id });
+ // Rows carrying a create-idempotency key leave a content-free TOMBSTONE instead
+ // of vanishing: a create whose response was lost retries with the same key, and
+ // if the owner deleted the schedule before that retry arrived, a hard delete
+ // would let the retry recreate the recurring work they just removed. The
+ // tombstone keeps only the identity fields (key, digest, owner) for a bounded
+ // window (TTL on erasedAt); the replay path answers "deleted" against it.
+ // ALLOWLIST, not a blocklist: everything except the replay-detection identity
+ // is unset, derived from the live schema paths so a field added later (tools,
+ // cron, anything) cannot silently survive into the tombstone. The deletion path
+ // promises that ONLY the idempotency identity remains.
+ const contentFields = [
+ ...new Set(Object.keys(Schedule().schema.paths).map((path) => path.split('.')[0])),
+ ].filter((field) => !TOMBSTONE_IDENTITY_FIELDS.has(field));
+ const tombstoned = await Schedule().updateOne(
+ { id, deleting: true, clientRequestId: { $exists: true } },
+ {
+ $set: { erased: true, erasedAt: new Date(), enabled: false },
+ $unset: Object.fromEntries(contentFields.map((field) => [field, 1])),
+ // Not a config edit; the tombstone must not surface in updated-time listings.
+ },
+ { timestamps: false },
+ );
+ if (tombstoned.matchedCount === 0) {
+ await Schedule().deleteOne({ id, deleting: true });
+ }
+ return true;
+ }
+
+ /** Cascade for account deletion: removes a user's schedules and their runs. */
+ async function deleteSchedulesByUser(userId: string | Types.ObjectId): Promise<void> {
+ // Delete RUNS before SCHEDULES so a partial failure is retryable: both are
+ // user-scoped and idempotent, so a crash after the runs delete leaves the
+ // schedules for a retry to re-delete (deleting schedules first would orphan the
+ // runs — a re-run finds no schedules and never removes the leftover run rows).
+ await ScheduleRun().deleteMany({ user: userId });
+ await Schedule().deleteMany({ user: userId });
+ }
+
+ return {
+ ensureScheduleIndexes,
+ createSchedule,
+ createScheduleWithSlot,
+ updateScheduleById,
+ deleteScheduleById,
+ deleteUnarmedSchedule,
+ getScheduleById,
+ getSchedulesByUser,
+ countSchedulesByUser,
+ claimDueSchedule,
+ acquireManualRunLease,
+ acquireResumeLease,
+ consumeResumeLease,
+ releaseLease,
+ releaseLeaseByHolder,
+ revalidateClaim,
+ advanceSchedule,
+ disableSchedule,
+ insertScheduleRun,
+ reserveStartedRun,
+ getCapacityOccupancy,
+ requestRunAbort,
+ getScheduleRunAbortState,
+ markRunResumeClaimed,
+ releaseRunResumeClaim,
+ markRunAbortPersisted,
+ setRunFireDetails,
+ countActiveRuns,
+ deleteScheduleRun,
+ markScheduleDeleting,
+ getActiveRunsForSchedule,
+ getActiveRunsForUser,
+ suspendUserSchedulesForDeletion,
+ restoreUserSchedulesFromDeletion,
+ getDeletingSchedules,
+ markEraseAttempted,
+ getScheduleByClientRequestId,
+ getDeletingScheduleIds,
+ getUnarmedSchedules,
+ armSchedule,
+ eraseScheduleIfDrained,
+ deleteSchedulesByUser,
+ getUnbookkeptRuns,
+ finalizeBookkeeping,
+ recordRunOutcome,
+ recordSkippedRun,
+ getRunsForReconciliation,
+ markRunsReconciled,
+ };
+}
diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts
index 06d6a818f8..1da5a3eb3a 100644
--- a/packages/data-schemas/src/models/index.ts
+++ b/packages/data-schemas/src/models/index.ts
@@ -1,4 +1,5 @@
import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
+import { createScheduleModel, createScheduleRunModel } from './schedule';
import { createSkillSyncCredentialModel } from './skillSyncCredential';
import { createAgentTriggerUserPurgeModel } from './triggerUserPurge';
import { createAgentTriggerDeliveryModel } from './triggerDelivery';
@@ -84,6 +85,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
AgentTriggerDelivery: ReturnType<typeof createAgentTriggerDeliveryModel>;
AgentTriggerLaneSequence: ReturnType<typeof createAgentTriggerLaneSequenceModel>;
AgentTriggerUserPurge: ReturnType<typeof createAgentTriggerUserPurgeModel>;
+ Schedule: ReturnType<typeof createScheduleModel>;
+ ScheduleRun: ReturnType<typeof createScheduleRunModel>;
} {
const models = {
User: createUserModel(mongoose),
@@ -126,6 +129,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
AgentTriggerDelivery: createAgentTriggerDeliveryModel(mongoose),
AgentTriggerLaneSequence: createAgentTriggerLaneSequenceModel(mongoose),
AgentTriggerUserPurge: createAgentTriggerUserPurgeModel(mongoose),
+ Schedule: createScheduleModel(mongoose),
+ ScheduleRun: createScheduleRunModel(mongoose),
};
/**
* Background index builds fail silently unless an 'index' listener is
diff --git a/packages/data-schemas/src/models/schedule.ts b/packages/data-schemas/src/models/schedule.ts
new file mode 100644
index 0000000000..67672aa251
--- /dev/null
+++ b/packages/data-schemas/src/models/schedule.ts
@@ -0,0 +1,20 @@
+import { Model } from 'mongoose';
+import type { IScheduleDocument, IScheduleRunDocument } from '~/types/schedule';
+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';
+import scheduleRunSchema from '~/schema/scheduleRun';
+import scheduleSchema from '~/schema/schedule';
+
+export function createScheduleModel(mongoose: typeof import('mongoose')): Model<IScheduleDocument> {
+ applyTenantIsolation(scheduleSchema);
+ return mongoose.models.Schedule || mongoose.model<IScheduleDocument>('Schedule', scheduleSchema);
+}
+
+export function createScheduleRunModel(
+ mongoose: typeof import('mongoose'),
+): Model<IScheduleRunDocument> {
+ applyTenantIsolation(scheduleRunSchema);
+ return (
+ mongoose.models.ScheduleRun ||
+ mongoose.model<IScheduleRunDocument>('ScheduleRun', scheduleRunSchema)
+ );
+}
diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.ts
index d9171d0099..84aa7d4ec0 100644
--- a/packages/data-schemas/src/schema/index.ts
+++ b/packages/data-schemas/src/schema/index.ts
@@ -35,3 +35,5 @@ export { default as configSchema } from './config';
export { default as triggerDeliverySchema } from './triggerDelivery';
export { default as triggerLaneSequenceSchema } from './triggerLaneSequence';
export { default as triggerUserPurgeSchema } from './triggerUserPurge';
+export { default as scheduleSchema } from './schedule';
+export { default as scheduleRunSchema } from './scheduleRun';
diff --git a/packages/data-schemas/src/schema/role.ts b/packages/data-schemas/src/schema/role.ts
index c931179631..9b55d7112c 100644
--- a/packages/data-schemas/src/schema/role.ts
+++ b/packages/data-schemas/src/schema/role.ts
@@ -79,6 +79,10 @@ const rolePermissionsSchema = new Schema(
[Permissions.SHARE]: { type: Boolean },
[Permissions.SHARE_PUBLIC]: { type: Boolean },
},
+ [PermissionTypes.SCHEDULES]: {
+ [Permissions.USE]: { type: Boolean },
+ [Permissions.CREATE]: { type: Boolean },
+ },
},
{ _id: false },
);
diff --git a/packages/data-schemas/src/schema/schedule.ts b/packages/data-schemas/src/schema/schedule.ts
new file mode 100644
index 0000000000..4fabb22065
--- /dev/null
+++ b/packages/data-schemas/src/schema/schedule.ts
@@ -0,0 +1,252 @@
+import { Schema } from 'mongoose';
+import type { IScheduleDocument } from '~/types/schedule';
+
+const scheduleSchema: Schema<IScheduleDocument> = new Schema(
+ {
+ id: {
+ type: String,
+ required: true,
+ index: true,
+ },
+ user: {
+ type: Schema.Types.ObjectId,
+ ref: 'User',
+ required: true,
+ index: true,
+ },
+ tenantId: {
+ type: String,
+ index: true,
+ },
+ name: {
+ type: String,
+ required: true,
+ maxlength: 256,
+ },
+ prompt: {
+ type: String,
+ required: true,
+ maxlength: 32000,
+ },
+ agent_id: {
+ type: String,
+ required: true,
+ },
+ cadence: {
+ frequency: {
+ type: String,
+ enum: ['hourly', 'daily', 'weekdays', 'weekly'],
+ required: true,
+ },
+ hour: { type: Number, min: 0, max: 23, required: true },
+ minute: { type: Number, min: 0, max: 59, required: true },
+ daysOfWeek: { type: [Number], default: undefined },
+ },
+ timezone: {
+ type: String,
+ required: true,
+ },
+ target: {
+ type: String,
+ enum: ['new'],
+ default: 'new',
+ required: true,
+ },
+ file_ids: {
+ type: [String],
+ default: undefined,
+ },
+ tools: {
+ type: [String],
+ default: undefined,
+ },
+ cron: {
+ type: String,
+ },
+ enabled: {
+ type: Boolean,
+ default: true,
+ required: true,
+ },
+ disabledReason: {
+ type: String,
+ enum: [
+ 'too_many_failures',
+ 'agent_deleted',
+ 'invalid_schedule',
+ 'permission_revoked',
+ 'insufficient_balance',
+ ],
+ },
+ nextRunAt: {
+ type: Date,
+ },
+ leaseUntil: {
+ type: Date,
+ },
+ leaseBy: {
+ type: String,
+ },
+ /**
+ * Per-claim fencing token. Set fresh on every lease acquisition (engine or
+ * manual) and rotated on every owner edit, so a stale/expired-lease worker's
+ * writes (disable/advance/release) and its pre-dispatch revalidation no-op
+ * once the schedule was re-claimed, edited, re-enabled, or deleted.
+ */
+ claimToken: {
+ type: String,
+ },
+ /**
+ * Owner-config generation. Bumped ONLY by an owner edit (updateScheduleById),
+ * atomically with the claimToken rotation. Distinct from claimToken, which also
+ * rotates on every lease acquisition. A run captures this at claim time so its
+ * terminal bookkeeping / auto-disable cannot act on config it never ran under.
+ */
+ configRevision: {
+ type: Number,
+ default: 0,
+ min: 0,
+ },
+ /**
+ * Soft-delete marker. A delete disables + marks the schedule `deleting` (so
+ * it is hidden from the owner and never re-claimed) and aborts in-flight
+ * runs; the reconciler erases it only once no run is still active, so a live
+ * loopback generation's evidence is never destroyed out from under it.
+ * Defaults false so the per-user `slot` partial unique index covers it.
+ */
+ deleting: {
+ type: Boolean,
+ default: false,
+ },
+ /**
+ * Reversible account-deletion suspension. Set at quiesce under a per-attempt token
+ * that snapshots the pre-suspension enabled/next-run state. A deletion cancelled by a
+ * controller failure restores exactly the rows carrying its token; a successful
+ * deletion hard-deletes the row (and this snapshot with it). Distinct from `deleting`:
+ * a suspended row is NOT eligible for erasure, so a failed attempt cannot lose it.
+ */
+ deletionSuspension: {
+ type: {
+ token: { type: String, required: true },
+ enabled: { type: Boolean, required: true },
+ nextRunAt: { type: Date },
+ },
+ default: undefined,
+ _id: false,
+ },
+ /** Erased tombstone: content is gone, only the create-idempotency identity
+ * remains for a bounded retry window (TTL below). A delayed create retry that
+ * matches this key must answer "deleted", not resurrect the recurring work. */
+ erased: {
+ type: Boolean,
+ },
+ erasedAt: {
+ type: Date,
+ },
+ /**
+ * Per-user occupancy slot in [0, maxPerUser). Assigned atomically via the
+ * partial unique index below so concurrent creates cannot exceed the cap: two
+ * racers claiming the same slot collide on the index and one retries the next
+ * free slot. Freed (excluded from the index) when the schedule is `deleting`.
+ */
+ slot: {
+ type: Number,
+ min: 0,
+ },
+ /** Client-supplied idempotency key for the CREATE that produced this row, unique
+ * per user via the partial index below. Creation commits the row and arms it in
+ * two writes, so a retry after a failure between them would otherwise mint a
+ * second recurring schedule; the index makes that retry collide and resolve to
+ * this row instead. */
+ clientRequestId: {
+ type: String,
+ },
+ /** Digest of the original create payload, stamped at insert and never edited, so a
+ * retry is matched against what was actually requested rather than a row a
+ * concurrent PATCH may have already reshaped. */
+ clientRequestDigest: {
+ type: String,
+ },
+ /** Last erasure-sweep attempt; rotates the `deleting` window (see eraseAttemptedAt
+ * ordering in getDeletingSchedules) so undrainable rows can't starve later ones. */
+ eraseAttemptedAt: {
+ type: Date,
+ },
+ lastRun: {
+ type: {
+ conversationId: { type: String },
+ status: { type: String, required: true },
+ error: { type: String },
+ firedAt: { type: Date, required: true },
+ scheduledFor: { type: Date },
+ },
+ default: undefined,
+ _id: false,
+ },
+ runCount: {
+ type: Number,
+ default: 0,
+ min: 0,
+ },
+ /**
+ * Bounded set of recently-counted occurrence timestamps. Per-occurrence
+ * idempotency guard for `recordRunOutcome` counter increments: a single
+ * scalar could be moved by an interleaved earlier occurrence, letting the
+ * reconciler double-count a crashed later one. Kept bounded via `$slice`.
+ */
+ countedFor: {
+ type: [Date],
+ default: undefined,
+ },
+ /** Newest occurrence whose outcome has been applied to the streak counters.
+ * `countedFor` makes counting idempotent per occurrence; this makes it ORDERED,
+ * so an older occurrence settling late cannot rebuild a streak a newer one
+ * cleared. Absent on rows written before it existed (fence disabled). */
+ countersAsOf: {
+ type: Date,
+ },
+ failureCount: {
+ type: Number,
+ default: 0,
+ min: 0,
+ },
+ balanceSkipCount: {
+ type: Number,
+ default: 0,
+ min: 0,
+ },
+ },
+ {
+ timestamps: true,
+ },
+);
+
+// Idempotency tombstones expire after the bounded retry window; live rows never match.
+scheduleSchema.index(
+ { erasedAt: 1 },
+ { expireAfterSeconds: 24 * 60 * 60, partialFilterExpression: { erased: true } },
+);
+scheduleSchema.index({ id: 1, tenantId: 1 }, { unique: true });
+scheduleSchema.index({ enabled: 1, nextRunAt: 1 });
+// Atomic per-user create cap: a live (non-deleting) schedule occupies a unique
+// slot, so concurrent creates cannot collectively exceed maxPerUser — a second
+// claimant of the same slot collides and retries the next free one. Scoped to
+// slot-bearing docs so the (non-user-facing) slotless create path is unaffected.
+scheduleSchema.index(
+ { user: 1, slot: 1 },
+ { unique: true, partialFilterExpression: { deleting: false, slot: { $exists: true } } },
+);
+
+// Makes a client's create RETRY idempotent: the second attempt collides here instead of
+// committing a second recurring schedule. Deliberately NOT filtered on `deleting` —
+// a key must stay claimed while its row is being erased, or a retry arriving mid-erase
+// would create a duplicate that outlives it.
+scheduleSchema.index(
+ { user: 1, clientRequestId: 1 },
+ { unique: true, partialFilterExpression: { clientRequestId: { $exists: true } } },
+);
+
+// Erasure sweeps read `deleting: true` ordered by last attempt (rotation).
+scheduleSchema.index({ deleting: 1, eraseAttemptedAt: 1 });
+
+export default scheduleSchema;
diff --git a/packages/data-schemas/src/schema/scheduleRun.ts b/packages/data-schemas/src/schema/scheduleRun.ts
new file mode 100644
index 0000000000..cf5e51a9a0
--- /dev/null
+++ b/packages/data-schemas/src/schema/scheduleRun.ts
@@ -0,0 +1,154 @@
+import { Schema } from 'mongoose';
+import type { IScheduleRunDocument } from '~/types/schedule';
+
+const SCHEDULE_RUN_TTL_SECONDS = 90 * 24 * 60 * 60;
+
+const scheduleRunSchema: Schema<IScheduleRunDocument> = new Schema(
+ {
+ scheduleId: {
+ type: String,
+ required: true,
+ },
+ user: {
+ type: Schema.Types.ObjectId,
+ ref: 'User',
+ required: true,
+ index: true,
+ },
+ tenantId: {
+ type: String,
+ index: true,
+ },
+ scheduledFor: {
+ type: Date,
+ required: true,
+ },
+ firedAt: {
+ type: Date,
+ },
+ conversationId: {
+ type: String,
+ },
+ /** Deterministic durable-trigger delivery key for this occurrence, stamped at
+ * reservation before enqueue so reconciliation can read the delivery's live/dead
+ * state rather than orphan-settling a deferred or dead-lettered run. */
+ deliveryKey: {
+ type: String,
+ },
+ /** Fresh while a RESUME of this paused run is mid-flight; a re-pause hand-off's
+ * writes are still landing, so quiesce must not settle on the paused job state.
+ * Cleared by the pause record (the hand-off completion) or aged out. */
+ resumeClaimedAt: {
+ type: Date,
+ },
+ status: {
+ type: String,
+ enum: [
+ 'started',
+ 'requires_action',
+ 'success',
+ 'error',
+ 'interrupted',
+ 'skipped_overlap',
+ 'skipped_balance',
+ ],
+ required: true,
+ },
+ error: {
+ type: String,
+ maxlength: 2048,
+ },
+ droppedFileIds: {
+ type: [String],
+ default: undefined,
+ },
+ durationMs: {
+ type: Number,
+ min: 0,
+ },
+ /** False on a terminal run whose schedule bookkeeping hasn't landed yet (crash-retry marker). */
+ bookkept: {
+ type: Boolean,
+ },
+ /** Set ONLY when the row reaches a terminal status; the TTL index below expires on
+ * this field, so live rows (which never carry it) never expire. */
+ settledAt: {
+ type: Date,
+ },
+ /** Global concurrency slot held while `started`. The unique partial index below
+ * turns fireConcurrency into a DB-enforced bound instead of a racy count. */
+ capacitySlot: {
+ type: Number,
+ min: 0,
+ },
+ /** When an abort was requested. The run keeps holding its capacity slot until the
+ * generation owner confirms settlement, so capacity is never freed early. */
+ abortRequestedAt: {
+ type: Date,
+ },
+ /** Who requested the abort ('stop' = interactive route, 'deletion' = delete/quiesce). */
+ abortSource: {
+ type: String,
+ enum: ['stop', 'deletion'],
+ },
+ /** Stamped once the interactive Stop route has persisted everything it writes;
+ * the generation owner defers settlement until this appears (bounded). */
+ abortPersistedAt: {
+ type: Date,
+ },
+ /** When reconciliation last examined this row. Orders the paused window so a full
+ * batch of still-live pauses cannot starve an abandoned row behind them. */
+ reconciledAt: {
+ type: Date,
+ },
+ /** The schedule's configRevision at claim time. Fences terminal bookkeeping and
+ * auto-disable from owner edits/re-enables that landed after this run started. */
+ configRevision: {
+ type: Number,
+ min: 0,
+ },
+ },
+ {
+ timestamps: true,
+ },
+);
+
+scheduleRunSchema.index({ scheduleId: 1, scheduledFor: 1 }, { unique: true });
+// TTL retention applies only to SETTLED rows: an approval window configured longer
+// than the retention period would otherwise let Mongo expire a still-live
+// `requires_action` run — its retained job could then resume with no run row (no
+// bookkeeping), and deletion quiescing could no longer discover or abort it.
+// Expressed as a PLAIN TTL index on `settledAt` (stamped only by terminal writes):
+// TTL expiry skips documents that lack the indexed date, so live rows never expire
+// without needing a partialFilterExpression — whose `$in` predicate Amazon
+// DocumentDB 5.0 cannot build (misc/documentdb/documentdb-compat.md), which would
+// leave initializeScheduleEngine refusing to arm on that target.
+scheduleRunSchema.index({ settledAt: 1 }, { expireAfterSeconds: SCHEDULE_RUN_TTL_SECONDS });
+// At most ONE active (`started`) run per schedule, enforced by the DB rather than a
+// read-then-write check: a second occurrence inserting while one is already active
+// fails with a duplicate-key error instead of racing.
+scheduleRunSchema.index(
+ { scheduleId: 1 },
+ { unique: true, partialFilterExpression: { status: 'started' } },
+);
+// GLOBAL fireConcurrency, enforced by the DB rather than a read-then-compare count.
+// A fire claims a slot in [0, fireConcurrency) in the SAME write that inserts the run;
+// a duplicate slot is rejected atomically, so two concurrent admissions of DIFFERENT
+// schedules can never both pass a cap-1 check.
+// Partial + $exists so legacy slotless rows (written before this field) never collide.
+scheduleRunSchema.index(
+ { capacitySlot: 1 },
+ {
+ unique: true,
+ partialFilterExpression: { status: 'started', capacitySlot: { $exists: true } },
+ },
+);
+scheduleRunSchema.index({ scheduleId: 1, firedAt: -1 });
+// Reconciliation sweeps by status; keeps `started` (capacity) fetch cheap and
+// prevents long-lived `requires_action` rows from starving the scan.
+scheduleRunSchema.index({ status: 1, firedAt: 1 });
+// The paused reconciliation window sorts on {reconciledAt, firedAt} within a status;
+// without this the round-robin rotation re-sorts the whole paused set every tick.
+scheduleRunSchema.index({ status: 1, reconciledAt: 1, firedAt: 1 });
+
+export default scheduleRunSchema;
diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.ts
index cecdab0a50..c3c9aecd61 100644
--- a/packages/data-schemas/src/types/index.ts
+++ b/packages/data-schemas/src/types/index.ts
@@ -30,6 +30,7 @@ export * from './prompts';
export * from './skill';
export * from './skillSync';
export * from './triggerDelivery';
+export * from './schedule';
/* Access Control */
export * from './accessRole';
export * from './aclEntry';
diff --git a/packages/data-schemas/src/types/role.ts b/packages/data-schemas/src/types/role.ts
index 2f6b6c813c..9a5807013d 100644
--- a/packages/data-schemas/src/types/role.ts
+++ b/packages/data-schemas/src/types/role.ts
@@ -77,6 +77,10 @@ export interface IRole extends Document {
[Permissions.SHARE]?: boolean;
[Permissions.SHARE_PUBLIC]?: boolean;
};
+ [PermissionTypes.SCHEDULES]?: {
+ [Permissions.USE]?: boolean;
+ [Permissions.CREATE]?: boolean;
+ };
};
tenantId?: string;
}
diff --git a/packages/data-schemas/src/types/schedule.ts b/packages/data-schemas/src/types/schedule.ts
new file mode 100644
index 0000000000..e806a82a69
--- /dev/null
+++ b/packages/data-schemas/src/types/schedule.ts
@@ -0,0 +1,118 @@
+import type {
+ ScheduleRunStatus,
+ ScheduleDisabledReason,
+ TScheduleCadence,
+} from 'librechat-data-provider';
+import type { Document, Types } from 'mongoose';
+
+export interface ISchedule {
+ _id?: Types.ObjectId;
+ id: string;
+ user: Types.ObjectId;
+ tenantId?: string;
+ name: string;
+ prompt: string;
+ agent_id: string;
+ cadence: TScheduleCadence;
+ timezone: string;
+ target: 'new';
+ file_ids?: string[];
+ tools?: string[];
+ cron?: string;
+ enabled: boolean;
+ disabledReason?: ScheduleDisabledReason;
+ nextRunAt?: Date;
+ leaseUntil?: Date;
+ leaseBy?: string;
+ claimToken?: string;
+ /** Owner-config generation; bumped only by an owner edit. */
+ configRevision?: number;
+ deleting?: boolean;
+ /** Reversible account-deletion suspension. Set at quiesce under a per-attempt token
+ * that snapshots the pre-suspension enabled/next-run state, so a deletion that is
+ * later cancelled restores exactly this row (fenced to its token) instead of leaving
+ * a live user with silently disabled schedules. Distinct from `deleting`, which marks
+ * a row for erasure. */
+ deletionSuspension?: {
+ token: string;
+ enabled: boolean;
+ nextRunAt?: Date;
+ };
+ erased?: boolean;
+ erasedAt?: Date;
+ slot?: number;
+ /** Client-supplied idempotency key of the create that produced this row. */
+ clientRequestId?: string;
+ /** Digest of the ORIGINAL create payload, stamped at insert and never edited.
+ * Replay matching compares against this rather than mutable schedule state, so a
+ * PATCH landing between the first attempt and its retry cannot fail the retry. */
+ clientRequestDigest?: string;
+ /** When an erasure sweep last attempted this soft-deleted row; orders the sweep
+ * window so undrainable rows cannot starve the ones behind them. */
+ eraseAttemptedAt?: Date;
+ lastRun?: {
+ conversationId?: string;
+ status: ScheduleRunStatus;
+ error?: string;
+ firedAt: Date;
+ /** The OCCURRENCE this projection came from; orders the card against delayed
+ * outcomes (a resumed pause, a reconciler replay) arriving after a newer run. */
+ scheduledFor?: Date;
+ };
+ runCount: number;
+ failureCount: number;
+ balanceSkipCount: number;
+ countedFor?: Date[];
+ /** Newest occurrence applied to the streak counters; orders them like the card. */
+ countersAsOf?: Date;
+ createdAt?: Date;
+ updatedAt?: Date;
+}
+
+export interface IScheduleDocument extends Omit<ISchedule, 'id' | '_id'>, Document {
+ id: string;
+}
+
+export interface IScheduleRun {
+ _id?: Types.ObjectId;
+ scheduleId: string;
+ user: Types.ObjectId;
+ tenantId?: string;
+ scheduledFor: Date;
+ firedAt?: Date;
+ conversationId?: string;
+ status: ScheduleRunStatus;
+ error?: string;
+ /** Deterministic durable-trigger delivery key for this occurrence, stamped at
+ * reservation (before enqueue). Lets reconciliation read the delivery's live/dead
+ * state instead of orphan-settling a jobless run that is merely deferred (Retry-After)
+ * or that dead-lettered before a generation ever started. */
+ deliveryKey?: string;
+ droppedFileIds?: string[];
+ durationMs?: number;
+ bookkept?: boolean;
+ /** Set only by terminal writes; the retention TTL index expires on this field. */
+ settledAt?: Date;
+ /** Global concurrency slot held while `started`. */
+ capacitySlot?: number;
+ /** When an abort was requested; capacity is held until settlement is confirmed. */
+ abortRequestedAt?: Date;
+ /** Who requested the abort: the interactive Stop route ('stop', which persists a
+ * partial response before the run may settle) or a deletion path ('deletion'). */
+ abortSource?: 'stop' | 'deletion';
+ /** Stamped by the interactive Stop route once every write it makes (checkpoint
+ * prune, partial-response save) has landed; the generation owner defers its
+ * terminal settlement until this appears so the run can never leave the active
+ * set while the route is still persisting. */
+ abortPersistedAt?: Date;
+ /** The schedule's configRevision at claim time. */
+ configRevision?: number;
+ /** When reconciliation last examined this row; orders the paused window so no row
+ * can be starved by a full batch of still-live pauses ahead of it. */
+ reconciledAt?: Date;
+ resumeClaimedAt?: Date;
+ createdAt?: Date;
+ updatedAt?: Date;
+}
+
+export interface IScheduleRunDocument extends Omit<IScheduleRun, '_id'>, Document {}