Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { isBalanceRefillDue, Permissions, PermissionTypes } from 'librechat-data-provider';
import { logger, runAsSystem, tenantStorage, isRuntimeDisabled } from '@librechat/data-schemas';
import {
DEFAULT_SCHEDULE_MCP_CONSENT_LIFETIME_HOURS,
isScheduleMCPAuthorizationFailure,
readScheduleMCPOutcomes,
getScheduleMCPDisabledReason,
projectScheduleMCPReceipt,
readScheduleMCPReceipts,
mergeScheduleMCPReceipts,
} from 'librechat-data-provider';
import type { ScheduleMethods, AppConfig, IBalance, IChatProject } from '@librechat/data-schemas';
import type { TCheckpointerConfig } from 'librechat-data-provider';
import type { ScheduleMCPOutcome } from 'librechat-data-provider';
import type { Types } from 'mongoose';
import type {
ScheduleEngineDeps,
ScheduleDeleteResult,
ScheduleMCPPreflight,
ScheduleLimits,
ScheduleUserContext,
FireableSchedule,
FireResult,
JobIdentity,
JobState,
ScheduleMCPFailureInput,
ScheduleMCPSettlementBoundary,
} from './types';
import type { SerializableJobData, ScheduleProviderOwner } from '../stream/interfaces/IJobStore';
import type { AgentCheckpointGeneration } from '../agents/checkpointer';
import type { ScheduleMCPExecution } from './authorization/execution';
import type { BalanceUpdateFields } from '../types/balance';
import type { GetAppConfigOptions } from '../app/service';
import {
resolveScheduleProjectId,
DEFAULT_SCHEDULE_LIMITS,
SCHEDULE_FILE_HOLD,
hasResumeHandoffInFlight,
retainedOutcome,
hasAbortInFlight,
} from './types';
import {
deleteAgentCheckpoint,
captureAgentCheckpointGeneration,
checkpointStorageConfigs,
} from '../agents/checkpointer';
import { isStopConfirmed, PROVIDER_DRAIN_TIMEOUT_MS } from '../stream/interfaces/IJobStore';
import { isShutdownInProgress, getShutdownSignal } from '../app/shutdown';
import { scheduleMCPFailurePriority } from '~/stream/scheduleFailure';
import { fireSchedule, BALANCE_SKIP_DISABLE_THRESHOLD } from './fire';
import { GenerationJobManager } from '../stream/GenerationJobManager';
import { ScheduledMCPPolicyError } from './authorization/policy';
import { buildBalanceUpdateFields } from '../middleware/balance';
import { getAppConfigOptionsFromUser } from '../app/service';
import { OboTokenResolutionError } from '../mcp/oauth/obo';
import { ScheduledMCPBearerError } from '../mcp/errors';
import { startScheduleErasureSweep } from './erasure';
import { getBalanceConfig } from '../app/config';
import { waitUntilDeadline } from '~/mcp/utils';
import { startScheduleEngine } from './engine';
import { withCapacitySlot } from './capacity';
import { isEnabled } from '../utils/common';
/** Recordable terminal/paused run outcome, as accepted by `recordRunOutcome`. */
type ScheduleRunOutcomeStatus = Parameters<ScheduleMethods['recordRunOutcome']>[0]['status'];
/** How a TERMINAL job status projects onto its run row. Mirrors the reconciler's
* mapping, so a run settled from a retained job reads the same either way. */
const TERMINAL_JOB_OUTCOMES: Record<string, ScheduleRunOutcomeStatus | undefined> = {
complete: 'success',
error: 'error',
aborted: 'interrupted',
};
/** Reason text recorded when account-deletion quiesce settles a run itself. */
const QUIESCE_SETTLE_ERRORS: Partial<Record<ScheduleRunOutcomeStatus, string>> = {
interrupted: 'Account deleted while awaiting approval',
error: 'Run ended in error',
};
/** Short schedule-document fence spanning capacity admission -> approval CAS. */
const SCHEDULE_RESUME_LEASE_MS = 60_000;
/**
* Whether this process may arm the scheduler at all.
*
* v1 is single-process by design, but the standard entrypoint runs
* `initializeScheduleEngine` in EVERY replica and nothing stops an operator from scaling
* it. That is only safe when replicas can see each other's generations: a shared stream
* store (Redis) gives every replica the same job view, so reconciliation and
* deletion-time aborts reach the run's real owner. With the process-local store a peer
* sees the globally visible `started` row but no job, and after the orphan cutoff it
* marks a still-running generation interrupted and frees its capacity slot.
*
* A single replica with the in-memory store is perfectly safe, but a process cannot
* observe its own replica count, so that case needs an explicit operator assertion.
*/
function isTopologySafeToArm(): boolean {
return GenerationJobManager.isRedis || isEnabled(process.env.SCHEDULES_SINGLE_PROCESS);
}
/** Whether a persisted job still carries a given scheduled occurrence's identity. */
function jobMatchesIdentity(
job: Pick<SerializableJobData, 'scheduleId' | 'scheduledFor'> & { createdAt?: number },
identity: JobIdentity,
): boolean {
if (
job.scheduleId !== identity.scheduleId ||
job.scheduledFor == null ||
(identity.createdAt != null && identity.createdAt !== job.createdAt)
) {
return false;
}
return new Date(job.scheduledFor).getTime() === new Date(identity.scheduledFor).getTime();
}
export interface RecordScheduleOutcomeInput {
scheduleId?: string;
scheduledFor?: string | Date;
/** Exact generation whose terminal evidence is being persisted. */
streamId?: string;
jobCreatedAt?: number;
status: ScheduleRunOutcomeStatus;
conversationId?: string;
checkpointNamespace?: string;
/** Erase the row's reserved conversationId (pre-start abort: no conversation exists). */
clearConversationId?: boolean;
error?: string;
}
export type ScheduleResumeClaimResult =
| { capacitySlot: number; claimToken: string; leaseBy: string }
| { conflict: 'capacity' | 'overlap' | 'not-paused' | 'inactive' };
/**
* Api-side dependencies the schedules service needs injected: model methods,
* config/balance access, and the owner-scoped agent access check. Everything
* else (job store and tenant context) lives in `@librechat/api` and is imported
* directly.
*/
export interface SchedulesServiceDeps {
preflightMCP: ScheduleMCPPreflight;
methods: ScheduleMethods & {
getRoleByName: (
role?: string,
) => Promise<{ permissions?: Record<string, Record<string, boolean | undefined>> } | null>;
getFiles: (
filter: unknown,
sort: unknown,
select: unknown,
) => Promise<Array<{
file_id: string;
filepath: string;
filename: string;
type: string;
height?: number;
width?: number;
source: string;
}> | null>;
/** Owner-scoped bounded TTL hold (`db.extendFilesTTL`-shaped). */
extendFilesTTL: (
fileIds: string[],
hold: { renewMs: number; maxLifetimeMs: number },
owner: { user: string; tenantId?: string | null },
) => Promise<number>;
};
getAppConfig: (options?: GetAppConfigOptions) => Promise<AppConfig | undefined>;
findUserById: (
userId: string | Types.ObjectId,
) => Promise<{ _id: Types.ObjectId; tenantId?: string; role?: string } | null>;
/** Reads the balance record together with the credits unexpired in-flight reservations hold. */
findBalance: (userId: string, options?: { applyReset?: boolean }) => Promise<IBalance | null>;
/**
* Upserts a balance record. `setOnInsert` carries fields that must ONLY apply to a
* document this call creates — chiefly the starting credit — so a record created by a
* concurrent charge is never overwritten with a fresh balance.
*/
upsertBalance: (
userId: string,
update: { set: Partial<BalanceUpdateFields>; setOnInsert: Partial<BalanceUpdateFields> },
) => Promise<IBalance | null>;
/**
* Compare-and-set initialization for an EXISTING balance whose `tokenCredits` is still
* null. Writes the starting credit (and any refill-config sync) only while
* `{ user, tokenCredits: null }` still matches — never an upsert — so a concurrent
* initializer/charge that already set credits between the read and this write is not
* clobbered. Returns the winning document, or `null` when the CAS did not match (the
* caller re-reads the winner rather than restoring credits).
*/
initializeNullBalance: (
userId: string,
update: { tokenCredits: number; sync: Partial<BalanceUpdateFields> },
) => Promise<IBalance | null>;
resolveAgentFireAccess: (
agentId: string,
user: ScheduleUserContext,
) => Promise<'ok' | 'missing' | 'forbidden'>;
/** Loads a chat project scoped to its owner, or null when it does not exist for
* them. Chat projects are user-owned, so this is both the existence check and the
* authorization check. */
getChatProject: (userId: string, projectId: string) => Promise<IChatProject | null>;
/** Positive process-termination proof for this exact provider segment, never a heartbeat timeout.
* Hosts without proof retain the obligation. This hook must return null for alive/unknown owners. */
confirmScheduleProviderOwnerLoss?: (
owner: ScheduleProviderOwner,
) => Promise<ScheduleProviderOwner | null>;
/** Whether this user's account deletion has begun. Fail-closed (unknown == true). */
isUserDeleting: (userId: string) => Promise<boolean>;
/** Shared durable trigger admission from the merged agent-trigger service. */
enqueueAgentTrigger: ScheduleEngineDeps['enqueueTrigger'];
/** Reads a durable trigger delivery by key from the merged agent-trigger service, for
* reconciliation's live-vs-dead delivery check. */
getTriggerDelivery: ScheduleEngineDeps['getTriggerDelivery'];
}
export interface SchedulesService {
getLimits: (user?: ScheduleUserContext) => Promise<ScheduleLimits>;
engineDeps: ScheduleEngineDeps;
fireScheduleNow: (
schedule: FireableSchedule,
limits: ScheduleLimits,
options?: { signal?: AbortSignal },
) => Promise<FireResult | null>;
recordScheduleOutcome: (input: RecordScheduleOutcomeInput) => Promise<boolean>;
recordMCPToolAuthFailure: (input: ScheduleMCPFailureInput) => Promise<boolean>;
getMCPReceiptRetryPolicy: (
input: ScheduleMCPFailureInput,
) => Promise<ScheduleMCPReceiptRetryPolicy | undefined>;
registerMCPSettlement: (boundary: ScheduleMCPSettlementBoundary) => void;
/**
* Stamps a scheduled run's interactive Stop BEFORE the abort is signalled, so the owner
* settlement barrier, reconciliation, and schedule/account deletion hold off settling or
* erasing until {@link acknowledgeScheduledStopPersistence}. Serialized: `'in_progress'`
* means a fresh Stop already owns the stamp and the caller must not signal a second abort;
* `false` means there is no active run to stop.
*/
beginScheduledStop: (input: {
scheduleId: string;
scheduledFor: string | Date;
}) => Promise<boolean | 'in_progress'>;
/**
* Releases the Stop settlement barrier once the route's partial-message/checkpoint writes
* have landed. Call ONLY after persistence succeeds — a failed persistence must leave the
* barrier unresolved so the run stays preserved and recovers via the stale-owner timeout.
*/
acknowledgeScheduledStopPersistence: (input: {
scheduleId: string;
scheduledFor: string | Date;
/** Optional terminal outcome to re-drive once the barrier clears, so a settlement the
* owner deferred past its poll budget still converges where no reconciler is armed. */
settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };
}) => Promise<void>;
/** Re-enters a paused occurrence into the DB-enforced global/same-schedule
* capacity set before its approval job is allowed to resume. */
claimScheduleResume: (
scheduleId: string,
scheduledFor: string | Date,
options?: { expectedConfigRevision?: number; automatic?: boolean },
) => Promise<ScheduleResumeClaimResult>;
/** Guarded rollback when the approval CAS did not consume the paused action. */
releaseScheduleResumeClaim: (
scheduleId: string,
scheduledFor: string | Date,
capacitySlot: number,
) => Promise<boolean>;
/** Atomically validates the live schedule generation after the approval CAS and
* releases its short-lived document fence. This is the resume linearization point. */
finalizeScheduleResumeClaim: (
scheduleId: string,
claimToken: string,
leaseBy: string,
options?: { expectedConfigRevision?: number; automatic?: boolean },
) => Promise<boolean>;
/** Releases only the schedule-document fence after an unconsumed/ambiguous approval CAS. */
releaseScheduleResumeFence: (scheduleId: string, leaseBy: string) => Promise<void>;
/**
* Whether a schedule is still live (exists and not soft-deleted). The loopback
* chat controller calls this right after creating the generation job to re-fence
* a fire against a delete/quiesce that landed in the claim -> POST window (when
* the reservation row exists but the job did not yet, so the deletion's abort
* missed it) — aborting before any messages are persisted.
*/
isScheduleLive: (
scheduleId: string,
expectedConfigRevision?: number,
options?: { automatic?: boolean; policy?: boolean; scheduledFor?: string | Date },
) => Promise<boolean>;
/** Soft-deletes an owner's schedule: stop claims, abort active runs, drain, erase. */
deleteScheduleForOwner: (scheduleId: string, userId: string) => Promise<ScheduleDeleteResult>;
/**
* Quiesces all of a user's schedules ahead of account deletion (reversible suspension +
* abort + drain). `token` identifies this deletion attempt and is what a later
* {@link restoreUserSchedulesFromDeletion} restores against. Returns whether the drain was
* CONFIRMED: false means at least one run could not be confirmed settled, and the caller
* must NOT proceed to destructive deletion — the durable barrier keeps refusing new work
* while a later pass finishes the cascade.
*/
quiesceUserSchedules: (userId: string, token: string) => Promise<boolean>;
/**
* Reverses a quiesce whose account deletion was cancelled, re-enabling and re-arming only
* the rows this exact attempt suspended. Safe to call even if quiesce never suspended a
* row (no matching token → no-op).
*/
restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise<void>;
/** Arms the scheduler for THIS process. v1 is single-process only; the clustered
* entrypoint does not start it. Returns undefined when index creation failed, or when
* the topology cannot be shown safe (process-local job store with no single-process
* assertion) — see isTopologySafeToArm. */
initializeScheduleEngine: () => Promise<ReturnType<typeof startScheduleEngine> | undefined>;
/** Starts bounded maintenance for an entrypoint that never arms the engine (the
* clustered worker). Idempotent per process and a no-op once the full engine is armed.
* Replays terminal permanent-MCP bookkeeping but never claims, fires, advances, or
* infers owner death from a process-local missing job. See startScheduleErasureSweep. */
initializeScheduleErasureSweep: () => void;
reconcileRetainedJobs: () => Promise<void>;
}
/** A replaced/foreign generation cannot receive an old denial or model-visible result. */
export class ScheduledMCPReceiptFencedError extends Error {
constructor() {
super('Scheduled generation is no longer current.');
this.name = 'AbortError';
}
}
export interface ScheduleMCPReceiptRetryPolicy {
baseMs?: number;
maxMs?: number;
}
const receiptFlights = new WeakMap<
SchedulesService['recordMCPToolAuthFailure'],
Map<string, Promise<boolean>>
>();
/** Scheduled authorization failures cannot become model errors before durable admission. */
export function recordScheduledMCPToolAuthFailure(
input: ScheduleMCPFailureInput,
getRecorder: () => SchedulesService['recordMCPToolAuthFailure'],
getRetryPolicy?: () => Promise<ScheduleMCPReceiptRetryPolicy | undefined>,
): Promise<boolean> {
const bearer =
input.error instanceof ScheduledMCPBearerError ||
input.error instanceof ScheduledMCPPolicyError;
const cause = input.error instanceof Error ? input.error.cause : undefined;
const missing = input.error instanceof OboTokenResolutionError ? input.error : cause;
if (
(!bearer &&
(!(missing instanceof OboTokenResolutionError) ||
missing.reason !== 'missing_upstream_provider')) ||
!input.streamId ||
input.jobCreatedAt == null ||
!input.userId ||
(input.error instanceof ScheduledMCPBearerError && !input.identity)
) {
return bearer ? Promise.reject(new ScheduledMCPReceiptFencedError()) : Promise.resolve(false);
}
if (isShutdownInProgress()) return Promise.reject(new ScheduledMCPReceiptFencedError());
let record: SchedulesService['recordMCPToolAuthFailure'];
try {
record = getRecorder();
} catch (error) {
return Promise.reject(error);
}
const key = JSON.stringify([
input.streamId,
input.jobCreatedAt,
input.userId,
input.identity,
input.serverName,
input.error instanceof ScheduledMCPBearerError || input.error instanceof ScheduledMCPPolicyError
? input.error.outcomes
: 'missing_upstream_provider',
]);
let flights = receiptFlights.get(record);
if (!flights) {
flights = new Map();
receiptFlights.set(record, flights);
}
const current = flights.get(key);
if (current) return current;
const flight = (async () => {
let interval = 250;
let cap = 30_000;
let configured = false;
const signal = getShutdownSignal();
for (;;) {
if (signal.aborted || isShutdownInProgress()) throw new ScheduledMCPReceiptFencedError();
try {
const result = await waitUntilDeadline(record(input), undefined, signal);
if (!result.settled) throw new ScheduledMCPReceiptFencedError();
if (result.value) return true;
} catch (error) {
if (error instanceof ScheduledMCPReceiptFencedError) throw error;
if (!bearer) {
logger.warn('[schedules] could not persist MCP authorization failure receipt:', error);
return false;
}
}
if (!bearer) return false;
if (!configured) {
const loaded = await waitUntilDeadline(
getRetryPolicy?.().catch(() => undefined) ?? Promise.resolve(undefined),
undefined,
signal,
);
if (!loaded.settled) throw new ScheduledMCPReceiptFencedError();
const policy = loaded.value;
const { baseMs, maxMs } = policy ?? {};
if (
typeof baseMs === 'number' &&
Number.isSafeInteger(baseMs) &&
baseMs >= 100 &&
baseMs <= 60_000
)
interval = baseMs;
if (
typeof maxMs === 'number' &&
Number.isSafeInteger(maxMs) &&
maxMs >= 100 &&
maxMs <= 600_000
)
cap = maxMs;
cap = Math.max(interval, cap);
configured = true;
}
// Equal jitter; no caller cancellation may turn an unrecorded denial into model continuation.
try {
await new Promise<void>((resolve, reject) => {
if (signal.aborted) {
reject(new ScheduledMCPReceiptFencedError());
return;
}
const finish = () => {
signal.removeEventListener('abort', abort);
resolve();
};
const timer = setTimeout(finish, Math.ceil(interval * (0.5 + Math.random() * 0.5)));
const abort = () => {
clearTimeout(timer);
signal.removeEventListener('abort', abort);
reject(new ScheduledMCPReceiptFencedError());
};
signal.addEventListener('abort', abort, { once: true });
});
} catch {
throw new ScheduledMCPReceiptFencedError();
}
interval = Math.min(cap, interval * 2);
}
})().finally(() => {
flights!.delete(key);
});
flights.set(key, flight);
return flight;
}
/** Captures host scope, never model arguments or mutable runnable config. */
export function createScheduledMCPPolicyRecorder(
execution: ScheduleMCPExecution | undefined,
job: { streamId?: string | null; jobCreatedAt?: number; userId?: string; tenantId?: string },
record: SchedulesService['recordMCPToolAuthFailure'],
): ((error: ScheduledMCPPolicyError) => Promise<boolean>) | undefined {
if (!execution) return;
const identity = Object.freeze({ ...execution.identity });
const scope = Object.freeze({ ...job });
return (error) => {
if (scope.userId !== identity.ownerId || (scope.tenantId ?? null) !== identity.tenantId)
return Promise.reject(new ScheduledMCPReceiptFencedError());
return record({
error,
identity,
streamId: scope.streamId ?? undefined,
jobCreatedAt: scope.jobCreatedAt,
userId: identity.ownerId,
serverName: error.outcomes[0].server,
});
};
}
/** Test-only overrides for the service's bounded waits (drains, barriers). */
export interface ScheduleServiceTimings {
drainTimeoutMs?: number;
drainPollMs?: number;
stopBarrierTimeoutMs?: number;
stopBarrierPollMs?: number;
}
/**
* Builds the scheduler service around api-side dependencies. Each call owns its
* own engine singleton and job-store-shared flag, so state never leaks between
* instances.
*/
export function createScheduleLimitsResolver(
getAppConfig: SchedulesServiceDeps['getAppConfig'],
): SchedulesService['getLimits'] {
/**
* Resolves schedule limits, honoring per-principal (role/user) config overrides
* when a user is supplied (routes pass req.user, the fire path passes the owner).
*/
return async function getLimits(user?: ScheduleUserContext): Promise<ScheduleLimits> {
// The BASE `interface.schedules: false` is a global stop and must win over any
// principal override. Without this a tenant/role/user override resolving to
// enabled would let the sidebar and CRUD handlers admit schedules that
// isGloballyDisabled() correctly refuses to ever fire.
// isRuntimeDisabled, NOT `=== false`: the base stop has TWO shapes (`false` and
// `{ use: false }`), and deepMerge turns a base `{ use: false, ... }` plus a principal
// override of `true` into `{ use: true, ... }`. A literal-false check missed that, so
// getLimits reported the feature enabled and Run Now dispatched straight through
// fireSchedule — bypassing the operator's object-form emergency stop. Same predicate
// the engine gate (isGloballyDisabled) already uses.
if (
user != null &&
isRuntimeDisabled((await getAppConfig({ baseOnly: true }))?.interfaceConfig?.schedules)
) {
return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
}
// NO principal means the DEPLOYMENT's config, so it must be read base-only. A bare
// getAppConfig() still resolves principals and picks up the tenant from the ALS
// context, and both callers of this form run inside one: fireSchedule clamps the
// global capacity allocator from within runInTenantContext(owner), and the engine
// tick budgets from within runAsSystem. Either would otherwise resolve a TENANT
// override as if it were the deployment-wide value, which is exactly what the
// global cap exists to prevent an override from widening.
const appConfig = user
? await getAppConfig(getAppConfigOptionsFromUser(user))
: await getAppConfig({ baseOnly: true });
// The env kill switch is a GLOBAL stop and must be visible everywhere limits are
// consulted (write handlers, fire path), not only at the engine tick.
if (isEnabled(process.env.SCHEDULES_DISABLED)) {
return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
}
const config = appConfig?.interfaceConfig?.schedules;
// EXPERIMENTAL, default-OFF (v1): scheduled chats are disabled unless an admin
// explicitly enables them. Absence, null, or `false` all resolve to disabled, so a
// deployment that never opts in never runs the scheduler. `true` uses the defaults;
// an object opts in unless it sets `use: false`.
if (config == null || config === false) {
return { ...DEFAULT_SCHEDULE_LIMITS, enabled: false };
}
if (config === true) {
return DEFAULT_SCHEDULE_LIMITS;
}
// A pinned project is itself a requirement: leaving `requireProject` to be set
// separately would let `projectId` alone resolve to "optional destination that
// happens to be forced", and a schedule created before the pin would keep firing
// with no project at all rather than being stopped for review.
const projectId = config.projectId?.trim() || undefined;
return {
enabled: config.use !== false,
...(config.mcpConsent && {
mcpConsent: {
enabled: config.mcpConsent.enabled === true,
maxLifetimeHours:
config.mcpConsent.maxLifetimeHours ?? DEFAULT_SCHEDULE_MCP_CONSENT_LIFETIME_HOURS,
},
}),
maxPerUser: config.maxPerUser ?? DEFAULT_SCHEDULE_LIMITS.maxPerUser,
minIntervalMinutes: config.minIntervalMinutes ?? DEFAULT_SCHEDULE_LIMITS.minIntervalMinutes,
autoDisableAfterFailures:
config.autoDisableAfterFailures ?? DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
admissionConcurrency:
config.admissionConcurrency ?? DEFAULT_SCHEDULE_LIMITS.admissionConcurrency,
fireConcurrency: config.fireConcurrency ?? DEFAULT_SCHEDULE_LIMITS.fireConcurrency,
mcpPreflightConcurrency:
config.mcpPreflightConcurrency ?? DEFAULT_SCHEDULE_LIMITS.mcpPreflightConcurrency,
mcpPreflightTimeoutMs:
config.mcpPreflightTimeoutMs ?? DEFAULT_SCHEDULE_LIMITS.mcpPreflightTimeoutMs,
requireProject: config.requireProject === true || projectId != null,
...(projectId != null && { projectId }),
};
};
}
export function createSchedulesService(
deps: SchedulesServiceDeps,
timings?: ScheduleServiceTimings,
): SchedulesService {
const { methods } = deps;
// Recovery evidence only: no authorization or bearer is cached here.
const pendingBearerFailures = new Map<
string,
Parameters<SchedulesService['recordMCPToolAuthFailure']>[0] & {
admitted: boolean;
outcomes: ScheduleMCPOutcome[];
}
>();
const mcpBoundaries = new Map<
string,
{ scheduleId: string; quiesce: () => Promise<void>; flight?: Promise<void> }
>();
function registerMCPSettlement(boundary: ScheduleMCPSettlementBoundary): void {
const key = receiptKey(boundary.streamId, boundary.jobCreatedAt);
if (mcpBoundaries.get(key)?.flight) throw new ScheduledMCPReceiptFencedError();
mcpBoundaries.set(key, { scheduleId: boundary.identity.scheduleId, quiesce: boundary.quiesce });
}
async function hasScheduleCleanupObligation(scope: {
scheduleId?: string;
userId?: string;
}): Promise<boolean> {
const store = GenerationJobManager.getJobStore();
if (!store) return false;
return !store.hasScheduleCleanupObligation || (await store.hasScheduleCleanupObligation(scope));
}
async function eraseSettledSchedule(scheduleId: string): Promise<boolean> {
for (const pending of pendingBearerFailures.values()) {
if (pending.identity?.scheduleId === scheduleId) return false;
}
if (await hasScheduleCleanupObligation({ scheduleId })) return false;
return methods.eraseScheduleIfDrained(scheduleId);
}
const receiptKey = (stream: string, epoch: number): string => JSON.stringify([stream, epoch]);
const outcomeKey = (outcome: ScheduleMCPOutcome): string =>
JSON.stringify([
outcome.server,
outcome.agentId,
outcome.status,
outcome.reason,
outcome.recovery,
outcome.detail,
outcome.automaticReplay,
]);
const mergeReceipts = mergeScheduleMCPReceipts;
const preferFailure = (
previous: ScheduleMCPOutcome | undefined,
next: ScheduleMCPOutcome,
): ScheduleMCPOutcome =>
scheduleMCPFailurePriority(next) > scheduleMCPFailurePriority(previous)
? next
: (previous ?? next);
const encodeReceipts = (outcomes: ScheduleMCPOutcome[]): string =>
`${getScheduleMCPDisabledReason(outcomes) ?? 'mcp_unavailable'}: ${JSON.stringify(outcomes)}`;
// Fail LOUDLY at construction, not per-fire. The JS adapter (api/server/services/
// Schedules) is not typechecked against SchedulesServiceDeps, so a missing dep would
// otherwise surface only as a `deps.X is not a function` deep inside a live fire —
// which is exactly how the deletion-barrier probe shipped unwired twice.
const REQUIRED_DEPS: Array<keyof SchedulesServiceDeps> = [
'methods',
'getAppConfig',
'findUserById',
'findBalance',
'upsertBalance',
'initializeNullBalance',
'resolveAgentFireAccess',
'getChatProject',
'isUserDeleting',
'enqueueAgentTrigger',
'getTriggerDelivery',
];
for (const key of REQUIRED_DEPS) {
if (deps[key] == null) {
throw new Error(`createSchedulesService: missing required dependency "${key}"`);
}
}
const getLimits = createScheduleLimitsResolver(deps.getAppConfig);
const MANUAL_RUN_LEASE_MS = 5 * 60 * 1000;
// Bounded wait for aborted scheduled runs to settle during account-deletion quiesce,
// before the message/conversation cascade runs. Long enough to cover a generation that
// already returned from the model finishing its persistence; capped so account deletion
// never blocks indefinitely on an unreachable peer-worker run.
const QUIESCE_DRAIN_TIMEOUT_MS = timings?.drainTimeoutMs ?? 10 * 1000;
const QUIESCE_DRAIN_POLL_MS = timings?.drainPollMs ?? 250;
// The owner's terminal settlement waits (briefly, bounded) for an interactive Stop to
// acknowledge its partial-message/checkpoint persistence before releasing capacity or
// erasing evidence. Short: the Stop route's persistence is a couple of writes; if it
// does not land in this window the stale-owner timeout (ABORT_OWNER_PRESUMED_ALIVE_MS)
// takes over as the bounded recovery, so the settlement never blocks indefinitely.
const STOP_BARRIER_TIMEOUT_MS = timings?.stopBarrierTimeoutMs ?? 5 * 1000;
const STOP_BARRIER_POLL_MS = timings?.stopBarrierPollMs ?? 100;
/**
* Whether a refill would top up this zero-credit balance record right now,
* mirroring the chat balance check's auto-refill eligibility (record-based).
*/
function isRefillEligible(record: IBalance | null | undefined): boolean {
return record != null && isBalanceRefillDue(record, new Date());
}
const engineDeps: ScheduleEngineDeps = {
reconcileRetainedJobs,
eraseSettledSchedule,
preflightMCP: deps.preflightMCP,
methods,
getLimits,
// On the BASE deps, not only the engine's per-pass wrapper: fireScheduleNow
// (manual Run Now) dispatches with these deps directly, and its POST must be
// gated by the same coordinator signal as the engine tick's.
isShuttingDown: isShutdownInProgress,
getUserContext: async (userId) => {
const user = await deps.findUserById(userId);
if (user == null) {
return null;
}
return { id: user._id.toString(), tenantId: user.tenantId, role: user.role };
},
hasScheduleAccess: async (user) => {
const role = await methods.getRoleByName(user.role);
return role?.permissions?.[PermissionTypes.SCHEDULES]?.[Permissions.USE] === true;
},
isOutOfBalance: async (user) => {
const appConfig = await deps.getAppConfig(getAppConfigOptionsFromUser(user));
const balanceConfig = getBalanceConfig(appConfig);
if (balanceConfig?.enabled !== true) {
return false;
}
let record = await deps.findBalance(user.id, { applyReset: false });
// Credits in-flight requests hold are unavailable to this fire as well: the chat
// balance check admits against the unreserved amount. Taken from this read because
// the initialization/sync writes below return the record without the total.
let reservedCredits = record?.reservedCredits ?? 0;
// Initialize/sync the record exactly as the chat's balance middleware would,
// so a new user's startBalance is applied before we read it (avoids skipping
// a schedule that an interactive chat would have allowed).
if (balanceConfig.startBalance != null) {
const updateFields = buildBalanceUpdateFields(balanceConfig, record, user.id);
if (Object.keys(updateFields).length > 0) {
// The read above and every write below are separate statements, and the credit
// field is an INITIALIZATION value: a concurrent charge that set-and-spent the
// record in between must never be handed back its starting balance. Each case
// fences the credit write so it can only create, never restore, credits. The
// refill-config fields are a genuine sync and never carry a credit.
const { user: initUser, tokenCredits, ...syncFields } = updateFields;
if (record == null) {
// ABSENT record: upsert. The credit rides `$setOnInsert` so a document created
// by a concurrent charge keeps its charged balance; the refill-config sync is
// a legitimate `$set`.
record = await deps.upsertBalance(user.id, {
set: syncFields,
setOnInsert: {
...(initUser != null ? { user: initUser } : {}),
...(tokenCredits != null ? { tokenCredits } : {}),
},
});
} else if (tokenCredits != null) {
// EXISTING record with a null credit: a blind `$set` would restore credits a
// concurrent initializer/charge already set and spent between the read and
// here. Initialize under a `{ tokenCredits: null }` CAS; on a miss re-read the
// winner's balance instead of overwriting it.
const initialized = await deps.initializeNullBalance(user.id, {
tokenCredits,
sync: syncFields,
});
record =
initialized ?? (await deps.findBalance(user.id, { applyReset: false })) ?? record;
} else if (Object.keys(syncFields).length > 0) {
// EXISTING record with a real credit: only refill-config sync remains, which
// never touches `tokenCredits` and is safe to `$set` directly.
record = await deps.upsertBalance(user.id, { set: syncFields, setOnInsert: {} });
}
}
}
if (
balanceConfig.autoRefillEnabled === true &&
record?.refillMode === 'reset' &&
isBalanceRefillDue(record, new Date())
) {
record = await deps.findBalance(user.id, { applyReset: true });
reservedCredits = record?.reservedCredits ?? 0;
}
const credits = (record?.tokenCredits ?? 0) - reservedCredits;
if (credits > 0) {
return false;
}
// At/below zero: an auto-refill user is only spared a pre-skip when a refill
// would actually fire now (mirrors the chat balance check's eligibility). If
// they aren't eligible yet, or the refill settings are incomplete, pre-skip as
// a balance skip — otherwise the zero-credit fire reaches the chat, is rejected
// there, and records a generic error that walks the schedule toward
// too_many_failures instead of skipped_balance/insufficient_balance.
if (balanceConfig.autoRefillEnabled === true && isRefillEligible(record)) {
return false;
}
return true;
},
// Mirrors the loopback chat route's authorization (role AGENTS:USE + resource
// VIEW with the manage:agents bypass); shared with the create/update precheck
// so the two never diverge.
agentAccess: (agentId, user) => deps.resolveAgentFireAccess(agentId, user),
// Ownership IS existence for a chat project, so one lookup answers both. Failing
// closed here would auto-disable a schedule on a transient Mongo blip, so a read
// error propagates instead: the fire fails and is retried like any other error.
projectAccess: async (projectId, user) =>
(await deps.getChatProject(user.id, projectId)) == null ? 'missing' : 'ok',
resolveFiles: async (fileIds, user) => {
// Renew the bounded upload hold at every fire preflight, BEST-EFFORT: the hold
// only has to bridge upload -> first consumption (a real send clears the TTL
// permanently), and a failed renewal must not fail the fire — at worst the hold
// lapses later and resolveFiles drops the reaped file (droppedFileIds records it).
await methods
.extendFilesTTL(fileIds, SCHEDULE_FILE_HOLD, { user: user.id, tenantId: user.tenantId })
.catch((err) => logger.warn('[schedules] attachment hold renewal failed:', err));
const files = await methods.getFiles(
{ file_id: { $in: fileIds }, user: user.id },
null,
'-text',
);
return (files ?? []).map((file) => ({
file_id: file.file_id,
filepath: file.filepath,
filename: file.filename,
type: file.type,
height: file.height,
width: file.width,
source: file.source,
}));
},
enqueueTrigger: deps.enqueueAgentTrigger,
getTriggerDelivery: deps.getTriggerDelivery,
runInTenantContext: (user, fn) =>
tenantStorage.run({ tenantId: user.tenantId, userId: user.id }, fn),
getJobStatus: async (conversationId) => {
const job = await GenerationJobManager.getJobStore()?.getJob(conversationId);
if (job == null) {
return null;
}
const pending = pendingBearerFailures.get(receiptKey(conversationId, job.createdAt));
const retained =
pending?.identity &&
job.scheduleId === pending.identity.scheduleId &&
job.userId === pending.identity.ownerId &&
(job.tenantId ?? null) === pending.identity.tenantId
? pending.outcomes.reduce(preferFailure, job.scheduleMCPFailure ?? pending.outcomes[0])
: job.scheduleMCPFailure;
return {
status: job.status,
createdAt: job.createdAt,
checkpointNamespace: job.checkpointNamespace,
scheduleId: job.scheduleId,
scheduledFor: job.scheduledFor,
createdEventEmitted: job.createdEventEmitted === true,
providerDrained: job.providerDrained,
terminalPersistencePending: job.terminalPersistencePending,
terminalHostActionPending: job.terminalHostActionPending,
preserveForScheduleReconcile: job.preserveForScheduleReconcile === true,
...(job.scheduleOutcome != null && { scheduleOutcome: job.scheduleOutcome }),
...(job.scheduleOutcomeError != null && {
scheduleOutcomeError: job.scheduleOutcomeError,
}),
...(retained && { scheduleMCPFailure: retained }),
};
},
abortScheduledJob: async (conversationId, identity, options) => {
const store = GenerationJobManager.getJobStore();
if (store == null) {
return false;
}
const job = await store.getJob(conversationId);
// A null/identity-mismatched job is NOT reachable from this replica: it may be
// a live generation on a peer worker's private in-memory store (unshared
// topology). Report false so the caller knows the abort was NOT delivered.
if (job == null || !jobMatchesIdentity(job, identity)) {
return false;
}
{
const retained = projectScheduleMCPReceipt(
{ status: 'interrupted', error: 'Schedule deleted' },
readScheduleMCPReceipts(job.scheduleOutcomeError),
);
await GenerationJobManager.updateMetadata(
conversationId,
{
preserveForScheduleReconcile: true,
scheduleOutcome: retained.status,
scheduleOutcomeError: retained.error,
},
job.createdAt,
);
}
// #14925's provider-drain contract is the authority here. It covers both
// locally owned and cross-replica generations and does not return until the
// exact provider segment can no longer persist user data. Calling abortJob on
// an already-terminal job is intentional: awaitProviderDrain still waits for
// trailing owner work even though no new abort transition is needed.
const aborted = await GenerationJobManager.abortJob(conversationId, {
expectedCreatedAt: job.createdAt,
awaitProviderDrain: true,
});
// Terminal-and-drained counts as delivered (see above); a replacement, a still-live
// run, or a job that vanished before the transition does not.
if (!isStopConfirmed(aborted)) {
return false;
}
// Read again after the exact provider/pause drain; evidence may have arrived during it.
const current = await store.getJob(conversationId);
if (
current?.createdAt !== job.createdAt ||
!jobMatchesIdentity(current, identity) ||
current.providerDrained === false ||
current.terminalPersistencePending === true ||
current.terminalHostActionPending === true
)
return false;
if (
readScheduleMCPReceipts(current.scheduleOutcomeError).length > 0 ||
pendingBearerFailures.has(receiptKey(conversationId, job.createdAt))
) {
const settled = await recordScheduleOutcome({
scheduleId: job.scheduleId!,
scheduledFor: job.scheduledFor!,
streamId: conversationId,
jobCreatedAt: job.createdAt,
conversationId: job.conversationId,
status: 'interrupted',
error: 'Schedule deleted while the run was active',
});
if (!settled) return false;
}
if (options?.preserve === false)
await engineDeps.clearReconciledJob(conversationId, {
...identity,
createdAt: job.createdAt,
});
return true;
},
clearReconciledJob: async (conversationId, identity) => {
const store = GenerationJobManager.getJobStore();
if (store == null) {
return;
}
const job = await store.getJob(conversationId);
// Only delete when the job still carries THIS run's identity, so a
// replacement generation occupying the same conversationId is never destroyed.
if (job == null || !jobMatchesIdentity(job, identity)) {
return;
}
if (
job.providerDrained === false ||
job.terminalPersistencePending === true ||
job.terminalHostActionPending === true ||
job.status === 'running' ||
job.status === 'requires_action'
)
return;
if (job.preserveForScheduleReconcile === true)
await GenerationJobManager.updateMetadata(
conversationId,
{ preserveForScheduleReconcile: false },
job.createdAt,
);
// CAS, not read-then-delete: the identity check above is a READ, and a replacement
// generation can land between it and the delete. Passing the observed createdAt
// makes the delete conditional on the job still being that exact generation, so
// the store itself rejects the write if one did.
await store.deleteJob(conversationId, job.createdAt);
},
// Counted in system scope so the cap is GLOBAL — a per-owner (tenant-scoped)
// count would let multiple tenants collectively exceed fireConcurrency.
countActiveRunsGlobal: () => runAsSystem(() => methods.countActiveRuns()),
isOwnerDeleting: (userId) => deps.isUserDeleting(userId),
isGloballyDisabled: async () => {
// Env first: an incident lever that must work even if the DB/config plane is the
// thing failing (a kill switch that needs a healthy DB is the one that fails when
// you need it).
if (isEnabled(process.env.SCHEDULES_DISABLED)) {
return true;
}
// BASE config only: DB principal overrides can narrow availability but must never
// widen past an operator's global stop, so `schedules: false` in librechat.yaml is
// genuinely non-overridable rather than emergent from the override filters.
// isRuntimeDisabled reads BOTH stop shapes (`false` and `{ use: false }`) — the
// same predicate the override merge preserves base stops with. A shape-blind
// check here made the object form disable getLimits but not this gate, so the
// engine kept claiming and fireSchedule ADVANCED each occurrence: a short
// maintenance stop silently dropped every occurrence it covered instead of
// leaving them due.
const base = await deps.getAppConfig({ baseOnly: true });
return isRuntimeDisabled(base?.interfaceConfig?.schedules);
},
// Occupancy is read in SYSTEM scope so the cap is global across tenants (the
// owner's tenant context would only see its own runs); the claim itself stays in
// the caller's context so the inserted row keeps correct tenant ownership.
withGlobalCapacitySlot: (cap, claim) =>
withCapacitySlot(cap, () => runAsSystem(() => methods.getCapacityOccupancy()), claim),
};
let engine: ReturnType<typeof startScheduleEngine> | undefined;
let erasureSweep: ReturnType<typeof startScheduleErasureSweep> | undefined;
/**
* Fallback cleanup for a process whose engine refused to arm (unsafe topology, or
* index creation failed): DELETE stays open on this entrypoint, so soft-deleted rows
* still accrue — and with no reconciler, an owner-death case (account deletion begun
* elsewhere, or a failed one-shot erase) would retain the hidden prompt forever.
* The engine's own reconcile pass covers this when armed, so never run both.
*/
function startErasureFallback(): void {
if (erasureSweep != null || engine != null) {
return;
}
erasureSweep = startScheduleErasureSweep({
methods,
getJobStatus: engineDeps.getJobStatus,
// Positive-evidence delivery convergence: a dead delivery settles its reservation
// and frees the global capacity slot even where no engine is armed.
getTriggerDelivery: engineDeps.getTriggerDelivery,
// Releases a run's retained terminal job once its outcome is durable, so preserved
// evidence (kept without `completedAt`, and therefore invisible to the store's
// finished-job sweep) cannot outlive the run it belonged to.
clearReconciledJob: engineDeps.clearReconciledJob,
abortScheduledJob: engineDeps.abortScheduledJob,
reconcileRetainedJobs,
eraseSettledSchedule,
// If topology itself prevented arming, this process's missing job says
// nothing about peer liveness. If only index creation failed, the topology
// proof still holds and the existing owner-death backstop remains valid.
canInferOwnerDeathFromMissingJob: isTopologySafeToArm(),
});
}
async function initializeScheduleEngine(): Promise<
ReturnType<typeof startScheduleEngine> | undefined
> {
if (engine != null) {
return engine;
}
// Always arm. The engine owns BOTH firing and reconciliation, and reconciliation must
// run even while firing is stopped: a previous process can leave `started` rows and
// preserved terminal jobs that would otherwise never settle until scheduling is
// re-enabled. Firing is gated separately — runTick refuses to claim while globally
// disabled, and getLimits reports disabled to the write handlers and the fire path —
// so "off" still means nothing fires, without stranding prior state.
// Deliberately not gated on the base config either: schedules can be enabled for a
// role/user/tenant while the base YAML stays default-off.
// Explicitly build the Schedule/ScheduleRun indexes first — the unique
// idempotency index and TTL retention index would otherwise never exist when
// MONGO_AUTO_INDEX is disabled (the production default). If this fails the
// unique {scheduleId, scheduledFor} guard may be absent, so leave the engine
// DISABLED rather than firing without duplicate protection — the app still
// runs; schedules simply don't fire until an operator resolves the index.
if (!isTopologySafeToArm()) {
logger.error(
"[schedules] scheduler NOT started: this process cannot see other replicas' generations. " +
'The job store is process-local (Redis is off), but nothing here proves this is the only ' +
'replica — and the standard server arms the scheduler in EVERY replica. A peer would ' +
'reconcile runs whose jobs it cannot see, eventually marking a still-running generation ' +
'interrupted and releasing its capacity, and deletions routed elsewhere could not abort ' +
'the generation at all. Enable a shared stream store (USE_REDIS_STREAMS), or set ' +
'SCHEDULES_SINGLE_PROCESS=true to assert this deployment runs exactly one replica. ' +
'Schedule writes are refused (503) until then.',
);
startErasureFallback();
return undefined;
}
try {
await runAsSystem(() => methods.ensureScheduleIndexes());
} catch (err) {
logger.error(
'[schedules] index creation failed — scheduler NOT started (fires need the unique idempotency index):',
err,
);
startErasureFallback();
return undefined;
}
engine = startScheduleEngine(engineDeps);
if (erasureSweep != null) {
// A later successful arm supersedes the fallback: the engine's reconcile pass
// owns erasure from here.
erasureSweep.stop();
erasureSweep = undefined;
}
return engine;
}
/**
* The clustered entrypoint's ONLY schedule maintenance. The fallback erases soft-deleted
* rows and converges terminal evidence, including permanent MCP bookkeeping interrupted
* between the run-row transition and card projection. It shares startErasureFallback's
* idempotent startup guard and topology-fenced owner-death policy, and never fires work.
*/
function initializeScheduleErasureSweep(): void {
startErasureFallback();
}
/**
* Manual run-now fire. Acquires the schedule lease to serialize concurrent
* run-now clicks (and to block against a background engine claim), then fires
* in manual mode so the next automatic occurrence is left untouched. Returns
* null when the lease is already held (a run is in progress).
*/
async function fireScheduleNow(
schedule: FireableSchedule,
limits: ScheduleLimits,
options?: { signal?: AbortSignal },
): Promise<FireResult | null> {
// The global stop means STOP: a manual run dispatches the same billed generation as
// an automatic one, so gating only the engine tick would leave Run Now wide open.
if (await engineDeps.isGloballyDisabled()) {
return { fired: false, skipped: 'disabled' as const };
}
const leased = await methods.acquireManualRunLease(
schedule.id,
schedule.user,
MANUAL_RUN_LEASE_MS,
);
if (leased == null) {
return null;
}
const claimToken = leased.claimToken;
try {
// Fire the FRESH leased row (post-image with the new claim token), not the
// snapshot the route read before the lease — an edit that committed in the
// window in between is reflected, so a stale prompt/agent is never dispatched.
return await fireSchedule(engineDeps, leased, limits, new Date(), {
manual: true,
signal: options?.signal,
});
} catch (err) {
const released =
claimToken != null
? await methods.releaseLease(schedule.id, claimToken).catch(() => false)
: false;
if (!released && leased.leaseBy != null) {
// An owner edit during the failed preflight rotates the token while preserving
// this unique holder. Clear only that holder; a takeover has a different value
// and is therefore never stripped by this exception safety net.
await methods.releaseLeaseByHolder(schedule.id, leased.leaseBy).catch(() => undefined);
}
throw err;
}
}
async function recordMCPToolAuthFailure(
input: Parameters<SchedulesService['recordMCPToolAuthFailure']>[0],
): Promise<boolean> {
const { error, streamId, jobCreatedAt, userId, serverName, identity } = input;
const cause = error instanceof Error ? error.cause : undefined;
const missing = error instanceof OboTokenResolutionError ? error : cause;
const bearer =
error instanceof ScheduledMCPBearerError || error instanceof ScheduledMCPPolicyError;
if (
(!bearer &&
(!(missing instanceof OboTokenResolutionError) ||
missing.reason !== 'missing_upstream_provider')) ||
!streamId ||
jobCreatedAt == null ||
!userId
)
return false;
const key = receiptKey(streamId, jobCreatedAt);
if (bearer && identity && identity.ownerId === userId) {
const previous = pendingBearerFailures.get(key);
const priority = (value: unknown): number =>
value instanceof ScheduledMCPBearerError || value instanceof ScheduledMCPPolicyError
? {
mcp_unavailable: 1,
mcp_reauth_required: 2,
mcp_configuration_missing: 3,
mcp_permission_denied: 4,
}[value.code]
: 0;
const incoming = error.outcomes;
const outcomes = mergeReceipts(previous?.outcomes ?? [], incoming);
const admitted = previous?.admitted === true && outcomes.length === previous.outcomes.length;
const retained = previous && priority(previous.error) > priority(error) ? previous : input;
pendingBearerFailures.set(key, {
...retained,
identity: Object.freeze({ ...identity }),
admitted,
outcomes,
});
}
const store = GenerationJobManager.getJobStore();
const job = await store?.getJob(streamId);
if (
job?.createdAt !== jobCreatedAt ||
job.userId !== userId ||
!job.scheduleId ||
!job.scheduledFor ||
!job.conversationId ||
(error instanceof ScheduledMCPBearerError && !identity) ||
(identity != null &&
(identity.ownerId !== job.userId ||
identity.scheduleId !== job.scheduleId ||
identity.tenantId !== (job.tenantId ?? null) ||
((error instanceof ScheduledMCPBearerError || job.agent_id != null) &&
identity.agentId !== job.agent_id)))
) {
pendingBearerFailures.delete(key);
if (bearer) throw new ScheduledMCPReceiptFencedError();
return false;
}
if (error instanceof ScheduledMCPPolicyError && !pendingBearerFailures.has(key)) {
pendingBearerFailures.set(key, {
...input,
identity: identity ?? {
scheduleId: job.scheduleId,
ownerId: job.userId,
tenantId: job.tenantId ?? null,
agentId: job.agent_id ?? '',
invocationMode: 'delegated',
},
admitted: false,
outcomes: error.outcomes,
});
}
const payload = {
scheduleId: job.scheduleId,
scheduledFor: new Date(job.scheduledFor),
conversationId: job.conversationId,
...(job.tenantId && { tenantId: job.tenantId }),
server: serverName,
};
if (!bearer) return methods.recordMCPToolAuthFailure(payload);
const pendingReceipt = pendingBearerFailures.get(key);
if (pendingReceipt?.admitted) return true;
const pendingError = pendingReceipt?.error;
const receiptError =
pendingError instanceof ScheduledMCPBearerError ||
pendingError instanceof ScheduledMCPPolicyError
? pendingError
: error;
const admittedOutcomes = mergeReceipts(
pendingReceipt?.outcomes ?? receiptError.outcomes,
error.outcomes,
);
const outcomes = mergeReceipts(
readScheduleMCPOutcomes(job.scheduleOutcomeError),
admittedOutcomes,
);
const encoded = encodeReceipts(outcomes);
const writes = await Promise.allSettled([
methods.recordMCPToolAuthFailure({
...payload,
outcomes: admittedOutcomes,
...(error instanceof ScheduledMCPPolicyError && { outcome: receiptError.outcomes[0] }),
}),
GenerationJobManager.updateMetadata(
streamId,
{
preserveForScheduleReconcile: true,
scheduleOutcome: 'error',
scheduleOutcomeError: encoded,
scheduleMCPFailure: admittedOutcomes.reduce(
(previous, next) =>
scheduleMCPFailurePriority(next) > scheduleMCPFailurePriority(previous)
? next
: previous,
job.scheduleMCPFailure ?? admittedOutcomes[0],
),
},
jobCreatedAt,
),
]);
const latest = await store?.getJob(streamId).catch(() => null);
const retained =
latest?.createdAt === jobCreatedAt &&
latest.scheduleId === job.scheduleId &&
latest.userId === userId &&
latest.scheduledFor === job.scheduledFor &&
readScheduleMCPOutcomes(latest.scheduleOutcomeError).some(
(item) => outcomeKey(item) === outcomeKey(receiptError.outcomes[0]),
);
if (
(writes[0].status === 'fulfilled' && writes[0].value) ||
(store?.durableScheduleReceipts === true && writes[1].status === 'fulfilled' && retained)
) {
const current = pendingBearerFailures.get(key);
if (current)
current.admitted =
mergeReceipts(admittedOutcomes, current.outcomes).length === admittedOutcomes.length;
return true;
}
return false;
}
async function getMCPReceiptRetryPolicy(
input: ScheduleMCPFailureInput,
): Promise<ScheduleMCPReceiptRetryPolicy | undefined> {
if (!input.identity || input.userId !== input.identity.ownerId) return;
const owner = await engineDeps.getUserContext(input.identity.ownerId);
if (!owner || (owner.tenantId ?? null) !== input.identity.tenantId) return;
return engineDeps.runInTenantContext(owner, async () => {
const config = await deps.getAppConfig(getAppConfigOptionsFromUser(owner));
const schedules = config?.interfaceConfig?.schedules;
return typeof schedules === 'object' ? schedules.mcpReceiptRetry : undefined;
});
}
/** The job outbox survives both owner death and a crash after Mongo bookkeeping. */
async function reconcileRetainedJobs(): Promise<void> {
const store = GenerationJobManager.getJobStore();
if (!store?.getScheduleReconcileJobs) return;
for (const held of await store.getScheduleReconcileJobs(100)) {
let job = held;
if (job.terminalPersistencePending === true) {
try {
// The manager owns the bounded, epoch-fenced stale final-write recovery.
await GenerationJobManager.getCleanupJob(job.streamId);
const current = await store.getJob(job.streamId);
if (current?.createdAt !== held.createdAt) continue;
job = current;
} catch (error) {
logger.warn('[schedules] retained terminal recovery deferred:', error);
continue;
}
}
if (
job.status === 'error' &&
job.error === 'Scheduled generation owner became unavailable' &&
job.providerDrained === false &&
job.providerExecutionId &&
job.scheduleId &&
job.scheduledFor &&
deps.confirmScheduleProviderOwnerLoss &&
store.recoverScheduleProviderOwnerLoss
) {
const owner: ScheduleProviderOwner = {
streamId: job.streamId,
createdAt: job.createdAt,
providerExecutionId: job.providerExecutionId,
scheduleId: job.scheduleId,
scheduledFor: job.scheduledFor,
userId: job.userId,
tenantId: job.tenantId ?? null,
lastActiveAt: job.lastActiveAt ?? job.createdAt,
};
try {
const observed = await waitUntilDeadline(
deps.confirmScheduleProviderOwnerLoss(Object.freeze(owner)),
Date.now() + PROVIDER_DRAIN_TIMEOUT_MS,
);
const proof = observed.settled ? observed.value : null;
if (
proof &&
Object.keys(owner).every(
(key) =>
owner[key as keyof ScheduleProviderOwner] ===
proof[key as keyof ScheduleProviderOwner],
) &&
(await store.recoverScheduleProviderOwnerLoss(owner))
) {
const current = await store.getJob(job.streamId);
if (!current || current.createdAt !== owner.createdAt) continue;
job = current;
}
} catch {
continue;
}
}
if (
!job.scheduleId ||
!job.scheduledFor ||
job.status === 'running' ||
job.status === 'requires_action' ||
job.providerDrained === false ||
job.terminalPersistencePending === true ||
job.terminalHostActionPending === true
)
continue;
const intended =
job.status === 'aborted'
? { status: 'interrupted' as const, error: undefined }
: retainedOutcome(job, job.status === 'complete' ? 'success' : 'error');
try {
await recordScheduleOutcome({
scheduleId: job.scheduleId,
scheduledFor: job.scheduledFor,
streamId: job.streamId,
jobCreatedAt: job.createdAt,
conversationId: job.conversationId,
status: intended.status,
error: intended.error,
});
} catch (error) {
logger.warn('[schedules] retained outcome recovery deferred:', error);
}
}
}
const OUTCOME_RETRY_ATTEMPTS = 3;
/**
* Completion hook: called from the agents controller finalize paths when the
* request carried a scheduleId. The caller deletes the job (`completeJob`) right
* after, destroying the only evidence the reconciler could use — so a transient
* Mongo failure here is RETRIED (bounded) before giving up, and the failure is
* surfaced to the caller (returns false) so it can keep the job when it matters.
*/
/**
* OWNER-SIDE Stop barrier. An interactive Stop stamps `abortSource: 'stop'` before it
* signals the abort, and only acknowledges (`abortPersistedAt`) once its partial-message
* and checkpoint writes have landed. A terminal settlement here in between would release
* the run's capacity — and let a concurrent schedule/account deletion erase the data —
* while that write is still in flight. Wait (bounded) for the acknowledgement. A resolved
* marker, a non-stop source, or a stamp gone stale (its route presumed dead) proceeds
* immediately, so the stale-owner timeout remains the bounded recovery path.
*/
async function waitForStopPersistence(
scheduleId: string,
scheduledFor: Date,
initialState?: Awaited<ReturnType<ScheduleMethods['getScheduleRunAbortState']>>,
): Promise<ScheduleMCPOutcome[] | null> {
let observed = initialState;
const deadline = Date.now() + STOP_BARRIER_TIMEOUT_MS;
for (;;) {
const state =
observed !== undefined
? observed
: await methods.getScheduleRunAbortState(scheduleId, scheduledFor);
observed = undefined;
if (
state == null ||
state.abortSource !== 'stop' ||
state.abortPersistedAt != null ||
!hasAbortInFlight(state, Date.now())
) {
// Cleared to settle: no Stop owns the run, it acknowledged, or its owner is past
// the stale cutoff and is presumed dead (the bounded recovery path).
return state?.mcp?.filter(isScheduleMCPAuthorizationFailure) ?? [];
}
if (Date.now() >= deadline) {
// Still an UNACKNOWLEDGED, FRESH Stop. The poll budget expiring proves nothing about
// the route's writes — slow checkpoint cleanup looks exactly like this — so treating
// the barrier as satisfied would terminalize the run and release its capacity (and
// its deletion/erasure barriers) while beforePublish may still be writing. DEFER
// instead: the acknowledgement settles it, or the stale-owner cutoff authorizes a
// later attempt.
return null;
}
await new Promise((resolve) => setTimeout(resolve, STOP_BARRIER_POLL_MS));
}
}
async function recordScheduleOutcome({
scheduleId,
scheduledFor,
streamId,
jobCreatedAt,
status,
conversationId,
checkpointNamespace,
clearConversationId,
error,
}: RecordScheduleOutcomeInput): Promise<boolean> {
if (!scheduleId || !scheduledFor) {
return true;
}
const boundaryKey =
streamId && jobCreatedAt != null ? receiptKey(streamId, jobCreatedAt) : undefined;
const boundary = boundaryKey && mcpBoundaries.get(boundaryKey);
if (boundary && boundary.scheduleId === scheduleId) {
try {
boundary.flight ??= boundary.quiesce();
await boundary.flight;
if (mcpBoundaries.get(boundaryKey!) === boundary) mcpBoundaries.delete(boundaryKey!);
} catch {
return false;
}
}
let terminal = status !== 'requires_action';
let mcp: ScheduleMCPOutcome[] = [];
const receipt =
streamId && jobCreatedAt != null ? receiptKey(streamId, jobCreatedAt) : undefined;
const pending = receipt && pendingBearerFailures.get(receipt);
let retiredPolicyGeneration = false;
if (pending && pending.identity?.scheduleId === scheduleId) {
if (!pending.admitted) {
try {
const current = await GenerationJobManager.getJobStore()?.getJob(streamId!);
retiredPolicyGeneration =
pending.error instanceof ScheduledMCPPolicyError &&
terminal &&
current != null &&
current.createdAt > jobCreatedAt! &&
current.userId === pending.identity.ownerId &&
(current.tenantId ?? null) === pending.identity.tenantId;
if (retiredPolicyGeneration) {
// A trusted denial may outlive its worker epoch; persist it only to the
// old occurrence, never to the successor's generation metadata.
if (
!(await methods.recordMCPToolAuthFailure({
scheduleId,
scheduledFor: new Date(scheduledFor),
conversationId: streamId!,
...(pending.identity.tenantId && { tenantId: pending.identity.tenantId }),
server: pending.serverName,
outcomes: pending.outcomes,
}))
)
return false;
pending.admitted = true;
} else if (!(await recordMCPToolAuthFailure(pending))) return false;
} catch {
return false;
}
}
const acknowledged = receipt ? pendingBearerFailures.get(receipt) : undefined;
if (!acknowledged?.admitted) return false;
mcp = mergeReceipts(mcp, acknowledged.outcomes);
}
if (streamId && jobCreatedAt != null) {
try {
const job = await GenerationJobManager.getJobStore()?.getJob(streamId);
if (
job?.createdAt === jobCreatedAt &&
job.scheduleId === scheduleId &&
new Date(job.scheduledFor ?? '').getTime() === new Date(scheduledFor).getTime()
) {
mcp = mergeReceipts(
mcp,
readScheduleMCPReceipts(job.scheduleOutcomeError),
job.scheduleMCPFailure ? [job.scheduleMCPFailure] : [],
);
}
} catch {
return false;
}
}
let pauseState: Awaited<ReturnType<ScheduleMethods['getScheduleRunAbortState']>> | undefined;
if (!terminal && !pending && mcp.length === 0) {
try {
// A peer can own the only denial. Ordinary pauses still do not wait on Stop.
pauseState = await methods.getScheduleRunAbortState(scheduleId, new Date(scheduledFor));
mcp = mergeReceipts(
mcp,
pauseState?.mcp?.filter((item) => isScheduleMCPAuthorizationFailure(item)) ?? [],
);
} catch {
return false;
}
}
if (terminal || pending || mcp.length > 0) {
// Honor an in-flight interactive Stop's persistence before terminalizing. A deferral
// is NOT a failure to record — the run is deliberately left active/preserved — but it
// must report "not settled" so callers with durable retry (the approval-expiry host
// action, reconciliation) re-drive it rather than assuming the outcome landed.
const observed = await waitForStopPersistence(scheduleId, new Date(scheduledFor), pauseState);
if (observed == null) {
logger.info(
`[schedules] deferring terminal settlement for ${scheduleId}: interactive Stop persistence is still unacknowledged`,
);
return false;
}
mcp = mergeReceipts(mcp, observed);
}
const retained = mcp.length > 0 ? mcp.reduce(preferFailure, mcp[0]) : undefined;
const missingAuth = mcp.length > 0;
if (missingAuth) terminal = true;
const effectiveStatus = missingAuth ? 'error' : status;
let effectiveError = error;
if (missingAuth)
effectiveError = mcp.some((item) => item.reason)
? encodeReceipts(mcp)
: 'MCP unattended authorization unavailable';
if (terminal && streamId && jobCreatedAt != null && !retiredPolicyGeneration) {
try {
await GenerationJobManager.updateMetadata(
streamId,
{
preserveForScheduleReconcile: true,
...(retained && { scheduleMCPFailure: retained }),
scheduleOutcome:
effectiveStatus === 'success' ||
effectiveStatus === 'error' ||
effectiveStatus === 'interrupted' ||
effectiveStatus === 'skipped_balance'
? effectiveStatus
: 'error',
...(effectiveError ? { scheduleOutcomeError: effectiveError } : {}),
},
jobCreatedAt,
);
} catch (err) {
logger.error('[schedules] failed to retain terminal outcome evidence:', err);
return false;
}
}
if (missingAuth && status === 'requires_action' && (!streamId || jobCreatedAt == null))
return false;
if (missingAuth && streamId && jobCreatedAt != null && !retiredPolicyGeneration) {
try {
const live = await GenerationJobManager.getJobStore()?.getJob(streamId);
if (
live?.createdAt !== jobCreatedAt ||
live.scheduleId !== scheduleId ||
new Date(live.scheduledFor ?? '').getTime() !== new Date(scheduledFor).getTime() ||
live.providerDrained === false ||
live.terminalPersistencePending === true
)
return false;
if (live.status === 'requires_action') {
// The abort owner parks accepted steers and re-enters settlement only after drain.
return await engineDeps.abortScheduledJob(
streamId,
{ scheduleId, scheduledFor, createdAt: jobCreatedAt },
{ preserve: true },
);
}
} catch (error) {
logger.warn('[schedules] denied generation drain deferred:', error);
return false;
}
}
for (let attempt = 1; attempt <= OUTCOME_RETRY_ATTEMPTS; attempt++) {
try {
// Resolve the owner's limits so auto-disable uses the same per-principal
// threshold as the fire path (not the global default).
const schedule = await methods.getScheduleById(scheduleId);
const owner = schedule ? await engineDeps.getUserContext(schedule.user) : null;
const limits = await getLimits(owner ?? undefined);
await methods.recordRunOutcome({
scheduleId,
scheduledFor: new Date(scheduledFor),
status: effectiveStatus,
clearConversationId,
conversationId,
...(status === 'requires_action' && checkpointNamespace != null
? { checkpointNamespace }
: {}),
error: effectiveError,
...(mcp.length > 0 ? { mcp } : {}),
autoDisableAfterFailures: limits.autoDisableAfterFailures,
balanceSkipDisableThreshold: BALANCE_SKIP_DISABLE_THRESHOLD,
});
if (terminal && streamId && jobCreatedAt != null && !retiredPolicyGeneration) {
try {
const current = await GenerationJobManager.getJobStore()?.getJob(streamId);
if (
current?.createdAt === jobCreatedAt &&
(current.providerDrained === false ||
current.terminalPersistencePending === true ||
current.terminalHostActionPending === true)
) {
// Its host callback may settle Mongo while independently retained cleanup stays pending.
return true;
}
const latest = receipt && pendingBearerFailures.get(receipt);
if (
latest &&
(!latest.admitted || mergeReceipts(mcp, latest.outcomes).length !== mcp.length)
)
return false;
await GenerationJobManager.updateMetadata(
streamId,
{ preserveForScheduleReconcile: false },
jobCreatedAt,
);
} catch (error) {
logger.warn('[schedules] failed to release terminal outcome evidence:', error);
return false;
}
// A paused occurrence can be aborted while its provider is already drained.
// In that case the manager's normal drain-time cleanup has already passed;
// reap the exact terminal job after the Mongo outcome becomes durable.
const store = GenerationJobManager.getJobStore();
const job = await store?.getJob(streamId).catch(() => null);
if (
job?.createdAt === jobCreatedAt &&
job.providerDrained !== false &&
job.terminalPersistencePending !== true &&
job.terminalHostActionPending !== true &&
job.status !== 'running' &&
job.status !== 'requires_action' &&
job.preserveForScheduleReconcile !== true
) {
await store?.deleteJob(streamId, jobCreatedAt).catch((err) => {
logger.warn('[schedules] failed to clear settled generation evidence:', err);
});
}
}
if (terminal && receipt) pendingBearerFailures.delete(receipt);
// ERASE-ON-SETTLE: whichever process records a run's terminal outcome also
// attempts the deferred erase of a deleting schedule. This is what makes a
// delete's `draining` state converge in EVERY topology — the clustered
// entrypoint runs no reconciler, so without this the hidden schedule (and its
// prompt, which has no TTL) survived its last run indefinitely there. A cheap
// guarded no-op for live schedules (the erase filters on `deleting: true`).
if (terminal) {
await eraseSettledSchedule(scheduleId).catch((err) => {
logger.warn(`[schedules] erase-on-settle failed for ${scheduleId}:`, err);
});
}
return true;
} catch (err) {
logger.error(
`[schedules] failed to record run outcome (attempt ${attempt}/${OUTCOME_RETRY_ATTEMPTS}):`,
err,
);
}
}
return false;
}
async function beginScheduledStop({
scheduleId,
scheduledFor,
}: {
scheduleId: string;
scheduledFor: string | Date;
}): Promise<boolean | 'in_progress'> {
if (!scheduleId || !scheduledFor) {
return false;
}
return methods.requestRunAbort(scheduleId, new Date(scheduledFor), 'stop');
}
async function acknowledgeScheduledStopPersistence({
scheduleId,
scheduledFor,
settle,
}: {
scheduleId: string;
scheduledFor: string | Date;
/** Terminal outcome to (re-)drive once the barrier clears. Supplied by the Stop route
* so a settlement its owner already DEFERRED past the poll budget still converges. */
settle?: { status: ScheduleRunOutcomeStatus; conversationId?: string; error?: string };
}): Promise<void> {
if (!scheduleId || !scheduledFor) {
return;
}
await methods.markRunAbortPersisted(scheduleId, new Date(scheduledFor));
if (settle == null) {
return;
}
// The owner calls recordScheduleOutcome ONCE. If its Stop barrier deferred (slow
// beforePublish), nothing would re-drive it where no schedule reconciler is armed —
// the run would stay `started`, its job preserved, and its global capacity slot held.
// Now that the barrier is acknowledged, settle from here; recordRunOutcome is
// match-guarded and idempotent, so a run the owner already settled is a no-op.
await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: settle.status,
conversationId: settle.conversationId,
error: settle.error,
}).catch((err) => logger.warn('[schedules] post-acknowledgement settlement failed:', err));
}
async function isScheduleLive(
scheduleId: string,
expectedConfigRevision?: number,
options?: { automatic?: boolean; policy?: boolean; scheduledFor?: string | Date },
): Promise<boolean> {
if (!scheduleId) {
return false;
}
const schedule = await methods.getScheduleById(scheduleId);
if (schedule == null) {
return false;
}
// An AUTOMATIC fire must still be wanted. A policy auto-disable (too many failures,
// insufficient balance) flips `enabled` WITHOUT touching configRevision, so the
// revision fence below cannot see it — an occurrence already in the claim-to-
// controller window would otherwise start a billed generation for a schedule that
// has just been switched off. Run Now is an explicit user action and stays allowed
// on a disabled schedule, matching fireScheduleNow.
if (options?.automatic === true && schedule.enabled === false) {
return false;
}
// REVISION FENCE at the admission boundary. Existence alone is not enough: an owner
// edit landing between the claim and this point means the dispatched prompt/agent
// came from a config the owner has since replaced, and nothing downstream would
// catch it because the run persists under the NEW schedule. Refuse before any
// message is written. Absent on either side disables the fence, so pre-existing
// schedules and older fires keep working.
if (
expectedConfigRevision != null &&
typeof schedule.configRevision === 'number' &&
schedule.configRevision !== expectedConfigRevision
) {
return false;
}
// LIVE dispatch policy, re-applied exactly as the fire path applies it. A pause
// can sit unanswered for hours, and an operator's global kill switch, a narrowed
// `interface.schedules` availability, or a revoked SCHEDULES:USE permission
// landing in that window touches neither the row nor its revision — none of the
// checks above can see it, and approving the pause would start a fresh billed
// continuation the operator believes is stopped. Applies to manual runs too: an
// emergency stop must stop those approvals as well.
if (options?.policy === true) {
if (await engineDeps.isGloballyDisabled()) {
return false;
}
const owner = await engineDeps.getUserContext(schedule.user);
if (owner == null) {
return false;
}
const limits = await engineDeps.runInTenantContext(owner, () => getLimits(owner));
if (!limits.enabled) {
return false;
}
if (!(await engineDeps.hasScheduleAccess(owner))) {
return false;
}
// Project policy belongs HERE rather than in the resume claim: this branch's
// refusal is already routed through abort-and-settle by both callers, so a
// policy stop settles the occurrence instead of leaving it at `requires_action`
// answering 409 to every approval attempt until it expires.
//
// Deliberately NARROW. It refuses only where there is no valid destination left:
// the requirement is on with nothing to satisfy it, or the schedule's own project
// is gone (which also unset it on the conversation). It does NOT refuse merely
// because an operator's pin moved to a different project — the paused
// conversation cannot be rebound (`chatProjectId` is excluded from the resume
// context, and the continuation reuses the same conversationId), so refusing
// would strand a pending approval over a pin that only governs where the NEXT
// run lands, which the fire path already redirects.
// Prefer the destination THIS OCCURRENCE recorded over the schedule-level value.
// A paused run does not block later occurrences (the single-active index covers
// `started` only), so after a pin moves, a subsequent fire rewrites the schedule
// row while the paused conversation stays where it was — validating the row would
// then check a project that conversation was never filed under.
//
// An ABSENT record falls back to the schedule: a pre-scope occurrence, or one
// whose row is gone, must never be read as evidence to stop a run.
const occurrence =
options?.scheduledFor != null
? await methods.getScheduleRunProject(scheduleId, options.scheduledFor)
: null;
// `recorded`, not the id: an occurrence that deliberately ran unscoped recorded a
// null, and validating the schedule's CURRENT value for it would admit a
// conversation that satisfies no present requirement. Only an unknown record — a
// row from before the field, or none at all — falls back.
const effectiveProject = occurrence?.recorded
? occurrence.chatProjectId
: resolveScheduleProjectId(limits, schedule.chatProjectId);
if (limits.requireProject && effectiveProject == null) {
return false;
}
if (
effectiveProject != null &&
(await engineDeps.projectAccess(effectiveProject, owner)) !== 'ok'
) {
return false;
}
}
return true;
}
async function claimScheduleResume(
scheduleId: string,
scheduledFor: string | Date,
options?: { expectedConfigRevision?: number; automatic?: boolean },
): Promise<ScheduleResumeClaimResult> {
const schedule = await methods.getScheduleById(scheduleId);
if (schedule == null) {
return { conflict: 'inactive' };
}
if (options?.automatic !== false && schedule.enabled === false) {
return { conflict: 'inactive' };
}
if (
options?.expectedConfigRevision != null &&
typeof schedule.configRevision === 'number' &&
schedule.configRevision !== options.expectedConfigRevision
) {
return { conflict: 'inactive' };
}
const owner = await engineDeps.getUserContext(schedule.user);
if (owner == null) {
return { conflict: 'inactive' };
}
return engineDeps.runInTenantContext(owner, async () => {
const [ownerLimits, deploymentLimits, globallyDisabled, hasAccess] = await Promise.all([
getLimits(owner),
getLimits(),
engineDeps.isGloballyDisabled(),
engineDeps.hasScheduleAccess(owner),
]);
if (!ownerLimits.enabled || globallyDisabled || !hasAccess) {
return { conflict: 'inactive' };
}
// Project policy is NOT re-checked here. It lives in isScheduleLive's `policy`
// branch, which both entry points consult first and whose refusal aborts and
// settles the occurrence; a second copy here would answer a bare 409 and strand
// the run at `requires_action` instead.
// FINAL schedule-side admission fence. Everything above is asynchronous and an
// owner edit/disable can land while it runs. Claim the schedule document under
// the expected config generation now, carry this lease through the approval CAS,
// and atomically consume it before provider execution begins.
const resumeLease = await methods.acquireResumeLease(
scheduleId,
options?.expectedConfigRevision,
options?.automatic !== false,
SCHEDULE_RESUME_LEASE_MS,
);
if (resumeLease?.claimToken == null || resumeLease.leaseBy == null) {
return { conflict: 'inactive' };
}
let retainResumeLease = false;
try {
const allocation = await engineDeps.withGlobalCapacitySlot(
Math.min(ownerLimits.fireConcurrency, deploymentLimits.fireConcurrency),
async (capacitySlot) => {
const attempt = await methods.markRunResumeClaimed(
scheduleId,
new Date(scheduledFor),
capacitySlot,
);
return 'conflict' in attempt && attempt.conflict === 'slot-taken'
? 'slot-taken'
: { claimed: attempt };
},
);
if (allocation === 'capacity') {
return { conflict: 'capacity' };
}
const claimed = allocation.claimed;
if ('conflict' in claimed) {
// withCapacitySlot consumes this internal collision sentinel by retrying the
// next free slot. Normalize defensively at the public boundary as well so a
// custom/test allocator can never leak an implementation-only conflict.
return {
conflict: claimed.conflict === 'slot-taken' ? 'capacity' : claimed.conflict,
};
}
retainResumeLease = true;
return {
capacitySlot: claimed.capacitySlot,
claimToken: resumeLease.claimToken,
leaseBy: resumeLease.leaseBy,
};
} finally {
if (!retainResumeLease) {
await methods.releaseLeaseByHolder(scheduleId, resumeLease.leaseBy);
}
}
});
}
function releaseScheduleResumeClaim(
scheduleId: string,
scheduledFor: string | Date,
capacitySlot: number,
): Promise<boolean> {
return methods.releaseRunResumeClaim(scheduleId, new Date(scheduledFor), capacitySlot);
}
async function finalizeScheduleResumeClaim(
scheduleId: string,
claimToken: string,
leaseBy: string,
options?: { expectedConfigRevision?: number; automatic?: boolean },
): Promise<boolean> {
const consumed = await methods.consumeResumeLease(
scheduleId,
claimToken,
leaseBy,
options?.automatic !== false,
options?.expectedConfigRevision,
);
if (!consumed) {
// An owner edit rotates claimToken but deliberately leaves the old holder's
// lease in place. Clear only that holder so the edited schedule is immediately
// usable; a takeover changed leaseBy and is therefore untouched.
await methods.releaseLeaseByHolder(scheduleId, leaseBy);
}
return consumed;
}
function releaseScheduleResumeFence(scheduleId: string, leaseBy: string): Promise<void> {
return methods.releaseLeaseByHolder(scheduleId, leaseBy);
}
/** Aborts an active run's loopback job (identity-guarded). Returns whether the
* abort was delivered (false when the job wasn't reachable — e.g. a peer worker's
* private store, or a transient error). */
async function abortActiveRun(
run: { scheduleId: string; scheduledFor: Date; conversationId?: string },
preserve: boolean,
options?: { stampRenewal?: boolean; settleAfterAbort?: boolean },
): Promise<boolean> {
if (!run.conversationId) {
return false;
}
// Record the abort REQUEST before signalling. This keeps the run holding its global
// capacity slot until its generation owner writes a terminal outcome (settlement),
// so an abort that has been asked for but not yet honored cannot free capacity for a
// new run while the old generation is still alive. The stamp is LOAD-BEARING: it is
// what makes a concurrent drain (and the reconciler) defer to the owner's settle,
// so if it cannot be made durable the abort must NOT be signalled — report
// undelivered and let the caller's unconfirmed/retry path re-drive both.
//
// `stampRenewal: false` skips the stamp entirely: callers pass it for CLEANUP
// aborts of a job they just observed terminal/absent (nothing live will be
// signalled). Renewing there would re-arm the 30-minute owner-death fence on
// every retry against a dead owner, so the run could never age into the
// reconciler's recovery.
if (options?.stampRenewal !== false) {
try {
await methods.requestRunAbort(run.scheduleId, run.scheduledFor, 'deletion');
} catch (err) {
logger.warn('[schedules] failed to record abort request; abort withheld:', err);
return false;
}
}
const before = await engineDeps.getJobStatus(run.conversationId).catch(() => null);
const generation = before && jobMatchesIdentity(before, run) ? before.createdAt : undefined;
const stopped = await engineDeps
.abortScheduledJob(
run.conversationId,
{ scheduleId: run.scheduleId, scheduledFor: run.scheduledFor, createdAt: generation },
{ preserve },
)
.catch((err) => {
logger.warn('[schedules] failed to abort run job on quiesce:', err);
return false;
});
if (!stopped || options?.settleAfterAbort === false) {
return stopped;
}
// awaitProviderDrain in abortScheduledJob is the positive persistence fence:
// the exact provider owner has unwound, so this row can now leave the active
// set without racing a late message write.
const recorded = await recordScheduleOutcome({
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
status: 'interrupted',
streamId: run.conversationId,
jobCreatedAt: generation,
conversationId: run.conversationId,
error: 'Schedule deleted while the run was active',
});
if (!recorded) {
return false;
}
await engineDeps
.clearReconciledJob(run.conversationId, {
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
})
.catch((err) => logger.warn('[schedules] failed to clear deleted run evidence:', err));
return true;
}
/** Resolve the owner's durable-checkpointer config (in their tenant context) so a
* paused-run checkpoint can be pruned on delete, mirroring the interactive abort. */
async function resolveOwnerCheckpointer(
ownerId: string | Types.ObjectId,
): Promise<TCheckpointerConfig | undefined> {
const owner = await engineDeps.getUserContext(ownerId);
if (owner == null) {
return undefined;
}
const appConfig = await engineDeps.runInTenantContext(owner, () =>
deps.getAppConfig(getAppConfigOptionsFromUser(owner)),
);
return appConfig?.endpoints?.agents?.checkpointer;
}
function deletionSettlementBlocked(job: JobState | null | undefined): boolean {
return (
job != null &&
(job.status === 'running' ||
job.providerDrained === false ||
job.terminalPersistencePending === true ||
job.terminalHostActionPending === true)
);
}
async function cleanupStillPending(
runs: Array<{ scheduleId: string; scheduledFor: Date; conversationId?: string }>,
): Promise<boolean> {
for (const run of runs) {
try {
const job = run.conversationId ? await engineDeps.getJobStatus(run.conversationId) : null;
if (job && jobMatchesIdentity(job, run) && deletionSettlementBlocked(job)) return true;
} catch {
return true;
}
}
return false;
}
/**
* Soft-deletes a schedule for its owner: disables + marks it `deleting` (so the
* engine can no longer claim it and it disappears from the owner's list), rotates
* the claim token to fence any in-flight worker, then DRAINS with the same evidence
* discipline as account-deletion quiesce — a run whose job is provably absent or
* settled is recorded and erased here, synchronously; a live generation is aborted
* and settles through its own outcome write (which erases on settle, so no
* reconciler is required in any topology). The result is honest: `unconfirmed`
* means at least one run could not be shown stopped, and the caller must not
* claim it was.
*/
async function deleteScheduleForOwner(
scheduleId: string,
userId: string,
): Promise<ScheduleDeleteResult> {
const schedule = await methods.markScheduleDeleting(scheduleId, userId);
if (schedule == null) {
return 'not_found';
}
const active = await methods.getActiveRunsForSchedule(scheduleId);
// Resolve the checkpointer config once (only when a paused run needs pruning) in
// the owner's tenant context, matching the interactive abort endpoint's prune.
const hasPausedRun = active.some(
(run) => run.status === 'requires_action' && run.conversationId != null,
);
// BEST-EFFORT: the prune this feeds is itself best-effort, and a lookup failure
// must not cost the aborts below.
const checkpointer = hasPausedRun
? await resolveOwnerCheckpointer(schedule.user).catch((err) => {
logger.warn(`[schedules] checkpointer lookup failed for delete ${scheduleId}:`, err);
return undefined;
})
: undefined;
const stores = hasPausedRun
? await checkpointStorageConfigs(userId, schedule.tenantId, checkpointer).catch((err) => {
logger.warn(
`[schedules] checkpoint storage lookup failed for delete ${scheduleId}:`,
err,
);
return [checkpointer];
})
: [];
let unconfirmed = 0;
const cleanupPending: typeof active = [];
for (const run of active) {
// UNKNOWN is not ABSENT — the same distinction the quiesce path draws. A lookup
// that succeeded and returned null is positive evidence no generation holds this
// conversation; a lookup that THREW is evidence of nothing.
const live = run.conversationId
? await engineDeps.getJobStatus(run.conversationId).then(
(job) => ({ known: true, job }),
() => ({ known: false, job: null }),
)
: { known: true, job: null };
const isThisGeneration =
live.job != null &&
jobMatchesIdentity(live.job, {
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
});
// Capture the paused run's checkpoint ids BEFORE any terminal transition below:
// the prune afterwards is scoped to exactly this set, so checkpoints a
// replacement turn writes after this point can never be swept up by it.
const checkpointNamespace =
isThisGeneration || live.job == null
? (live.job?.checkpointNamespace ?? run.checkpointNamespace)
: undefined;
const captured: Array<{
storage: TCheckpointerConfig | undefined;
generation: AgentCheckpointGeneration;
}> = [];
if (
run.status === 'requires_action' &&
run.conversationId &&
live.known &&
(live.job == null || isThisGeneration)
) {
for (const storage of stores) {
const generation = await captureAgentCheckpointGeneration(
run.conversationId,
storage,
checkpointNamespace == null ? {} : { checkpointNamespace },
);
if (generation != null) captured.push({ storage, generation });
}
}
// Same abort-in-flight deferral as quiesce: post-abort job state (status `aborted`,
// or absence once the abort deleted the job) appears before the owner has persisted
// and settled, so it is not evidence that the generation is done.
//
// A paused job whose run row is still `started` is a pause HAND-OFF in flight:
// the job reports requires_action the instant the run interrupts, while the
// controller's pause branch is still flushing this segment's writes and records
// the pause on the row only after them. Settling on the job state alone let the
// cascade complete before those writes landed, which then recreated the paused
// response for the deleted schedule. The controller's record (or the paused-
// window reconciler) flips the row to requires_action, so the deferral is
// bounded — a later pass settles it.
const pauseHandoffInFlight =
isThisGeneration &&
live.job?.status === 'requires_action' &&
(run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));
const settleable =
live.known &&
!hasAbortInFlight(run, Date.now()) &&
!pauseHandoffInFlight &&
!(isThisGeneration && deletionSettlementBlocked(live.job));
if (settleable) {
// Positive evidence nothing is generating: settle the row HERE so the erase
// below can proceed without any reconciler — the clustered entrypoint has
// none, and deferring these rows to it retained the deleted schedule (and its
// prompt) indefinitely in that topology. Settle BEFORE aborting: a retained
// job is the only evidence of a finished run whose outcome write failed.
const retainedOutcome = isThisGeneration
? TERMINAL_JOB_OUTCOMES[live.job!.status]
: undefined;
const settledStatus = retainedOutcome ?? 'interrupted';
const settled = await methods
.recordRunOutcome({
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
...projectScheduleMCPReceipt(
{
status: settledStatus,
mcp: run.mcp,
error: settledStatus === 'interrupted' ? 'Schedule deleted' : undefined,
},
isThisGeneration ? readScheduleMCPReceipts(live.job?.scheduleOutcomeError) : [],
),
conversationId: run.conversationId,
autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
})
.then(
() => true,
(err) => {
logger.warn(`[schedules] failed to settle run on delete ${scheduleId}:`, err);
return false;
},
);
if (settled) {
// Cleanup abort of a job just observed terminal/absent: nothing live gets
// signalled, so the stamp is NOT renewed — re-arming the owner-death fence
// on every delete retry would keep a dead owner's row from ever aging into
// the reconciler's recovery.
const cleaned = await abortActiveRun(run, false, {
stampRenewal: false,
settleAfterAbort: false,
});
if (!cleaned && isThisGeneration) cleanupPending.push(run);
} else {
unconfirmed += 1;
}
} else {
// A live generation (or an unknown one): abort it, preserving the job so its
// outcome survives, and require the DELIVERY to be confirmed. An abort that
// was not delivered leaves a generation that keeps producing and billing —
// reporting this delete as a success would claim otherwise.
const aborted = await abortActiveRun(run, true);
if (!aborted) {
unconfirmed += 1;
}
}
// HITL: prune the durable checkpoints of a run aborted while paused so a new turn
// in this conversation can't rehydrate the stale interrupt before the Mongo TTL
// reclaims it (thread_id === conversationId). Idempotent / no-op otherwise.
//
// Two guards against a REPLACEMENT owning the conversation: ownership is re-read
// fresh (the settle/abort awaits above are a window a replacement can claim the
// conversationId in), and the deletion is SCOPED to the checkpoint ids captured
// before the terminal transition — so even a replacement that slips in between
// this read and the delete only ever loses checkpoints that predate the abort,
// never its own. An unreadable store proves nothing, so it is left alone.
if (
run.status === 'requires_action' &&
run.conversationId &&
captured.some(({ generation }) => generation.checkpointIds.length > 0)
) {
const fresh = await engineDeps.getJobStatus(run.conversationId).then(
(job) => ({ known: true, job }),
() => ({ known: false, job: null }),
);
const freshIsThisGeneration =
fresh.job != null &&
jobMatchesIdentity(fresh.job, {
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
});
const ownsConversation = fresh.known && (fresh.job == null || freshIsThisGeneration);
if (ownsConversation) {
for (const { storage, generation } of captured) {
await deleteAgentCheckpoint(run.conversationId, storage, generation).catch(
() => undefined,
);
}
}
}
}
if (unconfirmed > 0) {
// The run row leaving the active set is the durable acknowledgement that its
// generation stopped and persisted (the owner settles LAST) — wait for it,
// bounded, before answering. Without this, every cross-replica delete whose
// abort delivery cannot be locally proven would answer 503 even though the
// owner settles moments later.
const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;
let remaining = unconfirmed;
while (Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));
remaining = (await methods.getActiveRunsForSchedule(scheduleId)).length;
if (remaining === 0) {
break;
}
}
if (remaining > 0) {
return 'unconfirmed';
}
}
if (await cleanupStillPending(cleanupPending)) return 'unconfirmed';
const erased = await eraseSettledSchedule(scheduleId).catch((err) => {
logger.warn(`[schedules] erase failed for ${scheduleId}:`, err);
return false;
});
if (erased) {
return 'deleted';
}
// eraseScheduleIfDrained already returns true when the row was concurrently
// erased or is absent. A false result therefore means the deleting row is still
// protected by a live lease/run, or the erase could not be confirmed.
return 'draining';
}
/**
* Quiesces every schedule of a user ahead of account deletion: REVERSIBLY suspends them
* under the attempt `token` (non-claimable, so no new occurrence fires while the cascade
* runs) and aborts the loopback jobs of any in-flight runs, so a scheduled generation
* cannot keep persisting messages after the account's messages/conversations are deleted.
* The suspension is reversible so a cancelled deletion can restore it (see
* restoreUserSchedulesFromDeletion) rather than stranding a live user with erased rows.
*/
async function quiesceUserSchedules(userId: string, token: string): Promise<boolean> {
await methods.suspendUserSchedulesForDeletion(userId, token);
const active = await methods.getActiveRunsForUser(userId);
const unconfirmed: string[] = [];
const cleanupPending: typeof active = [];
for (const run of active) {
// Current resumes promote the row back to `started` with a capacity slot, but a
// rolling deploy or crash-era row may still be `requires_action` while its job is
// already `running`. So the row status alone cannot prove a genuine pause. Read
// the live job BEFORE aborting, which settles or deletes it and would erase this
// evidence.
// UNKNOWN is not ABSENT. A lookup that THREW is evidence of nothing, while one
// that succeeded and returned null is positive evidence that no generation holds
// this conversation. Collapsing the two (a bare `.catch(() => null)`) would let a
// transient store failure read as "genuinely paused" and terminalize a row whose
// resumed generation is still running — the exact hazard this check exists for.
const live = run.conversationId
? await engineDeps.getJobStatus(run.conversationId).then(
(job) => ({ known: true, job }),
() => ({ known: false, job: null }),
)
: { known: true, job: null };
const isThisGeneration =
live.job != null &&
jobMatchesIdentity(live.job, {
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
});
// Settle only on POSITIVE evidence that nothing is generating: an identity-matched
// job that is not running, an identity MISMATCH (a replacement turn owns the
// conversation, so this occurrence's generation is already gone), or a confirmed
// absence. Anything unknown falls through and the drain waits for it.
//
// An abort IN FLIGHT is not such evidence, and this is the subtle case: `aborted`
// and post-abort ABSENCE both appear the instant abortJob wins its CAS, while the
// owner has yet to persist. Deferring to the owner's settle (see
// ABORT_SETTLE_GRACE_MS) is what stops a drain from being confirmed mid-write.
const abortInFlight = hasAbortInFlight(run, Date.now());
// Same pause hand-off deferral as the schedule delete path: a paused job whose
// row is still `started` has the controller's pause-branch writes in flight,
// and settling on the job state alone confirms the drain before they land.
const pauseHandoffInFlight =
isThisGeneration &&
live.job?.status === 'requires_action' &&
(run.status === 'started' || hasResumeHandoffInFlight(run, Date.now()));
const settleable =
live.known &&
!abortInFlight &&
!pauseHandoffInFlight &&
!(isThisGeneration && deletionSettlementBlocked(live.job));
// Aborts here never preserve for reconcile: account deletion hard-deletes these
// run rows, so no reconcile pass would ever finalize or clear a retained job.
const retainedOutcome = isThisGeneration
? TERMINAL_JOB_OUTCOMES[live.job!.status]
: undefined;
// `settleable` ALREADY means positive evidence that nothing is generating: an
// identity-matched job that is not running, an identity MISMATCH (a replacement
// turn owns the conversation), or a confirmed absence. Any extra condition on top
// of it strands exactly the cases it was computed to cover — a `started` row whose
// process died before creating its job reads as a confirmed absence, and the old
// clause left it active through the whole drain, so account deletion answered 503
// on every attempt until the 30-minute orphan sweep.
if (settleable) {
// SETTLE BEFORE ABORTING. The abort deletes the retained job, and for a run
// whose inline outcome write exhausted its retries that job is the ONLY evidence
// it finished — reading its status into a local is not the same as durably
// recording it. If this write fails, the abort below has not yet run, so the
// evidence survives for the next pass; the drain simply does not confirm and
// deletion defers, which is the safe direction.
// Either the retained job's own outcome, or — for a genuinely PAUSED run whose
// approval will never be consumed for a deleted account — `interrupted`. Without
// this a single paused run blocked the account's deletion permanently.
const settledStatus = retainedOutcome ?? 'interrupted';
const settledError = QUIESCE_SETTLE_ERRORS[settledStatus];
const settled = await methods
.recordRunOutcome({
scheduleId: run.scheduleId,
scheduledFor: run.scheduledFor,
...projectScheduleMCPReceipt(
{ status: settledStatus, mcp: run.mcp, error: settledError },
isThisGeneration ? readScheduleMCPReceipts(live.job?.scheduleOutcomeError) : [],
),
conversationId: run.conversationId,
autoDisableAfterFailures: DEFAULT_SCHEDULE_LIMITS.autoDisableAfterFailures,
})
.then(() => true)
.catch((err) => {
logger.warn('[schedules] failed to settle run on quiesce:', err);
return false;
});
// Only now is the job disposable. On a failed settle, leave it: the row is still
// active, so the drain reports unconfirmed and a later pass retries with its
// evidence intact. Cleanup abort of an observed-settled job — no stamp renewal,
// or every quiesce retry would re-arm the owner-death fence and a dead owner's
// row could never age into recovery.
if (settled) {
const cleaned = await abortActiveRun(run, false, {
stampRenewal: false,
settleAfterAbort: false,
});
if (!cleaned && isThisGeneration) cleanupPending.push(run);
}
continue;
}
// Not settleable here (a live generation, or an unknown job): abort it and let the
// bounded drain below wait for its own terminal outcome to land.
const aborted = await abortActiveRun(run, false);
if (!aborted && run.conversationId) {
unconfirmed.push(run.conversationId);
}
}
// WAIT (bounded) for the aborted generations to actually settle before the
// account-deletion cascade deletes messages/conversations: a run that already
// returned from the model can observe the abort but still persist its messages,
// which would otherwise resurrect data for the deleted account after the cascade
// ran. Poll the run rows (they leave the active set once their outcome is
// recorded) until drained or the deadline.
const deadline = Date.now() + QUIESCE_DRAIN_TIMEOUT_MS;
let remaining = active.length;
while (remaining > 0 && Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, QUIESCE_DRAIN_POLL_MS));
remaining = (await methods.getActiveRunsForUser(userId)).length;
}
// Surface anything that did not drain / could not be confirmed so the deletion
// cascade defers rather than destroying while a generation may still persist.
// Re-evaluate rather than trusting the initial abort-delivery result: an abort can
// report false because the job was briefly unreachable, yet that generation then
// finishes and records a terminal outcome during the drain poll. The run is genuinely
// settled at that point, so keeping its id in `unconfirmed` would defer account
// deletion forever. The DRAIN is the authority; delivery is only a hint.
const confirmed =
remaining === 0 &&
!(await cleanupStillPending(cleanupPending)) &&
!(await hasScheduleCleanupObligation({ userId }));
if (confirmed && unconfirmed.length > 0) {
logger.info(
`[schedules] ${unconfirmed.length} abort(s) were not confirmed delivered but their ` +
'runs settled during the drain; treating the quiesce as complete.',
);
}
if (!confirmed) {
logger.warn(
`[schedules] account-deletion quiesce did not confirm ${Math.max(remaining, unconfirmed.length, cleanupPending.length)} ` +
`in-flight scheduled run(s) settled${unconfirmed.length ? ` [${unconfirmed.join(', ')}]` : ''} ` +
'— a peer worker generation may still persist data. Guaranteed quiescing requires a ' +
'shared stream store (USE_REDIS_STREAMS).',
);
}
return confirmed;
}
/**
* Reverses a quiesce whose account deletion was cancelled (a controller failure that
* released the user-deletion fence). Re-enables and re-arms only the rows this exact
* attempt suspended; a schedule the owner independently deleted, or one a newer attempt
* re-suspended, is left as-is.
*/
async function restoreUserSchedulesFromDeletion(userId: string, token: string): Promise<void> {
await methods.restoreUserSchedulesFromDeletion(userId, token);
}
return {
getLimits,
engineDeps,
fireScheduleNow,
recordScheduleOutcome,
recordMCPToolAuthFailure,
getMCPReceiptRetryPolicy,
registerMCPSettlement,
beginScheduledStop,
acknowledgeScheduledStopPersistence,
claimScheduleResume,
releaseScheduleResumeClaim,
finalizeScheduleResumeClaim,
releaseScheduleResumeFence,
isScheduleLive,
deleteScheduleForOwner,
quiesceUserSchedules,
restoreUserSchedulesFromDeletion,
initializeScheduleEngine,
initializeScheduleErasureSweep,
reconcileRetainedJobs,
};
}