Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import type { ScheduleEngineDeps, ScheduleLimits, ScheduleUserContext } from './types';
import type { FireableSchedule } from './types';
import { AgentTriggerServiceUnavailableError } from '../agents/triggers/service';
import { getAgentTriggerIdempotencyKey } from '../agents/triggers/envelope';
import { AgentTriggerDeliveryError } from '../agents/triggers/delivery';
import { buildFireClientRequestId, fireSchedule } from './fire';
import { withCapacitySlot } from './capacity';
import { ScheduleMCPError } from './mcp';
const OWNER: ScheduleUserContext = { id: 'user-1', tenantId: 't1', role: 'USER' };
const LIMITS: ScheduleLimits = {
enabled: true,
maxPerUser: 10,
minIntervalMinutes: 60,
autoDisableAfterFailures: 5,
admissionConcurrency: 20,
fireConcurrency: 5,
mcpPreflightConcurrency: 3,
mcpPreflightTimeoutMs: 300_000,
requireProject: false,
};
function makeSchedule(overrides: Partial<FireableSchedule> = {}): FireableSchedule {
return {
id: 'sched-1',
user: 'user-1' as never,
tenantId: 't1',
name: 'Digest',
prompt: 'Summarize',
agent_id: 'agent-1',
cadence: { frequency: 'daily', hour: 8, minute: 0 },
timezone: 'America/New_York',
target: 'new',
enabled: true,
claimToken: 'ct-1',
leaseBy: 'inst-1',
runCount: 0,
failureCount: 0,
balanceSkipCount: 0,
...overrides,
} as FireableSchedule;
}
/** In-memory run store exercising the real insert/count/delete/idempotency interplay. */
function makeMethods() {
const runs = new Map<
string,
{ status: string; conversationId?: string; capacitySlot?: number; admissionOnly?: boolean }
>();
const calls = {
advance: 0,
releaseLease: 0,
disable: [] as string[],
recordOutcome: [] as { status: string }[],
skipped: [] as string[],
setFireDetails: 0,
};
const key = (id: string, when: Date) => `${id}:${when.toISOString()}`;
const methods = {
releaseLease: jest.fn(async () => {
calls.releaseLease += 1;
return true;
}),
advanceSchedule: jest.fn(async () => {
calls.advance += 1;
return true;
}),
disableSchedule: jest.fn(async (_id: string, reason: string) => {
calls.disable.push(reason);
}),
hasActiveRun: jest.fn(async (id: string) =>
[...runs.entries()].some(([k, r]) => k.startsWith(`${id}:`) && r.status === 'started'),
),
countActiveRuns: jest.fn(
async () => [...runs.values()].filter((r) => r.status === 'started').length,
),
insertScheduleRun: jest.fn(
async (data: { scheduleId: string; scheduledFor: Date; conversationId?: string }) => {
const k = key(data.scheduleId, data.scheduledFor);
if (runs.has(k)) {
return null; // unique {scheduleId, scheduledFor}
}
runs.set(k, { status: 'started', conversationId: data.conversationId });
return { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor };
},
),
// Mirrors the partial-unique-index semantics: same-occurrence row => 'duplicate';
// any OTHER started run for the schedule => 'overlap'; else reserve the slot.
reserveStartedRun: jest.fn(
async (data: {
scheduleId: string;
scheduledFor: Date;
conversationId?: string;
capacitySlot?: number;
admissionOnly?: boolean;
deliveryKey?: string;
}) => {
const k = key(data.scheduleId, data.scheduledFor);
if (runs.has(k)) {
// Mirrors the real method: a duplicate reports the EXISTING row's status so
// the caller can tell "still running" from "already finished".
return { conflict: 'duplicate' as const, existingStatus: runs.get(k)!.status };
}
// Mirrors the unique {capacitySlot} partial index (status:'started').
if (
data.capacitySlot != null &&
[...runs.values()].some(
(r) => r.status === 'started' && r.capacitySlot === data.capacitySlot,
)
) {
return { conflict: 'slot-taken' as const };
}
const overlap = [...runs.entries()].some(
([rk, r]) => rk.startsWith(`${data.scheduleId}:`) && r.status === 'started',
);
if (overlap) {
return { conflict: 'overlap' as const };
}
runs.set(k, {
status: 'started',
conversationId: data.conversationId,
capacitySlot: data.capacitySlot,
admissionOnly: data.admissionOnly,
});
return { run: { scheduleId: data.scheduleId, scheduledFor: data.scheduledFor } };
},
),
getCapacityOccupancy: jest.fn(async () => {
const takenSlots: number[] = [];
let unslotted = 0;
for (const r of runs.values()) {
if (r.status !== 'started') {
continue;
}
if (r.admissionOnly) {
continue;
}
if (typeof r.capacitySlot === 'number') {
takenSlots.push(r.capacitySlot);
} else {
unslotted += 1;
}
}
return { takenSlots, unslotted };
}),
revalidateClaim: jest.fn(async () => true),
releaseLeaseByHolder: jest.fn(async () => undefined),
deleteScheduleRun: jest.fn(
async (id: string, when: Date, _status?: string, expectedConversationId?: string) => {
const k = key(id, when);
const row = runs.get(k) as { conversationId?: string } | undefined;
// Mirrors the conversationId fence: a fire only deletes the row IT inserted.
if (expectedConversationId != null && row?.conversationId !== expectedConversationId) {
return;
}
runs.delete(k);
},
),
setRunFireDetails: jest.fn(async () => {
calls.setFireDetails += 1;
}),
recordSkippedRun: jest.fn(async (data: { status: string }) => {
calls.skipped.push(data.status);
}),
recordRunOutcome: jest.fn(
async (data: { scheduleId: string; scheduledFor: Date; status: string }) => {
const k = key(data.scheduleId, data.scheduledFor);
if (runs.has(k)) {
runs.set(k, { status: data.status });
}
calls.recordOutcome.push({ status: data.status });
},
),
};
return { methods, runs, calls };
}
function makeDeps(
methods: ReturnType<typeof makeMethods>['methods'],
over: Partial<ScheduleEngineDeps> = {},
): ScheduleEngineDeps {
return {
methods: methods as unknown as ScheduleEngineDeps['methods'],
getLimits: async () => LIMITS,
getUserContext: async () => OWNER,
isOutOfBalance: async () => false,
preflightMCP: jest.fn().mockResolvedValue([]),
agentAccess: async () => 'ok',
hasScheduleAccess: async () => true,
resolveFiles: async () => [],
enqueueTrigger: jest.fn(async () => undefined),
runInTenantContext: (_user, fn) => fn(),
getJobStatus: async () => null,
abortScheduledJob: async () => undefined,
clearReconciledJob: async () => undefined,
isOwnerDeleting: async () => false,
isGloballyDisabled: async () => false,
countActiveRunsGlobal: async () => methods.countActiveRuns(),
withGlobalCapacitySlot: (cap: number, claim: (slot: number) => Promise<unknown>) =>
withCapacitySlot(
cap,
() => methods.getCapacityOccupancy(),
claim as Parameters<typeof withCapacitySlot>[2],
),
...over,
} as ScheduleEngineDeps;
}
function mockFetch(impl: () => Promise<unknown> | never) {
global.fetch = jest.fn(impl as never) as never;
}
const okResponse = (conversationId = 'convo-1') =>
({
ok: true,
status: 200,
// The accept path answers with JSON; fire reads the body as text and JSON-parses it.
text: async () => JSON.stringify({ conversationId }),
}) as Response;
const dueAt = () => new Date(Date.now() - 60_000);
afterEach(() => jest.restoreAllMocks());
/**
* The chat route validates `clientRequestId` against `/^[A-Za-z0-9:_-]{1,128}$/`
* (CLIENT_REQUEST_ID_PATTERN in api/server/controllers/agents/request.js). An id
* outside that charset makes the route answer 400 INVALID_CLIENT_REQUEST_ID, which
* fails EVERY fire — so the encoding is a contract, not a formatting choice.
*/
describe('buildFireClientRequestId', () => {
const ROUTE_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
it('stays within the charset the chat route accepts', () => {
const id = buildFireClientRequestId(
'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',
new Date('2026-08-01T14:05:55.132Z'),
);
// A raw ISO instant carries a '.' in its milliseconds, which the route rejects.
expect(id).not.toContain('.');
expect(id).toMatch(ROUTE_PATTERN);
});
it('is deterministic per occurrence and distinct across occurrences', () => {
const scheduleId = 'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b';
const first = new Date('2026-08-01T14:05:55.132Z');
const second = new Date('2026-08-01T15:05:55.132Z');
expect(buildFireClientRequestId(scheduleId, first)).toBe(
buildFireClientRequestId(scheduleId, new Date(first.getTime())),
);
expect(buildFireClientRequestId(scheduleId, first)).not.toBe(
buildFireClientRequestId(scheduleId, second),
);
});
it('stays inside the 128-character cap', () => {
expect(
buildFireClientRequestId(
'sched_bf55e051-b26d-4ccc-b96c-93ffaafe1a5b',
new Date('2026-08-01T14:05:55.132Z'),
).length,
).toBeLessThanOrEqual(128);
});
});
describe('fireSchedule', () => {
it('preserves a claimed occurrence when the deployment switch turns off', async () => {
const { methods } = makeMethods();
const getLimits = jest.fn(async () => ({ ...LIMITS, enabled: false }));
const deps = makeDeps(methods, {
getLimits,
});
const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
expect(result).toMatchObject({ fired: false, skipped: 'superseded' });
expect(methods.advanceSchedule).not.toHaveBeenCalled();
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
expect(getLimits).toHaveBeenCalledWith();
});
it('fires the happy path and records fire details', async () => {
const { methods, runs } = makeMethods();
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.fired).toBe(true);
// The conversation id is pre-generated and recorded on the run row up front
// (so reconciliation can always find the occurrence's job), not read back from
// the POST response.
expect(result.conversationId).toMatch(/^[0-9a-f-]{36}$/);
expect([...runs.values()][0].conversationId).toBe(result.conversationId);
expect(methods.setRunFireDetails).toHaveBeenCalledTimes(1);
expect([...runs.values()][0].status).toBe('started');
});
it('refuses to fire a cadence the engine cannot read at all', async () => {
// A cron cadence reaches the fire path with a stored timezone croner cannot use,
// so the next run is uncomputable: disable without dispatching.
const { methods, runs } = makeMethods();
mockFetch(async () => okResponse());
const schedule = makeSchedule({
cadence: { frequency: 'cron', expression: '0 9 * * *' },
timezone: 'Not/AZone',
});
const result = await fireSchedule(makeDeps(methods), schedule, LIMITS, dueAt());
expect(result.fired).toBe(false);
expect(runs.size).toBe(0);
expect(methods.disableSchedule).toHaveBeenCalledWith('sched-1', 'invalid_schedule', 'ct-1');
});
it('releases the old holder when a post-enqueue owner edit fences the advance', async () => {
const { methods } = makeMethods();
const enqueueTrigger = jest.fn(async () => undefined);
methods.advanceSchedule.mockResolvedValueOnce(false);
const result = await fireSchedule(
makeDeps(methods, { enqueueTrigger }),
makeSchedule({ leaseBy: 'claim-holder-1' }),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(true);
expect(enqueueTrigger).toHaveBeenCalledTimes(1);
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'claim-holder-1');
});
it('stamps the deterministic deliveryKey on the reservation (== the enqueued key)', async () => {
const { methods } = makeMethods();
const enqueueTrigger = jest.fn<
ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
Parameters<ScheduleEngineDeps['enqueueTrigger']>
>(async () => undefined);
const result = await fireSchedule(
makeDeps(methods, { enqueueTrigger }),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(true);
const reservation = (methods.reserveStartedRun as jest.Mock).mock.calls[0][0];
const envelope = enqueueTrigger.mock.calls[0][0];
// Stored BEFORE enqueue and equal to the enqueued delivery's idempotency key, so
// reconciliation can read the durable delivery for THIS occurrence even if the enqueue
// committed ambiguously.
expect(reservation.deliveryKey).toEqual(expect.any(String));
expect(reservation.deliveryKey).toBe(getAgentTriggerIdempotencyKey(envelope));
});
it('marks a Run Now trigger as manual for downstream limiter policy', async () => {
const { methods } = makeMethods();
const deps = makeDeps(methods);
const preflight = jest.spyOn(deps, 'preflightMCP');
const enqueueTrigger = jest.fn<
ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
Parameters<ScheduleEngineDeps['enqueueTrigger']>
>(async () => undefined);
await fireSchedule({ ...deps, enqueueTrigger }, makeSchedule(), LIMITS, dueAt(), {
manual: true,
});
expect(preflight).toHaveBeenCalledWith(
'agent-1',
expect.anything(),
expect.objectContaining({ manual: true }),
);
const envelope = enqueueTrigger.mock.calls[0][0];
expect(envelope).toMatchObject({
mode: 'fire',
principal: { userId: 'user-1', role: 'USER', tenantId: 't1' },
event: {
type: 'schedule.occurrence',
source: { id: 'sched-1', type: 'schedule' },
},
run: { metadata: { manual: true } },
});
expect(enqueueTrigger).toHaveBeenCalledWith(envelope, { orderingKey: 'sched-1' });
});
it('marks an automatic occurrence as non-manual', async () => {
const { methods } = makeMethods();
const enqueueTrigger = jest.fn<
ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
Parameters<ScheduleEngineDeps['enqueueTrigger']>
>(async () => undefined);
await fireSchedule(makeDeps(methods, { enqueueTrigger }), makeSchedule(), LIMITS, dueAt());
expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({
run: { metadata: { manual: false } },
});
});
it('carries the claimed config revision and fire inputs on the durable trigger', async () => {
const { methods } = makeMethods();
const enqueueTrigger = jest.fn<
ReturnType<ScheduleEngineDeps['enqueueTrigger']>,
Parameters<ScheduleEngineDeps['enqueueTrigger']>
>(async () => undefined);
await fireSchedule(
makeDeps(methods, {
enqueueTrigger,
resolveFiles: async () => [{ file_id: 'file-1' }],
}),
makeSchedule({ configRevision: 7, file_ids: ['file-1'] } as never),
LIMITS,
dueAt(),
);
expect(enqueueTrigger.mock.calls[0][0]).toMatchObject({
input: 'Summarize',
target: { agentId: 'agent-1' },
run: {
timezone: 'America/New_York',
files: [{ file_id: 'file-1' }],
metadata: { manual: false, configRevision: 7 },
},
});
});
it('records a definite trigger admission rejection as an error', async () => {
const { methods, calls } = makeMethods();
const result = await fireSchedule(
makeDeps(methods, {
enqueueTrigger: async () => {
throw new AgentTriggerServiceUnavailableError('not initialized');
},
}),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(false);
expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
});
it('leaves an ambiguous persistence failure reconcilable', async () => {
const { methods, runs, calls } = makeMethods();
const result = await fireSchedule(
makeDeps(methods, {
enqueueTrigger: async () => {
throw new Error('Mongo result unknown');
},
}),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(false);
expect(calls.recordOutcome).toHaveLength(0);
expect([...runs.values()][0].status).toBe('started');
expect(calls.advance).toBe(1);
});
it('records an invalid durable delivery as a definite error', async () => {
const { methods, runs, calls } = makeMethods();
const result = await fireSchedule(
makeDeps(methods, {
enqueueTrigger: async () => {
throw new AgentTriggerDeliveryError('invalid delivery');
},
}),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(false);
expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
expect([...runs.values()][0].status).toBe('error');
});
it('does not orphan a run when file resolution fails', async () => {
const { methods, runs, calls } = makeMethods();
mockFetch(async () => okResponse());
const deps = makeDeps(methods, {
resolveFiles: async () => {
throw new Error('db down');
},
});
const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt());
expect(result.fired).toBe(false);
expect(runs.size).toBe(0); // no run row created
// Automatic fire KEEPS the claim lease as a backoff (nextRunAt untouched → the
// occurrence retries when the lease expires) so a transient file error can't get
// this row re-claimed every tick and starve others.
expect(calls.releaseLease).toBe(0);
expect(global.fetch).not.toHaveBeenCalled();
});
it('releases the lease on a manual run-now when file resolution fails', async () => {
const { methods, runs } = makeMethods();
// An owner edit can rotate the token while the file lookup is in flight while
// deliberately preserving this fire's unique lease holder.
methods.releaseLease.mockResolvedValueOnce(false);
mockFetch(async () => okResponse());
const deps = makeDeps(methods, {
resolveFiles: async () => {
throw new Error('db down');
},
});
const result = await fireSchedule(deps, makeSchedule({ file_ids: ['f1'] }), LIMITS, dueAt(), {
manual: true,
});
expect(result.fired).toBe(false);
expect(runs.size).toBe(0);
// Run-now releases so the user can retry immediately (no misleading lease-held 409).
expect(methods.releaseLease).toHaveBeenCalledWith('sched-1', 'ct-1');
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
expect(global.fetch).not.toHaveBeenCalled();
});
it('refuses the fire at the global capacity cap WITHOUT inserting a run', async () => {
const { methods, runs } = makeMethods();
// All 5 slots taken → the allocator finds no free slot and never inserts.
for (let i = 0; i < 5; i++) {
runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });
}
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('capacity');
expect(global.fetch).not.toHaveBeenCalled();
// Slot-based capacity is decided BEFORE the write, so there is nothing to roll back.
expect([...runs.values()].filter((r) => r.status === 'started')).toHaveLength(5);
expect(methods.reserveStartedRun).not.toHaveBeenCalled();
expect(methods.deleteScheduleRun).not.toHaveBeenCalled();
});
it('completes readiness before requesting generation capacity', async () => {
const { methods } = makeMethods();
const order: string[] = [];
const deps = makeDeps(methods, {
preflightMCP: async () => {
order.push('readiness');
return [];
},
withGlobalCapacitySlot: async (_cap, claim) => {
order.push('generation-capacity');
const attempt = await claim(0);
return attempt === 'slot-taken' ? 'capacity' : attempt;
},
});
mockFetch(async () => okResponse());
await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
expect(order).toEqual(['readiness', 'generation-capacity']);
});
it('does not let a principal override widen the global capacity cap', async () => {
const { methods, runs } = makeMethods();
// The single deployment-wide slot is already occupied.
runs.set('other-0:x', { status: 'started', capacitySlot: 0 });
mockFetch(async () => okResponse());
const deps = makeDeps(methods, {
// Base allows 1 concurrent scheduled generation; this owner's role/tenant
// override raises their limit to 5.
getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 5 : 1 }),
});
// Run Now resolves the OWNER's limits and skips the tick's base-config budget, so
// without the clamp the override would hand out slots 1-4 and run five billed
// generations against a deployment that advertises a cap of one.
const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('capacity');
expect(global.fetch).not.toHaveBeenCalled();
expect(methods.reserveStartedRun).not.toHaveBeenCalled();
expect(methods.releaseLease).toHaveBeenCalledWith('sched-1', 'ct-1');
});
it('still honors an owner override that is STRICTER than the deployment cap', async () => {
const { methods, runs } = makeMethods();
runs.set('other-0:x', { status: 'started', capacitySlot: 0 });
mockFetch(async () => okResponse());
const deps = makeDeps(methods, {
getLimits: async (user) => ({ ...LIMITS, fireConcurrency: user ? 1 : 5 }),
});
// Only WIDENING is the defect; a tenant that wants less concurrency than the
// deployment allows must still get it.
const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('capacity');
});
it('claims a free slot and never exceeds the cap when slots collide', async () => {
const { methods, runs } = makeMethods();
// Slots 0 and 2 are taken; the allocator must land the fire on slot 1.
runs.set('other-a:x', { status: 'started', capacitySlot: 0 });
runs.set('other-b:x', { status: 'started', capacitySlot: 2 });
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.fired).toBe(true);
const own = [...runs.entries()].find(([k]) => k.startsWith('sched-1:'));
expect(own?.[1].capacitySlot).toBe(1);
});
it('re-fires cleanly next tick once capacity frees, exactly once', async () => {
const { methods, runs } = makeMethods();
for (let i = 0; i < 5; i++) {
runs.set(`other-${i}:x`, { status: 'started', capacitySlot: i });
}
const enqueueTrigger = jest.fn(async () => undefined);
const deps = makeDeps(methods, { enqueueTrigger });
const schedule = makeSchedule();
const when = dueAt();
// Tick 1: every slot taken → refused before any insert.
const first = await fireSchedule(deps, schedule, LIMITS, when);
expect(first.skipped).toBe('capacity');
// Capacity frees up before the next tick.
runs.delete('other-0:x');
// Tick 2: same occurrence re-claimed → now fires, exactly one live run.
const second = await fireSchedule(deps, schedule, LIMITS, when);
expect(second.fired).toBe(true);
expect(methods.reserveStartedRun).toHaveBeenCalledTimes(1); // only the successful tick inserts
expect(
[...runs.entries()].filter(([k, r]) => k.startsWith('sched-1:') && r.status === 'started'),
).toHaveLength(1);
expect(enqueueTrigger).toHaveBeenCalledTimes(1);
});
it('does not reserve a run when the claim already lapsed during preflight', async () => {
const { methods, runs } = makeMethods();
// The preflight (user/config/permission/balance/attachment queries) outlasted the
// 5-minute lease and another worker re-claimed the occurrence.
(methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('superseded');
// Reserving here would win the occurrence's unique row: the FRESH claimer would then
// see `duplicate` and advance without firing, while this worker's own revalidation
// fails and rollbackReservation deliberately retains the row (leaseBy changed). The
// occurrence would be lost with its capacity slot held until the orphan sweep.
expect(methods.reserveStartedRun).not.toHaveBeenCalled();
expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
expect(global.fetch).not.toHaveBeenCalled();
// The lease is handed back by holder so the fresh claimer is not left waiting.
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('sched-1', 'inst-1');
});
/**
* A failed revalidation must NEVER advance. In every true supersession (takeover,
* owner edit) the claim token rotated, so an advance would no-op anyway — the only
* case where it can land is a PURE lease expiry with no takeover, where the token
* never rotated. advanceSchedule checks no lease, so that advance moved nextRunAt
* past an occurrence nothing had fired: a slow preflight silently DROPPED it
* instead of leaving it due for the next claim to retry.
*/
it('leaves the occurrence due when only the lease expired (no takeover)', async () => {
const { methods } = makeMethods();
// Same claim token, merely past leaseUntil: revalidateClaim fails on the lease
// predicate alone while the token-fenced advance filter would still MATCH.
(methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('superseded');
expect(methods.advanceSchedule).not.toHaveBeenCalled();
expect(global.fetch).not.toHaveBeenCalled();
});
it('never advances from any superseded branch, including post-reserve', async () => {
const { methods } = makeMethods();
(methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('superseded');
expect(methods.advanceSchedule).not.toHaveBeenCalled();
});
/**
* The lease can expire between the pre-reserve revalidation and the reservation
* itself: the capacity allocator reads occupancy in between, and the preflight
* before it can already have outlasted the 5-minute lease. Gating the rollback on
* lease ownership stranded the row — the new holder's own reserve saw `duplicate`
* and advanced past the occurrence without firing it, while this row held a global
* capacity slot until the 30-minute orphan sweep.
*/
it('deletes its own undispatched reservation even after a lease takeover', async () => {
const { methods, runs } = makeMethods();
// Valid at the pre-reserve check, superseded by the pre-POST one: that ordering IS
// the scenario, since a claim already dead before reserving never reserves at all.
(methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('superseded');
// Nothing was dispatched for this row, so it is unambiguously this fire's garbage.
expect(methods.deleteScheduleRun).toHaveBeenCalledWith(
'sched-1',
expect.any(Date),
'started',
expect.any(String),
);
expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
expect(global.fetch).not.toHaveBeenCalled();
});
it('rechecks shutdown immediately before dispatch and rolls back the reservation', async () => {
const { methods, runs } = makeMethods();
// The coordinator flips AFTER the pre-reservation gate: the deployment-limit
// read, capacity allocation, and claim revalidation all run in between and can
// overlap SIGTERM. Without the recheck the POST lands on a closing listener —
// booked as an error (or a refused connect that advances the occurrence).
const isShuttingDown = jest.fn().mockReturnValueOnce(false).mockReturnValue(true);
mockFetch(async () => okResponse());
const result = await fireSchedule(
makeDeps(methods, { isShuttingDown }),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.skipped).toBe('superseded');
expect(global.fetch).not.toHaveBeenCalled();
// Rolled back, not advanced: the occurrence stays due for the restarted process.
expect(methods.advanceSchedule).not.toHaveBeenCalled();
expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
});
it('deletes the reserved run when the schedule was hard-deleted mid-fire', async () => {
const { methods, runs } = makeMethods();
// Account deletion hard-deleted the schedule after this fire reserved its run.
(methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('superseded');
// The orphaned reservation (no schedule left to own it) is deleted, not leaked.
expect(methods.deleteScheduleRun).toHaveBeenCalledWith(
'sched-1',
expect.any(Date),
'started',
expect.any(String),
);
expect([...runs.entries()].some(([k]) => k.startsWith('sched-1:'))).toBe(false);
});
it('records an unavailable trigger service as a definite error', async () => {
const { methods, runs, calls } = makeMethods();
const result = await fireSchedule(
makeDeps(methods, {
enqueueTrigger: async () => {
throw new AgentTriggerServiceUnavailableError('not ready');
},
}),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.fired).toBe(false);
expect(calls.recordOutcome).toEqual([{ status: 'error' }]);
expect([...runs.values()][0].status).toBe('error');
});
it('skips overlap when a prior run is still active', async () => {
const { methods, runs, calls } = makeMethods();
runs.set('sched-1:prior', { status: 'started' });
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('overlap');
expect(calls.skipped).toEqual(['skipped_overlap']);
expect(global.fetch).not.toHaveBeenCalled();
});
it('disables and stops firing when agent VIEW access was revoked', async () => {
const { methods, calls } = makeMethods();
const deps = makeDeps(methods, { agentAccess: async () => 'forbidden' });
const result = await fireSchedule(deps, makeSchedule(), LIMITS, dueAt());
expect(result.skipped).toBe('permission_revoked');
expect(calls.disable).toEqual(['permission_revoked']);
});
it('skips a duplicate occurrence (idempotency claim already held)', async () => {
const { methods, runs } = makeMethods();
runs.set(`sched-1:${dueAt().toISOString()}`, { status: 'requires_action' });
mockFetch(async () => okResponse());
const when = new Date(dueAt().getTime());
runs.set(`sched-1:${when.toISOString()}`, { status: 'requires_action' });
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
expect(result.skipped).toBe('duplicate');
expect(global.fetch).not.toHaveBeenCalled();
});
/**
* A balance skip is not a no-op: it stamps the card and walks the balance-skip streak
* toward auto-disable. The preflight above it (user, config, permission and balance
* lookups) can outlast the 5-minute lease, so writing it under a dead claim is a write
* on behalf of a fire that no longer owns the occurrence.
*/
it('does not record a balance skip under a superseded claim', async () => {
const { methods, calls } = makeMethods();
(methods.revalidateClaim as jest.Mock).mockResolvedValue(false);
mockFetch(async () => okResponse());
const result = await fireSchedule(
makeDeps(methods, { isOutOfBalance: async () => true }),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.skipped).toBe('superseded');
expect(methods.recordSkippedRun).not.toHaveBeenCalled();
expect(calls.skipped).toEqual([]);
});
it('records the balance skip normally while the claim is still valid', async () => {
const { methods, calls } = makeMethods();
mockFetch(async () => okResponse());
const result = await fireSchedule(
makeDeps(methods, { isOutOfBalance: async () => true }),
makeSchedule(),
LIMITS,
dueAt(),
);
expect(result.skipped).toBe('balance');
expect(calls.skipped).toEqual(['skipped_balance']);
});
/**
* A `duplicate` means ANOTHER worker holds this occurrence's row — not that the
* occurrence is done. Advancing past it hands the occurrence away: if that other
* worker is a stale lease holder whose own revalidation then fails, it rolls its
* undispatched row back and NOTHING ever fires the occurrence. Leaving nextRunAt
* alone keeps it claimable; the worker that actually dispatches is the one that
* advances, and the claim's lease provides the retry backoff.
*/
/**
* A settled-but-unadvanced occurrence (its fire was accepted, the post-accept advance
* failed) leaves nextRunAt pointing at it. Refusing to advance on `duplicate` then
* makes every future claim re-pick the same finished occurrence — a permanent stall.
*/
it('advances past a duplicate whose run already settled', async () => {
const { methods, runs, calls } = makeMethods();
const when = new Date(dueAt().getTime());
runs.set(`sched-1:${when.toISOString()}`, { status: 'success', conversationId: 'done' });
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
expect(result.skipped).toBe('duplicate');
expect(global.fetch).not.toHaveBeenCalled();
expect(calls.advance).toBe(1);
});
it('does not advance past an occurrence another worker is holding', async () => {
const { methods, runs, calls } = makeMethods();
const when = new Date(dueAt().getTime());
// A peer already reserved this occurrence's row.
runs.set(`sched-1:${when.toISOString()}`, {
status: 'started',
conversationId: 'peer-convo',
});
mockFetch(async () => okResponse());
const result = await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when);
expect(result.skipped).toBe('duplicate');
expect(global.fetch).not.toHaveBeenCalled();
expect(calls.advance).toBe(0);
});
it('releases the lease on a duplicate for run-now so the user can retry', async () => {
const { methods, runs, calls } = makeMethods();
const when = new Date(dueAt().getTime());
runs.set(`sched-1:${when.toISOString()}`, { status: 'started' });
mockFetch(async () => okResponse());
await fireSchedule(makeDeps(methods), makeSchedule(), LIMITS, when, { manual: true });
// Automatic claims keep the lease as backoff; a manual click must not be told
// "already in progress" for the full lease TTL.
expect(calls.releaseLease).toBe(1);
expect(calls.advance).toBe(0);
});
});
it('settles an unavailable MCP occurrence without dispatching a generation', async () => {
const { methods } = makeMethods();
const failure = new ScheduleMCPError([{ server: 'Notion', status: 'mcp_reauth_required' }]);
const deps = makeDeps(methods, {
preflightMCP: async () => {
throw failure;
},
});
const capacitySpy = jest.spyOn(deps, 'withGlobalCapacitySlot');
const result = await fireSchedule(deps, makeSchedule(), LIMITS, new Date('2026-09-09T12:00:00Z'));
expect(result).toMatchObject({ fired: false, mcp: failure.outcomes });
expect(capacitySpy).not.toHaveBeenCalled();
expect(methods.reserveStartedRun).toHaveBeenCalledWith(
expect.objectContaining({
admissionOnly: true,
error: failure.message,
mcp: failure.outcomes,
}),
);
expect(deps.enqueueTrigger).not.toHaveBeenCalled();
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({
error: failure.message,
status: 'error',
clearConversationId: true,
mcp: failure.outcomes,
}),
);
});
it('does not settle or disable from a failed MCP preflight after losing its claim', async () => {
const { methods } = makeMethods();
(methods.revalidateClaim as jest.Mock).mockResolvedValueOnce(true).mockResolvedValue(false);
const deps = makeDeps(methods, {
preflightMCP: async () => {
throw new ScheduleMCPError([{ server: 'Notion', status: 'mcp_reauth_required' }]);
},
});
const result = await fireSchedule(deps, makeSchedule(), LIMITS, new Date('2026-09-09T12:00:00Z'));
expect(result.skipped).toBe('superseded');
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
expect(methods.advanceSchedule).not.toHaveBeenCalled();
});
it('does not invent a server outcome for infrastructure preflight failures', async () => {
const { methods } = makeMethods();
const deps = makeDeps(methods, {
preflightMCP: async () => {
throw new Error('role database unavailable with private details');
},
});
const result = await fireSchedule(deps, makeSchedule(), LIMITS, new Date('2026-09-09T12:00:00Z'));
expect(result).toMatchObject({
fired: false,
error: 'MCP preflight unavailable',
mcpPreflightUnavailable: true,
});
expect(result.mcp).toBeUndefined();
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ error: 'MCP preflight unavailable' }),
);
});
it('bounds MCP preflight by the claim lease and the stricter concurrency config', async () => {
const { methods } = makeMethods();
const leaseUntil = new Date(Date.now() + 10_000);
const failure = new ScheduleMCPError([{ server: 'Notion', status: 'mcp_unavailable' }]);
const preflightMCP = jest.fn(async () => {
throw failure;
});
const deps = makeDeps(methods, {
preflightMCP,
getLimits: async (user) => ({
...LIMITS,
mcpPreflightConcurrency: user == null ? 2 : 5,
}),
});
await fireSchedule(deps, makeSchedule({ leaseUntil }), LIMITS, new Date('2026-09-09T12:00:00Z'));
expect(preflightMCP).toHaveBeenCalledWith(
'agent-1',
OWNER,
expect.objectContaining({
scheduleId: 'sched-1',
concurrency: 2,
deadlineMs: leaseUntil.getTime(),
}),
);
});
it('bounds MCP preflight by the stricter owner and deployment timeout', async () => {
const { methods } = makeMethods();
const startedAt = Date.now();
let deadlineMs: number | undefined;
const preflightMCP: ScheduleEngineDeps['preflightMCP'] = async (_agentId, _user, options) => {
deadlineMs = options?.deadlineMs;
return [];
};
const deps = makeDeps(methods, {
preflightMCP,
getLimits: async (user) => ({
...LIMITS,
mcpPreflightTimeoutMs: user == null ? 12_000 : 20_000,
}),
});
await fireSchedule(
deps,
makeSchedule({ leaseUntil: new Date(startedAt + 240_000) }),
LIMITS,
new Date('2026-09-09T12:00:00Z'),
);
expect(deadlineMs).toBeGreaterThanOrEqual(startedAt + 11_900);
expect(deadlineMs).toBeLessThanOrEqual(Date.now() + 12_000);
});
it('cancels MCP preflight before reserving or recording a run', async () => {
const { methods } = makeMethods();
const controller = new AbortController();
const deps = makeDeps(methods, {
preflightMCP: async (_agentId, _user, options) => {
expect(options?.signal).toBe(controller.signal);
controller.abort(new Error('schedule engine stopped'));
throw controller.signal.reason;
},
});
const result = await fireSchedule(
deps,
makeSchedule(),
LIMITS,
new Date('2026-09-09T12:00:00Z'),
{ signal: controller.signal },
);
expect(result).toMatchObject({ fired: false, skipped: 'superseded' });
// Readiness now runs before generation reservation, so cancellation has no
// started row or capacity slot to roll back.
expect(methods.deleteScheduleRun).not.toHaveBeenCalled();
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
expect(methods.advanceSchedule).not.toHaveBeenCalled();
});
it('does not enqueue when Run Now is cancelled during final claim validation', async () => {
const { methods } = makeMethods();
const controller = new AbortController();
let releaseValidation: () => void = () => undefined;
let markValidationStarted: () => void = () => undefined;
const validationStarted = new Promise<void>((resolve) => {
markValidationStarted = resolve;
});
const validationGate = new Promise<void>((resolve) => {
releaseValidation = resolve;
});
let validations = 0;
(methods.revalidateClaim as jest.Mock).mockImplementation(async () => {
validations += 1;
if (validations === 2) {
markValidationStarted();
await validationGate;
}
return true;
});
const deps = makeDeps(methods);
const pending = fireSchedule(deps, makeSchedule(), LIMITS, new Date('2026-09-09T12:00:00Z'), {
manual: true,
signal: controller.signal,
});
await validationStarted;
controller.abort(new Error('Run Now request closed'));
releaseValidation();
const result = await pending;
expect(result).toMatchObject({ fired: false, skipped: 'superseded' });
expect(methods.deleteScheduleRun).toHaveBeenCalled();
expect(deps.enqueueTrigger).not.toHaveBeenCalled();
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
});