Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js';
import { logger, createModels, createMethods } from '@librechat/data-schemas';
import type { SchedulesServiceDeps } from './service';
import {
createSchedulesService,
recordScheduledMCPToolAuthFailure,
ScheduledMCPReceiptFencedError,
} from './service';
import { ScheduledMCPPolicyError } from './authorization/policy';
import { OboTokenResolutionError } from '../mcp/oauth/obo';
import { ScheduledMCPBearerError } from '../mcp/errors';
import { isShutdownInProgress } from '../app/shutdown';
/** Swappable per test: null keeps the no-job-store harness the drain tests rely on. */
let mockJobStore: { getJob: jest.Mock; deleteJob?: jest.Mock; updateJob?: jest.Mock } | null = null;
jest.mock('../agents/checkpointer', () => ({
checkpointStorageConfigs: jest.fn(async (_user, _tenant, cfg) => [cfg]),
deleteAgentCheckpoint: jest.fn(async () => undefined),
// Non-empty by default so the scoped prune has something to delete in tests.
captureAgentCheckpointGeneration: jest.fn(async (threadId: string) => ({
threadId,
checkpointIds: ['ck-1'],
})),
}));
const checkpointerModule = jest.requireMock('../agents/checkpointer') as {
checkpointStorageConfigs: jest.Mock;
deleteAgentCheckpoint: jest.Mock;
captureAgentCheckpointGeneration: jest.Mock;
};
jest.mock('../stream/GenerationJobManager', () => ({
GenerationJobManager: {
// No configured job store by default: abortScheduledJob returns false, so the drain
// loop is driven purely by getActiveRunsForUser (the run rows).
getJobStore: () =>
mockJobStore && { ...mockJobStore, hasScheduleCleanupObligation: async () => false },
abortJob: jest.fn(),
updateMetadata: jest.fn(async () => undefined),
isRedis: false,
},
}));
type ActiveRun = {
scheduleId: string;
scheduledFor: Date;
conversationId?: string;
checkpointNamespace?: string;
status?: string;
};
let recordRunOutcome: jest.Mock;
function makeService(
getActiveRunsForUser: jest.Mock<Promise<ActiveRun[]>, [string]>,
getAppConfig?: SchedulesServiceDeps['getAppConfig'],
enqueueAgentTrigger: SchedulesServiceDeps['enqueueAgentTrigger'] = jest.fn(async () => undefined),
): ReturnType<typeof createSchedulesService> {
recordRunOutcome = jest.fn(async () => undefined);
const methods = {
suspendUserSchedulesForDeletion: jest.fn(async () => undefined),
restoreUserSchedulesFromDeletion: jest.fn(async () => undefined),
getActiveRunsForUser,
countActiveRuns: jest.fn(async () => 0),
requestRunAbort: jest.fn(async () => true),
getScheduleRunAbortState: jest.fn(async () => null),
recordMCPToolAuthFailure: jest.fn(async () => true),
markRunAbortPersisted: jest.fn(async () => undefined),
recordRunOutcome,
};
const deps = {
methods,
getAppConfig: getAppConfig ?? jest.fn(async () => ({})),
findUserById: jest.fn(async () => null),
findBalance: jest.fn(async () => null),
upsertBalance: jest.fn(async () => null),
initializeNullBalance: jest.fn(async () => null),
preflightMCP: jest.fn().mockResolvedValue([]),
resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
getChatProject: jest.fn(async () => ({ _id: 'proj-1' })),
isUserDeleting: jest.fn(async () => false),
enqueueAgentTrigger,
getTriggerDelivery: jest.fn(async () => null),
} as unknown as SchedulesServiceDeps;
// Short bounded waits so fail-closed drain paths resolve in test time.
return createSchedulesService(deps, {
drainTimeoutMs: 400,
drainPollMs: 25,
stopBarrierTimeoutMs: 400,
stopBarrierPollMs: 25,
});
}
const run = (): ActiveRun => ({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
});
describe('shutdown wiring', () => {
it('carries the coordinator signal on the BASE deps so Run Now is gated too', async () => {
const service = makeService(jest.fn(async (_userId: string) => []));
// fireScheduleNow dispatches with engineDeps directly (not the engine's
// per-pass wrapper), so the shutdown gate must live on the base deps or a
// manual Run Now POSTs into a closing listener.
expect(service.engineDeps.isShuttingDown).toBe(isShutdownInProgress);
});
});
describe('manual Run Now lease cleanup', () => {
it('releases the old holder when an owner edit fences cleanup after a preflight throw', async () => {
const service = makeService(jest.fn(async (_userId: string) => []));
const leased = {
id: 's1',
user: 'user-1',
tenantId: 't1',
name: 'Digest',
prompt: 'Summarize',
agent_id: 'agent-1',
cadence: { frequency: 'daily', hour: 8, minute: 0 },
timezone: 'America/New_York',
target: 'new',
enabled: true,
claimToken: 'manual-token',
leaseBy: 'manual-holder',
runCount: 0,
failureCount: 0,
balanceSkipCount: 0,
} as never;
const methods = service.engineDeps.methods as unknown as {
acquireManualRunLease: jest.Mock;
releaseLease: jest.Mock;
releaseLeaseByHolder: jest.Mock;
};
methods.acquireManualRunLease = jest.fn(async () => leased);
// Simulates an edit rotating claimToken while deliberately preserving leaseBy.
methods.releaseLease = jest.fn(async () => false);
methods.releaseLeaseByHolder = jest.fn(async () => undefined);
service.engineDeps.getUserContext = jest.fn(async () => {
throw new Error('user lookup failed');
});
await expect(
service.fireScheduleNow(leased, {
enabled: true,
maxPerUser: 10,
minIntervalMinutes: 60,
autoDisableAfterFailures: 5,
admissionConcurrency: 20,
fireConcurrency: 5,
mcpPreflightConcurrency: 3,
mcpPreflightTimeoutMs: 300_000,
requireProject: false,
}),
).rejects.toThrow('user lookup failed');
expect(methods.releaseLease).toHaveBeenCalledWith('s1', 'manual-token');
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'manual-holder');
});
});
describe('balance initialization', () => {
const balanceConfig = {
interfaceConfig: {},
balance: { enabled: true, startBalance: 20000 },
} as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;
function serviceWithBalance(
existing: Record<string, unknown> | null,
overrides: {
initializeNullBalance?: jest.Mock;
findBalance?: jest.Mock;
} = {},
) {
const upsertBalance = jest.fn(async () => ({ tokenCredits: 20000 }));
const initializeNullBalance =
overrides.initializeNullBalance ?? jest.fn(async () => ({ tokenCredits: 20000 }));
const findBalance = overrides.findBalance ?? jest.fn(async () => existing);
const service = createSchedulesService({
methods: {} as unknown as SchedulesServiceDeps['methods'],
getAppConfig: (async () => balanceConfig) as SchedulesServiceDeps['getAppConfig'],
findUserById: jest.fn(async () => null),
findBalance,
upsertBalance,
initializeNullBalance,
preflightMCP: jest.fn().mockResolvedValue([]),
resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
getChatProject: jest.fn(async () => ({ _id: 'proj-1' })),
isUserDeleting: jest.fn(async () => false),
enqueueAgentTrigger: jest.fn(async () => undefined),
getTriggerDelivery: jest.fn(async () => null),
} as unknown as SchedulesServiceDeps);
return { service, upsertBalance, initializeNullBalance, findBalance };
}
const updateFrom = (spy: jest.Mock) =>
(
spy.mock.calls[0] as unknown as [
string,
{ set: Record<string, unknown>; setOnInsert: Record<string, unknown> },
]
)[1];
const casUpdateFrom = (spy: jest.Mock) =>
(
spy.mock.calls[0] as unknown as [
string,
{ tokenCredits: number; sync: Record<string, unknown> },
]
)[1];
/**
* The balance READ and this write are separate statements. A concurrent charge that
* creates the record in between would be overwritten by a blind `$set`, handing back
* credits the user had already spent.
*/
it('initializes the starting credit via setOnInsert, never $set', async () => {
const { service, upsertBalance } = serviceWithBalance(null);
await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
expect(upsertBalance).toHaveBeenCalledTimes(1);
const update = updateFrom(upsertBalance);
expect(update.setOnInsert).toMatchObject({ tokenCredits: 20000 });
expect(update.set).not.toHaveProperty('tokenCredits');
});
/**
* The record was observed with a null credit, but the read and the write are separate
* statements: a concurrent initializer/charge could have set and spent it in between.
* The credit must go through a `{ tokenCredits: null }` CAS, never a blind `$set` that
* would restore spent credits.
*/
it('initializes an EXISTING null credit through the CAS, never a blind $set', async () => {
const { service, upsertBalance, initializeNullBalance } = serviceWithBalance({
autoRefillEnabled: false,
});
await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
expect(initializeNullBalance).toHaveBeenCalledTimes(1);
const cas = casUpdateFrom(initializeNullBalance);
expect(cas.tokenCredits).toBe(20000);
// The credit write goes ONLY through the CAS, never the unconditional upsert path.
expect(upsertBalance).not.toHaveBeenCalled();
});
/**
* When the CAS misses — a concurrent initializer/charge won — the preflight must re-read
* the winner's balance rather than restore the starting credit.
*/
it('re-reads the winner on a CAS miss and never restores spent credits', async () => {
const initializeNullBalance = jest.fn(async () => null);
// First read observes a null credit; the winner's re-read returns the spent balance.
const findBalance = jest
.fn()
.mockResolvedValueOnce({ autoRefillEnabled: false, tokenCredits: null })
.mockResolvedValueOnce({ tokenCredits: 5 });
const { service } = serviceWithBalance(null, { initializeNullBalance, findBalance });
const outOfBalance = await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
expect(initializeNullBalance).toHaveBeenCalledTimes(1);
// Re-read after the miss (the two isOutOfBalance reads), and the small winner balance
// wins: > 0 credits means the user is not pre-skipped for balance.
expect(findBalance).toHaveBeenCalledTimes(2);
expect(outOfBalance).toBe(false);
});
it.each([
['all of its credits are held by in-flight requests', 100, true],
['part of its credits are free', 50, false],
])('pre-skips a record only when %s', async (_case, reservedCredits, outOfBalance) => {
const { service } = serviceWithBalance({
tokenCredits: 100,
reservedCredits,
autoRefillEnabled: false,
});
await expect(service.engineDeps.isOutOfBalance({ id: 'user-1' } as never)).resolves.toBe(
outOfBalance,
);
});
/**
* A stale record whose credit is already set but whose refill config drifted still syncs
* that config, and the sync must not widen into a credit write.
*/
it('syncs refill config on a credited record without writing tokenCredits', async () => {
const refillConfig = {
interfaceConfig: {},
balance: {
enabled: true,
startBalance: 20000,
autoRefillEnabled: true,
refillIntervalValue: 30,
refillIntervalUnit: 'days',
refillAmount: 10000,
},
} as unknown as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>;
const upsertBalance = jest.fn(async () => ({ tokenCredits: 100 }));
const initializeNullBalance = jest.fn(async () => null);
const service = createSchedulesService({
methods: {} as unknown as SchedulesServiceDeps['methods'],
getAppConfig: (async () => refillConfig) as SchedulesServiceDeps['getAppConfig'],
findUserById: jest.fn(async () => null),
findBalance: jest.fn(async () => ({
tokenCredits: 100,
autoRefillEnabled: false,
lastRefill: new Date('2026-01-01T00:00:00.000Z'),
})),
upsertBalance,
initializeNullBalance,
preflightMCP: jest.fn().mockResolvedValue([]),
resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
getChatProject: jest.fn(async () => ({ _id: 'proj-1' })),
isUserDeleting: jest.fn(async () => false),
enqueueAgentTrigger: jest.fn(async () => undefined),
getTriggerDelivery: jest.fn(async () => null),
} as unknown as SchedulesServiceDeps);
await service.engineDeps.isOutOfBalance({ id: 'user-1' } as never);
// Credit is already set, so the CAS path is never taken.
expect(initializeNullBalance).not.toHaveBeenCalled();
expect(upsertBalance).toHaveBeenCalledTimes(1);
const update = updateFrom(upsertBalance);
expect(update.set).toMatchObject({ autoRefillEnabled: true });
expect(update.set).not.toHaveProperty('tokenCredits');
expect(update.setOnInsert).toEqual({});
});
});
describe('deleteScheduleForOwner', () => {
beforeEach(() => {
// Module-level mock: without this it accumulates calls across tests and the
// prune assertions below read a previous test's invocation.
checkpointerModule.deleteAgentCheckpoint.mockClear();
});
/**
* markScheduleDeleting runs FIRST and is one-shot: it matches only a not-yet-deleting
* row, so a retry answers 404. Anything between it and the aborts that can throw
* therefore strands the schedule — hidden and fenced, but with its paused job still
* resumable and its checkpoint unpruned, until the 25-hour abandonment sweep.
*/
it('still aborts a paused run when the checkpointer config cannot be resolved', async () => {
const pausedRun = {
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'requires_action',
};
const service = makeService(
jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]),
// The checkpointer lookup resolves the owner's config; a transient failure here
// must not cost the abort.
jest.fn(async () => {
throw new Error('config plane down');
}) as unknown as SchedulesServiceDeps['getAppConfig'],
);
const methods = service.engineDeps.methods as unknown as {
markScheduleDeleting: jest.Mock;
getActiveRunsForSchedule: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
};
methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
methods.getActiveRunsForSchedule = jest.fn(async () => [pausedRun]);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
// The owner must RESOLVE, or the checkpointer lookup short-circuits before it ever
// reads the config and the throw under test never happens.
(service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(
async () => ({ id: 'user-1', tenantId: 't1', role: 'USER' }),
);
const abortJob = jest.fn(async () => ({ success: true }));
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = abortJob;
// A paused (not running) job is positive evidence: settled synchronously, erased.
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
expect(abortJob).toHaveBeenCalled();
});
/** Shared double set for the drain-discipline tests below. */
function makeDeleteHarness(run: Partial<ActiveRun> & { scheduleId: string }) {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
markScheduleDeleting: jest.Mock;
getActiveRunsForSchedule: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
recordRunOutcome: jest.Mock;
getScheduleById: jest.Mock;
};
methods.markScheduleDeleting = jest.fn(async () => ({ id: run.scheduleId, user: 'user-1' }));
methods.getActiveRunsForSchedule = jest.fn(async () => [run]);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
methods.recordRunOutcome = jest.fn(async () => undefined);
methods.getScheduleById = jest.fn(async () => ({ user: 'user-1' }));
return { service, methods };
}
it.each([undefined, 'lcg:v2:owner:generation'])(
'passes the matching paused job namespace (%s) through the raw projection to capture',
async (checkpointNamespace) => {
const scheduledFor = '2026-01-01T00:00:00.000Z';
const { service } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date(scheduledFor),
conversationId: 'c1',
status: 'requires_action',
});
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
createdAt: 1,
scheduleId: 's1',
scheduledFor,
checkpointNamespace,
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: true }));
checkpointerModule.captureAgentCheckpointGeneration.mockClear();
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
expect(checkpointerModule.captureAgentCheckpointGeneration).toHaveBeenCalledWith(
'c1',
undefined,
checkpointNamespace == null ? {} : { checkpointNamespace },
);
},
);
it('uses the durable paused namespace after job-store loss', async () => {
const namespace = 'retained-owned-namespace';
const { service } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
checkpointNamespace: namespace,
status: 'requires_action',
});
mockJobStore = { getJob: jest.fn(async () => null) } as unknown as typeof mockJobStore;
checkpointerModule.captureAgentCheckpointGeneration.mockClear();
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
expect(checkpointerModule.captureAgentCheckpointGeneration).toHaveBeenCalledWith(
'c1',
undefined,
{ checkpointNamespace: namespace },
);
});
it('captures the retained namespace in every recorded store after job loss and a config change', async () => {
const namespace = 'retained-owned-namespace';
const stores = [
{ type: 'mongo', checkpointCollectionName: 'old-checkpoints' },
{ type: 'mongo', checkpointCollectionName: 'current-checkpoints' },
];
checkpointerModule.checkpointStorageConfigs.mockResolvedValueOnce(stores);
const { service } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
checkpointNamespace: namespace,
status: 'requires_action',
});
mockJobStore = { getJob: jest.fn(async () => null) } as unknown as typeof mockJobStore;
checkpointerModule.captureAgentCheckpointGeneration.mockClear();
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
for (const storage of stores) {
expect(checkpointerModule.captureAgentCheckpointGeneration).toHaveBeenCalledWith(
'c1',
storage,
{ checkpointNamespace: namespace },
);
expect(checkpointerModule.deleteAgentCheckpoint).toHaveBeenCalledWith(
'c1',
storage,
expect.objectContaining({ checkpointIds: ['ck-1'] }),
);
}
});
it('settles a pause hand-off after the exact provider drain is confirmed', async () => {
const { service, methods } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
});
// The job can report requires_action before the run row records the pause. The
// delete path must abort through #14925's provider-drain barrier before settling,
// so no trailing controller write can race the outcome.
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({
success: true,
signalDelivered: true,
jobData: { status: 'requires_action' },
}));
await service.deleteScheduleForOwner('s1', 'user-1');
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ status: 'interrupted', conversationId: 'c1' }),
);
});
it('does not report success when the abort of a live run is not delivered', async () => {
const { service, methods } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
});
// A RUNNING identity-matched job: not settleable, must be aborted...
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
deleteJob: jest.fn(async () => true),
} as unknown as typeof mockJobStore;
// ...and the abort delivery FAILS (job store write rejected).
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => {
throw new Error('store unreachable');
});
// The generation may still be producing and billing; claiming success would say
// it was stopped. The schedule stays hidden and fenced; the delete is idempotent.
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();
});
it('treats an unreadable job store as unknown, not absent', async () => {
const { service, methods } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
});
mockJobStore = {
getJob: jest.fn(async () => {
throw new Error('redis gone');
}),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => {
throw new Error('redis gone');
});
// With the store unreadable NOTHING is proven: the row must not be settled as
// an orphan (its generation may be live) and the delete must not read as done.
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
});
it('settles a provably job-less run synchronously and erases without a reconciler', async () => {
const { service, methods } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
});
// Confirmed absence: the lookup SUCCEEDS and returns null (a crashed fire, or a
// stale row from a previous topology). The clustered entrypoint has no reconciler,
// so deferring this row to one retained the deleted schedule indefinitely there.
mockJobStore = { getJob: jest.fn(async () => null) } as unknown as typeof mockJobStore;
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
);
expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');
});
/**
* The durable checkpoint is keyed by conversationId ALONE, so pruning it when a
* REPLACEMENT turn owns the conversation strips the resume state of a live generation
* that has nothing to do with this schedule — the same hazard the interactive abort
* route refuses with a 409.
*/
it('does not prune the checkpoint when a replacement owns the conversation', async () => {
const { service } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'requires_action',
});
// A job IS present, but it carries a DIFFERENT occurrence's identity: this
// conversation now belongs to someone else's turn.
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 99,
scheduleId: 'a-different-schedule',
scheduledFor: new Date('2030-01-01T00:00:00.000Z').toISOString(),
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: true }));
await service.deleteScheduleForOwner('s1', 'user-1');
expect(checkpointerModule.deleteAgentCheckpoint).not.toHaveBeenCalled();
});
it('prunes the checkpoint of a paused run it does own', async () => {
const { service } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'requires_action',
});
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
createdAt: 1,
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: true }));
await service.deleteScheduleForOwner('s1', 'user-1');
// SCOPED to the checkpoint ids captured before the terminal transition, so a
// replacement's later checkpoints can never be swept up by this prune.
expect(checkpointerModule.deleteAgentCheckpoint).toHaveBeenCalledWith('c1', undefined, {
threadId: 'c1',
checkpointIds: ['ck-1'],
});
});
it('reports draining when a live run was aborted but has not yet settled', async () => {
const { service, methods } = makeDeleteHarness({
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
});
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
deleteJob: jest.fn(async () => true),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: true }));
// The run row is still active, so the erase declines; settlement (and the
// erase-on-settle it triggers) belongs to the aborted generation's outcome write.
methods.eraseScheduleIfDrained = jest.fn(async () => false);
// Public reads intentionally hide deleting rows. That absence must not turn a
// declined erasure into a false `deleted` response.
methods.getScheduleById.mockResolvedValue(null);
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('draining');
});
afterEach(() => {
mockJobStore = null;
jest.restoreAllMocks();
});
});
describe('erase-on-settle', () => {
/**
* Whichever process records a run's terminal outcome also attempts the deferred
* erase of a deleting schedule. This is what makes a delete's `draining` state
* converge in EVERY topology — the clustered entrypoint runs no reconciler, so
* without it the hidden schedule (and its prompt, which has no TTL) survived its
* last run indefinitely there.
*/
it('attempts the deferred erase after recording a terminal outcome', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
recordRunOutcome: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => null);
methods.recordRunOutcome = jest.fn(async () => undefined);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
await expect(
service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'success',
}),
).resolves.toBe(true);
expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('s1');
});
it('does not erase on a pause, which is not a settlement', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
recordRunOutcome: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => null);
methods.recordRunOutcome = jest.fn(async () => undefined);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'requires_action',
});
expect(methods.eraseScheduleIfDrained).not.toHaveBeenCalled();
});
});
describe('scheduled OBO tool failure settlement', () => {
const occurrence = '2026-09-09T12:00:00.000Z';
const failure = {
server: 'Graph',
status: 'mcp_configuration_missing' as const,
detail: 'unattended_auth_required' as const,
};
function setup() {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>(async () => []));
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
getScheduleRunAbortState: jest.Mock;
recordMCPToolAuthFailure: jest.Mock;
recordRunOutcome: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => null);
methods.eraseScheduleIfDrained = jest.fn(async () => false);
methods.getScheduleRunAbortState = jest.fn(async () => ({ status: 'started', mcp: [failure] }));
const store = {
durableScheduleReceipts: false,
updateJob: jest.fn(async () => undefined),
getJob: jest.fn(async () => ({
createdAt: 42,
scheduleId: 's1',
scheduledFor: occurrence,
conversationId: 'c1',
userId: 'owner',
tenantId: 'tenant-1',
status: 'running',
})),
};
mockJobStore = store;
return { service, methods, store };
}
it.each(['Mongo receipt', 'job evidence', 'neither'] as const)(
'retains a verified bearer denial using %s and prevents success settlement',
async (storageMode) => {
const { service, methods, store } = setup();
store.durableScheduleReceipts = storageMode === 'job evidence';
const job: {
createdAt: number;
scheduleId: string;
scheduledFor: string;
conversationId: string;
userId: string;
tenantId: string;
status: string;
agent_id: string;
scheduleOutcomeError?: string;
preserveForScheduleReconcile?: boolean;
} = {
createdAt: 42,
scheduleId: 's1',
scheduledFor: occurrence,
conversationId: 'c1',
userId: 'owner',
tenantId: 'tenant-1',
status: 'running',
agent_id: 'root',
};
store.getJob.mockImplementation(async () => job);
methods.getScheduleRunAbortState.mockResolvedValue({ status: 'started', mcp: [] });
const error = new ScheduledMCPBearerError('consent_revoked', 'Files', 'child');
methods.recordMCPToolAuthFailure.mockImplementation(async () => {
if (storageMode === 'job evidence' || storageMode === 'neither')
throw new Error('receipt store unavailable');
methods.getScheduleRunAbortState.mockResolvedValue({
status: 'started',
mcp: error.outcomes,
});
return true;
});
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
const originalUpdate = manager.updateMetadata;
const update = jest.fn(async (_stream, changes) => {
if (storageMode === 'Mongo receipt' || storageMode === 'neither')
throw new Error('job store unavailable');
Object.assign(job, changes);
});
manager.updateMetadata = update;
const input = {
error,
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: 'tenant-1',
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
await expect(service.recordMCPToolAuthFailure(input)).resolves.toBe(
storageMode !== 'neither',
);
// Restore metadata writes so terminal retry can retain Mongo-only evidence.
if (storageMode === 'Mongo receipt')
update.mockImplementation(async (_stream, changes) => {
Object.assign(job, changes);
});
const settled = await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
streamId: 'c1',
jobCreatedAt: 42,
status: 'success',
});
expect(settled).toBe(storageMode !== 'neither');
if (storageMode === 'neither') expect(methods.recordRunOutcome).not.toHaveBeenCalled();
else
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({
status: 'error',
mcp: expect.arrayContaining([
expect.objectContaining({
server: 'Files',
reason: 'consent_revoked',
recovery: 'authorize',
agentId: 'child',
}),
]),
}),
);
manager.updateMetadata = originalUpdate;
},
);
it('refuses a bearer failure receipt for a replaced epoch, owner, tenant or enrolled root', async () => {
const { service, methods, store } = setup();
store.getJob.mockImplementation(async () => ({
createdAt: 42,
scheduleId: 's1',
scheduledFor: occurrence,
conversationId: 'c1',
userId: 'owner',
tenantId: 'tenant-1',
status: 'running',
agent_id: 'root',
}));
const source = {
error: new ScheduledMCPBearerError('tool_policy_denied', 'Files'),
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: 'tenant-1',
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
for (const input of [
{ ...source, jobCreatedAt: 43 },
{ ...source, userId: 'other' },
{ ...source, identity: { ...source.identity, tenantId: 'other' } },
{ ...source, identity: { ...source.identity, agentId: 'child' } },
])
await expect(service.recordMCPToolAuthFailure(input)).rejects.toMatchObject({
name: 'AbortError',
});
expect(methods.recordMCPToolAuthFailure).not.toHaveBeenCalled();
});
it('does not return a model-visible bearer error while durable writes are unavailable', async () => {
const input = {
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: 'tenant-1',
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
let durable = false;
let returned = false;
const persist = jest.fn(async () => durable);
const pending = recordScheduledMCPToolAuthFailure(input, () => persist).then(() => {
returned = true;
});
await new Promise((resolve) => setTimeout(resolve, 20));
expect(returned).toBe(false);
durable = true;
await pending;
expect(returned).toBe(true);
});
it('does not initialize the schedule service for unrelated MCP errors', async () => {
const getRecorder = jest.fn();
await expect(
recordScheduledMCPToolAuthFailure(
{
error: new Error('Ordinary MCP failure'),
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Graph',
},
getRecorder,
),
).resolves.toBe(false);
expect(getRecorder).not.toHaveBeenCalled();
});
it.each(['job lookup', 'run write'] as const)(
'keeps receipt persistence best-effort when the %s fails',
async (failurePoint) => {
const { service, methods, store } = setup();
const persistenceError = new Error('Storage temporarily unavailable');
if (failurePoint === 'job lookup') {
store.getJob.mockRejectedValueOnce(persistenceError);
} else {
methods.recordMCPToolAuthFailure.mockRejectedValueOnce(persistenceError);
}
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const missing = new OboTokenResolutionError('missing_upstream_provider', 'No credentials');
const input = {
error: missing,
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Graph',
};
await expect(
recordScheduledMCPToolAuthFailure(input, () => service.recordMCPToolAuthFailure),
).resolves.toBe(false);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining('could not persist'),
persistenceError,
);
warn.mockRestore();
},
);
it('records the public-safe policy denial for the exact scheduled generation', async () => {
const { service, methods } = setup();
const error = new ScheduledMCPPolicyError('tool_policy_denied', 'Graph', 'child');
const input = { error, streamId: 'c1', jobCreatedAt: 42, userId: 'owner', serverName: 'Graph' };
await expect(
recordScheduledMCPToolAuthFailure(input, () => service.recordMCPToolAuthFailure),
).resolves.toBe(true);
expect(methods.recordMCPToolAuthFailure).toHaveBeenCalledWith(
expect.objectContaining({
outcome: {
server: 'Graph',
agentId: 'child',
status: 'mcp_permission_denied',
reason: 'tool_policy_denied',
recovery: 'configure',
automaticReplay: false,
},
}),
);
await expect(
service.recordMCPToolAuthFailure({ ...input, jobCreatedAt: 43 }),
).rejects.toMatchObject({ name: 'AbortError' });
});
it.each(['consent_revoked', 'tool_policy_denied', 'credential_rejected'] as const)(
'settles a handled %s tool denial as an error with the durable diagnosis',
async (reason) => {
const { service, methods } = setup();
const error = new ScheduledMCPPolicyError(reason, 'Graph', 'child');
const input = {
error,
streamId: 'c1',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Graph',
};
await expect(
recordScheduledMCPToolAuthFailure(input, () => service.recordMCPToolAuthFailure),
).resolves.toBe(true);
methods.getScheduleRunAbortState.mockResolvedValue({
status: 'started',
mcp: error.outcomes,
});
await expect(
service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
status: 'success',
conversationId: 'c1',
streamId: 'c1',
jobCreatedAt: 42,
}),
).resolves.toBe(true);
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ status: 'error', mcp: error.outcomes }),
);
expect(
jest.requireMock('../stream/GenerationJobManager').GenerationJobManager.updateMetadata,
).toHaveBeenCalledWith('c1', expect.objectContaining({ scheduleOutcome: 'error' }), 42);
},
);
it('records a typed tool failure only for the matching scheduled generation and owner', async () => {
const { service, methods, store } = setup();
const source = new OboTokenResolutionError('missing_upstream_provider', 'No credentials');
const error = Object.assign(new McpError(ErrorCode.InternalError, 'OBO failed'), {
cause: source,
});
const input = { error, streamId: 'c1', jobCreatedAt: 42, userId: 'owner', serverName: 'Graph' };
await expect(service.recordMCPToolAuthFailure(input)).resolves.toBe(true);
expect(methods.recordMCPToolAuthFailure).toHaveBeenCalledWith({
scheduleId: 's1',
scheduledFor: new Date(occurrence),
conversationId: 'c1',
tenantId: 'tenant-1',
server: 'Graph',
});
expect(store.getJob).toHaveBeenCalledWith('c1');
for (const wrong of [{ jobCreatedAt: 43 }, { userId: 'another' }]) {
await expect(service.recordMCPToolAuthFailure({ ...input, ...wrong })).resolves.toBe(false);
}
await expect(
service.recordMCPToolAuthFailure({ ...input, error: new Error('No credentials') }),
).resolves.toBe(false);
store.getJob.mockResolvedValueOnce({
createdAt: 42,
userId: 'owner',
scheduleId: '',
scheduledFor: occurrence,
conversationId: 'c1',
tenantId: 'tenant-1',
status: 'running',
});
await expect(service.recordMCPToolAuthFailure(input)).resolves.toBe(false);
expect(methods.recordMCPToolAuthFailure).toHaveBeenCalledTimes(1);
});
it('settles a resumed run with missing MCP credentials as an error with old-client-readable guidance', async () => {
const { service, methods } = setup();
methods.getScheduleRunAbortState.mockResolvedValue({ status: 'started', mcp: [failure] });
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
status: 'requires_action',
});
expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
await expect(
service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
status: 'success',
conversationId: 'c1',
streamId: 'c1',
jobCreatedAt: 42,
}),
).resolves.toBe(true);
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({
status: 'error',
mcp: [failure],
error: 'MCP unattended authorization unavailable',
}),
);
expect(
jest.requireMock('../stream/GenerationJobManager').GenerationJobManager.updateMetadata,
).toHaveBeenCalledWith('c1', expect.objectContaining({ scheduleOutcome: 'error' }), 42);
});
it('retains an error rather than a balance skip for a known missing OBO provider', async () => {
const { service, methods } = setup();
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
status: 'skipped_balance',
conversationId: 'c1',
streamId: 'c1',
jobCreatedAt: 42,
});
expect(methods.recordRunOutcome).toHaveBeenLastCalledWith(
expect.objectContaining({
status: 'error',
error: 'MCP unattended authorization unavailable',
mcp: [failure],
}),
);
expect(
jest.requireMock('../stream/GenerationJobManager').GenerationJobManager.updateMetadata,
).toHaveBeenCalledWith('c1', expect.objectContaining({ scheduleOutcome: 'error' }), 42);
});
it('keeps ordinary completion and missing-run outcomes unchanged', async () => {
const { service, methods } = setup();
methods.getScheduleRunAbortState.mockResolvedValueOnce({
status: 'started',
mcp: [{ server: 'Graph', status: 'ready' }],
});
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: occurrence,
status: 'success',
});
expect(methods.recordRunOutcome).toHaveBeenLastCalledWith(
expect.objectContaining({ status: 'success' }),
);
});
});
describe('interactive Stop persistence barrier', () => {
function outcomeService() {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
recordRunOutcome: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
getScheduleRunAbortState: jest.Mock;
requestRunAbort: jest.Mock;
markRunAbortPersisted: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => null);
methods.recordRunOutcome = jest.fn(async () => undefined);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
return { service, methods };
}
it('beginScheduledStop stamps a serialized stop; acknowledge marks persistence', async () => {
const { service, methods } = outcomeService();
methods.requestRunAbort = jest.fn(async () => 'in_progress' as const);
methods.markRunAbortPersisted = jest.fn(async () => undefined);
const stamp = await service.beginScheduledStop({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
});
expect(stamp).toBe('in_progress');
expect(methods.requestRunAbort).toHaveBeenCalledWith('s1', expect.any(Date), 'stop');
await service.acknowledgeScheduledStopPersistence({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
});
expect(methods.markRunAbortPersisted).toHaveBeenCalledWith('s1', expect.any(Date));
});
it('owner terminal settlement waits until the Stop persistence is acknowledged', async () => {
const { service, methods } = outcomeService();
const fresh = new Date();
let reads = 0;
// Unresolved fresh stop for the first two reads, then acknowledged.
methods.getScheduleRunAbortState = jest.fn(async () => {
reads += 1;
return {
status: 'started',
abortSource: 'stop',
abortRequestedAt: fresh,
...(reads > 2 ? { abortPersistedAt: new Date() } : {}),
};
});
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'success',
});
// Polled until the acknowledgement landed, and only THEN recorded the outcome.
expect(reads).toBeGreaterThanOrEqual(3);
expect(methods.recordRunOutcome).toHaveBeenCalled();
expect(methods.getScheduleRunAbortState.mock.invocationCallOrder[0]).toBeLessThan(
methods.recordRunOutcome.mock.invocationCallOrder[0],
);
});
it('does not wait on a non-stop abort source', async () => {
const { service, methods } = outcomeService();
methods.getScheduleRunAbortState = jest.fn(async () => ({
status: 'started',
abortSource: 'deletion',
abortRequestedAt: new Date(),
}));
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'interrupted',
});
expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
expect(methods.recordRunOutcome).toHaveBeenCalled();
});
it('does not wait on a STALE stop stamp (route presumed dead)', async () => {
const { service, methods } = outcomeService();
// Older than ABORT_OWNER_PRESUMED_ALIVE_MS: the owner presumes the route dead.
const stale = new Date(Date.now() - 31 * 60_000);
methods.getScheduleRunAbortState = jest.fn(async () => ({
status: 'started',
abortSource: 'stop',
abortRequestedAt: stale,
}));
await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'error',
});
expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
expect(methods.recordRunOutcome).toHaveBeenCalled();
});
/**
* The poll budget expiring proves nothing about the Stop route's writes (slow checkpoint
* cleanup looks identical), so the barrier must DEFER rather than terminalize the run and
* release its capacity mid-persistence.
*/
it('defers settlement when a fresh Stop never acknowledges within the poll budget', async () => {
const { service, methods } = outcomeService();
methods.getScheduleRunAbortState = jest.fn(async () => ({
status: 'started',
abortSource: 'stop',
abortRequestedAt: new Date(),
}));
const settled = await service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'interrupted',
});
// Reported NOT settled so a durable-retry caller re-drives it, and the run is left
// active rather than terminalized while persistence may still be in flight.
expect(settled).toBe(false);
expect(methods.recordRunOutcome).not.toHaveBeenCalled();
});
it('does not gate a pause (requires_action) on the Stop barrier', async () => {
const { service, methods } = outcomeService();
methods.getScheduleRunAbortState = jest.fn(async () => ({
status: 'started',
abortSource: 'stop',
abortRequestedAt: new Date(),
mcp: [],
}));
await expect(
service.recordScheduleOutcome({
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
status: 'requires_action',
}),
).resolves.toBe(true);
// Receipt discovery does not poll a fresh Stop for an ordinary pause.
expect(methods.getScheduleRunAbortState).toHaveBeenCalledTimes(1);
expect(methods.recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ status: 'requires_action' }),
);
});
});
describe('attachment hold renewal', () => {
it('renews the bounded upload hold at fire preflight, best-effort', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
extendFilesTTL: jest.Mock;
getFiles: jest.Mock;
};
// The hold bridges upload -> first consumption; a renewal failure must not fail
// the fire (the file resolves now; at worst the hold lapses later).
methods.extendFilesTTL = jest.fn(async () => {
throw new Error('mongo down');
});
methods.getFiles = jest.fn(async () => [
{ file_id: 'f1', filepath: '/f1', filename: 'a.png', type: 'image/png', source: 'local' },
]);
const files = await service.engineDeps.resolveFiles(['f1'], { id: 'user-1', tenantId: 't1' });
expect(methods.extendFilesTTL).toHaveBeenCalledWith(
['f1'],
expect.objectContaining({ renewMs: expect.any(Number), maxLifetimeMs: expect.any(Number) }),
{ user: 'user-1', tenantId: 't1' },
);
expect(files).toHaveLength(1);
});
});
describe('isScheduleLive policy recheck', () => {
const liveRow = { id: 's1', user: 'u1', enabled: true } as never;
it('refuses a resume while the operator kill switch is up', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as { getScheduleById: jest.Mock };
methods.getScheduleById = jest.fn(async () => liveRow);
process.env.SCHEDULES_DISABLED = 'true';
try {
// The row-level checks alone pass; only the policy recheck sees the switch.
await expect(service.isScheduleLive('s1')).resolves.toBe(true);
// A pause can sit for hours; an approval must not start a billed continuation
// the operator believes is stopped.
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
} finally {
delete process.env.SCHEDULES_DISABLED;
}
});
it('refuses a resume when the owner lost SCHEDULES:USE', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
getRoleByName: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => liveRow);
methods.getRoleByName = jest.fn(async () => ({ permissions: { SCHEDULES: { USE: false } } }));
(service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(
async () => ({ id: 'u1', tenantId: 't1', role: 'USER' }),
);
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
});
/**
* Project policy rides this branch on purpose: BOTH callers route its refusal through
* abort-and-settle, so a policy stop settles the occurrence rather than leaving it at
* `requires_action` answering 409 to every approval until it expires.
*/
describe('project policy', () => {
function makeProjectService(
row: Record<string, unknown>,
schedulesConfig: Record<string, unknown>,
project: unknown = { _id: 'proj-1' },
over: { run?: { recorded: boolean; chatProjectId?: string } | null } = {},
) {
const service = makeService(
jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]),
jest.fn(async () => ({
interfaceConfig: { schedules: { use: true, ...schedulesConfig } },
})) as unknown as SchedulesServiceDeps['getAppConfig'],
);
const methods = service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
getRoleByName: jest.Mock;
};
methods.getScheduleById = jest.fn(async () => ({
id: 's1',
user: 'u1',
enabled: true,
...row,
}));
methods.getRoleByName = jest.fn(async () => ({ permissions: { SCHEDULES: { USE: true } } }));
(methods as unknown as { getScheduleRunProject: jest.Mock }).getScheduleRunProject = jest.fn(
async () => ('run' in over ? over.run : null),
);
(service.engineDeps as unknown as { getUserContext: jest.Mock }).getUserContext = jest.fn(
async () => ({ id: 'u1', tenantId: 't1', role: 'USER' }),
);
(service.engineDeps as unknown as { projectAccess: jest.Mock }).projectAccess = jest.fn(
async () => (project == null ? 'missing' : 'ok'),
);
return service;
}
it('refuses when the destination project was deleted', async () => {
const service = makeProjectService({ chatProjectId: 'proj-gone' }, {}, null);
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
});
it('refuses an unscoped schedule once the owner requires a project', async () => {
const service = makeProjectService({}, { requireProject: true });
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(false);
});
/**
* A pin governs where the NEXT run lands, and the fire path already redirects those.
* The paused conversation cannot be rebound — `chatProjectId` is excluded from the
* resume context and the continuation reuses the same conversationId — so refusing
* here would strand a pending approval over a destination it can never reach.
*/
it('admits a paused run whose pin moved to a different project', async () => {
const service = makeProjectService({ chatProjectId: 'proj-old' }, { projectId: 'proj-new' });
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(true);
});
it('admits a scoped schedule whose project is still owned', async () => {
const service = makeProjectService({ chatProjectId: 'proj-1' }, {});
await expect(service.isScheduleLive('s1', undefined, { policy: true })).resolves.toBe(true);
});
/**
* A paused run does NOT block later occurrences (the single-active index covers
* `started` only), so a fire after a pin move rewrites the schedule row while the
* paused conversation stays where it was filed. The occurrence's own record is
* what must be validated.
*/
it('validates the occurrence record over a schedule row a later fire moved', async () => {
const service = makeProjectService(
{ chatProjectId: 'proj-new' },
{ projectId: 'proj-new' },
null,
{ run: { recorded: true, chatProjectId: 'proj-paused' } },
);
const access = (service.engineDeps as unknown as { projectAccess: jest.Mock }).projectAccess;
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(false);
expect(access).toHaveBeenCalledWith('proj-paused', expect.anything());
});
it('admits when the occurrence record itself is still live', async () => {
const service = makeProjectService(
{ chatProjectId: 'proj-new' },
{},
{ _id: 'x' },
{
run: { recorded: true, chatProjectId: 'proj-paused' },
},
);
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(true);
});
/** An absent record is not evidence to stop a run: pre-scope occurrences and rows
* that are simply gone fall back to the schedule-level resolution. */
it('falls back to the schedule when the occurrence recorded nothing', async () => {
const service = makeProjectService({ chatProjectId: 'proj-gone' }, {}, null, { run: null });
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(false);
});
/**
* A run that DELIBERATELY went unscoped recorded that decision. Falling back to the
* schedule's current project for it would admit a conversation satisfying no
* present requirement — the fallback is for UNKNOWN records only.
*/
it('refuses a recorded-unscoped occurrence once a project became required', async () => {
const service = makeProjectService(
{ chatProjectId: 'proj-new' },
{ requireProject: true },
{ _id: 'x' },
{
run: { recorded: true },
},
);
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(false);
});
/** A pre-scope row is UNKNOWN, not unscoped, and keeps today's behaviour. */
it('falls back for an unrecorded pre-scope occurrence', async () => {
const service = makeProjectService(
{ chatProjectId: 'proj-live' },
{ requireProject: true },
{ _id: 'x' },
{
run: { recorded: false },
},
);
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(true);
});
/**
* The INITIAL start runs the same policy branch, and its run row is reserved before
* the loopback request is dispatched — so a pin introduced while that request sat
* queued must not be validated in place of the destination whose envelope was
* already built. Same call shape as the resume path.
*/
it('refuses an initial start whose occurrence was reserved unscoped', async () => {
const service = makeProjectService(
{ chatProjectId: 'proj-pinned' },
{ projectId: 'proj-pinned' },
{ _id: 'x' },
{ run: { recorded: true } },
);
await expect(
service.isScheduleLive('s1', undefined, {
policy: true,
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
).resolves.toBe(false);
});
it('leaves the non-policy recheck untouched', async () => {
const service = makeProjectService({ chatProjectId: 'proj-gone' }, {}, null);
await expect(service.isScheduleLive('s1')).resolves.toBe(true);
});
});
});
describe('quiesceUserSchedules drain wait', () => {
afterEach(() => {
mockJobStore = null;
jest.useRealTimers();
jest.restoreAllMocks();
});
it('polls the active runs until they drain, then returns', async () => {
jest.useFakeTimers();
const active = [run()];
const getActive = jest
.fn<Promise<ActiveRun[]>, [string]>()
.mockResolvedValueOnce(active) // initial collection for abort
.mockResolvedValueOnce(active) // first poll: still settling
.mockResolvedValue([]); // subsequent polls: drained
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
// Each poll waits one interval; advance twice so the loop observes the drain.
await jest.advanceTimersByTimeAsync(250);
await jest.advanceTimersByTimeAsync(250);
// The rows DRAINED, so the runs are genuinely settled even though this harness has
// no job store and the aborts could not be confirmed delivered. The drain is the
// authority; an undelivered abort whose run then settled must not defer deletion
// forever.
await expect(pending).resolves.toBe(true);
// Initial read + at least one poll that observed a non-empty set + the empty one.
expect(getActive.mock.calls.length).toBeGreaterThanOrEqual(3);
});
it('stops waiting at the bounded deadline when runs never drain', async () => {
jest.useFakeTimers();
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([run()]); // never drains
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
// Advance past the full bounded timeout; the loop must give up, not hang, and must
// report the drain as UNCONFIRMED so deletion defers rather than destroying.
await jest.advanceTimersByTimeAsync(10_000);
await expect(pending).resolves.toBe(false);
// It polled repeatedly (bounded by the deadline) and surfaced the un-drained runs.
expect(getActive.mock.calls.length).toBeGreaterThan(1);
expect(warn).toHaveBeenCalled();
});
/**
* A `started` row whose job is already terminal is the preserve-for-reconcile case:
* the inline outcome write exhausted its retries, so the retained job is the ONLY
* evidence the run finished. Account deletion deletes that job (nothing will ever
* reconcile it once the rows are hard-deleted), so it has to project the job's
* terminal state onto the row first — otherwise the row stays active, the bounded
* drain can never confirm, and deletion defers until the 30-minute orphan cutoff.
*/
it('does not settle a pause hand-off still in flight (started row, paused job)', async () => {
jest.useFakeTimers();
const started = { ...run(), status: 'started' };
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]);
const service = makeService(getActive);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
createdAt: 1,
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
})),
} as unknown as typeof mockJobStore;
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
// The pause hand-off's writes are still in flight; confirming the drain here
// let the destructive cascade run before they landed. Deferral is bounded: the
// controller (or the paused-window reconciler) flips the row to requires_action
// and the pending-deletion sweep retries.
await expect(pending).resolves.toBe(false);
expect(recordRunOutcome).not.toHaveBeenCalled();
});
it('settles a started run from its retained terminal job before dropping the evidence', async () => {
jest.useFakeTimers();
const started = { ...run(), status: 'started' };
const getActive = jest
.fn<Promise<ActiveRun[]>, [string]>()
.mockResolvedValueOnce([started])
.mockResolvedValue([]);
const service = makeService(getActive);
// deleteJob MUST be present: without it the store call throws, the delete never
// happens, and this test would pass while proving nothing about the ordering.
const deleteJob = jest.fn(async () => undefined);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'complete',
createdAt: 1,
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
})),
deleteJob,
} as unknown as typeof mockJobStore;
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(500);
await expect(pending).resolves.toBe(true);
expect(recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 's1', status: 'success' }),
);
// The retained job is the ONLY evidence this run finished, so it may not be deleted
// until the outcome is durably recorded.
expect(deleteJob).toHaveBeenCalled();
expect(recordRunOutcome.mock.invocationCallOrder[0]).toBeLessThan(
deleteJob.mock.invocationCallOrder[0],
);
});
/**
* `aborted` is the abort REQUEST landing, not the generation finishing: abortJob wins
* its status CAS before the owner unwinds, and both owner paths settle LAST, after
* saveMessage. Settling on that evidence confirms the drain while the partial write is
* still in flight — account deletion then destroys the user's data and the owner
* writes a message back for the account that no longer exists.
*/
it('does not settle a run whose abort is still in flight', async () => {
jest.useFakeTimers();
const aborting = {
...run(),
status: 'started',
abortRequestedAt: new Date(Date.now() - 1000),
};
const service = makeService(
jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([aborting]),
);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'aborted',
createdAt: 1,
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
})),
deleteJob: jest.fn(async () => undefined),
} as unknown as typeof mockJobStore;
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
// Unconfirmed, so the caller defers deletion (503 + Retry-After) instead of
// destroying data the owner is still writing.
await expect(pending).resolves.toBe(false);
expect(recordRunOutcome).not.toHaveBeenCalled();
});
/**
* The other direction: an abort whose owner never came back is presumed dead past the
* grace, or the account could never be deleted at all.
*/
it('settles a run whose abort request has outlived the owner grace', async () => {
jest.useFakeTimers();
const abandoned = {
...run(),
status: 'started',
abortRequestedAt: new Date(Date.now() - 45 * 60 * 1000),
};
const getActive = jest
.fn<Promise<ActiveRun[]>, [string]>()
.mockResolvedValueOnce([abandoned])
.mockResolvedValue([]);
const service = makeService(getActive);
mockJobStore = {
getJob: jest.fn(async () => null),
deleteJob: jest.fn(async () => undefined),
} as unknown as typeof mockJobStore;
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(500);
await expect(pending).resolves.toBe(true);
expect(recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
);
});
it('keeps the retained job when settling it fails', async () => {
jest.useFakeTimers();
const started = { ...run(), status: 'started' };
const service = makeService(
jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([started]),
);
const deleteJob = jest.fn(async () => undefined);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'complete',
createdAt: 1,
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z').toISOString(),
})),
deleteJob,
} as unknown as typeof mockJobStore;
recordRunOutcome.mockRejectedValue(new Error('mongo down'));
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(11_000);
// The row never settles, so the drain cannot confirm and deletion defers.
await expect(pending).resolves.toBe(false);
// Evidence intact for the next pass.
expect(deleteJob).not.toHaveBeenCalled();
});
it('does not wait when the user has no active runs', async () => {
jest.useFakeTimers();
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
const service = makeService(getActive);
// Nothing to abort and nothing to drain, so the quiesce is trivially CONFIRMED and
// the deletion cascade may proceed to its destructive steps.
await expect(service.quiesceUserSchedules('user-1', 'attempt-1')).resolves.toBe(true);
// Only the initial collection read; the drain loop is skipped for an empty set.
expect(getActive).toHaveBeenCalledTimes(1);
});
/** A `requires_action` row whose HITL approval was resumed through the chat UI. */
const pausedRun = (): ActiveRun => ({ ...run(), status: 'requires_action' });
it('does not terminalize a paused row whose resumed generation is still running', async () => {
jest.useFakeTimers();
// Backward-compatibility case from a pre-capacity resume (or rolling deploy):
// the job is running while the durable row still reads paused.
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
createdAt: 1,
})),
};
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
// Terminalizing here would drop the row out of the active set, so the drain would
// report nothing to wait for and the destructive cascade could delete messages the
// resumed generation is still able to persist.
await expect(pending).resolves.toBe(false);
expect(recordRunOutcome).not.toHaveBeenCalled();
});
it('still terminalizes a genuinely paused row so deletion is not blocked forever', async () => {
jest.useFakeTimers();
// Paused for approval, no live generation behind it.
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'requires_action',
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
createdAt: 1,
})),
};
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
await pending;
// An approval that will never be consumed must not keep the account undeletable.
expect(recordRunOutcome).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 's1', status: 'interrupted' }),
);
});
it('does not terminalize a paused row when the job lookup FAILS', async () => {
jest.useFakeTimers();
mockJobStore = {
getJob: jest.fn(async () => {
throw new Error('redis unavailable');
}),
};
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
// A thrown lookup is evidence of NOTHING. Reading it as "genuinely paused" would
// terminalize a row whose resumed generation may still be running, and the drain
// would then permit the destructive cascade.
await expect(pending).resolves.toBe(false);
expect(recordRunOutcome).not.toHaveBeenCalled();
});
it('terminalizes a paused row when a replacement turn took over the conversation', async () => {
jest.useFakeTimers();
// A replacement user turn reuses the conversationId but strips the scheduled
// identity, so this running job is NOT this occurrence's generation.
mockJobStore = {
getJob: jest.fn(async () => ({ status: 'running', createdAt: 1 })),
};
const getActive = jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([pausedRun()]);
const service = makeService(getActive);
const pending = service.quiesceUserSchedules('user-1', 'attempt-1');
await jest.advanceTimersByTimeAsync(10_000);
await pending;
expect(recordRunOutcome).toHaveBeenCalled();
});
});
describe('global kill switch', () => {
const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
afterEach(() => {
delete process.env.SCHEDULES_DISABLED;
});
it('is off by default', async () => {
const service = makeService(noRuns());
expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
});
it('trips on the SCHEDULES_DISABLED env lever without reading config', async () => {
process.env.SCHEDULES_DISABLED = 'true';
// Throwing getAppConfig proves the env lever works even when the config plane is
// unhealthy — the case where a config-dependent kill switch would fail.
const getAppConfig = jest.fn(async () => {
throw new Error('config plane down');
}) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
expect(getAppConfig).not.toHaveBeenCalled();
});
it('trips on `interface.schedules: false` read from the BASE config only', async () => {
const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
options?.baseOnly === true
? { interfaceConfig: { schedules: false } }
: // A principal-merged view that re-enables must NOT be consulted: the global
// stop is base-only so no role/user/tenant override can widen past it.
{ interfaceConfig: { schedules: true } },
) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });
});
it('does not trip when only a principal-merged config disables it', async () => {
// Per-principal availability is NOT the global stop; the engine keeps claiming so
// other principals still fire, and the fire path skips this owner's occurrences.
const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
options?.baseOnly === true
? { interfaceConfig: { schedules: true } }
: { interfaceConfig: { schedules: false } },
) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
});
it('trips on the object form `{ use: false }` exactly like the boolean stop', async () => {
// Both stop shapes must FREEZE occurrences (engine stops claiming, nothing
// advances). A shape-blind gate left the engine claiming while getLimits refused
// fires, so the disabled path ADVANCED each occurrence — a short maintenance stop
// silently dropped everything it covered instead of leaving it due.
const getAppConfig = jest.fn(async () => ({
interfaceConfig: { schedules: { use: false, maxPerUser: 5 } },
})) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
expect(await service.engineDeps.isGloballyDisabled()).toBe(true);
});
it('does not trip on the object form while `use` stays enabled', async () => {
const getAppConfig = jest.fn(async () => ({
interfaceConfig: { schedules: { use: true, maxPerUser: 5 } },
})) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
expect(await service.engineDeps.isGloballyDisabled()).toBe(false);
});
});
describe('scheduled resume capacity', () => {
function makeResumeService(
occupancy: { takenSlots: number[]; unslotted: number },
over: {
scheduleProjectId?: string;
projectConfig?: Record<string, unknown>;
project?: unknown;
} = {},
) {
const methods = {
getScheduleById: jest.fn(async () => ({
id: 's1',
user: 'user-1',
enabled: true,
configRevision: 3,
...(over.scheduleProjectId != null && { chatProjectId: over.scheduleProjectId }),
})),
getRoleByName: jest.fn(async () => ({ permissions: { SCHEDULES: { USE: true } } })),
getCapacityOccupancy: jest.fn(async () => occupancy),
acquireResumeLease: jest.fn(async () => ({
id: 's1',
claimToken: 'resume-token',
leaseBy: 'resume:resume-token',
})),
consumeResumeLease: jest.fn(async () => true),
releaseLeaseByHolder: jest.fn(async () => undefined),
markRunResumeClaimed: jest.fn(async (_id, _scheduledFor, capacitySlot) => ({
capacitySlot,
})),
releaseRunResumeClaim: jest.fn(async () => true),
};
const service = createSchedulesService({
methods: methods as unknown as SchedulesServiceDeps['methods'],
getAppConfig: jest.fn(async () => ({
interfaceConfig: {
schedules: {
use: true,
maxPerUser: 10,
minIntervalMinutes: 60,
autoDisableAfterFailures: 5,
admissionConcurrency: 20,
fireConcurrency: 1,
mcpPreflightConcurrency: 3,
mcpPreflightTimeoutMs: 300_000,
...(over.projectConfig ?? {}),
},
},
})),
findUserById: jest.fn(async () => ({ _id: 'user-1', role: 'USER' })),
findBalance: jest.fn(async () => null),
upsertBalance: jest.fn(async () => null),
initializeNullBalance: jest.fn(async () => null),
preflightMCP: jest.fn().mockResolvedValue([]),
resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
getChatProject: jest.fn(async () => ('project' in over ? over.project : { _id: 'proj-1' })),
isUserDeleting: jest.fn(async () => false),
enqueueAgentTrigger: jest.fn(async () => undefined),
getTriggerDelivery: jest.fn(async () => null),
} as unknown as SchedulesServiceDeps);
return { service, methods };
}
it('promotes a pause through the global slot allocator and releases by exact slot', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
const scheduledFor = '2026-08-17T12:00:00.000Z';
await expect(service.claimScheduleResume('s1', scheduledFor)).resolves.toEqual({
capacitySlot: 0,
claimToken: 'resume-token',
leaseBy: 'resume:resume-token',
});
expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', undefined, true, 60_000);
expect(methods.markRunResumeClaimed).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);
await expect(
service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {
expectedConfigRevision: 3,
automatic: true,
}),
).resolves.toBe(true);
expect(methods.consumeResumeLease).toHaveBeenCalledWith(
's1',
'resume-token',
'resume:resume-token',
true,
3,
);
await expect(service.releaseScheduleResumeClaim('s1', scheduledFor, 0)).resolves.toBe(true);
expect(methods.releaseRunResumeClaim).toHaveBeenCalledWith('s1', new Date(scheduledFor), 0);
});
it('leaves the paused row untouched when deployment capacity is full', async () => {
const { service, methods } = makeResumeService({ takenSlots: [0], unslotted: 0 });
await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({
conflict: 'capacity',
});
expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');
});
it('normalizes an exhausted slot-collision retry to public capacity conflict', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
methods.markRunResumeClaimed.mockResolvedValue({ conflict: 'slot-taken' } as never);
await expect(service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z')).resolves.toEqual({
conflict: 'capacity',
});
});
it('refuses a resume whose schedule revision changed before the capacity handoff', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
await expect(
service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
expectedConfigRevision: 2,
automatic: true,
}),
).resolves.toEqual({ conflict: 'inactive' });
expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
});
it('refuses an edit that wins during async policy checks at the final schedule CAS', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
methods.acquireResumeLease.mockResolvedValue(null as never);
await expect(
service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
expectedConfigRevision: 3,
automatic: true,
}),
).resolves.toEqual({ conflict: 'inactive' });
expect(methods.acquireResumeLease).toHaveBeenCalledWith('s1', 3, true, 60_000);
expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
});
it('releases the old holder when the post-approval schedule handoff loses', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
methods.consumeResumeLease.mockResolvedValue(false);
await expect(
service.finalizeScheduleResumeClaim('s1', 'resume-token', 'resume:resume-token', {
expectedConfigRevision: 3,
automatic: true,
}),
).resolves.toBe(false);
expect(methods.releaseLeaseByHolder).toHaveBeenCalledWith('s1', 'resume:resume-token');
});
it('refuses a disabled automatic resume but preserves explicit Run Now semantics', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
methods.getScheduleById.mockResolvedValue({
id: 's1',
user: 'user-1',
enabled: false,
configRevision: 3,
});
await expect(
service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
expectedConfigRevision: 3,
automatic: true,
}),
).resolves.toEqual({ conflict: 'inactive' });
await expect(
service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
expectedConfigRevision: 3,
automatic: false,
}),
).resolves.toEqual({
capacitySlot: 0,
claimToken: 'resume-token',
leaseBy: 'resume:resume-token',
});
});
it('refuses a resume when the owner lost schedule access before the capacity handoff', async () => {
const { service, methods } = makeResumeService({ takenSlots: [], unslotted: 0 });
methods.getRoleByName.mockResolvedValue({ permissions: { SCHEDULES: { USE: false } } });
await expect(
service.claimScheduleResume('s1', '2026-08-17T12:00:00.000Z', {
expectedConfigRevision: 3,
automatic: true,
}),
).resolves.toEqual({ conflict: 'inactive' });
expect(methods.markRunResumeClaimed).not.toHaveBeenCalled();
});
});
describe('deployment-wide limits', () => {
const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
it('resolves a principal-less getLimits from the BASE config only', async () => {
const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) =>
options?.baseOnly === true
? {
interfaceConfig: {
schedules: { use: true, fireConcurrency: 1, mcpPreflightConcurrency: 3 },
},
}
: // The principal/tenant-merged view. A bare getAppConfig() resolves THIS,
// including whatever tenant the ALS context happens to carry.
{
interfaceConfig: {
schedules: { use: true, fireConcurrency: 5, mcpPreflightConcurrency: 3 },
},
},
) as unknown as SchedulesServiceDeps['getAppConfig'];
const service = makeService(noRuns(), getAppConfig);
const limits = await service.getLimits();
// NO principal must mean the DEPLOYMENT's config. Both callers of this form run
// inside a tenant context (fireSchedule clamps the global capacity allocator from
// within runInTenantContext(owner); the engine tick budgets from within
// runAsSystem), so resolving the merged view would let a tenant override widen the
// very global cap it is clamped against.
expect(limits.fireConcurrency).toBe(1);
expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true });
});
});
describe('durable trigger wiring', () => {
const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
it('uses the shared trigger service instead of owning a second loopback transport', async () => {
const enqueueAgentTrigger = jest.fn(async () => ({ status: 'queued' }));
const service = makeService(noRuns(), undefined, enqueueAgentTrigger);
const envelope = { version: 1, mode: 'fire' } as never;
await service.engineDeps.enqueueTrigger(envelope, { orderingKey: 'schedule-1' });
expect(enqueueAgentTrigger).toHaveBeenCalledWith(envelope, {
orderingKey: 'schedule-1',
});
});
});
describe('admission revision fence', () => {
const noRuns = () => jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]);
function serviceWithSchedule(schedule: { configRevision?: number; enabled?: boolean } | null) {
const service = makeService(noRuns());
(
service.engineDeps.methods as unknown as {
getScheduleById: jest.Mock;
}
).getScheduleById = jest.fn(async () => schedule);
return service;
}
it('admits when the claimed revision still matches', async () => {
const service = serviceWithSchedule({ configRevision: 3 });
expect(await service.isScheduleLive('sched-1', 3)).toBe(true);
});
it('REFUSES when an owner edit moved the revision on after the claim', async () => {
// The fire was claimed under revision 3; the owner edited since (now 4). Persisting
// would write the OLD prompt/agent into the edited schedule's history.
const service = serviceWithSchedule({ configRevision: 4 });
expect(await service.isScheduleLive('sched-1', 3)).toBe(false);
});
/**
* A policy auto-disable flips `enabled` WITHOUT touching configRevision (an older
* paused occurrence can resume, fail and cross the threshold while a newer occurrence
* is already in the claim-to-controller window), so the revision fence cannot see it.
*/
it('refuses an AUTOMATIC fire once the schedule was disabled', async () => {
const service = serviceWithSchedule({ configRevision: 3, enabled: false });
expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(false);
});
it('still admits Run Now on a disabled schedule', async () => {
// An explicit user action, matching fireScheduleNow's own relaxation.
const service = serviceWithSchedule({ configRevision: 3, enabled: false });
expect(await service.isScheduleLive('sched-1', 3, { automatic: false })).toBe(true);
expect(await service.isScheduleLive('sched-1', 3)).toBe(true);
});
it('admits an automatic fire while the schedule is still enabled', async () => {
const service = serviceWithSchedule({ configRevision: 3, enabled: true });
expect(await service.isScheduleLive('sched-1', 3, { automatic: true })).toBe(true);
});
it('refuses a schedule that is gone regardless of revision', async () => {
const service = serviceWithSchedule(null);
expect(await service.isScheduleLive('sched-1', 3)).toBe(false);
});
it('stays permissive when either side has no revision (pre-existing rows)', async () => {
expect(await serviceWithSchedule({}).isScheduleLive('sched-1', 3)).toBe(true);
expect(await serviceWithSchedule({ configRevision: 4 }).isScheduleLive('sched-1')).toBe(true);
});
});
describe('abort stamp is load-bearing (withheld abort on stamp failure)', () => {
it('withholds the abort and reports unconfirmed when the stamp cannot be made durable', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
markScheduleDeleting: jest.Mock;
getActiveRunsForSchedule: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
requestRunAbort: jest.Mock;
};
methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
methods.getActiveRunsForSchedule = jest.fn(async () => [
{
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
},
]);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
// The stamp is what makes concurrent drains and the reconciler defer to the
// owner's settle; signalling an abort without it re-opens the drain-mid-write
// window the stamp exists to close.
methods.requestRunAbort = jest.fn(async () => {
throw new Error('mongo down');
});
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: true }));
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('unconfirmed');
expect(manager.abortJob).not.toHaveBeenCalled();
});
it('reports an exact generation that remains active as not stopped', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({
success: false,
failureReason: 'job_still_active',
}));
const delivered = await service.engineDeps.abortScheduledJob(
'c1',
{ scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
{ preserve: true },
);
expect(delivered).toBe(false);
});
});
describe('deleteScheduleForOwner waits for the settle acknowledgement', () => {
it('converges when an unconfirmed abort settles during the bounded drain', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const methods = service.engineDeps.methods as unknown as {
markScheduleDeleting: jest.Mock;
getActiveRunsForSchedule: jest.Mock;
eraseScheduleIfDrained: jest.Mock;
};
methods.markScheduleDeleting = jest.fn(async () => ({ id: 's1', user: 'user-1' }));
// Active at the abort pass, drained by the first drain poll: the owner settled.
methods.getActiveRunsForSchedule = jest
.fn()
.mockResolvedValueOnce([
{
scheduleId: 's1',
scheduledFor: new Date('2026-01-01T00:00:00.000Z'),
conversationId: 'c1',
status: 'started',
},
])
.mockResolvedValue([]);
methods.eraseScheduleIfDrained = jest.fn(async () => true);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 1,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
// Delivery cannot be locally proven (peer-owned generation): honest false...
manager.abortJob = jest.fn(async () => ({ success: true, signalDelivered: false }));
// ...but the run row leaving the active set is the durable acknowledgement, so
// the delete converges instead of answering a spurious 503.
await expect(service.deleteScheduleForOwner('s1', 'user-1')).resolves.toBe('deleted');
});
});
describe('provider-drained schedule aborts', () => {
it('waits on the exact terminal generation instead of using the removed re-signal path', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'aborted',
createdAt: 7,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: false, failureReason: 'already_settled' }));
const delivered = await service.engineDeps.abortScheduledJob(
'c1',
{ scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
{ preserve: true },
);
expect(manager.abortJob).toHaveBeenCalledWith('c1', {
expectedCreatedAt: 7,
awaitProviderDrain: true,
});
expect(delivered).toBe(true);
});
it.each(['generation_replaced', 'job_still_active', 'job_not_found'] as const)(
'reports %s as an undelivered abort',
async (failureReason) => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const deleteJob = jest.fn(async () => true);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'running',
createdAt: 7,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
deleteJob,
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: false, failureReason }));
const delivered = await service.engineDeps.abortScheduledJob(
'c1',
{ scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
{ preserve: false },
);
expect(delivered).toBe(false);
// Never destroy evidence for a generation this call did not stop.
expect(deleteJob).not.toHaveBeenCalled();
},
);
it('deletes terminal evidence only after the exact provider drain is confirmed', async () => {
const service = makeService(jest.fn<Promise<ActiveRun[]>, [string]>().mockResolvedValue([]));
const deleteJob = jest.fn(async () => true);
mockJobStore = {
getJob: jest.fn(async () => ({
status: 'aborted',
createdAt: 7,
scheduleId: 's1',
scheduledFor: '2026-01-01T00:00:00.000Z',
})),
deleteJob,
} as unknown as typeof mockJobStore;
const manager = jest.requireMock('../stream/GenerationJobManager').GenerationJobManager;
manager.abortJob = jest.fn(async () => ({ success: false, failureReason: 'already_settled' }));
const delivered = await service.engineDeps.abortScheduledJob(
'c1',
{ scheduleId: 's1', scheduledFor: '2026-01-01T00:00:00.000Z' },
{ preserve: false },
);
expect(delivered).toBe(true);
expect(manager.abortJob).toHaveBeenCalledWith('c1', {
expectedCreatedAt: 7,
awaitProviderDrain: true,
});
expect(deleteJob).toHaveBeenCalledWith('c1', 7);
});
});
describe('scheduled receipt retry load', () => {
it('backs off instead of polling durable stores four times per second', async () => {
jest.useFakeTimers();
const input = {
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'backoff',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
let durable = false;
const persist = jest.fn(async () => durable);
const pending = recordScheduledMCPToolAuthFailure(input, () => persist);
try {
await jest.advanceTimersByTimeAsync(10_000);
expect(persist.mock.calls.length).toBeLessThanOrEqual(7);
durable = true;
await jest.advanceTimersByTimeAsync(30_000);
await expect(pending).resolves.toBe(true);
} finally {
durable = true;
await jest.runAllTimersAsync();
await pending;
jest.useRealTimers();
}
});
it('does not issue receipt writes during shutdown or return model-visible success', async () => {
const shutdown = jest
.spyOn(jest.requireActual('../app/shutdown'), 'isShutdownInProgress')
.mockReturnValue(true);
const persist = jest.fn(async () => true);
try {
await expect(
recordScheduledMCPToolAuthFailure(
{
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'shutdown',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated',
},
},
() => persist,
),
).rejects.toMatchObject({ name: 'AbortError' });
expect(persist).not.toHaveBeenCalled();
} finally {
shutdown.mockRestore();
}
});
});
it('coalesces only identical receipt retries and honors a capped host policy', async () => {
jest.useFakeTimers();
const randomness = jest.spyOn(Math, 'random').mockReturnValue(1);
const input = {
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'coalesced',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
let durable = false;
const record = jest.fn(async () => durable);
const config = jest.fn(async () => ({ baseMs: 1000, maxMs: 2000 }));
const first = recordScheduledMCPToolAuthFailure(input, () => record, config);
const second = recordScheduledMCPToolAuthFailure(input, () => record, config);
try {
expect(second).toBe(first);
await jest.advanceTimersByTimeAsync(3000);
expect(record).toHaveBeenCalledTimes(3);
expect(config).toHaveBeenCalledTimes(1);
durable = true;
await jest.advanceTimersByTimeAsync(2000);
await expect(first).resolves.toBe(true);
} finally {
durable = true;
await jest.runAllTimersAsync();
await first;
randomness.mockRestore();
jest.useRealTimers();
}
});
it('interrupts a pending receipt backoff immediately when shutdown starts', async () => {
const controller = new AbortController();
const signal = jest
.spyOn(jest.requireActual('../app/shutdown'), 'getShutdownSignal')
.mockReturnValue(controller.signal);
const record = jest.fn(async () => false);
const pending = recordScheduledMCPToolAuthFailure(
{
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'abort-wait',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated',
},
},
() => record,
async () => ({ baseMs: 60_000, maxMs: 60_000 }),
);
try {
await new Promise((resolve) => setImmediate(resolve));
const stopped = pending.catch((error) => error);
controller.abort();
await expect(stopped).resolves.toMatchObject({ name: 'AbortError' });
expect(record).toHaveBeenCalledTimes(1);
} finally {
controller.abort();
signal.mockRestore();
}
});
it('stops a stalled receipt attempt on shutdown without acknowledging persistence', async () => {
const controller = new AbortController();
const signal = jest
.spyOn(jest.requireActual('../app/shutdown'), 'getShutdownSignal')
.mockReturnValue(controller.signal);
let release!: (value: boolean) => void;
const unresolved = new Promise<boolean>((resolve) => {
release = resolve;
});
const record = jest.fn(() => unresolved);
const pending = recordScheduledMCPToolAuthFailure(
{
error: new ScheduledMCPBearerError('consent_revoked', 'Files'),
streamId: 'stalled-shutdown',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated',
},
},
() => record,
);
try {
const stopped = pending.catch((error) => error);
controller.abort();
await expect(stopped).resolves.toMatchObject({ name: 'AbortError' });
expect(record).toHaveBeenCalledTimes(1);
} finally {
release(false);
controller.abort();
signal.mockRestore();
}
});
it.each(['false', 'throw'] as const)(
'holds A3 policy failures behind durable admission when the recorder returns %s',
async (mode) => {
jest.useFakeTimers();
const error = new ScheduledMCPPolicyError('tool_policy_denied', 'Files', 'child');
const input = {
error,
streamId: 'policy-admission',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
let durable = false,
returned = false;
const record = jest.fn(async () => {
if (!durable && mode === 'throw') throw new Error('Receipt unavailable');
return durable;
});
const first = recordScheduledMCPToolAuthFailure(input, () => record);
const duplicate = recordScheduledMCPToolAuthFailure(input, () => record);
const observed = first.then((value) => {
returned = true;
return value;
});
try {
await jest.advanceTimersByTimeAsync(1000);
expect(returned).toBe(false);
expect(duplicate).toBe(first);
expect(record.mock.calls.length).toBeGreaterThan(1);
durable = true;
await jest.advanceTimersByTimeAsync(30_000);
await expect(observed).resolves.toBe(true);
} finally {
durable = true;
await jest.runAllTimersAsync();
await Promise.all([first, duplicate, observed]);
jest.useRealTimers();
}
},
);
it('does not coalesce different A3 diagnoses for the same exact generation', async () => {
jest.useFakeTimers();
let durable = false;
const record = jest.fn(async () => durable);
const input = {
streamId: 'policy-diagnoses',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated' as const,
},
};
const first = recordScheduledMCPToolAuthFailure(
{ ...input, error: new ScheduledMCPPolicyError('tool_policy_denied', 'Files', 'child') },
() => record,
);
const second = recordScheduledMCPToolAuthFailure(
{ ...input, error: new ScheduledMCPPolicyError('binding_mismatch', 'Files', 'child') },
() => record,
);
try {
expect(second).not.toBe(first);
durable = true;
await jest.advanceTimersByTimeAsync(30_000);
await expect(first).resolves.toBe(true);
await expect(second).resolves.toBe(true);
} finally {
durable = true;
await jest.runAllTimersAsync();
await Promise.all([first, second]);
jest.useRealTimers();
}
});
it('fences a foreign or replaced A3 receipt instead of retrying an impossible admission', async () => {
const record = jest.fn(async () => {
throw new ScheduledMCPReceiptFencedError();
});
await expect(
recordScheduledMCPToolAuthFailure(
{
error: new ScheduledMCPPolicyError('tool_policy_denied', 'Files', 'child'),
streamId: 'retired-policy',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated',
},
},
() => record,
),
).rejects.toMatchObject({ name: 'AbortError' });
expect(record).toHaveBeenCalledTimes(1);
});
it('interrupts an A3 admission wait at shutdown without a model-visible policy result', async () => {
const controller = new AbortController();
const signal = jest
.spyOn(jest.requireActual('../app/shutdown'), 'getShutdownSignal')
.mockReturnValue(controller.signal);
const record = jest.fn(async () => false);
const pending = recordScheduledMCPToolAuthFailure(
{
error: new ScheduledMCPPolicyError('tool_policy_denied', 'Files', 'child'),
streamId: 'policy-shutdown',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Files',
identity: {
scheduleId: 's1',
ownerId: 'owner',
tenantId: null,
agentId: 'root',
invocationMode: 'delegated',
},
},
() => record,
async () => ({ baseMs: 60_000, maxMs: 60_000 }),
);
const outcome = pending.catch((error) => error);
try {
await new Promise((resolve) => setImmediate(resolve));
controller.abort();
await expect(outcome).resolves.toMatchObject({ name: 'AbortError' });
expect(record).toHaveBeenCalledTimes(1);
} finally {
controller.abort();
await outcome;
signal.mockRestore();
}
});
describe('scheduled reset policy synchronization', () => {
let mongo: MongoMemoryServer;
beforeAll(async () => {
mongo = await MongoMemoryServer.create();
createModels(mongoose);
await mongoose.connect(mongo.getUri());
});
afterAll(async () => {
await mongoose.disconnect();
await mongo.stop();
});
beforeEach(async () => {
await mongoose.connection.dropDatabase();
});
it.each([
['increased', 2000, true],
['decreased', 400, true],
['unchanged', 1000, true],
['disabled', 1000, false],
])(
'applies the %s policy before resetting a scheduled balance',
async (_case, refillAmount, autoRefillEnabled) => {
const user = new mongoose.Types.ObjectId();
const lastRefill = new Date('2020-01-01');
await mongoose.models.Balance.create({
user,
tokenCredits: 500,
autoRefillEnabled: true,
refillMode: 'reset',
refillAmount: 1000,
refillIntervalValue: 1,
refillIntervalUnit: 'weeks',
lastRefill,
reservations: [{ id: 'held', amount: 450, expiresAt: new Date(Date.now() + 60_000) }],
reservedCredits: 450,
});
const db = createMethods(mongoose);
const findBalance: SchedulesServiceDeps['findBalance'] = (userId, options) =>
db.findBalanceByUser(userId, { includeReservedCredits: true, ...options });
const service = createSchedulesService({
methods: {} as SchedulesServiceDeps['methods'],
getAppConfig: async () =>
({
balance: {
enabled: true,
startBalance: 1000,
autoRefillEnabled,
refillMode: 'reset',
refillAmount,
refillIntervalValue: 1,
refillIntervalUnit: 'weeks',
},
}) as Awaited<ReturnType<SchedulesServiceDeps['getAppConfig']>>,
findBalance,
upsertBalance: (userId, { set, setOnInsert }) =>
db.upsertBalanceFields(userId, set, setOnInsert),
initializeNullBalance: jest.fn(async () => null),
findUserById: jest.fn(async () => null),
preflightMCP: jest.fn().mockResolvedValue([]),
resolveAgentFireAccess: jest.fn(async () => 'ok' as const),
getChatProject: jest.fn(async () => null),
isUserDeleting: jest.fn(async () => false),
enqueueAgentTrigger: jest.fn(async () => undefined),
getTriggerDelivery: jest.fn(async () => null),
});
await expect(service.engineDeps.isOutOfBalance({ id: user.toString() })).resolves.toBe(
autoRefillEnabled && refillAmount <= 450,
);
const record = await db.findBalanceByUser(user.toString());
expect(record?.tokenCredits).toBe(autoRefillEnabled ? refillAmount : 500);
expect(record?.lastRefill).toEqual(autoRefillEnabled ? expect.any(Date) : lastRefill);
const ledger = await mongoose.models.Transaction.find({
user,
context: 'balanceReset',
}).lean();
expect(ledger).toHaveLength(autoRefillEnabled ? 1 : 0);
if (autoRefillEnabled) {
expect(ledger[0].rawAmount).toBe(refillAmount - 500);
}
},
);
});