Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import type { Agents } from 'librechat-data-provider';
import {
GenerationPublicationFencedError,
PAUSE_PERSISTENCE_TIMEOUT_ERROR,
} from '~/stream/interfaces/IJobStore';
import { ApprovalLifecycle, PendingActionExpiredError } from '~/stream/ApprovalLifecycle';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { buildToolApprovalPayload } from '~/agents/hitl/__fixtures__/approval';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { buildPendingAction } from '~/agents/hitl/policy';
jest.spyOn(console, 'log').mockImplementation();
describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', () => {
let manager: GenerationJobManagerClass;
let jobStore: InMemoryJobStore;
let eventTransport: InMemoryEventTransport;
beforeEach(() => {
jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
eventTransport = new InMemoryEventTransport();
manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport,
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
});
afterEach(async () => {
await manager.destroy();
});
function buildAction(streamId: string, overrides: Partial<Agents.PendingAction> = {}) {
const payload = buildToolApprovalPayload([
{ name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_abc' },
]);
const action = buildPendingAction(payload, {
streamId,
conversationId: streamId,
runId: 'run-1',
responseMessageId: 'msg-1',
});
return { ...action, ...overrides };
}
describe('pause', () => {
test('running → requires_action, persisting the pending record', async () => {
const streamId = 'stream-pause';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
expect(await manager.approvals.pause(streamId, action)).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('requires_action');
const pending = await manager.approvals.peek(streamId);
expect(pending?.actionId).toBe(action.actionId);
expect(pending?.payload.type).toBe('tool_approval');
if (pending?.payload.type === 'tool_approval') {
expect(pending.payload.action_requests[0].name).toBe('shell');
}
});
test('refuses an action whose deadline passed before the pause transition', async () => {
const streamId = 'stream-expired-before-pause';
await manager.createJob(streamId, 'user-1');
await expect(
manager.approvals.pause(streamId, buildAction(streamId, { expiresAt: Date.now() - 1 })),
).rejects.toBeInstanceOf(PendingActionExpiredError);
await expect(manager.getJob(streamId)).resolves.toMatchObject({ status: 'running' });
await expect(manager.approvals.peek(streamId)).resolves.toBeNull();
});
test('refuses an action that expires while the pause transition is waiting on storage', async () => {
const streamId = 'stream-expired-during-pause';
await manager.createJob(streamId, 'user-1');
const expiresAt = Date.now() + 1000;
const originalTransition = jobStore.transitionStatus.bind(jobStore);
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt - 1);
const transition = jest
.spyOn(jobStore, 'transitionStatus')
.mockImplementationOnce(async (...args) => {
now.mockReturnValue(expiresAt);
return originalTransition(...args);
});
try {
await expect(
manager.approvals.pause(streamId, buildAction(streamId, { expiresAt })),
).rejects.toBeInstanceOf(PendingActionExpiredError);
} finally {
transition.mockRestore();
now.mockRestore();
}
await expect(manager.getJob(streamId)).resolves.toMatchObject({ status: 'running' });
await expect(manager.approvals.peek(streamId)).resolves.toBeNull();
});
test('a later ask retains a legacy answer for ordered cross-replica reconstruction', async () => {
const streamId = 'stream-repause-retains-legacy-answer';
await manager.createJob(streamId, 'user-1');
const firstAction = buildAction(streamId);
expect(await manager.approvals.pause(streamId, firstAction)).toBe(true);
expect(
await manager.approvals.resolve(streamId, firstAction.actionId, {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
},
],
}),
).toBe(true);
expect(
await manager.approvals.pause(
streamId,
buildAction(streamId, {
payload: {
type: 'ask_user_question',
question: { question: 'Approve deployment?' },
tool_call_id: 'ask-2',
},
}),
),
).toBe(true);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'requires_action',
metadata: {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
},
],
},
});
});
test('a later tool approval retains a legacy answer for cross-replica reconstruction', async () => {
const streamId = 'stream-repause-retains-legacy-answer';
await manager.createJob(streamId, 'user-1');
const firstAction = buildAction(streamId);
expect(await manager.approvals.pause(streamId, firstAction)).toBe(true);
expect(
await manager.approvals.resolve(streamId, firstAction.actionId, {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
},
],
}),
).toBe(true);
expect(await manager.approvals.pause(streamId, buildAction(streamId))).toBe(true);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'requires_action',
metadata: {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
},
],
},
});
});
test('a later pause retains an exact-ID answer for cross-replica reconstruction', async () => {
const streamId = 'stream-repause-retains-exact-answer';
await manager.createJob(streamId, 'user-1');
const firstAction = buildAction(streamId);
expect(await manager.approvals.pause(streamId, firstAction)).toBe(true);
expect(
await manager.approvals.resolve(streamId, firstAction.actionId, {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
toolCallId: 'ask-1',
},
],
}),
).toBe(true);
expect(await manager.approvals.pause(streamId, buildAction(streamId))).toBe(true);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'requires_action',
metadata: {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
toolCallId: 'ask-1',
},
],
},
});
});
test('persists discovered tools in the same transition that makes the pause visible', async () => {
const streamId = 'stream-pause-discoveries';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
expect(
await manager.approvals.pause(streamId, action, {
discoveredTools: ['save_issue_mcp_linear'],
authoredSkills: [
{ agentId: 'a', name: 'new-skill', id: '507f1f77bcf86cd799439011', primed: true },
],
}),
).toBe(true);
const paused = await manager.getJob(streamId);
expect(paused?.status).toBe('requires_action');
expect(paused?.metadata.discoveredTools).toEqual(['save_issue_mcp_linear']);
expect(paused?.metadata.authoredSkills).toEqual([
{ agentId: 'a', name: 'new-skill', id: '507f1f77bcf86cd799439011', primed: true },
]);
});
test('persists activity phase state in the same transition as the pause', async () => {
const streamId = 'stream-pause-activity-phase';
await manager.createJob(streamId, 'user-1');
const activityPhaseSnapshot = {
version: 1 as const,
generated: 0,
activityCount: 1,
failedActivityCount: 0,
partialActivityCount: 0,
agentIds: [],
activities: [
{
label: 'Inspected the deployment',
status: 'success' as const,
startIndex: 0,
},
],
assistantContext: ['Checking the remaining replicas'],
pendingReasoning: [],
};
expect(
await manager.approvals.pause(streamId, buildAction(streamId), {
activityPhaseSnapshot,
}),
).toBe(true);
const paused = await manager.getJob(streamId);
expect(paused?.metadata.activityPhaseSnapshot).toEqual(activityPhaseSnapshot);
});
test('persists compaction guidance in the same transition as the pause', async () => {
const streamId = 'stream-pause-compaction-index';
await manager.createJob(streamId, 'user-1');
const compactionSemanticIndex = {
version: 1 as const,
entries: [
{
type: 'activity_phase' as const,
sourceMessageId: 'assistant-history',
sourceContentIndex: 1,
revision: 1,
status: 'committed' as const,
text: 'Verified the release state',
},
],
};
expect(
await manager.approvals.pause(streamId, buildAction(streamId), {
compactionSemanticIndex,
}),
).toBe(true);
const paused = await manager.getJob(streamId);
expect(paused?.metadata.compactionSemanticIndex).toEqual(compactionSemanticIndex);
});
test('persists context meta in the same transition as the pause', async () => {
const streamId = 'stream-pause-context-meta';
await manager.createJob(streamId, 'user-1');
const contextMeta = {
calibrationRatio: 1.25,
encoding: 'claude',
fading: { v: 1 as const, budgetTokens: 50_000, masked: true },
fadingTiers: [{ agentId: 'agent-123', v: 1 as const, budgetTokens: 50_000, masked: true }],
};
expect(await manager.approvals.pause(streamId, buildAction(streamId), { contextMeta })).toBe(
true,
);
const paused = await manager.getJob(streamId);
expect(paused?.metadata.contextMeta).toEqual(contextMeta);
});
test('clears the previous pause context meta when a re-pause has none', async () => {
const streamId = 'stream-pause-context-meta-cleared';
await manager.createJob(streamId, 'user-1');
const firstAction = buildAction(streamId);
const contextMeta = {
calibrationRatio: 1.25,
encoding: 'claude',
fading: { v: 1 as const, budgetTokens: 50_000, masked: true },
fadingTiers: [{ agentId: 'agent-123', v: 1 as const, budgetTokens: 50_000, masked: true }],
};
expect(await manager.approvals.pause(streamId, firstAction, { contextMeta })).toBe(true);
expect(await manager.approvals.resolve(streamId, firstAction.actionId)).toBe(true);
expect(await manager.approvals.pause(streamId, buildAction(streamId))).toBe(true);
const repaused = await manager.getJob(streamId);
expect(repaused?.status).toBe('requires_action');
expect(repaused?.metadata.contextMeta).toBeUndefined();
});
test('does not write a stale pause or discoveries onto a replacement job', async () => {
const streamId = 'stream-pause-replaced';
const original = await manager.createJob(streamId, 'user-1');
const replacement = await manager.createJob(streamId, 'user-1');
expect(
await manager.approvals.pause(streamId, buildAction(streamId), {
discoveredTools: ['stale_tool'],
expectedCreatedAt: original.createdAt,
}),
).toBe(false);
const liveJob = await manager.getJob(streamId);
expect(liveJob?.createdAt).toBe(replacement.createdAt);
expect(liveJob?.status).toBe('running');
expect(liveJob?.metadata.discoveredTools).toBeUndefined();
});
test('commits discoveries and the response barrier in the visible pause transition', async () => {
const streamId = 'stream-pause-discoveries-and-persistence';
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
discoveredTools: ['save_issue_mcp_linear'],
persistencePending: true,
}),
).resolves.toBe(true);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'requires_action',
metadata: {
discoveredTools: ['save_issue_mcp_linear'],
terminalPersistencePending: true,
},
});
});
test('holds abort behind the paused-response persistence barrier', async () => {
const streamId = 'stream-pause-persistence-abort-race';
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
let abortSettled = false;
const aborting = manager
.abortJob(streamId, { expectedCreatedAt: job.createdAt })
.then((result) => {
abortSettled = true;
return result;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(abortSettled).toBe(false);
await expect(
manager.approvals.ownsPausePersistence(streamId, action.actionId, job.createdAt),
).resolves.toBe(true);
await expect(
manager.approvals.finishPausePersistence(streamId, action.actionId, job.createdAt),
).resolves.toBe(true);
await expect(aborting).resolves.toMatchObject({
success: true,
finalEvent: expect.objectContaining({ aborted: true }),
});
});
test('holds approval resume behind the paused-response persistence barrier', async () => {
const streamId = 'stream-pause-persistence-resume-race';
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
let resumeSettled = false;
const resuming = manager.approvals
.resolve(streamId, action.actionId, undefined, job.createdAt)
.then((result) => {
resumeSettled = true;
return result;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(resumeSettled).toBe(false);
await expect(
manager.approvals.finishPausePersistence(streamId, action.actionId, job.createdAt),
).resolves.toBe(true);
await expect(resuming).resolves.toBe(true);
await expect(manager.getJobStatus(streamId)).resolves.toBe('running');
});
test('failed pause persistence atomically beats a waiting resume and parks steers', async () => {
const streamId = 'stream-pause-persistence-failure-race';
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
const waitingSteer = {
steerId: 'steer-waiting-on-failed-pause',
text: 'preserve me after the failed pause',
userId: 'user-1',
createdAt: Date.now(),
};
await expect(manager.steering.enqueue(streamId, waitingSteer, job.createdAt)).resolves.toBe(
1,
);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
let resumeSettled = false;
const resuming = manager.approvals
.resolve(streamId, action.actionId, undefined, job.createdAt)
.then((result) => {
resumeSettled = true;
return result;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(resumeSettled).toBe(false);
await expect(
manager.failPausePersistence(
streamId,
action.actionId,
'paused response was not persisted',
job.createdAt,
),
).resolves.toBe(true);
await expect(resuming).resolves.toBe(false);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'error',
error: 'paused response was not persisted',
});
const failedStoredJob = await jobStore.getJob(streamId);
expect(failedStoredJob?.pendingAction).toBeUndefined();
expect(failedStoredJob?.pendingActionId).toBeUndefined();
expect(failedStoredJob?.terminalPersistencePending).toBeUndefined();
expect(failedStoredJob?.terminalPersistenceStartedAt).toBeUndefined();
await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);
await expect(manager.steering.claim(streamId, { userId: 'user-1' })).resolves.toEqual([
expect.objectContaining({
steerId: waitingSteer.steerId,
text: waitingSteer.text,
}),
]);
});
test('failed pause persistence is fenced to the exact action and generation', async () => {
const streamId = 'stream-pause-persistence-failure-fence';
const predecessor = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId, { actionId: 'reused-pause-action' });
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: predecessor.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
await expect(
manager.failPausePersistence(
streamId,
'different-action',
'must not win',
predecessor.createdAt,
),
).resolves.toBe(false);
await expect(
manager.approvals.ownsPausePersistence(streamId, action.actionId, predecessor.createdAt),
).resolves.toBe(true);
await new Promise<void>((resolve) => setTimeout(resolve, 2));
const replacement = await manager.createJob(streamId, 'user-1');
await expect(
manager.failPausePersistence(
streamId,
action.actionId,
'stale predecessor failure',
predecessor.createdAt,
),
).resolves.toBe(false);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
createdAt: replacement.createdAt,
status: 'running',
});
});
test('fails a crashed pause writer closed after the bounded barrier deadline', async () => {
const streamId = 'stream-stale-pause-persistence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
try {
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
const waitingSteer = {
steerId: 'stale-pause-steer',
text: 'recover me instead of resuming unsafe history',
userId: 'user-1',
createdAt: 1_000,
};
await expect(manager.steering.enqueue(streamId, waitingSteer, job.createdAt)).resolves.toBe(
1,
);
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
now.mockReturnValue(31_001);
await expect(
manager.approvals.resolve(streamId, action.actionId, undefined, job.createdAt),
).resolves.toBe(false);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'error',
error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
});
const failedJob = await jobStore.getJob(streamId);
expect(failedJob?.pendingAction).toBeUndefined();
expect(failedJob?.pendingActionId).toBeUndefined();
expect(failedJob?.terminalPersistencePending).toBeUndefined();
expect(failedJob?.terminalPersistenceStartedAt).toBeUndefined();
expect(job.abortController.signal.aborted).toBe(true);
expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
await expect(manager.steering.claim(streamId, { userId: 'user-1' })).resolves.toEqual([
expect.objectContaining({ steerId: waitingSteer.steerId, text: waitingSteer.text }),
]);
subscription?.unsubscribe();
} finally {
now.mockRestore();
}
});
test('relays a store-only pause timeout into the matching attached runtime once', async () => {
const streamId = 'stream-remote-stale-pause-persistence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
try {
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
const broadcast = jest.spyOn(eventTransport, 'emitError');
const clearContentState = jest.spyOn(jobStore, 'clearContentState');
expect(job.abortController.signal.aborted).toBe(false);
now.mockReturnValue(31_001);
// Model another replica/store cleanup winning the terminal CAS without
// access to this manager's in-process runtime.
await jobStore.cleanup();
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'error',
error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
});
expect(onError).not.toHaveBeenCalled();
expect(clearContentState).not.toHaveBeenCalled();
const managerWithCleanup = manager as unknown as { cleanup(): Promise<void> };
await managerWithCleanup.cleanup();
expect(job.abortController.signal.aborted).toBe(true);
expect(onError).toHaveBeenCalledTimes(1);
expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
expect(broadcast).toHaveBeenCalledWith(
streamId,
PAUSE_PERSISTENCE_TIMEOUT_ERROR,
job.createdAt,
);
expect(clearContentState).toHaveBeenCalledWith(streamId, job.createdAt);
await managerWithCleanup.cleanup();
expect(onError).toHaveBeenCalledTimes(1);
subscription?.unsubscribe();
} finally {
now.mockRestore();
}
});
test('notifies the attached runtime when a locally-won timeout row is immediately evicted', async () => {
const streamId = 'stream-zero-ttl-stale-pause-persistence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
try {
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
const originalGetJob = jobStore.getJob.bind(jobStore);
jest.spyOn(jobStore, 'getJob').mockImplementation(async (...args) => {
const stored = await originalGetJob(...args);
return stored?.status === 'error' ? null : stored;
});
now.mockReturnValue(31_001);
await expect(
manager.approvals.resolve(streamId, action.actionId, undefined, job.createdAt),
).resolves.toBe(false);
expect(job.abortController.signal.aborted).toBe(true);
expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
subscription?.unsubscribe();
} finally {
now.mockRestore();
}
});
test('returns false when the job is already terminal', async () => {
const streamId = 'stream-pause-dead';
await manager.createJob(streamId, 'user-1');
await manager.completeJob(streamId, 'terminated mid-flight');
expect(await manager.approvals.pause(streamId, buildAction(streamId))).toBe(false);
// a late interrupt must NOT resurrect a terminal job into requires_action
expect(await manager.getJobStatus(streamId)).not.toBe('requires_action');
});
test('returns false when the job does not exist', async () => {
expect(await manager.approvals.pause('nonexistent', buildAction('nonexistent'))).toBe(false);
});
test('replacing a paused generation removes its approval and rejects a late decision', async () => {
const streamId = 'stream-replace-paused-generation';
const predecessor = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
const replacement = await manager.createJob(streamId, 'user-1');
expect(replacement.createdAt).not.toBe(predecessor.createdAt);
expect(await manager.approvals.peek(streamId)).toBeNull();
expect((await manager.getResumeState(streamId))?.pendingAction).toBeUndefined();
expect(await manager.approvals.resolve(streamId, action.actionId)).toBe(false);
expect(await manager.getJobStatus(streamId)).toBe('running');
});
test('a predecessor interrupt cannot pause a replacement generation', async () => {
const streamId = 'stream-pause-epoch-fence';
const predecessor = await manager.createJob(streamId, 'user-1');
const replacement = await manager.createJob(streamId, 'user-1');
await expect(
manager.approvals.pause(streamId, buildAction(streamId), {
expectedCreatedAt: predecessor.createdAt,
}),
).resolves.toBe(false);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
createdAt: replacement.createdAt,
status: 'running',
});
});
test('client-facing copies omit resumeContext/requestFingerprint; the stored record keeps them', async () => {
const streamId = 'stream-redact';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId, {
requestFingerprint: 'fp-hash',
resumeContext: {
endpoint: 'agents',
model_parameters: { temperature: 0.7 },
},
});
expect(await manager.approvals.pause(streamId, action)).toBe(true);
const resumeState = await manager.getResumeState(streamId);
expect(resumeState?.pendingAction?.actionId).toBe(action.actionId);
expect(resumeState?.pendingAction?.resumeContext).toBeUndefined();
expect(resumeState?.pendingAction?.requestFingerprint).toBeUndefined();
// The resume route replays from the job store, which must retain the full record.
const job = await manager.getJob(streamId);
expect(job?.metadata.pendingAction?.resumeContext).toEqual(action.resumeContext);
expect(job?.metadata.pendingAction?.requestFingerprint).toBe('fp-hash');
});
});
describe('peek', () => {
test('returns null for jobs not in requires_action', async () => {
const streamId = 'stream-running';
await manager.createJob(streamId, 'user-1');
expect(await manager.approvals.peek(streamId)).toBeNull();
});
test('returns null when the job does not exist', async () => {
expect(await manager.approvals.peek('nonexistent')).toBeNull();
});
test('treats a past-expiresAt record as gone (lazy expiry)', async () => {
const streamId = 'stream-expired-peek';
await manager.createJob(streamId, 'user-1');
const expiresAt = Date.now() + 1000;
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
try {
expect(await manager.approvals.peek(streamId)).toBeNull();
} finally {
now.mockRestore();
}
});
});
describe('resolve', () => {
test('requires_action → running, clearing the record, returns true once', async () => {
const streamId = 'stream-resolve';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
expect(await manager.approvals.resolve(streamId)).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('running');
expect(await manager.approvals.peek(streamId)).toBeNull();
});
test('clears the predecessor Event Actor suspension projection on resume', async () => {
const streamId = 'stream-resolve-event-actor-suspension';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { providerExecutionId: 'provider-paused' },
});
const pausedProviderExecutionId = job.metadata.providerExecutionId!;
const action = buildAction(streamId);
expect(
await manager.beginProviderExecution(streamId, job.createdAt, pausedProviderExecutionId),
).toBe(true);
await manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
agentEventSuspension: { version: 1, suspensionId: 'suspension-1', attempt: 0 },
});
expect(
await manager.approvals.resolve(
streamId,
action.actionId,
{ providerExecutionId: 'provider-resume', providerDrained: true },
job.createdAt,
),
).toBe(true);
await expect(manager.getJob(streamId)).resolves.toMatchObject({
status: 'running',
metadata: { providerExecutionId: 'provider-resume' },
});
expect((await manager.getJob(streamId))?.metadata.agentEventSuspension).toBeUndefined();
expect((await manager.getJob(streamId))?.metadata.providerExecutionStartedId).toBeUndefined();
expect(await manager.beginProviderExecution(streamId, job.createdAt, 'provider-resume')).toBe(
true,
);
expect((await manager.getJob(streamId))?.metadata.providerExecutionStartedId).toBe(
'provider-resume',
);
});
test('a concurrent double-resolve wins exactly once (race-safe)', async () => {
const streamId = 'stream-double-resolve';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
const results = await Promise.all([
manager.approvals.resolve(streamId),
manager.approvals.resolve(streamId),
]);
// Exactly one caller may drive the run — the other must be rejected.
expect(results.filter(Boolean)).toHaveLength(1);
expect(await manager.getJobStatus(streamId)).toBe('running');
});
test('publishes the new owner capability in the same CAS that reopens steering', async () => {
const streamId = 'stream-resolve-capability';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { preemptCapable: true },
});
await manager.steering.enqueue(streamId, {
steerId: 'waiting-steer',
text: 'wait for the new owner',
userId: 'user-1',
createdAt: Date.now(),
});
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
expect(
await manager.approvals.resolve(streamId, action.actionId, {
preemptCapable: false,
}),
).toBe(true);
expect((await manager.getJob(streamId))?.metadata.preemptCapable).toBe(false);
await expect(manager.steering.arm(streamId, 'waiting-steer', job.createdAt)).resolves.toBe(
'incapable',
);
});
test('persists user-submitted provenance in the same CAS that resumes the job', async () => {
const streamId = 'stream-resolve-provenance';
const job = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
const userSubmittedMessageFieldPaths = [
{ path: '/content/0/tool_call/output', field: 'decision_response' as const },
];
expect(
await manager.approvals.resolve(
streamId,
action.actionId,
{
userSubmittedPaths: ['/content/0/tool_call/args'],
userSubmittedMessageFieldPaths,
},
job.createdAt,
),
).toBe(true);
const resumed = await manager.getJob(streamId);
expect(resumed?.status).toBe('running');
expect(resumed?.metadata.userSubmittedPaths).toEqual(['/content/0/tool_call/args']);
expect(resumed?.metadata.userSubmittedMessageFieldPaths).toEqual(
userSubmittedMessageFieldPaths,
);
await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
preResumeProvenance: { userSubmittedPaths: [], userSubmittedMessageFieldPaths: [] },
});
});
test('returns false when the job is not paused', async () => {
const streamId = 'stream-resolve-running';
await manager.createJob(streamId, 'user-1');
expect(await manager.approvals.resolve(streamId)).toBe(false);
});
test('rejects a resolve whose actionId no longer matches (stale-decision guard)', async () => {
const streamId = 'stream-stale-action';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
// A decision targeting a different action must not resume this one.
expect(await manager.approvals.resolve(streamId, 'some-other-action-id')).toBe(false);
expect(await manager.getJobStatus(streamId)).toBe('requires_action');
// The matching actionId resolves it.
expect(await manager.approvals.resolve(streamId, action.actionId)).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('running');
});
test('does not resolve a replacement paused on the same action id', async () => {
const streamId = 'stream-resolve-epoch-fence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
try {
const predecessor = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId, { actionId: 'reused-action-id' });
await manager.approvals.pause(streamId, action);
now.mockReturnValue(2000);
const replacement = await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, action);
expect(
await manager.approvals.resolve(
streamId,
action.actionId,
undefined,
predecessor.createdAt,
),
).toBe(false);
expect(await manager.getJob(streamId)).toMatchObject({
createdAt: replacement.createdAt,
status: 'requires_action',
});
} finally {
now.mockRestore();
}
});
test('an expired pending action expires instead of resuming', async () => {
const streamId = 'stream-resolve-expired';
await manager.createJob(streamId, 'user-1');
const expiresAt = Date.now() + 1000;
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
try {
expect(await manager.approvals.resolve(streamId)).toBe(false);
expect(await manager.getJobStatus(streamId)).toBe('aborted');
} finally {
now.mockRestore();
}
});
});
describe('expire', () => {
test('requires_action → aborted, clearing the record, returns true once', async () => {
const streamId = 'stream-expire';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
expect(await manager.approvals.expire(streamId)).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('aborted');
expect(await manager.approvals.peek(streamId)).toBeNull();
// idempotent — a second expire does not fire again
expect(await manager.approvals.expire(streamId)).toBe(false);
});
test('returns false when the job is not paused', async () => {
const streamId = 'stream-expire-running';
await manager.createJob(streamId, 'user-1');
expect(await manager.approvals.expire(streamId)).toBe(false);
});
test('a mismatched expectedActionId no-ops (protects a re-paused action from a stale sweep)', async () => {
const streamId = 'stream-expire-mismatch';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId, { actionId: 'action-A' }));
// A sweep that observed an OLDER (now-resolved) action must not abort the current
// pause — its CAS only fires when the live pendingActionId still matches.
expect(await manager.approvals.expire(streamId, 'stale-other-action')).toBe(false);
expect(await manager.getJobStatus(streamId)).toBe('requires_action');
// The matching id still expires it.
expect(await manager.approvals.expire(streamId, 'action-A')).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('aborted');
});
test('sets completedAt so terminal cleanup can reclaim the job', async () => {
const streamId = 'stream-expire-completed';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
expect(await manager.approvals.expire(streamId)).toBe(true);
const job = await manager.getJob(streamId);
expect(job?.status).toBe('aborted');
expect(job?.completedAt).toBeGreaterThan(0);
});
});
describe('expireApproval notification', () => {
test('delivers the exact expired generation to the host lifecycle hook', async () => {
const streamId = 'stream-expire-host-hook';
const job = await manager.createJob(streamId, 'user-1');
await manager.updateMetadata(streamId, {
scheduleId: 'schedule-1',
scheduledFor: '2026-08-17T12:00:00.000Z',
});
await manager.approvals.pause(streamId, buildAction(streamId));
const onApprovalExpired = jest.fn(async () => undefined);
manager.setApprovalExpiredHandler(onApprovalExpired);
expect(await manager.expireApproval(streamId)).toBe(true);
expect(onApprovalExpired).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
createdAt: job.createdAt,
status: 'aborted',
scheduleId: 'schedule-1',
scheduledFor: '2026-08-17T12:00:00.000Z',
}),
);
});
test('expires a durable paused job even when no process-local runtime survived', async () => {
const streamId = 'stream-expire-ownerless-host-hook';
const job = await jobStore.createJob(streamId, 'user-1', streamId, undefined, {
scheduleId: 'schedule-ownerless',
scheduledFor: '2026-08-17T12:00:00.000Z',
});
const lifecycle = new ApprovalLifecycle(jobStore);
const expiresAt = Date.now() + 1_000;
await lifecycle.pause(streamId, buildAction(streamId, { expiresAt }));
const onApprovalExpired = jest.fn(async () => undefined);
manager.setApprovalExpiredHandler(onApprovalExpired);
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
try {
await (
manager as unknown as { expireStaleApprovals: () => Promise<void> }
).expireStaleApprovals();
} finally {
now.mockRestore();
}
expect(onApprovalExpired).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
createdAt: job.createdAt,
scheduleId: 'schedule-ownerless',
status: 'aborted',
}),
);
await expect(jobStore.getJob(streamId)).resolves.toMatchObject({ status: 'aborted' });
});
test('does not expire a replacement paused on the same action id', async () => {
const streamId = 'stream-expire-epoch-fence';
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
try {
const predecessor = await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId, { actionId: 'reused-action-id' });
await manager.approvals.pause(streamId, action);
now.mockReturnValue(2000);
const replacement = await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, action);
expect(await manager.expireApproval(streamId, action.actionId, predecessor.createdAt)).toBe(
false,
);
expect(await manager.getJob(streamId)).toMatchObject({
createdAt: replacement.createdAt,
status: 'requires_action',
});
} finally {
now.mockRestore();
}
});
test('publishes a generation-tagged expiry to local and remote subscribers', async () => {
const streamId = 'stream-expire-local-notification';
const job = await manager.createJob(streamId, 'user-1');
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await manager.approvals.pause(streamId, buildAction(streamId));
const broadcast = jest.spyOn(eventTransport, 'emitError');
expect(await manager.expireApproval(streamId)).toBe(true);
expect(onError).toHaveBeenCalledWith('Approval expired before a decision was made');
expect(broadcast).toHaveBeenCalledWith(
streamId,
'Approval expired before a decision was made',
job.createdAt,
);
subscription?.unsubscribe();
});
test('does not invoke local expiry fallback when publication is fenced', async () => {
const streamId = 'stream-expire-publication-fenced';
const job = await manager.createJob(streamId, 'user-1');
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await manager.approvals.pause(streamId, buildAction(streamId));
jest.spyOn(eventTransport, 'emitError').mockImplementation(() => {
throw new GenerationPublicationFencedError('error', streamId, job.createdAt);
});
expect(await manager.expireApproval(streamId)).toBe(true);
expect(onError).not.toHaveBeenCalled();
subscription?.unsubscribe();
});
test('delivers the stored expiry error to a late subscriber', async () => {
const streamId = 'stream-expire-late-subscriber';
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
expect(await manager.approvals.expire(streamId)).toBe(true);
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await new Promise<void>((resolve) => setImmediate(resolve));
expect(onError).toHaveBeenCalledWith('Approval expired before a decision was made');
subscription?.unsubscribe();
});
test('notifies the observed runtime when zero terminal TTL removes the job hash', async () => {
const streamId = 'stream-expire-zero-terminal-ttl';
await manager.createJob(streamId, 'user-1');
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await manager.approvals.pause(streamId, buildAction(streamId));
const originalGetJob = jobStore.getJob.bind(jobStore);
jest.spyOn(jobStore, 'getJob').mockImplementation(async (...args) => {
const job = await originalGetJob(...args);
return job?.status === 'aborted' ? null : job;
});
expect(await manager.expireApproval(streamId)).toBe(true);
expect(onError).toHaveBeenCalledWith('Approval expired before a decision was made');
subscription?.unsubscribe();
});
test('notifies after the manager pre-read fails and the expiry CAS removes the job hash', async () => {
const streamId = 'stream-expire-read-failure-zero-terminal-ttl';
await manager.createJob(streamId, 'user-1');
const onError = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
await manager.approvals.pause(streamId, buildAction(streamId));
const originalGetJob = jobStore.getJob.bind(jobStore);
jest
.spyOn(jobStore, 'getJob')
.mockRejectedValueOnce(new Error('transient read failure'))
.mockImplementation(async (...args) => {
const job = await originalGetJob(...args);
return job?.status === 'aborted' ? null : job;
});
expect(await manager.expireApproval(streamId)).toBe(true);
expect(onError).toHaveBeenCalledWith('Approval expired before a decision was made');
subscription?.unsubscribe();
});
test('does not notify or mutate a replacement created after the expiry CAS', async () => {
const streamId = 'stream-expire-replacement-notification';
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
const originalTransition = jobStore.transitionStatus.bind(jobStore);
let signalExpired: (() => void) | undefined;
const expired = new Promise<void>((resolve) => {
signalExpired = resolve;
});
let releaseTransition: (() => void) | undefined;
const transitionGate = new Promise<void>((resolve) => {
releaseTransition = resolve;
});
jest.spyOn(jobStore, 'transitionStatus').mockImplementation(async (...args) => {
const transitioned = await originalTransition(...args);
if (args[1].to === 'aborted' && transitioned) {
signalExpired?.();
await transitionGate;
}
return transitioned;
});
try {
await manager.createJob(streamId, 'user-1');
await manager.approvals.pause(streamId, buildAction(streamId));
const expiring = manager.expireApproval(streamId);
await expired;
now.mockReturnValue(2000);
await manager.createJob(streamId, 'user-1');
const replacementError = jest.fn();
const replacementSubscription = await manager.subscribe(
streamId,
() => undefined,
undefined,
replacementError,
);
releaseTransition?.();
await expect(expiring).resolves.toBe(true);
await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
createdAt: 2000,
status: 'running',
});
expect(replacementError).not.toHaveBeenCalled();
replacementSubscription?.unsubscribe();
} finally {
releaseTransition?.();
now.mockRestore();
}
});
});
describe('durable approval-expiry host action', () => {
const sweep = (m: GenerationJobManagerClass) =>
(m as unknown as { expireStaleApprovals: () => Promise<void> }).expireStaleApprovals();
async function pauseScheduled(streamId: string) {
const job = await manager.createJob(streamId, 'user-1');
await manager.updateMetadata(streamId, {
scheduleId: 's1',
scheduledFor: '2026-08-17T12:00:00.000Z',
});
await manager.approvals.pause(streamId, buildAction(streamId));
return job;
}
test('retains the marker on hook failure and retries it on a later cleanup pass', async () => {
const streamId = 'stream-host-retry';
await pauseScheduled(streamId);
const handler = jest
.fn<Promise<void>, [string, { scheduleId?: string }]>()
.mockRejectedValueOnce(new Error('mongo down'))
.mockResolvedValue(undefined);
manager.setApprovalExpiredHandler(handler);
// First expiry wins its CAS, but the host hook FAILS: the durable marker is retained.
expect(await manager.expireApproval(streamId)).toBe(true);
expect(handler).toHaveBeenCalledTimes(1);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(1);
// A later cleanup pass re-enumerates the aborted job and retries; it now succeeds
// and clears the marker.
await sweep(manager);
expect(handler).toHaveBeenCalledTimes(2);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
});
test('a fresh manager (restart / other replica) retries the pending host action', async () => {
const streamId = 'stream-host-restart';
const job = await pauseScheduled(streamId);
manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
expect(await manager.expireApproval(streamId)).toBe(true);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
// Another replica / a restarted process: a fresh manager over the SAME durable store,
// with NO local runtime for this stream.
const other = new GenerationJobManagerClass();
other.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
other.initialize();
const succeeding = jest.fn().mockResolvedValue(undefined);
other.setApprovalExpiredHandler(succeeding);
await sweep(other);
expect(succeeding).toHaveBeenCalledWith(
streamId,
expect.objectContaining({ createdAt: job.createdAt, scheduleId: 's1' }),
);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
await other.destroy();
});
test('a successful acknowledgement prevents duplicate work on later sweeps', async () => {
const streamId = 'stream-host-ack';
await pauseScheduled(streamId);
const handler = jest.fn().mockResolvedValue(undefined);
manager.setApprovalExpiredHandler(handler);
expect(await manager.expireApproval(streamId)).toBe(true);
expect(handler).toHaveBeenCalledTimes(1);
expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
await sweep(manager);
expect(handler).toHaveBeenCalledTimes(1); // no duplicate invocation
});
test('passes the committed aborted state to an event outcome handler', async () => {
const streamId = 'stream-host-event-expiry-state';
await manager.createJob(streamId, 'user-1');
await manager.updateMetadata(streamId, { agentEventDeliveryKey: 'delivery-expired' });
await manager.approvals.pause(streamId, buildAction(streamId));
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
expect(await manager.expireApproval(streamId)).toBe(true);
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
status: 'aborted',
error: 'Approval expired before a decision was made',
terminalHostActionPending: true,
}),
expect.any(Array),
expect.any(Array),
);
});
test('clearTerminalHostAction is identity-fenced against a replacement generation', async () => {
const streamId = 'stream-host-fence';
const job = await pauseScheduled(streamId);
manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
await manager.expireApproval(streamId);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
// A clear fenced to a DIFFERENT generation must not clear this one's action.
await jobStore.clearTerminalHostAction(streamId, job.createdAt + 1);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBe(true);
// The exact generation clears it.
await jobStore.clearTerminalHostAction(streamId, job.createdAt);
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
});
test('terminalizes and delivers the terminal state even when the first hook fails', async () => {
const streamId = 'stream-host-notify';
await pauseScheduled(streamId);
manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('down')));
// The CAS + terminal notification proceed regardless of the host hook outcome.
expect(await manager.expireApproval(streamId)).toBe(true);
expect(await manager.getJobStatus(streamId)).toBe('aborted');
});
test('retention is refreshed on each retry so a long host outage keeps the evidence', async () => {
const streamId = 'stream-host-refresh';
await pauseScheduled(streamId);
manager.setApprovalExpiredHandler(jest.fn().mockRejectedValue(new Error('mongo down')));
expect(await manager.expireApproval(streamId)).toBe(true);
const before = (await jobStore.getJob(streamId))?.terminalHostActionRefreshedAt;
// A later cleanup pass re-enumerates it for retry; the hook still fails.
await sweep(manager);
const after = await jobStore.getJob(streamId);
// Still pending, and its retention basis moved forward — so the bounded retention
// window measures from the LAST retry, not from when the approval expired.
expect(after?.terminalHostActionPending).toBe(true);
expect(after?.terminalHostActionRefreshedAt).toEqual(expect.any(Number));
if (before != null) {
expect(after!.terminalHostActionRefreshedAt!).toBeGreaterThanOrEqual(before);
}
});
test('a non-scheduled job with a no-op host hook accumulates no marker', async () => {
const streamId = 'stream-host-nonsched';
await manager.createJob(streamId, 'user-1'); // no schedule metadata
await manager.approvals.pause(streamId, buildAction(streamId));
// The schedule adapter is installed but owns no action for a non-scheduled job.
manager.setApprovalExpiredHandler(
jest.fn(async (_streamId: string, job: { scheduleId?: string }) => {
if (!job.scheduleId) {
return;
}
}),
);
expect(await manager.expireApproval(streamId)).toBe(true);
// Marked atomically then cleared on the no-op success — nothing lingers.
expect((await jobStore.getJob(streamId))?.terminalHostActionPending).toBeUndefined();
expect(await jobStore.getTerminalHostActionJobs()).toHaveLength(0);
});
});
describe('facade integration', () => {
test('requires_action drops the running count but keeps the user-active set', async () => {
const streamId = 'stream-counts';
await manager.createJob(streamId, 'user-counts');
const before = await manager.getJobCountByStatus();
expect(before.running).toBe(1);
expect(before.requires_action).toBe(0);
await manager.approvals.pause(streamId, buildAction(streamId));
const after = await manager.getJobCountByStatus();
expect(after.running).toBe(0);
expect(after.requires_action).toBe(1);
// Pending-approval jobs still occupy the user's conversation slot.
expect(await manager.getActiveJobIdsForUser('user-counts')).toContain(streamId);
});
test('getActiveJobIdsForUser excludes terminal jobs but includes requires_action', async () => {
await manager.createJob('s-running', 'user-mix');
await manager.createJob('s-paused', 'user-mix');
await manager.createJob('s-done', 'user-mix');
await manager.approvals.pause('s-paused', buildAction('s-paused'));
await manager.completeJob('s-done');
const active = await manager.getActiveJobIdsForUser('user-mix');
expect(active.sort()).toEqual(['s-paused', 's-running']);
});
test('excludes a pending-approval job whose prompt has expired', async () => {
const streamId = 'stream-expired-active';
await manager.createJob(streamId, 'user-exp');
const expiresAt = Date.now() + 1000;
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
try {
// Still requires_action, but the prompt is past expiry → no longer active.
expect(await manager.getActiveJobIdsForUser('user-exp')).not.toContain(streamId);
} finally {
now.mockRestore();
}
});
});
});
describe('InMemoryJobStore — approval expiry cleanup', () => {
test('guards status transitions against a replaced job epoch', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const job = await store.createJob('epoch-guard', 'u1');
expect(
await store.transitionStatus('epoch-guard', {
from: 'running',
to: 'error',
expectCreatedAt: job.createdAt + 1,
}),
).toBe(false);
expect((await store.getJob('epoch-guard'))?.status).toBe('running');
expect(
await store.transitionStatus('epoch-guard', {
from: 'running',
to: 'error',
expectCreatedAt: job.createdAt,
}),
).toBe(true);
expect((await store.getJob('epoch-guard'))?.status).toBe('error');
});
test('cleanup() finalizes a past-expiry approval, retaining it for host-hook retry', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
const job = await store.createJob('s1', 'u1');
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
{ streamId: 's1', ttlMs: -1000 },
);
await store.transitionStatus('s1', {
from: 'running',
to: 'requires_action',
patch: { pendingAction: action, pendingActionId: action.actionId },
});
// A past-expiry approval is finalized (aborted) but RETAINED with a pending host-action
// marker so the manager relay can still run its lifecycle hook (the store cannot know
// whether one is owed); it is enumerable for that retry.
await store.cleanup();
const aborted = await store.getJob('s1');
expect(aborted?.status).toBe('aborted');
expect(aborted?.terminalHostActionPending).toBe(true);
expect(await store.getTerminalHostActionJobs()).toHaveLength(1);
// Once the host action acknowledges, the marker clears and the next cleanup reclaims it.
await store.clearTerminalHostAction('s1', job.createdAt);
await store.cleanup();
expect(await store.getJob('s1')).toBeNull();
});
test('cleanup() preserves a fresh pause-persistence barrier before expiring the action', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
try {
const job = await store.createJob('fresh-pause-barrier', 'u1');
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
{ streamId: 'fresh-pause-barrier', ttlMs: -1_000 },
);
await store.transitionStatus('fresh-pause-barrier', {
from: 'running',
to: 'requires_action',
expectCreatedAt: job.createdAt,
patch: {
pendingAction: action,
pendingActionId: `pause-persistence:${action.actionId}`,
terminalPersistencePending: true,
terminalPersistenceStartedAt: 1_000,
},
});
now.mockReturnValue(11_000);
await store.cleanup();
await expect(store.getJob('fresh-pause-barrier')).resolves.toMatchObject({
status: 'requires_action',
terminalPersistencePending: true,
});
now.mockReturnValue(31_001);
await store.cleanup();
await expect(store.getJob('fresh-pause-barrier')).resolves.toBeNull();
} finally {
now.mockRestore();
}
});
test('cleanup() fails a stale pause-persistence barrier closed and parks steers', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
const store = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
try {
const job = await store.createJob('stale-pause-barrier', 'u1');
await store.updateJob(
'stale-pause-barrier',
{ agentEventDeliveryKey: 'trigger-stale-pause' },
job.createdAt,
);
await store.enqueueSteer(
'stale-pause-barrier',
{
steerId: 'stale-cleanup-steer',
text: 'park me when the pause writer disappears',
userId: 'u1',
createdAt: 1_000,
},
job.createdAt,
);
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
{ streamId: 'stale-pause-barrier' },
);
await store.transitionStatus('stale-pause-barrier', {
from: 'running',
to: 'requires_action',
expectCreatedAt: job.createdAt,
patch: {
pendingAction: action,
pendingActionId: `pause-persistence:${action.actionId}`,
terminalPersistencePending: true,
terminalPersistenceStartedAt: 1_000,
},
});
now.mockReturnValue(31_001);
await store.cleanup();
await expect(store.getJob('stale-pause-barrier')).resolves.toMatchObject({
status: 'error',
error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
terminalHostActionPending: true,
});
const failedJob = await store.getJob('stale-pause-barrier');
expect(failedJob?.pendingAction).toBeUndefined();
expect(failedJob?.pendingActionId).toBeUndefined();
expect(failedJob?.terminalPersistencePending).toBeUndefined();
expect(failedJob?.terminalPersistenceStartedAt).toBeUndefined();
await expect(store.peekSteers('stale-pause-barrier', job.createdAt)).resolves.toEqual([]);
await expect(store.claimParkedSteers('stale-pause-barrier', 'u1')).resolves.toEqual(
expect.any(String),
);
} finally {
now.mockRestore();
}
});
test('cleanup() retains a terminal persistence barrier when normal retention is zero', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
const completedAt = Date.now();
const job = await store.createJob('terminal-persistence-barrier', 'u1');
await store.transitionStatus('terminal-persistence-barrier', {
from: 'running',
to: 'aborted',
expectCreatedAt: job.createdAt,
patch: {
completedAt,
terminalPersistencePending: true,
terminalPersistenceStartedAt: completedAt,
},
});
await store.cleanup();
await expect(store.getJob('terminal-persistence-barrier')).resolves.toMatchObject({
status: 'aborted',
terminalPersistencePending: true,
});
await expect(
store.finalizeTerminalPersistence('terminal-persistence-barrier', job.createdAt, 'final'),
).resolves.toBe(true);
await expect(store.getJob('terminal-persistence-barrier')).resolves.toMatchObject({
terminalPersistencePending: false,
finalEvent: 'final',
});
});
});
describe('ApprovalLifecycle ownership callbacks', () => {
test('notifies ownership changes only after successful lifecycle transitions', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const callbacks = {
onPaused: jest.fn(),
onResumed: jest.fn(),
onExpired: jest.fn(),
};
const lifecycle = new ApprovalLifecycle(store, callbacks);
const streamId = 'ownership-callbacks';
const job = await store.createJob(streamId, 'u1');
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
{ streamId },
);
expect(await lifecycle.pause(streamId, action)).toBe(true);
expect(await lifecycle.pause(streamId, action)).toBe(false);
expect(callbacks.onPaused).toHaveBeenCalledTimes(1);
expect(callbacks.onPaused).toHaveBeenCalledWith(streamId, job.createdAt);
expect(await lifecycle.resolve(streamId, action.actionId)).toBe(true);
expect(await lifecycle.resolve(streamId, action.actionId)).toBe(false);
expect(callbacks.onResumed).toHaveBeenCalledTimes(1);
expect(callbacks.onResumed).toHaveBeenCalledWith(streamId, job.createdAt);
const nextAction = { ...action, actionId: 'next-action' };
expect(await lifecycle.pause(streamId, nextAction)).toBe(true);
expect(await lifecycle.expire(streamId, nextAction.actionId)).toBe(true);
expect(await lifecycle.expire(streamId, nextAction.actionId)).toBe(false);
expect(callbacks.onExpired).toHaveBeenCalledTimes(1);
expect(callbacks.onExpired).toHaveBeenCalledWith(streamId, job.createdAt);
});
test('does not resume a replacement that appeared after the pending action was read', async () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const onResumed = jest.fn();
const lifecycle = new ApprovalLifecycle(store, { onResumed });
const streamId = 'resolve-replacement-guard';
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
try {
await store.createJob(streamId, 'u1');
const action = buildPendingAction(
buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
{ streamId },
);
await lifecycle.pause(streamId, action);
const observedJob = await store.getJob(streamId);
if (!observedJob) {
throw new Error('Expected paused job');
}
now.mockReturnValue(2000);
const replacement = await store.createJob(streamId, 'u1');
await store.transitionStatus(streamId, {
from: 'running',
to: 'requires_action',
patch: { pendingAction: action, pendingActionId: action.actionId },
});
jest.spyOn(store, 'getJob').mockResolvedValueOnce(observedJob);
expect(await lifecycle.resolve(streamId, action.actionId)).toBe(false);
expect(await store.getJob(streamId)).toMatchObject({
createdAt: replacement.createdAt,
status: 'requires_action',
});
expect(onResumed).not.toHaveBeenCalled();
} finally {
now.mockRestore();
}
});
});
describe('GenerationJobManager HITL resume metadata (round 19)', () => {
let manager: GenerationJobManagerClass;
beforeEach(() => {
manager = new GenerationJobManagerClass();
manager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
});
afterEach(async () => {
await manager.destroy();
});
function buildAction(streamId: string) {
const payload = buildToolApprovalPayload([
{ name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_abc' },
]);
return buildPendingAction(payload, {
streamId,
conversationId: streamId,
runId: 'run-1',
responseMessageId: 'msg-1',
});
}
// H1: round-18 captured discoveredTools but the metadata allowlists (updateMetadata,
// Redis deserialize, buildJobFacade) dropped it, so resume replayed `undefined`.
test('updateMetadata persists discoveredTools and the job facade exposes them', async () => {
const streamId = 'stream-discovered';
await manager.createJob(streamId, 'user-1');
await manager.updateMetadata(streamId, { discoveredTools: ['deep_tool', 'other_tool'] });
const job = await manager.getJob(streamId);
expect(job?.metadata.discoveredTools).toEqual(['deep_tool', 'other_tool']);
});
test('updateMetadata exposes a paused legacy-event fence through the job facade', async () => {
const streamId = 'stream-legacy-event';
await manager.createJob(streamId, 'user-1');
await manager.updateMetadata(streamId, {
agentEventLegacyTurnToken: 'legacy-hitl-token',
});
const job = await manager.getJob(streamId);
expect(job?.metadata.agentEventLegacyTurnToken).toBe('legacy-hitl-token');
});
// H4: a pause that lands AFTER the resume snapshot but before the subscription must
// still reach the client. subscribeWithResume re-reads the live job and surfaces it.
test('subscribeWithResume surfaces a pause that the resume snapshot missed', async () => {
const streamId = 'stream-race';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
// Simulate the snapshot being taken BEFORE the pause: drop pendingAction from the
// resume state even though the live job is now requires_action.
const realGetResumeState = manager.getResumeState.bind(manager);
jest.spyOn(manager, 'getResumeState').mockImplementation(async (id: string) => {
const state = await realGetResumeState(id);
return state ? { ...state, pendingAction: undefined } : state;
});
const result = await manager.subscribeWithResume(streamId, () => {});
const pending = result.pendingEvents.find(
(e) => 'event' in e && e.event === 'on_pending_action',
);
expect(pending).toBeDefined();
expect((pending as unknown as { data: { actionId: string } }).data.actionId).toBe(
action.actionId,
);
result.subscription?.unsubscribe();
});
// H4 negative: when the snapshot already carried the action, the re-read is skipped
// (the client gets it via resumeState.pendingAction) — no duplicate pending event.
test('does not re-surface the pending action when the snapshot already carried it', async () => {
const streamId = 'stream-norace';
await manager.createJob(streamId, 'user-1');
const action = buildAction(streamId);
await manager.approvals.pause(streamId, action);
const result = await manager.subscribeWithResume(streamId, () => {});
const pendingCount = result.pendingEvents.filter(
(e) => 'event' in e && e.event === 'on_pending_action',
).length;
expect(pendingCount).toBe(0);
expect(result.resumeState?.pendingAction?.actionId).toBe(action.actionId);
result.subscription?.unsubscribe();
});
});