Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import type { AgentEventActorDetachedAction } from '@librechat/data-schemas';
import {
createAgentEventActorDetachedActionLifecycle,
createAgentEventDetachedResumeHandler,
} from '~/agents/triggers/detachedAction';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { buildToolApprovalPayload } from '~/agents/hitl/__fixtures__/approval';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { buildPendingAction } from '~/agents/hitl/policy';
describe('GenerationJobManager terminal host actions', () => {
let manager: GenerationJobManagerClass;
let store: InMemoryJobStore;
beforeEach(() => {
manager = new GenerationJobManagerClass();
store = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
manager.configure({
jobStore: store,
eventTransport: new InMemoryEventTransport(),
cleanupOnComplete: false,
});
manager.initialize();
});
afterEach(async () => {
await manager.destroy();
});
it.each(['ordinary', 'detached'])(
'all deletion callers discover %s terminal host actions outside owner membership',
async (lane) => {
const job = await manager.createJob('pending-host', 'user-1', 'conversation');
await store.transitionStatus('pending-host', {
from: 'running',
to: 'complete',
expectCreatedAt: job.createdAt,
patch: { completedAt: Date.now(), providerDrained: true, terminalHostActionPending: true },
});
Object.assign(store, { getCleanupJobIdsByUser: undefined });
const terminal = (await store.getJob('pending-host'))!;
jest.spyOn(store, 'getRetainedJobIdsByUser').mockResolvedValue([]);
const rows = [
terminal,
{ ...terminal, streamId: 'foreign-user', userId: 'user-2' },
{ ...terminal, streamId: 'foreign-tenant', tenantId: 'tenant-b' },
];
jest
.spyOn(store, 'getTerminalHostActionJobs')
.mockResolvedValue(lane === 'ordinary' ? rows : []);
Object.assign(store, {
getDetachedAgentEventTerminalHostActionJobs: jest
.fn()
.mockResolvedValue(lane === 'detached' ? rows : []),
});
expect(await manager.getAccountCleanupJobIdsForUser('user-1', 'tenant-a')).toEqual([
'pending-host',
]);
expect(await manager.getCleanupBlockingJobIdsForUser('user-1', 'tenant-a')).toEqual([
'pending-host',
]);
expect(
await manager.getCleanupBlockingJobIdsForConversations(
'user-1',
['conversation'],
'tenant-a',
),
).toEqual(['pending-host']);
expect(
await manager.getCleanupBlockingJobIdsForConversations('user-1', ['unrelated'], 'tenant-a'),
).toEqual([]);
expect((await manager.getJob('pending-host'))?.metadata?.terminalHostActionPending).toBe(
true,
);
},
);
it('owner queries retain terminal obligations through active polling without global scans', async () => {
const job = await manager.createJob('owner-host', 'user-1', 'conversation');
await store.transitionStatus('owner-host', {
from: 'running',
to: 'complete',
expectCreatedAt: job.createdAt,
patch: { completedAt: Date.now(), providerDrained: true, terminalHostActionPending: true },
});
const global = jest.spyOn(store, 'getTerminalHostActionJobs');
expect(await manager.getActiveJobIdsForUser('user-1')).toEqual([]);
expect(await manager.getCleanupBlockingJobIdsForUser('user-1')).toEqual(['owner-host']);
expect(global).not.toHaveBeenCalled();
await store.clearTerminalHostAction('owner-host', job.createdAt);
expect(await manager.getCleanupBlockingJobIdsForUser('user-1')).toEqual([]);
});
it('durable cleanup preserves stale terminal-persistence recovery', async () => {
const job = await manager.createJob('stale-persistence', 'user-1', 'conversation');
await store.transitionStatus('stale-persistence', {
from: 'running',
to: 'complete',
expectCreatedAt: job.createdAt,
patch: { completedAt: 1, terminalPersistenceStartedAt: 1, terminalPersistencePending: true },
});
expect(await manager.getCleanupJob('stale-persistence')).toMatchObject({
createdAt: job.createdAt,
metadata: { terminalPersistencePending: false },
});
});
it('durable cleanup still sees a replacement when runtime attachment loses its epoch', async () => {
const first = await store.createJob('remote', 'user-1', 'conversation');
const read = store.getJob.bind(store);
const replacement = await store.createJob('remote', 'user-1', 'conversation');
jest.spyOn(store, 'getJob').mockResolvedValueOnce(first).mockImplementation(read);
expect(await manager.getJob('remote')).toBeUndefined();
expect(await manager.getCleanupJob('remote')).toMatchObject({
createdAt: replacement.createdAt,
status: 'running',
metadata: { userId: 'user-1' },
});
});
it('verifies detached terminal outbox persistence against the exact generation', async () => {
const streamId = 'conversation-detached-outbox';
const job = await manager.createJob(streamId, 'user-1', streamId);
const evidence = {
version: 1 as const,
deliveryKey: 'trigger-outbox',
generationCreatedAt: job.createdAt,
taskId: 'task-outbox',
idempotencyKey: 'a'.repeat(64),
status: 'succeeded' as const,
result: 'accepted',
observedAt: job.createdAt + 1,
};
await expect(
manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),
).resolves.toBe(true);
await expect(store.getJob(streamId)).resolves.toMatchObject({
agentEventDetachedTerminalEvidence: evidence,
});
const replacement = await store.createJob(streamId, 'user-1', streamId);
expect(replacement.createdAt).not.toBe(job.createdAt);
await expect(
manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),
).resolves.toBe(false);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty(
'agentEventDetachedTerminalEvidence',
);
});
it('runs detached launch, completion, and continuation through the in-memory adapter', async () => {
const streamId = 'conversation-memory-detached-action';
const invocationId = 'trigger-memory-detached-action';
const enqueueContinuation = jest.fn().mockResolvedValue(undefined);
let action: AgentEventActorDetachedAction = {
version: 1 as const,
invocationId,
expectedToolName: 'submit_move',
toolName: 'submit_move',
toolCallId: 'call-memory-detached-action',
turnId: 'response-memory:0',
taskId: `event_actor_${'a'.repeat(64)}`,
idempotencyKey: 'a'.repeat(64),
launchAttempt: 1,
status: 'reserved',
reservedAt: new Date(),
observedAt: new Date(),
recoveryAfter: new Date(Date.now() + 60_000),
};
const resumeDetachedAction = createAgentEventDetachedResumeHandler({
getAgentTriggerDelivery: jest.fn().mockResolvedValue({
user: 'user-1',
envelope: {
mode: 'continue',
principal: { userId: 'user-1' },
target: {
agentId: 'agent-1',
bindingId: 'binding-1',
conversationId: streamId,
parentMessageId: 'message-1',
sourceKeyId: 'source-key-1',
},
expectedAction: { toolName: 'submit_move' },
},
}),
enqueueAgentTrigger: enqueueContinuation,
requestId: () => 'memory-detached-continuation',
});
manager.setTerminalHostActionHandler(async (_streamId, job) => {
if (action.status !== 'succeeded') {
throw new Error('detached action is still running');
}
await resumeDetachedAction({
streamId,
job,
handlingGenerationCreatedAt: job.createdAt,
suspension: {} as never,
action,
});
});
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: {
agentEventDeliveryKey: invocationId,
agentEventBindingId: 'binding-1',
agentEventExpectedAction: { toolName: 'submit_move' },
},
});
const lifecycle = createAgentEventActorDetachedActionLifecycle(
{
user: 'user-1',
bindingId: 'binding-1',
conversationId: streamId,
generationCreatedAt: job.createdAt,
turnCreatedAt: job.createdAt,
invocationId,
expectedAction: { toolName: 'submit_move' },
},
{
storeMode: () => manager.detachedAgentEventActionStoreMode,
reserveAgentEventActorDetachedAction: jest.fn(async () => ({
status: 'reserved' as const,
action,
})),
markAgentEventActorDetachedActionRunning: jest.fn(async () => {
action = { ...action, status: 'running', observedAt: new Date() };
return { status: 'applied' as const, action };
}),
settleAgentEventActorDetachedAction: jest.fn(async (input) => {
action = {
...action,
status: 'succeeded',
result: input.result,
observedAt: input.observedAt,
};
return { status: 'applied' as const, action };
}),
persistTerminalEvidence: async (evidence) => {
const persisted = await manager.persistAgentEventDetachedTerminalEvidence(
streamId,
job.createdAt,
evidence,
);
if (!persisted) {
throw new Error('failed to stage detached terminal evidence');
}
},
onTerminal: async () => {
await manager.retryTerminalHostAction(streamId, job.createdAt);
},
},
);
expect(manager.detachedAgentEventActionStoreMode).toBe('process_local');
await expect(
lifecycle.reserve({
toolName: 'submit_move',
toolCallId: action.toolCallId,
turnId: action.turnId,
arguments: {},
}),
).resolves.toMatchObject({ status: 'reserved', taskId: action.taskId });
await expect(
lifecycle.markRunning({ taskId: action.taskId, idempotencyKey: action.idempotencyKey }),
).resolves.toBe(true);
expect(lifecycle.readSuspension()).toMatchObject({
kind: 'internal_completion',
actionId: action.taskId,
});
await expect(manager.completeJob(streamId, undefined, job.createdAt)).resolves.toBe(true);
expect(enqueueContinuation).not.toHaveBeenCalled();
await expect(store.getJob(streamId)).resolves.toMatchObject({
terminalHostActionPending: true,
});
await expect(
lifecycle.settle({
taskId: action.taskId,
idempotencyKey: action.idempotencyKey,
status: 'succeeded',
result: 'move accepted',
}),
).resolves.toBe(true);
await lifecycle.wake({ taskId: action.taskId, idempotencyKey: action.idempotencyKey });
expect(enqueueContinuation).toHaveBeenCalledWith(
expect.objectContaining({
mode: 'continue',
deliveryId: `detached_completion:${action.taskId}`,
event: expect.objectContaining({
type: 'librechat.event_actor.detached_completion',
payload: expect.objectContaining({ invocationId, taskId: action.taskId }),
}),
}),
{ requiredWorkerCapability: 'event_actor_detached_action_v1' },
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('runs and acknowledges a generation-fenced host action before terminal cleanup', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const job = await manager.createJob('conversation-1', 'user-1', 'conversation-1', {
initialMetadata: {
agentEventDeliveryKey: 'trigger_1',
agentEventExpectedAction: { toolName: 'submit_move' },
},
});
await manager.emitChunk('conversation-1', {
event: 'on_run_step',
data: {
id: 'step-1',
index: 0,
type: 'tool_calls',
status: 'in_progress',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-1', name: 'submit_move', args: { gameId: 'game-1' } }],
},
},
});
await manager.emitChunk('conversation-1', {
event: 'on_run_step_completed',
data: {
result: {
id: 'step-1',
index: 0,
type: 'tool_call',
tool_call: { id: 'call-1', name: 'submit_move', output: 'ok' },
},
},
});
await expect(manager.completeJob('conversation-1', undefined, job.createdAt)).resolves.toBe(
true,
);
expect(handler).toHaveBeenCalledWith(
'conversation-1',
expect.objectContaining({
createdAt: job.createdAt,
agentEventDeliveryKey: 'trigger_1',
status: 'complete',
}),
[
expect.objectContaining({
id: 'step-1',
status: 'completed',
stepDetails: expect.objectContaining({
tool_calls: [
expect.objectContaining({
id: 'call-1',
args: { gameId: 'game-1' },
output: 'ok',
}),
],
}),
}),
],
expect.any(Array),
);
await expect(store.getJob('conversation-1')).resolves.not.toHaveProperty(
'terminalHostActionPending',
);
});
it('retries an exact pending terminal host action when late evidence arrives', async () => {
const handler = jest
.fn()
.mockRejectedValueOnce(new Error('detached action is still running'))
.mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-late-terminal-evidence';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger-late-evidence' },
});
await expect(manager.completeJob(streamId, undefined, job.createdAt)).resolves.toBe(true);
await expect(store.getJob(streamId)).resolves.toMatchObject({
terminalHostActionPending: true,
});
await expect(manager.retryTerminalHostAction(streamId, job.createdAt + 1)).resolves.toBe(false);
expect(handler).toHaveBeenCalledTimes(1);
await expect(manager.retryTerminalHostAction(streamId, job.createdAt)).resolves.toBe(true);
expect(handler).toHaveBeenCalledTimes(2);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('waits for the provider evidence fence and retains a failed tool close', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-provider-evidence';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: {
agentEventDeliveryKey: 'trigger_provider_evidence',
},
});
const providerExecutionId = job.metadata.providerExecutionId!;
await expect(
manager.beginProviderExecution(streamId, job.createdAt, providerExecutionId),
).resolves.toBe(true);
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-failed',
index: 0,
type: 'tool_calls',
status: 'in_progress',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-failed', name: 'submit_move', args: {} }],
},
},
});
await manager.emitChunk(streamId, {
event: 'on_run_step_completed',
data: {
result: {
id: 'step-failed',
index: 0,
type: 'tool_call',
tool_call: {
id: 'call-failed',
name: 'submit_move',
output: 'Error: [submit_move] tool call failed: unavailable',
},
},
},
});
await manager.emitChunk(streamId, {
event: 'on_run_step_closed',
data: {
id: 'step-failed',
index: 0,
type: 'tool_calls',
status: 'failed',
closed_at: Date.now(),
},
});
await expect(manager.completeJob(streamId, undefined, job.createdAt)).resolves.toBe(true);
expect(handler).not.toHaveBeenCalled();
await expect(store.getJob(streamId)).resolves.toMatchObject({
providerDrained: false,
terminalHostActionPending: true,
});
await expect(
manager.markProviderExecutionDrained(streamId, job.createdAt, providerExecutionId),
).resolves.toBe(true);
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({ providerDrained: true }),
[
expect.objectContaining({
id: 'step-failed',
status: 'failed',
stepDetails: expect.objectContaining({
tool_calls: [expect.objectContaining({ executionStatus: 'error' })],
}),
}),
],
expect.any(Array),
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('routes a post-drain approval expiry through the schedule-specific hook', async () => {
const approvalHandler = jest.fn().mockResolvedValue(undefined);
const genericHandler = jest.fn().mockResolvedValue(undefined);
manager.setApprovalExpiredHandler(approvalHandler);
manager.setTerminalHostActionHandler(genericHandler);
const streamId = 'conversation-expired-approval-drain';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { scheduleId: 'schedule-1' },
});
const providerExecutionId = job.metadata.providerExecutionId!;
await manager.beginProviderExecution(streamId, job.createdAt, providerExecutionId);
await expect(
store.transitionStatus(streamId, {
from: 'running',
to: 'aborted',
expectCreatedAt: job.createdAt,
patch: {
completedAt: Date.now(),
error: 'Approval expired before a decision was made',
terminalHostActionPending: true,
},
}),
).resolves.toBe(true);
await manager.markProviderExecutionDrained(streamId, job.createdAt, providerExecutionId);
expect(approvalHandler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({ scheduleId: 'schedule-1', providerDrained: true }),
);
expect(genericHandler).not.toHaveBeenCalled();
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('recovers a terminal host action after its provider owner is lost', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(10_000);
try {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-lost-provider-owner';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger_lost_provider' },
});
await manager.beginProviderExecution(
streamId,
job.createdAt,
job.metadata.providerExecutionId!,
);
await manager.completeJob(streamId, undefined, job.createdAt);
expect(handler).not.toHaveBeenCalled();
now.mockReturnValue(40_001);
await (
manager as unknown as { expireStaleApprovals: () => Promise<void> }
).expireStaleApprovals();
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({ providerDrained: true }),
expect.any(Array),
expect.any(Array),
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
} finally {
now.mockRestore();
}
});
it('serializes run-step snapshots so an earlier write cannot erase completion evidence', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-run-step-order';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger_run_step_order' },
});
let releaseFirstWrite: (() => void) | undefined;
const firstWrite = new Promise<void>((resolve) => {
releaseFirstWrite = resolve;
});
const writes: Array<Array<{ status?: string }>> = [];
Object.assign(store, { saveRunSteps: jest.fn() });
jest
.spyOn(store as InMemoryJobStore & { saveRunSteps: jest.Mock }, 'saveRunSteps')
.mockImplementation(async (_id, steps) => {
writes.push(structuredClone(steps));
if (writes.length === 1) {
await firstWrite;
}
});
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-ordered',
index: 0,
type: 'tool_calls',
status: 'in_progress',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-ordered', name: 'submit_move', args: {} }],
},
},
});
await manager.emitChunk(streamId, {
event: 'on_run_step_completed',
data: {
result: {
id: 'step-ordered',
index: 0,
type: 'tool_call',
tool_call: { id: 'call-ordered', name: 'submit_move', output: 'ok' },
},
},
});
const completing = manager.completeJob(streamId, undefined, job.createdAt);
await new Promise<void>((resolve) => setImmediate(resolve));
expect(handler).not.toHaveBeenCalled();
releaseFirstWrite?.();
await expect(completing).resolves.toBe(true);
expect(writes[writes.length - 1]).toEqual([expect.objectContaining({ status: 'completed' })]);
expect(handler).toHaveBeenCalledTimes(1);
});
it('retains a failed host action and lets a later cleanup owner retry it', async () => {
manager.setTerminalHostActionHandler(jest.fn().mockRejectedValue(new Error('mongo down')));
const job = await manager.createJob('conversation-2', 'user-1', 'conversation-2', {
initialMetadata: {
agentEventDeliveryKey: 'trigger_2',
agentEventExpectedAction: { toolName: 'submit_move' },
},
});
await manager.emitChunk('conversation-2', {
event: 'on_run_step_completed',
data: {
result: {
id: 'step-2',
index: 0,
type: 'tool_calls',
status: 'completed',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-2', name: 'submit_move', output: 'ok' }],
},
},
},
});
await manager.completeJob('conversation-2', undefined, job.createdAt);
await expect(store.getJob('conversation-2')).resolves.toMatchObject({
terminalHostActionPending: true,
});
const recovered = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(recovered);
await (
manager as unknown as {
cleanup: () => Promise<void>;
}
).cleanup();
expect(recovered).toHaveBeenCalledWith(
'conversation-2',
expect.objectContaining({ createdAt: job.createdAt, agentEventDeliveryKey: 'trigger_2' }),
[expect.objectContaining({ id: 'step-2', status: 'completed' })],
expect.any(Array),
);
await expect(store.getJob('conversation-2')).resolves.not.toHaveProperty(
'terminalHostActionPending',
);
});
it('recovers detached completion jobs from the capability-isolated terminal lane', async () => {
const streamId = 'conversation-detached-terminal-lane';
const invocationKey = 'trigger-original-invocation';
const completionDeliveryKey = 'trigger-internal-completion';
const invocationGenerationCreatedAt = 42;
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: {
agentEventDeliveryKey: completionDeliveryKey,
agentEventInvocationKey: invocationKey,
agentEventInvocationGenerationCreatedAt: invocationGenerationCreatedAt,
},
});
await store.transitionStatus(streamId, {
from: 'running',
to: 'complete',
expectCreatedAt: job.createdAt,
patch: { completedAt: Date.now(), terminalHostActionPending: true },
});
const terminalJob = (await store.getJob(streamId))!;
jest.spyOn(store, 'getTerminalHostActionJobs').mockResolvedValue([]);
Object.assign(store, {
getDetachedAgentEventTerminalHostActionJobs: jest.fn().mockResolvedValue([terminalJob]),
});
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
await (manager as unknown as { cleanup: () => Promise<void> }).cleanup();
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
agentEventDeliveryKey: completionDeliveryKey,
agentEventInvocationKey: invocationKey,
agentEventInvocationGenerationCreatedAt: invocationGenerationCreatedAt,
}),
expect.any(Array),
expect.any(Array),
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('retains completed run-step evidence through repeated host-action failures', async () => {
const handler = jest
.fn()
.mockRejectedValueOnce(new Error('mongo down'))
.mockRejectedValueOnce(new Error('mongo still down'))
.mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const job = await manager.createJob('conversation-retry', 'user-1', 'conversation-retry', {
initialMetadata: { agentEventDeliveryKey: 'trigger_retry' },
});
await manager.emitChunk('conversation-retry', {
event: 'on_run_step_completed',
data: {
result: {
id: 'step-retry',
index: 0,
type: 'tool_calls',
status: 'completed',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-retry', name: 'submit_move', output: 'ok' }],
},
},
},
});
await manager.completeJob('conversation-retry', undefined, job.createdAt);
const cleanup = () =>
(
manager as unknown as {
cleanup: () => Promise<void>;
}
).cleanup();
await cleanup();
await expect(store.getJob('conversation-retry')).resolves.toMatchObject({
terminalHostActionPending: true,
});
await cleanup();
expect(handler).toHaveBeenLastCalledWith(
'conversation-retry',
expect.objectContaining({ agentEventDeliveryKey: 'trigger_retry' }),
[expect.objectContaining({ id: 'step-retry', status: 'completed' })],
expect.any(Array),
);
await expect(store.getJob('conversation-retry')).resolves.not.toHaveProperty(
'terminalHostActionPending',
);
});
it('does not let a stale buffered step replace completed durable evidence', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-completed-evidence';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger_completed' },
});
const completedStep = {
id: 'step-shared',
index: 0,
type: 'tool_calls',
status: 'completed',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-shared', name: 'submit_move', output: 'ok' }],
},
};
store.setGraph(streamId, { contentData: [completedStep] } as never, job.createdAt);
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: { ...completedStep, status: 'in_progress' },
});
await manager.completeJob(streamId, undefined, job.createdAt);
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({ agentEventDeliveryKey: 'trigger_completed' }),
[expect.objectContaining({ id: 'step-shared', status: 'completed' })],
expect.any(Array),
);
});
it('marks an aborted event generation for terminal handling', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const job = await manager.createJob('conversation-3', 'user-1', 'conversation-3', {
initialMetadata: { agentEventDeliveryKey: 'trigger_3' },
});
await expect(
manager.abortJob('conversation-3', { expectedCreatedAt: job.createdAt }),
).resolves.toMatchObject({ success: true });
expect(handler).toHaveBeenCalledWith(
'conversation-3',
expect.objectContaining({
createdAt: job.createdAt,
agentEventDeliveryKey: 'trigger_3',
status: 'aborted',
}),
[],
expect.any(Array),
);
await expect(store.getJob('conversation-3')).resolves.not.toHaveProperty(
'terminalHostActionPending',
);
});
it('settles a bound generation when graceful shutdown terminalizes it', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-shutdown';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger_shutdown' },
});
await (
manager as unknown as { finalizeOwnedJobsForShutdown: () => Promise<void> }
).finalizeOwnedJobsForShutdown();
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
createdAt: job.createdAt,
agentEventDeliveryKey: 'trigger_shutdown',
status: 'error',
}),
[],
expect.any(Array),
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
});
it('settles a bound generation when pause persistence times out', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
const streamId = 'conversation-pause-timeout';
const job = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { agentEventDeliveryKey: 'trigger_pause_timeout' },
});
const action = buildPendingAction(
buildToolApprovalPayload([
{ name: 'submit_move', arguments: { gameId: 'game-1' }, tool_call_id: 'call-1' },
]),
{ streamId, conversationId: streamId },
);
await expect(
manager.approvals.pause(streamId, action, {
expectedCreatedAt: job.createdAt,
persistencePending: true,
}),
).resolves.toBe(true);
now.mockReturnValue(31_001);
await (manager as unknown as { cleanup: () => Promise<void> }).cleanup();
expect(handler).toHaveBeenCalledWith(
streamId,
expect.objectContaining({
agentEventDeliveryKey: 'trigger_pause_timeout',
status: 'error',
}),
[],
expect.any(Array),
);
await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
now.mockRestore();
});
});