Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import {
createMethods,
createModels,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
} from '@librechat/data-schemas';
import type { IMessage } from '@librechat/data-schemas';
import type { AgentTriggerFetch } from './triggers/host';
import {
createBackgroundToolCompletionWakeupResolver,
createBackgroundToolDeadClaimRecovery,
BACKGROUND_TOOL_COMPLETION_SOURCE,
} from './backgroundCompletionWakeup';
import {
backgroundTaskRegistry,
runCheckBackgroundTask,
getBackgroundCodeDelivery,
} from './background';
import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { createAgentTriggerExecutionHost } from './triggers/host';
import { prepareAgentTriggerDelivery } from './triggers/delivery';
import { claimBackgroundToolResult } from './backgroundClaims';
let mongo: MongoMemoryServer;
let methods: ReturnType<typeof createMethods>;
let userId: string;
const conversationId = 'receipt-replay';
const parentMessageId = 'receipt-parent';
const agentId = 'agent_parent_1';
const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3;
beforeAll(async () => {
mongo = await MongoMemoryServer.create();
await mongoose.connect(mongo.getUri());
createModels(mongoose);
methods = createMethods(mongoose);
}, 60_000);
afterAll(async () => {
await mongoose.disconnect();
await mongo.stop();
});
beforeEach(async () => {
await Promise.all(
['AgentTriggerDelivery', 'AgentTriggerLaneSequence', 'Conversation', 'Message'].map((name) =>
mongoose.models[name].deleteMany({}),
),
);
userId = new mongoose.Types.ObjectId().toString();
await mongoose.models.Conversation.create({
conversationId,
user: userId,
tenantId: 'tenant-1',
endpoint: 'agents',
agent_id: agentId,
});
await mongoose.models.Message.create({
messageId: parentMessageId,
conversationId,
user: userId,
tenantId: 'tenant-1',
parentMessageId: 'user-parent',
isCreatedByUser: false,
unfinished: false,
endpoint: 'agents',
content: [],
});
});
async function ready(taskId: string, persistReceipt = true) {
const envelope = createAgentTriggerEnvelope({
mode: 'continue',
requestId: taskId,
deliveryId: taskId,
receivedAt: Date.now(),
principal: { id: userId, tenantId: 'tenant-1' },
event: {
id: taskId,
type: 'background-tool.completion',
occurredAt: Date.now(),
source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },
payload: { taskId, toolCallId: taskId, toolName: 'tool' },
},
target: { conversationId, parentMessageId, agentId },
input: 'waiting',
});
const row = prepareAgentTriggerDelivery(envelope, {
orderingKey: taskId,
requiredWorkerCapability: capability,
});
await methods.enqueueAgentTriggerDelivery(row);
if (persistReceipt)
await methods.persistAgentBackgroundToolResult({
deliveryKey: row.deliveryKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
result: { status: 'completed', output: taskId, settledAt: new Date() },
});
return envelope;
}
function owner(deliveryKey: string) {
return {
deliveryKey,
userId,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
tenantId: 'tenant-1',
conversationId,
parentMessageId,
agentId,
};
}
it('replays exactly the admitted input after a lost response and a definitely rejected retry', async () => {
const root = await ready('one');
const sibling = await ready('two');
const inputs: string[] = [];
const consumed: string[] = [];
let firstInput: string | undefined;
let calls = 0;
const fetcher: AgentTriggerFetch = async (_url, init) => {
const body = JSON.parse(String(init?.body)) as { text: string };
inputs.push(body.text);
if (++calls === 1) {
firstInput = body.text;
consumed.push('one', 'two');
throw Object.assign(new Error('lost admitted response'), { code: 'ECONNRESET' });
}
if (calls === 2)
throw Object.assign(new Error('retry did not connect'), { code: 'ECONNREFUSED' });
if (body.text !== firstInput) consumed.push('three');
return new Response(
JSON.stringify({
status: 'started',
streamId: conversationId,
conversationId,
generationCreatedAt: 100,
}),
{ status: 200 },
);
};
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
const host = createAgentTriggerExecutionHost({
prepareContinue: resolve,
fetch: fetcher,
mintToken: () => 'test',
getBaseUrl: () => 'http://localhost',
});
const options = { requiredWorkerCapability: capability };
await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'ambiguous' });
const late = await ready('three');
await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'definite' });
await expect(host.dispatch(root, options)).resolves.toMatchObject({ status: 'started' });
expect(inputs).toEqual([firstInput, firstInput, firstInput]);
expect(consumed).toEqual(['one', 'two']);
await expect(host.dispatch(sibling, options)).resolves.toMatchObject({ status: 'settled' });
const lateEnvelope = late.mode === 'continue' ? late : undefined;
if (lateEnvelope == null) throw new Error('Expected continuation');
const prepared = await resolve(lateEnvelope, {
idempotencyKey: getAgentTriggerIdempotencyKey(late),
...options,
});
expect(prepared?.status === 'ready' && prepared.input).toContain('three');
});
it('cleans a late predecessor projection without clearing its successor receipt batch', async () => {
const root = await ready('one');
const deliveryKey = getAgentTriggerIdempotencyKey(root);
const scope = owner(deliveryKey);
const old = await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
if (old.status !== 'acquired') throw new Error('Expected batch');
expect(
await methods.releaseAgentBackgroundToolResultClaims({
...scope,
claimId: deliveryKey,
batchId: old.batchId,
}),
).toBe(true);
await mongoose.models.Message.updateOne(
{ messageId: parentMessageId, user: userId },
{
$set: {
content: [
{
type: 'tool_call',
tool_call: {
id: 'one',
output: 'one',
backgroundTask: {
taskId: 'one',
toolName: 'tool',
status: 'completed',
completionWakeup: true,
completionReceipt: true,
},
},
},
],
},
},
);
// A predecessor's delayed projection CAS lands after its cleanup completed.
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'wakeup',
claimId: deliveryKey,
batchId: old.batchId,
limit: 1,
});
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
await expect(
resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),
).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING' });
const successor = await methods.getAgentBackgroundToolResultBatch(scope);
expect(successor?.batchId).not.toBe(old.batchId);
await expect(
resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),
).resolves.toMatchObject({ status: 'ready' });
expect((await methods.getAgentBackgroundToolResultBatch(scope))?.batchId).toBe(
successor?.batchId,
);
});
it('confirms a dead owner with native admission proof instead of re-presenting its results', async () => {
const root = await ready('one');
const sibling = await ready('two');
const deliveryKey = getAgentTriggerIdempotencyKey(root);
const scope = owner(deliveryKey);
await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey },
{ $set: { capabilityStatus: 'dead' } },
);
const recover = createBackgroundToolDeadClaimRecovery(
async (key, sourceId, reason, options) =>
methods.retireAgentTriggerDelivery({
deliveryKey: key,
sourceId,
reason,
settledAt: new Date(),
...options,
}),
methods.releaseBackgroundToolResultClaims,
async () => null,
async () => 'started',
methods.releaseAgentBackgroundToolResultClaims,
methods,
async () => ({ generationId: conversationId, generationCreatedAt: 100 }),
);
expect(
await recover({ userId, conversationId, messageId: parentMessageId, claimId: deliveryKey }),
).toBe(false);
const follower = await methods.claimAgentBackgroundToolResultBatch({
...owner(getAgentTriggerIdempotencyKey(sibling)),
limit: 8,
maxMetadataChars: 16000,
});
expect(follower).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
});
async function project(taskId: string) {
await mongoose.models.Message.updateOne(
{ messageId: parentMessageId, user: userId },
{
$push: {
content: {
type: 'tool_call',
tool_call: {
id: taskId,
output: `durable-${taskId}`,
backgroundTask: {
taskId,
toolName: 'tool',
status: 'completed',
settledAt: new Date(),
completionWakeup: true,
completionReceipt: true,
},
},
},
},
},
);
}
it('reconstructs a failed receipt write from the successful terminal message projection', async () => {
const root = await ready('one', false);
const deliveryKey = getAgentTriggerIdempotencyKey(root);
await project('one');
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey },
{ $set: { producerLeaseUntil: new Date(0) } },
);
const fetcher: AgentTriggerFetch = async (_url, init) => {
expect(JSON.parse(String(init?.body)).text).toContain('durable-one');
return new Response(
JSON.stringify({
status: 'started',
conversationId,
streamId: conversationId,
generationCreatedAt: 100,
}),
{ status: 200 },
);
};
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
const host = createAgentTriggerExecutionHost({
prepareContinue: resolve,
fetch: fetcher,
mintToken: () => 'test',
getBaseUrl: () => 'http://localhost',
});
await expect(
host.dispatch(root, { requiredWorkerCapability: capability }),
).resolves.toMatchObject({ status: 'started' });
expect(
await methods.getAgentBackgroundToolResult({
deliveryKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
}),
).toMatchObject({ output: 'durable-one', resultClaim: { appliedAt: expect.any(Date) } });
});
it('does not settle a root when a speculative manual poll and automatic owner mutually yield', async () => {
const root = await ready('one');
await ready('two');
await project('one');
const key = getAgentTriggerIdempotencyKey(root);
let entered: () => void = () => undefined;
let captured: () => void = () => undefined;
let resume: () => void = () => undefined;
const automaticEntered = new Promise<void>((resolve) => {
entered = resolve;
});
const snapshotTaken = new Promise<void>((resolve) => {
captured = resolve;
});
const manualMayYield = new Promise<void>((resolve) => {
resume = resolve;
});
let lookups = 0;
const getter = async (input: Parameters<typeof methods.getAgentBackgroundToolResultClaim>[0]) => {
if (++lookups === 1) return methods.getAgentBackgroundToolResultClaim(input);
await automaticEntered;
const snapshot = await methods.getAgentBackgroundToolResultClaim(input);
captured();
await manualMayYield;
return snapshot;
};
const claim = methods.claimBackgroundToolResults.bind(methods);
const automatic = jest
.spyOn(methods, 'claimBackgroundToolResults')
.mockImplementation(async (input) => {
if (input.kind === 'wakeup' && input.taskId === 'one') {
entered();
await snapshotTaken;
}
return claim(input);
});
const manual = claimBackgroundToolResult(methods, getter, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
});
// Allow the initial manual projection CAS to commit before receipt selection.
const deadline = Date.now() + 3000;
while (
(await mongoose.models.Message.findOne({
messageId: parentMessageId,
'content.tool_call.backgroundTask.resultClaim.claimId': 'manual-poll',
}).lean()) == null
) {
if (Date.now() > deadline) throw new Error('Manual claim did not arrive');
await new Promise((resolve) => setTimeout(resolve, 5));
}
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
await expect(
resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability }),
).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', deferWithoutAttempt: true });
resume();
expect(await manual).toMatchObject({ status: 'claimed' });
automatic.mockRestore();
const retry = await resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability });
expect(retry?.status === 'ready' && retry.input).toContain('one');
expect(retry?.status === 'ready' && retry.input).toContain('two');
});
it('settles a root only after a manual poll has completed receipt reconciliation', async () => {
const root = await ready('one');
await project('one');
expect(
(
await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
})
).status,
).toBe('acquired');
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
await expect(
resolve(root, {
idempotencyKey: getAgentTriggerIdempotencyKey(root),
requiredWorkerCapability: capability,
}),
).resolves.toEqual({ status: 'settled' });
});
it.each([false, true])(
'rolls back an unconfirmed manual claim with generation identity %s',
async (withGeneration) => {
const root = await ready('one');
await project('one');
const confirmation = jest
.spyOn(mongoose.models.Message, 'updateOne')
.mockImplementationOnce(() => {
throw new Error('confirmation unavailable');
});
await expect(
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
...(withGeneration && { generationId: 'manual-generation' }),
}),
).rejects.toThrow('confirmation unavailable');
confirmation.mockRestore();
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
const prepared = await resolve(root, {
idempotencyKey: getAgentTriggerIdempotencyKey(root),
requiredWorkerCapability: capability,
});
expect(prepared?.status === 'ready' && prepared.input).toContain('"result":"one"');
},
);
it('resolves a committed manual confirmation whose write reply was lost', async () => {
const root = await ready('one');
await project('one');
const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);
const lostReply = jest
.spyOn(mongoose.models.Message, 'updateOne')
.mockImplementationOnce((...args) => {
const query = update(...args);
const execute = query.exec.bind(query);
jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
await execute();
throw new Error('lost confirmation reply');
});
return query;
});
const consumed = await claimBackgroundToolResult(
methods,
methods.getAgentBackgroundToolResultClaim,
{
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
},
);
expect(consumed).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });
lostReply.mockRestore();
expect(
await methods.releaseBackgroundToolResultClaims({
userId,
conversationId,
messageId: parentMessageId,
taskIds: ['one'],
kind: 'manual',
claimId: 'manual-poll',
onlyIfUnreconciled: true,
}),
).toBe(false);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
await expect(
resolve(root, {
idempotencyKey: getAgentTriggerIdempotencyKey(root),
requiredWorkerCapability: capability,
}),
).resolves.toEqual({ status: 'settled' });
});
it('recovers abandoned manual ownership after both confirmation and rollback failed', async () => {
const root = await ready('one');
await ready('two');
await project('one');
const confirmation = jest
.spyOn(methods, 'confirmBackgroundToolResultClaim')
.mockRejectedValueOnce(new Error('confirmation unavailable'));
const rollback = jest
.spyOn(methods, 'releaseBackgroundToolResultClaims')
.mockRejectedValueOnce(new Error('rollback unavailable'));
await expect(
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
}),
).rejects.toThrow('rollback unavailable');
confirmation.mockRestore();
rollback.mockRestore();
let active = true;
const getGenerationJob = async () =>
active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null;
const recover = createBackgroundToolDeadClaimRecovery(
async () => false,
methods.releaseBackgroundToolResultClaims,
getGenerationJob,
async () => 'unavailable',
);
const recoverDeadClaim = jest.fn(recover);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob,
recoverDeadClaim,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
const key = getAgentTriggerIdempotencyKey(root);
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey: key },
{ $set: { capabilityClaimToken: 'exact-queue-lease' } },
);
const context = {
idempotencyKey: key,
requiredWorkerCapability: capability,
deliveryClaimToken: 'exact-queue-lease',
};
await expect(resolve(root, context)).rejects.toMatchObject({ code: 'PARENT_NOT_READY' });
expect(recoverDeadClaim).not.toHaveBeenCalled();
active = false;
await expect(resolve(root, context)).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
deferWithoutAttempt: true,
});
expect(recoverDeadClaim).toHaveBeenCalledWith(
expect.objectContaining({
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
}),
);
const prepared = await resolve(root, context);
expect(prepared?.status === 'ready' && prepared.input).toContain('one');
expect(prepared?.status === 'ready' && prepared.input).toContain('two');
});
it('does not use a different manual owner as confirmation read-back proof', async () => {
await ready('one');
await project('one');
const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);
const replaced = jest
.spyOn(mongoose.models.Message, 'updateOne')
.mockImplementationOnce((...args) => {
const query = update(...args);
jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
await update(
{ user: userId, messageId: parentMessageId },
{
$set: {
'content.0.tool_call.backgroundTask.resultClaim': {
kind: 'manual',
claimId: 'successor-poll',
receiptReconciled: true,
},
},
},
);
throw new Error('predecessor confirmation failed');
});
return query;
});
await expect(
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
}),
).rejects.toThrow('predecessor confirmation failed');
replaced.mockRestore();
expect(
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'wakeup',
claimId: 'automatic',
}),
).toMatchObject({
status: 'claimed',
claim: { claimId: 'successor-poll', receiptReconciled: true },
});
});
it.each(['missing-receipt', 'batch-root', 'batch-sibling'] as const)(
'preserves a manual handoff committed after the %s recovery snapshot',
async (scenario) => {
const polled = await ready('one', scenario !== 'missing-receipt');
const other = scenario === 'batch-sibling' ? await ready('two') : undefined;
const root = other ?? polled;
await project('one');
let confirmationReached: () => void = () => undefined;
let confirmNow: () => void = () => undefined;
let recoveryReached: () => void = () => undefined;
let recoverNow: () => void = () => undefined;
const confirmationEntered = new Promise<void>((resolve) => {
confirmationReached = resolve;
});
const confirmationBarrier = new Promise<void>((resolve) => {
confirmNow = resolve;
});
const recoveryEntered = new Promise<void>((resolve) => {
recoveryReached = resolve;
});
const recoveryBarrier = new Promise<void>((resolve) => {
recoverNow = resolve;
});
const confirm = methods.confirmBackgroundToolResultClaim.bind(methods);
const pausedConfirmation = jest
.spyOn(methods, 'confirmBackgroundToolResultClaim')
.mockImplementationOnce(async (input) => {
confirmationReached();
await confirmationBarrier;
return confirm(input);
});
const manual = claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
});
await confirmationEntered;
let manualActive = true;
const release = jest.fn(methods.releaseBackgroundToolResultClaims);
const recover = createBackgroundToolDeadClaimRecovery(
async () => false,
release,
async () =>
manualActive
? { status: 'running', metadata: { responseMessageId: 'manual-generation' } }
: null,
async () => 'unavailable',
);
const recoverDeadClaim = jest.fn(async (input: Parameters<typeof recover>[0]) => {
recoveryReached();
await recoveryBarrier;
return recover(input);
});
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
recoverDeadClaim,
});
if (root.mode !== 'continue') throw new Error('Expected continuation');
const key = getAgentTriggerIdempotencyKey(root);
await mongoose.models.AgentTriggerDelivery.updateMany(
{ deliveryKey: { $in: [key, getAgentTriggerIdempotencyKey(polled)] } },
{
$set: {
capabilityStatus: 'leased',
capabilityClaimToken: 'exact-queue-lease',
capabilityLeaseBy: 'automatic-worker',
capabilityLeaseUntil: new Date(Date.now() + 60_000),
},
},
);
const context = {
idempotencyKey: key,
requiredWorkerCapability: capability,
deliveryClaimToken: 'exact-queue-lease',
};
const automatic = resolve(root, context);
await recoveryEntered;
confirmNow();
expect(await manual).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });
// The automatic queue lease won manual retirement. The poll returned its
// result and its generation ended while recovery held a stale snapshot.
expect(
await methods.retireAgentTriggerDelivery({
deliveryKey: getAgentTriggerIdempotencyKey(polled),
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
settledAt: new Date(),
reason: 'manual poll',
onlyIfUnclaimed: true,
requireTransition: true,
}),
).toBe(false);
manualActive = false;
recoverNow();
await expect(automatic).rejects.toMatchObject({
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
deferWithoutAttempt: true,
});
expect(recoverDeadClaim).toHaveBeenCalledWith(
expect.objectContaining({ onlyIfUnreconciled: true }),
);
expect(release).toHaveBeenCalledWith(
expect.objectContaining({ kind: 'manual', claimId: 'manual-poll', onlyIfUnreconciled: true }),
);
const claim = await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'wakeup',
claimId: 'probe',
});
expect(claim).toMatchObject({
status: 'claimed',
claim: { claimId: 'manual-poll', receiptReconciled: true },
});
const pollEnvelope = polled.mode === 'continue' ? polled : undefined;
if (pollEnvelope == null) throw new Error('Expected polled continuation');
await expect(
resolve(pollEnvelope, {
idempotencyKey: getAgentTriggerIdempotencyKey(polled),
requiredWorkerCapability: capability,
}),
).resolves.toEqual({ status: 'settled' });
if (other != null) {
const row = await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(polled));
if (row == null) throw new Error('Expected polled delivery');
expect(
await methods.completeAgentTriggerDelivery({
id: row.id,
workerId: 'automatic-worker',
claimToken: 'exact-queue-lease',
attempt: 1,
settledAt: new Date(),
result: { status: 'settled', mode: 'continue', conversationId },
}),
).toBe(true);
const remaining = await resolve(root, context);
if (remaining?.status !== 'ready') throw new Error('Expected undelivered sibling');
expect(remaining.input).toContain('"background_task_id":"two"');
expect(remaining.input).not.toContain('"background_task_id":"one"');
await remaining.beginDispatch?.();
}
pausedConfirmation.mockRestore();
},
);
it('keeps explicit manual recovery able to reopen a committed claim after its generation ends', async () => {
await ready('one');
await project('one');
await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'manual-poll',
generationId: 'manual-generation',
});
let active = true;
const recover = createBackgroundToolDeadClaimRecovery(
async () => false,
methods.releaseBackgroundToolResultClaims,
async () =>
active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null,
async () => 'unavailable',
);
const input = {
userId,
conversationId,
messageId: parentMessageId,
claimId: 'manual-poll',
kind: 'manual' as const,
generationId: 'manual-generation',
};
expect(await recover(input)).toBe(false);
active = false;
expect(await recover({ ...input, onlyIfUnreconciled: true })).toBe(false);
expect(await recover(input)).toBe(true);
expect(
(
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'manual',
claimId: 'next-poll',
})
).status,
).toBe('acquired');
});
it.each(['same-generation', 'later-generation'] as const)(
'restores manual reconciliation after %s final persistence',
async (scenario) => {
const generationId = scenario === 'same-generation' ? parentMessageId : 'manual-generation';
const created = backgroundTaskRegistry.create({
userId,
conversationId,
toolCallId: 'manual-final-call',
toolName: 'tool',
messageId: parentMessageId,
harvestStarted: true,
});
if ('atCapacity' in created) throw new Error('Unexpected task capacity');
const taskId = created.task.id;
const root = await ready(taskId);
await project(taskId);
await mongoose.models.Message.updateOne(
{ user: userId, messageId: parentMessageId },
{ $set: { unfinished: true } },
);
const original = await mongoose.models.Message.findOne({
user: userId,
messageId: parentMessageId,
}).lean<Pick<IMessage, 'content'>>();
backgroundTaskRegistry.complete(userId, conversationId, taskId, { content: 'manual-result' });
backgroundTaskRegistry.finishHarvest(userId, conversationId, taskId);
backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, taskId, {
renew: async () => true,
retire: async (reason, options) =>
methods.retireAgentTriggerDelivery({
deliveryKey: getAgentTriggerIdempotencyKey(root),
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
settledAt: new Date(),
reason,
...options,
}),
});
const polled = JSON.parse(
await runCheckBackgroundTask({
userId,
conversationId,
args: { background_task_id: taskId },
toolCallId: 'confirmed-poll',
runId: 'manual-run',
generationId,
claimBackgroundToolResult: (input) =>
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
}),
);
expect(polled).toMatchObject({ status: 'completed', result: 'manual-result' });
expect(backgroundTaskRegistry.get(userId, conversationId, taskId)?.resultClaim).toMatchObject({
receiptReconciled: true,
});
// Final full persistence uses content captured before manual confirmation.
await mongoose.models.Message.updateOne(
{ user: userId, messageId: parentMessageId },
{ $set: { unfinished: false, content: original?.content } },
);
const restored = getBackgroundCodeDelivery({
userId,
conversationId,
args: { background_task_id: taskId },
});
expect(restored?.backgroundTask?.resultClaim).toMatchObject({ receiptReconciled: true });
await methods.updateToolCallResult({
userId,
conversationId,
messageId: parentMessageId,
toolCallId: taskId,
output: 'manual-result',
backgroundTask: restored?.backgroundTask,
});
const release = jest.fn(methods.releaseBackgroundToolResultClaims);
const recover = createBackgroundToolDeadClaimRecovery(
async () => false,
release,
async () => null,
async () => 'unavailable',
);
const claim = await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId,
kind: 'wakeup',
claimId: 'automatic-probe',
});
if (claim.status !== 'claimed' || claim.claim == null)
throw new Error('Expected preserved manual claim');
expect(claim.claim).toMatchObject({ receiptReconciled: true });
expect(
await recover({
userId,
conversationId,
messageId: parentMessageId,
claimId: claim.claim.claimId,
kind: 'manual',
generationId,
onlyIfUnreconciled: true,
}),
).toBe(false);
expect(
(await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(root)))?.status,
).toBe('succeeded');
},
);
it.each(['admitted', 'unpublished'] as const)(
'uses native recovery proof after custom-store %s job cleanup',
async (state) => {
const root = await ready('one');
const sibling = await ready('two');
const key = getAgentTriggerIdempotencyKey(root);
await methods.claimAgentBackgroundToolResultBatch({
...owner(key),
limit: 8,
maxMetadataChars: 16000,
});
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
// Historical lookup is optional; the native claim/fence CAS still exists.
Reflect.set(store, 'getIdempotencyClaim', undefined);
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: store,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
try {
const claimed = await manager.claimGeneration(userId, key, conversationId, conversationId, 2);
if (state === 'admitted') {
const job = await manager.createJob(conversationId, userId, conversationId, {
idempotencyClientRequestId: key,
idempotencyClaimToken: claimed.existing!.claimToken,
initialMetadata: { generationProtocolVersion: 2 },
});
expect(await store.deleteJob(conversationId, job.createdAt)).toBe(true);
}
expect(
await manager.getGenerationAdmissionEvidence(userId, key, conversationId, conversationId),
).toBeNull();
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey: key },
{ $set: { capabilityStatus: 'dead' } },
);
const release = jest.fn(methods.releaseAgentBackgroundToolResultClaims);
const recover = createBackgroundToolDeadClaimRecovery(
async (deliveryKey, sourceId, reason, options) =>
methods.retireAgentTriggerDelivery({
deliveryKey,
sourceId,
reason,
settledAt: new Date(),
...options,
}),
methods.releaseBackgroundToolResultClaims,
async () => null,
({ userId, conversationId, claimId }) =>
manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),
release,
methods,
(...args) => manager.getGenerationAdmissionEvidence(...args),
);
expect(
await recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),
).toBe(state === 'unpublished');
if (state === 'admitted') {
expect(release).not.toHaveBeenCalled();
expect(
await methods.claimAgentBackgroundToolResultBatch({
...owner(getAgentTriggerIdempotencyKey(sibling)),
limit: 8,
maxMetadataChars: 16000,
}),
).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
const batch = await methods.getAgentBackgroundToolResultBatch(owner(key));
expect(batch?.proofCopiedAt).toBeInstanceOf(Date);
} else {
expect(release).toHaveBeenCalledWith(expect.objectContaining({ recoveryFenced: true }));
expect(
(
await methods.claimAgentBackgroundToolResultBatch({
...owner(getAgentTriggerIdempotencyKey(sibling)),
limit: 8,
maxMetadataChars: 16000,
})
).status,
).toBe('acquired');
}
} finally {
await manager.destroy();
}
},
);
it.each(['lookup', 'confirmation', 'confirmation-unknown'] as const)(
'preserves a committed manual replay when %s is unavailable',
async (phase) => {
const root = await ready('one');
await project('one');
const key = getAgentTriggerIdempotencyKey(root);
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey: key },
{ $set: { capabilityStatus: 'leased', capabilityClaimToken: 'automatic-lease' } },
);
const input = {
userId,
conversationId,
taskId: 'one',
kind: 'manual' as const,
claimId: 'committed-poll',
generationId: 'manual-generation',
};
const delivered = await claimBackgroundToolResult(
methods,
methods.getAgentBackgroundToolResultClaim,
input,
);
expect(delivered).toMatchObject({ status: 'acquired' });
let restore: () => void = () => undefined;
try {
if (phase === 'lookup') {
await expect(
claimBackgroundToolResult(
methods,
async () => {
throw new Error('receipt lookup unavailable');
},
input,
),
).rejects.toThrow();
} else if (phase === 'confirmation') {
const failure = jest
.spyOn(mongoose.models.Message, 'updateOne')
.mockImplementationOnce(() => {
throw new Error('confirmation write unavailable');
});
restore = () => {
failure.mockRestore();
};
// Exact committed read-back succeeds even though this replay's write fails.
await expect(
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
...input,
generationId: undefined,
}),
).resolves.toEqual(delivered);
} else {
const failure = jest
.spyOn(methods, 'confirmBackgroundToolResultClaim')
.mockRejectedValueOnce(new Error('confirmation outcome unavailable'));
restore = () => {
failure.mockRestore();
};
await expect(
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
).rejects.toThrow('confirmation outcome unavailable');
}
} finally {
restore();
}
const competitor = await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'one',
kind: 'wakeup',
claimId: 'automatic-probe',
});
expect(competitor).toMatchObject({
status: 'claimed',
claim: { claimId: input.claimId, generationId: 'manual-generation', receiptReconciled: true },
});
expect(
await methods.releaseBackgroundToolResultClaims({
userId,
conversationId,
messageId: parentMessageId,
taskIds: ['one'],
kind: 'manual',
claimId: input.claimId,
onlyIfUnreconciled: true,
}),
).toBe(false);
if (root.mode !== 'continue') throw new Error('Expected continuation');
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
await expect(
resolve(root, {
idempotencyKey: key,
requiredWorkerCapability: capability,
deliveryClaimToken: 'automatic-lease',
}),
).resolves.toEqual({ status: 'settled' });
},
);
it('repairs native started-fence confirmation from an applied follower after a retired owner crashed', async () => {
const root = await ready('one');
const sibling = await ready('two');
const key = getAgentTriggerIdempotencyKey(root);
const claimed = await methods.claimAgentBackgroundToolResultBatch({
...owner(key),
limit: 8,
maxMetadataChars: 16000,
});
if (claimed.status !== 'acquired') throw new Error('Expected frozen batch');
await mongoose.models.AgentTriggerDelivery.updateOne(
{ deliveryKey: key },
{ $set: { capabilityStatus: 'dead' } },
);
const recover = createBackgroundToolDeadClaimRecovery(
async (deliveryKey, sourceId, reason, options) =>
methods.retireAgentTriggerDelivery({
deliveryKey,
sourceId,
reason,
settledAt: new Date(),
...options,
}),
methods.releaseBackgroundToolResultClaims,
async () => null,
async () => 'started',
methods.releaseAgentBackgroundToolResultClaims,
methods,
async () => null,
);
const Delivery = mongoose.models.AgentTriggerDelivery;
const copyFailure = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
throw new Error('proof copy lost');
});
await expect(
recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),
).rejects.toThrow('proof copy lost');
copyFailure.mockRestore();
expect(await Delivery.findOne({ deliveryKey: key }).lean()).toMatchObject({
status: 'succeeded',
});
expect(await Delivery.findOne({ deliveryKey: key }).lean()).not.toHaveProperty('expiresAt');
if (sibling.mode !== 'continue') throw new Error('Expected sibling');
const resolve = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: async () => null,
});
await expect(
resolve(sibling, {
idempotencyKey: getAgentTriggerIdempotencyKey(sibling),
requiredWorkerCapability: capability,
}),
).resolves.toEqual({ status: 'settled' });
expect(
(await methods.getAgentBackgroundToolResultBatch(owner(key)))?.proofCopiedAt,
).toBeInstanceOf(Date);
expect(await Delivery.findOne({ deliveryKey: key }).lean()).toHaveProperty('expiresAt');
expect(
await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'one' }),
).toMatchObject({ appliedAt: expect.any(Date) });
expect(
await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'two' }),
).toMatchObject({ appliedAt: expect.any(Date) });
});
it.each([
['local', false],
['reconstructed', false],
['local', true],
['reconstructed', true],
] as const)(
'recovers a released physical projection through a %s poll after successor retirement %s',
async (pollPath, replaceOwner) => {
const root = await ready('stale-task');
const key = getAgentTriggerIdempotencyKey(root);
const scope = owner(key);
await project('stale-task');
const old = await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
if (old.status !== 'acquired') throw new Error('Expected predecessor batch');
let pause: () => void = () => undefined;
let resume: () => void = () => undefined;
const paused = new Promise<void>((resolve) => {
pause = resolve;
});
const barrier = new Promise<void>((resolve) => {
resume = resolve;
});
const Message = mongoose.models.Message;
const update = Message.findOneAndUpdate.bind(Message);
const lateProjection = jest
.spyOn(Message, 'findOneAndUpdate')
.mockImplementationOnce((...args) => {
const query = update(...args);
const execute = query.exec.bind(query);
jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
pause();
await barrier;
return execute();
});
return query;
});
const delayed = methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'stale-task',
kind: 'wakeup',
claimId: key,
batchId: old.batchId,
limit: 1,
});
await paused;
expect(
await methods.releaseAgentBackgroundToolResultClaims({
...scope,
claimId: key,
batchId: old.batchId,
}),
).toBe(true);
if (replaceOwner) {
const successor = await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
if (successor.status !== 'acquired') throw new Error('Expected successor batch');
expect(successor.batchId).not.toBe(old.batchId);
expect(
await methods.releaseAgentBackgroundToolResultClaims({
...scope,
claimId: key,
batchId: successor.batchId,
}),
).toBe(true);
}
expect(
await methods.retireAgentTriggerDelivery({
deliveryKey: key,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
reason: 'unadmitted owner retired',
settledAt: new Date(),
onlyIfUnclaimed: true,
}),
).toBe(true);
resume();
await expect(delayed).resolves.toMatchObject({ status: 'acquired' });
lateProjection.mockRestore();
expect(await methods.getAgentBackgroundToolResultBatch(scope)).toBeNull();
// A broad logical-owner release would also erase these unrelated claims.
await project('other-epoch');
await project('manual-task');
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'other-epoch',
kind: 'wakeup',
claimId: key,
batchId: 'unrelated-epoch',
limit: 1,
});
await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
userId,
conversationId,
messageId: parentMessageId,
taskId: 'manual-task',
kind: 'manual',
claimId: key,
});
if (pollPath === 'local') {
const created = backgroundTaskRegistry.create({
taskId: 'stale-task',
userId,
conversationId,
messageId: parentMessageId,
toolCallId: 'stale-task',
toolName: 'tool',
});
if ('atCapacity' in created) throw new Error('Unexpected capacity');
backgroundTaskRegistry.complete(userId, conversationId, created.task.id, {
content: 'durable-stale-task',
});
backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, created.task.id);
}
const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: store,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
try {
const release = jest.fn(methods.releaseBackgroundToolResultClaims);
const recover = createBackgroundToolDeadClaimRecovery(
async (deliveryKey, sourceId, reason, options) =>
methods.retireAgentTriggerDelivery({
deliveryKey,
sourceId,
reason,
settledAt: new Date(),
...options,
}),
release,
async () => null,
({ userId, conversationId, claimId }) =>
manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),
methods.releaseAgentBackgroundToolResultClaims,
methods,
(...args) => manager.getGenerationAdmissionEvidence(...args),
);
const recoverDeadBackgroundToolClaim = jest.fn(recover);
const polled = JSON.parse(
await runCheckBackgroundTask({
userId,
conversationId,
args: { background_task_id: 'stale-task' },
toolCallId: 'stale-poll',
runId: 'recovery-run',
generationId: 'recovery-generation',
claimBackgroundToolResult: (input) =>
claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
recoverDeadBackgroundToolClaim,
}),
);
expect(polled).toMatchObject({ status: 'completed', result: 'durable-stale-task' });
expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith(
expect.objectContaining({ claimId: key, batchId: old.batchId }),
);
expect(release).toHaveBeenCalledWith(
expect.objectContaining({ kind: 'wakeup', claimId: key, batchId: old.batchId }),
);
for (const [taskId, claim] of [
['other-epoch', { kind: 'wakeup', claimId: key, batchId: 'unrelated-epoch' }],
['manual-task', { kind: 'manual', claimId: key, receiptReconciled: true }],
] as const) {
expect(
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId,
kind: 'wakeup',
claimId: 'probe',
}),
).toMatchObject({ status: 'claimed', claim });
}
} finally {
await manager.destroy();
}
},
);
it('does not release a stale epoch while a successor receipt batch owns the logical delivery', async () => {
const root = await ready('stale-task');
const key = getAgentTriggerIdempotencyKey(root);
const scope = owner(key);
await project('stale-task');
const old = await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
if (old.status !== 'acquired') throw new Error('Expected predecessor');
await methods.releaseAgentBackgroundToolResultClaims({
...scope,
claimId: key,
batchId: old.batchId,
});
const successor = await methods.claimAgentBackgroundToolResultBatch({
...scope,
limit: 8,
maxMetadataChars: 16000,
});
if (successor.status !== 'acquired') throw new Error('Expected successor');
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'stale-task',
kind: 'wakeup',
claimId: key,
batchId: successor.batchId,
limit: 1,
});
const release = jest.fn(methods.releaseBackgroundToolResultClaims);
const retire = jest.fn(async () => true);
const fence = jest.fn(async () => 'fenced' as const);
const recover = createBackgroundToolDeadClaimRecovery(
retire,
release,
async () => null,
fence,
methods.releaseAgentBackgroundToolResultClaims,
methods,
);
expect(
await recover({
userId,
conversationId,
messageId: parentMessageId,
claimId: key,
batchId: old.batchId,
}),
).toBe(false);
expect(retire).not.toHaveBeenCalled();
expect(fence).not.toHaveBeenCalled();
expect(release).not.toHaveBeenCalled();
expect(
await methods.getAgentBackgroundToolResultClaim({ ...scope, taskId: 'stale-task' }),
).toMatchObject({ batchId: successor.batchId });
expect(
await methods.claimBackgroundToolResults({
userId,
conversationId,
messageId: parentMessageId,
taskId: 'stale-task',
kind: 'manual',
claimId: 'probe',
}),
).toMatchObject({ status: 'claimed', claim: { claimId: key, batchId: successor.batchId } });
});