Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

45.7 KB
Raw Blame
import { randomUUID } from 'node:crypto';
import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';
import {
  AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
  AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
  AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
} from '@librechat/data-schemas';
import type {
  AgentTriggerProducerLeaseStatus,
  AgentTriggerDeliveryMethods,
  ConversationMethods,
  IMessage,
  MessageMethods,
} from '@librechat/data-schemas';
import type {
  BackgroundToolDeadClaimRecovery,
  PendingBackgroundCompletion,
  BackgroundToolWakeupAdmission,
  BackgroundToolWakeupRegistration,
  BackgroundToolWakeupRetireOptions,
  PendingBackgroundCompletionControls,
} from './backgroundCompletion';
import type {
  AgentTriggerContinuePreparation,
  AgentTriggerExecutionHostDeps,
} from './triggers/host';
import type { ScheduleMCPCompletionLookup } from '~/schedules/authorization/continuation';
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
import type { AgentTriggerDispatchContext } from './triggers/dispatch';
import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
import { resolveScheduleMCPCompletion } from '~/schedules/authorization/continuation';
import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';
import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion';
import { SUBAGENT_COMPLETION_SOURCE } from './subagentCompletionWakeup';
import { createAgentTriggerEnvelope } from './triggers/envelope';
import { AgentTriggerExecutionError } from './triggers/host';
import { truncateMiddle } from '~/utils';

const WAKEUP_ADMISSION_DELAY_MS = 250;
const MAX_WAKEUP_RESULT_CHARS = 24 * 1024;
export const BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS: number = 16 * 1024;
const MESSAGE_SELECT = 'messageId parentMessageId isCreatedByUser createdAt';
export const BACKGROUND_TOOL_COMPLETION_SOURCE = 'background-tool-completion';
const EVENT_TYPE = 'background-tool.completion';

export type EnqueueBackgroundToolCompletion = (
  envelope: unknown,
  options?: AgentTriggerEnqueueOptions,
) => Promise<{ deliveryKey: string }>;

export type RetireBackgroundToolCompletion = (
  deliveryKey: string,
  sourceId: string,
  reason: string,
  options?: BackgroundToolWakeupRetireOptions,
) => Promise<boolean>;

export type RenewBackgroundToolCompletionProducerLease = (
  deliveryKey: string,
  sourceId: string,
  leaseUntil: Date,
) => Promise<boolean>;

export type PersistBackgroundToolCompletionResult = (
  deliveryKey: string,
  sourceId: string,
  result: {
    status: 'completed' | 'error' | 'cancelled';
    output: string;
    settledAt: Date;
  },
) => Promise<boolean>;

type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
  Pick<
    MessageMethods,
    'getMessages' | 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
  > & {
    getAgentTriggerDeliveryProducerLease(params: {
      deliveryKey: string;
      sourceId: string;
      now: Date;
    }): Promise<AgentTriggerProducerLeaseStatus>;
    getAgentBackgroundToolResult?(params: { deliveryKey: string; sourceId: string }): Promise<{
      status: 'completed' | 'error' | 'cancelled';
      output: string;
      settledAt: Date;
    } | null>;
    persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
    beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch'];
    claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch'];
    confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch'];
    claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
    getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
    releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
  };

interface GenerationState {
  status?: unknown;
  metadata?: {
    idempotencyClientRequestId?: unknown;
    responseMessageId?: unknown;
    terminalPersistencePending?: unknown;
  };
}

export interface BackgroundToolCompletionWakeupResolverDeps {
  getScheduleMCPCompletionState?: ScheduleMCPCompletionLookup;
  methods: WakeupMethods;
  getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
  getResultBatchSize?: () => number | undefined;
  recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
  getGenerationAdmissionEvidence?: (
    userId: string,
    clientRequestId: string,
    streamId: string,
    conversationId?: string,
  ) => Promise<{ generationId: string; generationCreatedAt: number } | null>;
  /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
  getWaitMaxIntervalMs?: () => number | undefined;
}

function executionError(
  message: string,
  options: {
    code: string;
    retryable: boolean;
    deferWithoutAttempt?: boolean;
    status?: number;
    retryAfter?: string;
  },
): AgentTriggerExecutionError {
  return new AgentTriggerExecutionError(message, {
    mode: 'continue',
    certainty: 'definite',
    ...options,
  });
}

function payloadRegistration(
  envelope: AgentContinueTriggerEnvelope,
): Pick<BackgroundToolWakeupRegistration, 'taskId' | 'toolCallId' | 'toolName'> | null | undefined {
  if (
    envelope.event.source.type !== 'internal' ||
    envelope.event.source.id !== BACKGROUND_TOOL_COMPLETION_SOURCE ||
    envelope.event.type !== EVENT_TYPE
  ) {
    return;
  }
  const payload = envelope.event.payload;
  if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) {
    return null;
  }
  const { taskId, toolCallId, toolName } = payload;
  if (
    typeof taskId !== 'string' ||
    taskId.length === 0 ||
    taskId.length > 256 ||
    typeof toolCallId !== 'string' ||
    toolCallId.length === 0 ||
    toolCallId.length > 256 ||
    typeof toolName !== 'string' ||
    toolName.length === 0 ||
    toolName.length > 256
  ) {
    return null;
  }
  return { taskId, toolCallId, toolName };
}

function isParentActive(job: GenerationState | null): boolean {
  return (
    job?.status === 'running' ||
    job?.status === 'requires_action' ||
    job?.metadata?.terminalPersistencePending === true
  );
}

/** A running or approval-paused parent can stay busy for hours; one that has
 * settled and is only finishing terminal persistence clears within moments. */
function isParentWorking(job: GenerationState | null): boolean {
  return job?.status === 'running' || job?.status === 'requires_action';
}

function timestamp(message: Pick<IMessage, 'createdAt'>): number {
  const value = message.createdAt;
  if (value instanceof Date) {
    return value.getTime();
  }
  const parsed = value == null ? Number.NaN : new Date(value).getTime();
  return Number.isFinite(parsed) ? parsed : 0;
}

function latestAssistantDescendant(messages: IMessage[], anchorId: string): string | undefined {
  const byId = new Map(messages.map((message) => [message.messageId, message]));
  if (!byId.has(anchorId)) {
    return;
  }
  const memo = new Map<string, boolean>([[anchorId, true]]);
  const reachesAnchor = (message: IMessage, visiting = new Set<string>()): boolean => {
    const known = memo.get(message.messageId);
    if (known != null) {
      return known;
    }
    if (visiting.has(message.messageId)) {
      memo.set(message.messageId, false);
      return false;
    }
    visiting.add(message.messageId);
    const parent =
      typeof message.parentMessageId === 'string' ? byId.get(message.parentMessageId) : undefined;
    const reachable = parent != null && reachesAnchor(parent, visiting);
    visiting.delete(message.messageId);
    memo.set(message.messageId, reachable);
    return reachable;
  };
  const descendants = messages
    .filter((message) => message.isCreatedByUser === false && reachesAnchor(message))
    .sort((left, right) => {
      const time = timestamp(left) - timestamp(right);
      return time === 0 ? left.messageId.localeCompare(right.messageId) : time;
    });
  return descendants[descendants.length - 1]?.messageId;
}

function fitWakeupResult(output: string, serializedBudget: number): string {
  if (serializedBudget <= 0 || output.length === 0) {
    return '';
  }
  let low = 0;
  let high = Math.min(output.length, MAX_WAKEUP_RESULT_CHARS);
  let fitted = '';
  while (low <= high) {
    const limit = Math.floor((low + high) / 2);
    const candidate = truncateMiddle(output, limit);
    /** The aggregate limit applies after JSON escaping, not just to raw tool
     * text. Subtract the empty string's two quote characters because the
     * fixed payload budget below already includes `result: ""`. */
    const cost = JSON.stringify(candidate).length - 2;
    if (cost <= serializedBudget) {
      fitted = candidate;
      low = limit + 1;
    } else {
      high = limit - 1;
    }
  }
  return fitted;
}

function buildWakeupInput(
  results: Array<{
    taskId: string;
    toolCallId: string;
    toolName: string;
    status: 'completed' | 'error' | 'cancelled';
    output: string;
  }>,
): string {
  const header =
    results.length === 1
      ? 'A background tool task has finished. Continue using its durable result below.'
      : `${results.length} background tool tasks have finished. Continue using their durable results below.`;
  const payload = results.map(backgroundResultMetadata);
  let remaining = Math.max(
    0,
    BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - header.length - 1 - JSON.stringify(payload).length,
  );
  for (let index = 0; index < results.length; index++) {
    const slots = results.length - index;
    const share = Math.floor(remaining / slots);
    const fitted = fitWakeupResult(results[index]?.output ?? '', share);
    payload[index]!.result = fitted;
    remaining -= JSON.stringify(fitted).length - 2;
  }
  return `${header}\n${JSON.stringify(payload)}`;
}

/** Resolves a pre-registered delivery only after its result is durably
 * readable. The message claim elects automatic delivery against manual polls
 * and returns a bounded sibling batch for the continuation input. */
export function createBackgroundToolCompletionWakeupResolver({
  methods,
  getGenerationJob,
  getResultBatchSize,
  getWaitMaxIntervalMs,
  getScheduleMCPCompletionState,
  recoverDeadClaim,
  getGenerationAdmissionEvidence,
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
  AgentTriggerExecutionHostDeps['prepareContinue']
> {
  const waitingRetry = (receivedAt: number): string =>
    waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
  const resolve = async (
    envelope: AgentContinueTriggerEnvelope,
    context: AgentTriggerDispatchContext,
  ): Promise<AgentTriggerContinuePreparation | undefined> => {
    const registration = payloadRegistration(envelope);
    if (registration === undefined) {
      return;
    }
    if (registration === null) {
      throw executionError('The background tool completion payload is invalid.', {
        code: 'INVALID_BACKGROUND_TOOL_WAKEUP',
        retryable: false,
      });
    }
    let parentJob: GenerationState | null;
    try {
      parentJob = await getGenerationJob(envelope.target.conversationId);
    } catch (error) {
      throw executionError(
        `Parent generation state is temporarily unavailable: ${
          error instanceof Error ? error.message : String(error)
        }`,
        { code: 'PARENT_STATE_UNAVAILABLE', retryable: true },
      );
    }
    if (
      isParentActive(parentJob) &&
      parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey
    ) {
      throw executionError('The parent generation has not settled yet.', {
        code: 'PARENT_NOT_READY',
        retryable: true,
        status: 409,
        retryAfter: isParentWorking(parentJob) ? waitingRetry(envelope.receivedAt) : '1',
        deferWithoutAttempt: true,
      });
    }
    const userId = envelope.principal.userId;
    const parent = await methods.getConvo(userId, envelope.target.conversationId);
    if (parent == null || parent.tenantId !== envelope.principal.tenantId) {
      throw executionError('The parent conversation is no longer available.', {
        code: 'PARENT_NOT_FOUND',
        retryable: false,
        status: 404,
      });
    }
    const payload = envelope.event.payload;
    const scheduleMCPIdentity = await resolveScheduleMCPCompletion(
      {
        ownerId: userId,
        tenantId: envelope.principal.tenantId ?? null,
        scheduleMCPIdentity:
          payload && typeof payload === 'object' && 'scheduleMCPIdentity' in payload
            ? payload.scheduleMCPIdentity
            : undefined,
      },
      getScheduleMCPCompletionState,
    );
    const parentMessages = await methods.getMessages(
      { user: userId, conversationId: envelope.target.conversationId },
      MESSAGE_SELECT,
      { sort: { createdAt: 1, _id: 1 } },
    );
    const parentMessageId = latestAssistantDescendant(
      parentMessages,
      envelope.target.parentMessageId,
    );
    if (parentMessageId == null) {
      throw executionError('The parent conversation branch is no longer available.', {
        code: 'PARENT_NOT_FOUND',
        retryable: false,
        status: 404,
      });
    }
    const batchScope = {
      deliveryKey: context.idempotencyKey,
      sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
      userId,
      ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
      conversationId: envelope.target.conversationId,
      parentMessageId: envelope.target.parentMessageId,
      agentId: envelope.target.agentId,
      ...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }),
    };
    const batch =
      context.requiredWorkerCapability != null &&
      context.requiredWorkerCapability !==
        AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
        ? undefined
        : await methods.claimAgentBackgroundToolResultBatch?.({
            ...batchScope,
            limit: getResultBatchSize?.() ?? 8,
            maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
          });
    if (batch?.status === 'claimed') {
      if (batch.ownerStatus === 'applied') return { status: 'settled' };
      if (batch.ownerStatus === 'recoverable') {
        await recoverDeadClaim?.({
          userId,
          conversationId: envelope.target.conversationId,
          messageId: envelope.target.parentMessageId,
          claimId: batch.claimId,
          batchId: batch.batchId,
        });
      }
      throw executionError('The background result is owned by another completion delivery.', {
        code: 'BACKGROUND_TOOL_BATCH_PENDING',
        retryable: true,
        deferWithoutAttempt: true,
        retryAfter: '1',
      });
    }
    if (batch?.status === 'not_ready') {
      if (batch.waitingForResult === true) {
        // A producer can fail its receipt write while its terminal message
        // projection succeeds. Reconstruct from that already-durable result.
        const projection = await methods.claimBackgroundToolResults({
          userId,
          conversationId: envelope.target.conversationId,
          messageId: envelope.target.parentMessageId,
          taskId: registration.taskId,
          agentId: envelope.target.agentId,
          kind: 'wakeup',
          claimId: context.idempotencyKey,
          limit: 1,
        });
        if (
          projection.status === 'claimed' &&
          projection.claim?.kind === 'manual' &&
          projection.claim.receiptReconciled === true
        )
          return { status: 'settled' };
        if (projection.status === 'acquired') {
          const result = projection.results.find((result) => result.taskId === registration.taskId);
          const releaseProjection = async (): Promise<void> => {
            if (
              !(await methods.releaseBackgroundToolResultClaims({
                userId,
                conversationId: envelope.target.conversationId,
                messageId: envelope.target.parentMessageId,
                taskIds: [registration.taskId],
                kind: 'wakeup',
                claimId: context.idempotencyKey,
                allowMissingMessage: true,
              }))
            ) {
              throw new Error('Background projection reconstruction release was not confirmed');
            }
          };
          let restored = false;
          try {
            if (result != null)
              restored =
                (await methods.persistAgentBackgroundToolResult?.({
                  deliveryKey: context.idempotencyKey,
                  sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
                  result: {
                    status: result.status,
                    output: truncateMiddle(
                      result.output,
                      AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
                    ),
                    settledAt: result.settledAt ?? new Date(envelope.event.occurredAt),
                  },
                })) === true;
          } finally {
            await releaseProjection();
          }
          if (restored) return resolve(envelope, context);
          throw executionError('The background result receipt is being reconstructed.', {
            code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
            retryable: true,
            deferWithoutAttempt: true,
            retryAfter: '1',
          });
        }
        if (
          projection.status === 'claimed' &&
          projection.claim?.kind === 'manual' &&
          projection.claim.generationId != null
        ) {
          await recoverDeadClaim?.({
            userId,
            conversationId: envelope.target.conversationId,
            messageId: envelope.target.parentMessageId,
            claimId: projection.claim.claimId,
            kind: 'manual',
            generationId: projection.claim.generationId,
            onlyIfUnreconciled: true,
          });
        }
        if (projection.status === 'claimed')
          throw executionError('The background projection is being reconciled.', {
            code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
            retryable: true,
            deferWithoutAttempt: true,
            retryAfter: '1',
          });

        const producer = await methods.getAgentTriggerDeliveryProducerLease({
          deliveryKey: context.idempotencyKey,
          sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
          now: new Date(),
        });
        if (producer.status === 'expired') {
          throw executionError('The process-local background tool executor was lost.', {
            code: 'BACKGROUND_TOOL_PRODUCER_LOST',
            retryable: false,
          });
        }
      }
      throw executionError('The background result batch is not durable yet.', {
        code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
        retryable: true,
        deferWithoutAttempt: true,
        retryAfter: waitingRetry(envelope.receivedAt),
      });
    }
    if (batch?.status === 'acquired') {
      const owned = { ...batchScope, batchId: batch.batchId };
      const dispatchId = randomUUID();
      let dispatchBegan = false;
      const release = async (): Promise<boolean> => {
        const admitted = await getGenerationAdmissionEvidence?.(
          userId,
          context.idempotencyKey,
          envelope.target.conversationId,
          envelope.target.conversationId,
        );
        if (admitted != null) {
          await methods.confirmAgentBackgroundToolResultBatch?.(owned);
          return false;
        }
        // Absence is not proof. The storage CAS permits only a first definite
        // failure or a never-dispatched plan, never a retry of an ambiguous POST.
        const released = await methods.releaseAgentBackgroundToolResultClaims?.({
          sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
          userId,
          conversationId: envelope.target.conversationId,
          parentMessageId: envelope.target.parentMessageId,
          claimId: context.idempotencyKey,
          batchId: batch.batchId,
          ...(dispatchBegan && { dispatchId }),
          ...(context.deliveryClaimToken != null && {
            deliveryClaimToken: context.deliveryClaimToken,
          }),
        });
        return released === true;
      };
      // A receipt may precede its projection. Reconcile every member, not just
      // the root, against a concurrent manual poll or late message persistence.
      const projections = await Promise.all(
        batch.results.map((result) =>
          methods.claimBackgroundToolResults({
            userId,
            conversationId: envelope.target.conversationId,
            messageId: envelope.target.parentMessageId,
            taskId: result.taskId,
            agentId: envelope.target.agentId,
            kind: 'wakeup',
            claimId: context.idempotencyKey,
            batchId: batch.batchId,
            limit: 1,
          }),
        ),
      );
      if (
        projections.some(
          (projection) =>
            projection.status === 'claimed' || projection.status === 'outcome_unknown',
        )
      ) {
        for (let index = 0; index < projections.length; index++) {
          const stale = projections[index];
          if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue;
          const taskId = batch.results[index]!.taskId;
          const receipt = await methods.getAgentBackgroundToolResultClaim?.({
            sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
            userId,
            conversationId: envelope.target.conversationId,
            parentMessageId: envelope.target.parentMessageId,
            taskId,
          });
          if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId)
            continue;
          await methods.releaseBackgroundToolResultClaims({
            userId,
            conversationId: envelope.target.conversationId,
            messageId: envelope.target.parentMessageId,
            taskIds: [taskId],
            kind: 'wakeup',
            claimId: stale.claim.claimId,
            ...(stale.claim.batchId != null && { batchId: stale.claim.batchId }),
            allowMissingMessage: true,
          });
          throw executionError('A stale background projection is being reconciled.', {
            code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
            retryable: true,
            deferWithoutAttempt: true,
            retryAfter: '1',
          });
        }
        const released = await release();
        const rootProjection =
          projections[batch.results.findIndex((result) => result.taskId === registration.taskId)];
        if (
          released &&
          rootProjection?.status === 'claimed' &&
          rootProjection.claim?.kind === 'manual' &&
          rootProjection.claim.receiptReconciled === true
        ) {
          return { status: 'settled' };
        }
        if (released && recoverDeadClaim != null) {
          await Promise.all(
            projections.map(async (projection) => {
              if (
                projection.status !== 'claimed' ||
                projection.claim?.kind !== 'manual' ||
                projection.claim.receiptReconciled === true ||
                projection.claim.generationId == null
              )
                return;
              await recoverDeadClaim({
                userId,
                conversationId: envelope.target.conversationId,
                messageId: envelope.target.parentMessageId,
                claimId: projection.claim.claimId,
                kind: 'manual',
                generationId: projection.claim.generationId,
                onlyIfUnreconciled: true,
              });
            }),
          );
        }
        throw executionError('Background result ownership is being reconciled.', {
          code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
          retryable: true,
          deferWithoutAttempt: true,
          retryAfter: '1',
        });
      }
      return {
        status: 'ready',
        parentMessageId,
        ...(scheduleMCPIdentity && { scheduleMCPIdentity }),
        ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
        input: buildWakeupInput(batch.results),
        releaseOnDefiniteFailure: async () => {
          await release();
        },
        beginDispatch: async () => {
          dispatchBegan = true;
          if (
            !(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId }))
          ) {
            throw executionError('The background batch preparation was superseded.', {
              code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED',
              retryable: true,
              deferWithoutAttempt: true,
            });
          }
        },
        settleOnAdmission: async (result) => {
          if (
            result.status === 'settled' &&
            getGenerationAdmissionEvidence != null &&
            (await getGenerationAdmissionEvidence(
              userId,
              context.idempotencyKey,
              envelope.target.conversationId,
              envelope.target.conversationId,
            )) == null
          ) {
            throw new Error('A recovery fence does not prove background admission');
          }
          if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) {
            throw new Error('Background result batch admission was not confirmed');
          }
        },
      };
    }
    const claim = await methods.claimBackgroundToolResults({
      userId,
      conversationId: envelope.target.conversationId,
      messageId: envelope.target.parentMessageId,
      taskId: registration.taskId,
      agentId: envelope.target.agentId,
      kind: 'wakeup',
      claimId: context.idempotencyKey,
      limit: getResultBatchSize?.() ?? 8,
      maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
    });
    if (claim.status === 'claimed') {
      const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
        sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
        userId,
        conversationId: envelope.target.conversationId,
        parentMessageId: envelope.target.parentMessageId,
        taskId: registration.taskId,
      });
      if (receiptOwner?.claimId === context.idempotencyKey) {
        const released = await methods.releaseAgentBackgroundToolResultClaims?.({
          sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
          userId,
          conversationId: envelope.target.conversationId,
          parentMessageId: envelope.target.parentMessageId,
          claimId: context.idempotencyKey,
        });
        if (released === false)
          throw new Error('Background receipt claim release was not confirmed');
        throw executionError('Background result ownership is being reconciled.', {
          code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
          retryable: true,
          deferWithoutAttempt: true,
          retryAfter: '1',
        });
      }
      return { status: 'settled' };
    }
    if (claim.status === 'outcome_unknown') {
      throw executionError('The process-local background tool outcome is unknown.', {
        code: 'BACKGROUND_TOOL_OUTCOME_UNKNOWN',
        retryable: false,
      });
    }
    if (claim.status === 'acquired') {
      const taskIds = claim.results.map((result) => result.taskId);
      const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
        sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
        userId,
        conversationId: envelope.target.conversationId,
        parentMessageId: envelope.target.parentMessageId,
        taskId: registration.taskId,
      });
      if (receiptOwner != null && receiptOwner.claimId !== context.idempotencyKey) {
        const released = await methods.releaseBackgroundToolResultClaims({
          userId,
          conversationId: envelope.target.conversationId,
          messageId: envelope.target.parentMessageId,
          taskIds,
          kind: 'wakeup',
          claimId: context.idempotencyKey,
        });
        if (!released) throw new Error('Background projection claim release was not confirmed');
        return { status: 'settled' };
      }
      const input = buildWakeupInput(claim.results);
      return {
        status: 'ready',
        parentMessageId,
        ...(scheduleMCPIdentity && { scheduleMCPIdentity }),
        ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
        input,
        releaseOnDefiniteFailure: async () => {
          const released = await methods.releaseBackgroundToolResultClaims({
            userId,
            conversationId: envelope.target.conversationId,
            messageId: envelope.target.parentMessageId,
            kind: 'wakeup',
            claimId: context.idempotencyKey,
          });
          if (!released) throw new Error('Background projection claim release was not confirmed');
          const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
            sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
            userId,
            conversationId: envelope.target.conversationId,
            parentMessageId: envelope.target.parentMessageId,
            claimId: context.idempotencyKey,
          });
          if (receiptReleased === false)
            throw new Error('Background receipt claim release was not confirmed');
        },
      };
    }
    const receiptClaim = await methods.claimAgentBackgroundToolResults?.({
      deliveryKey: context.idempotencyKey,
      sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
      userId,
      conversationId: envelope.target.conversationId,
      parentMessageId: envelope.target.parentMessageId,
      agentId: envelope.target.agentId,
      claimId: context.idempotencyKey,
      limit: 1,
    });
    if (receiptClaim?.status === 'claimed') {
      return { status: 'settled' };
    }
    if (receiptClaim?.status === 'acquired') {
      /** Reconcile with a parent projection that may have appeared while the
       * receipt CAS was in flight. A manual poll that already owns the message
       * wins; otherwise stamp this same automatic owner before dispatch. */
      const projectedClaim = await methods.claimBackgroundToolResults({
        userId,
        conversationId: envelope.target.conversationId,
        messageId: envelope.target.parentMessageId,
        taskId: registration.taskId,
        agentId: envelope.target.agentId,
        kind: 'wakeup',
        claimId: context.idempotencyKey,
        limit: 1,
      });
      if (projectedClaim.status === 'claimed') {
        const released = await methods.releaseAgentBackgroundToolResultClaims?.({
          sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
          userId,
          conversationId: envelope.target.conversationId,
          parentMessageId: envelope.target.parentMessageId,
          claimId: context.idempotencyKey,
        });
        if (released === false)
          throw new Error('Background receipt claim release was not confirmed');
        throw executionError('Background result ownership is being reconciled.', {
          code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
          retryable: true,
          deferWithoutAttempt: true,
          retryAfter: '1',
        });
      }
      return {
        status: 'ready',
        parentMessageId,
        ...(scheduleMCPIdentity && { scheduleMCPIdentity }),
        ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
        input: buildWakeupInput(receiptClaim.results),
        releaseOnDefiniteFailure: async () => {
          const projectionReleased = await methods.releaseBackgroundToolResultClaims({
            userId,
            conversationId: envelope.target.conversationId,
            messageId: envelope.target.parentMessageId,
            kind: 'wakeup',
            claimId: context.idempotencyKey,
          });
          if (projectedClaim.status === 'acquired' && !projectionReleased) {
            throw new Error('Background projection claim release was not confirmed');
          }
          const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
            sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
            userId,
            conversationId: envelope.target.conversationId,
            parentMessageId: envelope.target.parentMessageId,
            claimId: context.idempotencyKey,
          });
          if (receiptReleased === false)
            throw new Error('Background receipt claim release was not confirmed');
        },
      };
    }
    const receipt =
      methods.claimAgentBackgroundToolResults == null
        ? await methods.getAgentBackgroundToolResult?.({
            deliveryKey: context.idempotencyKey,
            sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
          })
        : null;
    if (receipt != null) {
      return {
        status: 'ready',
        parentMessageId,
        ...(scheduleMCPIdentity && { scheduleMCPIdentity }),
        ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
        input: buildWakeupInput([
          { ...registration, status: receipt.status, output: receipt.output },
        ]),
      };
    }
    let producerLease: AgentTriggerProducerLeaseStatus;
    try {
      producerLease = await methods.getAgentTriggerDeliveryProducerLease({
        deliveryKey: context.idempotencyKey,
        sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
        now: new Date(),
      });
    } catch (error) {
      throw executionError(
        `Background tool producer liveness is temporarily unavailable: ${
          error instanceof Error ? error.message : String(error)
        }`,
        { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
      );
    }
    if (producerLease.status === 'expired') {
      throw executionError('The process-local background tool executor was lost.', {
        code: 'BACKGROUND_TOOL_PRODUCER_LOST',
        retryable: false,
      });
    }
    /** A live lease proves the invocation or its durable persistence retry
     * still has an owner. Missing remains defer-only for compatibility with
     * completion rows admitted before producer leases existed. */
    throw executionError('The background tool result is not durable yet.', {
      code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
      retryable: true,
      status: 409,
      retryAfter: waitingRetry(envelope.receivedAt),
      deferWithoutAttempt: true,
    });
  };
  return resolve;
}

/** Lists and discards a conversation's undelivered background completions from the
 * durable delivery store, which outlives the process-local task registry: a result
 * dispatched in an earlier turn, on another replica, or before a restart is still
 * going to arrive, and the owner must be able to see and stop that. */
export function createPendingBackgroundCompletions(deps: {
  list: (input: {
    user: string;
    conversationId: string;
    sourceId: string;
    taskId?: string;
  }) => Promise<{
    completions: Array<PendingBackgroundCompletion & { deliveryKey: string }>;
    dead: Array<PendingBackgroundCompletion & { deliveryKey: string }>;
    truncated: boolean;
  }>;
  listTaskIds: (input: {
    user: string;
    conversationId: string;
    sourceId: string;
  }) => Promise<{ taskIds: string[]; truncated: boolean }>;
  retire: RetireBackgroundToolCompletion;
}): PendingBackgroundCompletionControls {
  const read = (input: { userId: string; conversationId: string; taskId?: string }) =>
    deps.list({
      user: input.userId,
      conversationId: input.conversationId,
      sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
      ...(input.taskId != null && { taskId: input.taskId }),
    });
  return {
    list: async (input) => {
      const { completions, dead, truncated } = await read(input);
      const project = ({
        taskId,
        toolCallId,
        toolName,
        dispatchedAt,
        result,
        claimedByWakeup,
      }: PendingBackgroundCompletion): PendingBackgroundCompletion => ({
        taskId,
        toolCallId,
        toolName,
        dispatchedAt,
        ...(result != null && { result }),
        claimedByWakeup,
      });
      return {
        completions: completions.map(project),
        dead: dead.map(project),
        complete: !truncated,
      };
    },
    discard: async (input) => {
      const [completion] = (await read(input)).completions;
      if (completion == null) {
        return 'not_pending';
      }
      if (completion.result == null) {
        return 'running';
      }
      if (completion.claimedByWakeup) {
        return 'delivering';
      }
      /** Unclaimed-only: once a resolver owns the delivery its continuation can no
       * longer be withdrawn, so that race, including one it already finished,
       * reports as delivering rather than discarded. */
      const retired = await deps.retire(
        completion.deliveryKey,
        BACKGROUND_TOOL_COMPLETION_SOURCE,
        'background result discarded by its owner',
        { onlyIfUnclaimed: true, requireTransition: true },
      );
      return retired ? 'discarded' : 'delivering';
    },
    listSubagentWakeups: async (input) => {
      const { taskIds, truncated } = await deps.listTaskIds({
        user: input.userId,
        conversationId: input.conversationId,
        sourceId: SUBAGENT_COMPLETION_SOURCE,
      });
      return { taskIds, complete: !truncated };
    },
    settleClaimed: async (input) => {
      const [completion] = (await read(input)).completions;
      if (completion == null) {
        return false;
      }
      return deps.retire(
        completion.deliveryKey,
        BACKGROUND_TOOL_COMPLETION_SOURCE,
        'completion claimed by manual poll',
        { onlyIfUnclaimed: true },
      );
    },
  };
}

/** Pre-registers the ordered completion delivery before external tool work starts. */
export function createBackgroundToolCompletionWakeupHandler(
  enqueue: EnqueueBackgroundToolCompletion,
  retire: RetireBackgroundToolCompletion,
  renewProducerLease: RenewBackgroundToolCompletionProducerLease,
  persistResult?: PersistBackgroundToolCompletionResult,
  expedite?: (deliveryKey: string) => void,
  getReceiptBatchingEnabled?: () => boolean,
): (
  registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
  return async (registration) => {
    const parentAgentId = registration.parentAgentId?.trim();
    if (parentAgentId == null || parentAgentId === '' || isEphemeralAgentId(parentAgentId)) {
      return false;
    }
    const envelope = createAgentTriggerEnvelope({
      mode: 'continue',
      requestId: randomUUID(),
      deliveryId: registration.taskId,
      receivedAt: Date.now(),
      principal: {
        id: registration.userId,
        ...(registration.tenantId == null ? {} : { tenantId: registration.tenantId }),
      },
      event: {
        id: registration.taskId,
        type: EVENT_TYPE,
        occurredAt: registration.createdAt,
        source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },
        payload: {
          scheduleMCPIdentity: registration.scheduleMCPIdentity ?? null,
          taskId: registration.taskId,
          toolCallId: registration.toolCallId,
          toolName: registration.toolName,
        },
      },
      target: {
        agentId: parentAgentId,
        conversationId: registration.conversationId,
        parentMessageId: registration.parentMessageId,
      },
      input: 'A background tool task is waiting to complete.',
    });
    const admitted = await enqueue(envelope, {
      /** Pending work gets a task-local lane so a slow tool cannot block an
       * independently completed sibling. The generation admission fence and
       * atomic result claim serialize the actual continuations. */
      orderingKey: `background-tool-completion:${registration.conversationId}:${registration.taskId}`,
      availableAt: new Date(
        Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
      ),
      requiredWorkerCapability:
        getReceiptBatchingEnabled?.() === true
          ? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
          : AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
      producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
    });
    return {
      renew: () =>
        renewProducerLease(
          admitted.deliveryKey,
          BACKGROUND_TOOL_COMPLETION_SOURCE,
          new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
        ),
      ...(persistResult == null
        ? {}
        : {
            persistResult: (result) =>
              persistResult(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, result),
          }),
      retire: (reason, options) =>
        options == null
          ? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
          : retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason, options),
      ...(expedite == null ? {} : { expedite: () => expedite(admitted.deliveryKey) }),
    };
  };
}

/** Reopens every result owned by one dead automatic batch. Generation state
 * first fences an admitted continuation that is still running/finalizing;
 * delivery retirement then proves no retry remains, and releasing by the
 * batch-root claim identity makes sibling recovery independent of which task
 * originally admitted that delivery. */
export function createBackgroundToolDeadClaimRecovery(
  retire: RetireBackgroundToolCompletion,
  releaseClaims: WakeupMethods['releaseBackgroundToolResultClaims'],
  getGenerationJob: (conversationId: string) => Promise<GenerationState | null | undefined>,
  fenceGenerationClaim: (input: {
    userId: string;
    conversationId: string;
    claimId: string;
  }) => Promise<'fenced' | 'started' | 'unavailable'>,
  releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
  batchRecovery?: Pick<
    AgentTriggerDeliveryMethods,
    'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch'
  >,
  getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'],
): BackgroundToolDeadClaimRecovery {
  return async ({
    userId,
    conversationId,
    messageId,
    claimId,
    kind,
    generationId,
    batchId,
    onlyIfUnreconciled,
  }) => {
    if (kind === 'manual') {
      if (generationId == null || generationId.length === 0) {
        return false;
      }
      const generation = await getGenerationJob(conversationId);
      if (generation?.metadata?.responseMessageId === generationId && isParentActive(generation)) {
        return false;
      }
      /** Releasing a dead manual DELIVERY only re-presents an already durable
       * terminal result. It never retries the completed tool mutation. */
      return releaseClaims({
        userId,
        conversationId,
        messageId,
        kind: 'manual',
        claimId,
        ...(onlyIfUnreconciled === true && { onlyIfUnreconciled }),
      });
    }
    const owner = {
      deliveryKey: claimId,
      sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
      userId,
      conversationId,
      parentMessageId: messageId,
    };
    const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner);
    if (batchId != null && batch != null && batch.batchId !== batchId) return false;
    if (batch?.appliedAt != null) {
      await batchRecovery?.confirmAgentBackgroundToolResultBatch({
        ...owner,
        batchId: batch.batchId,
      });
      return false;
    }
    const confirmAdmission = async (): Promise<boolean> => {
      const evidence = await getAdmissionEvidence?.(
        userId,
        claimId,
        conversationId,
        conversationId,
      );
      if (evidence == null) return false;
      if (batch != null)
        await batchRecovery?.confirmAgentBackgroundToolResultBatch({
          ...owner,
          batchId: batch.batchId,
        });
      return true;
    };
    if (batch != null && (await confirmAdmission())) return false;
    const claimGenerationIsActive = async (): Promise<boolean> => {
      const generation = await getGenerationJob(conversationId);
      return (
        generation?.metadata?.idempotencyClientRequestId === claimId && isParentActive(generation)
      );
    };
    if (await claimGenerationIsActive()) {
      return false;
    }
    const retired = await retire(
      claimId,
      BACKGROUND_TOOL_COMPLETION_SOURCE,
      'dead background completion batch recovered by manual poll',
      { onlyIfDead: true },
    );
    if (!retired) {
      return false;
    }
    /** Retirement closes further delivery retries. The idempotency-claim CAS
     * closes the remaining claim-to-job-publication window: recovery either
     * installs a started tombstone that invalidates a delayed creator's token,
     * or observes that job creation already won. */
    const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });
    if (generationFence === 'unavailable') return false;
    if (generationFence === 'started' && batch != null) {
      // The native claim CAS proves admission even when a custom store cannot
      // read historical claims or the live generation has been cleaned up.
      await batchRecovery?.confirmAgentBackgroundToolResultBatch({
        ...owner,
        batchId: batch.batchId,
      });
      return false;
    }
    if (await claimGenerationIsActive()) {
      return false;
    }
    const receiptBatchId = batch?.batchId ?? batchId;
    const receiptReleased = await releaseReceiptClaims?.({
      sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
      userId,
      conversationId,
      parentMessageId: messageId,
      claimId,
      ...(receiptBatchId != null && { batchId: receiptBatchId }),
      recoveryFenced: true,
    });
    if (receiptReleased === false) return false;
    if (batch != null) return receiptReleased === true;
    return releaseClaims({
      userId,
      conversationId,
      messageId,
      allowMissingMessage: true,
      kind: 'wakeup',
      claimId,
      ...(receiptBatchId != null && { batchId: receiptBatchId }),
    });
  };
}