Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
21.8 KB
Raw Blame
import { AgentStateMachine } from './agent'
import { LlmErrorCode } from './core/errors'
import type { Attachment } from './core/types'
import type { TaskLifecycle } from './lifecycle/taskLifecycle'
import type { TaskRecord } from './lifecycle/types'
import { logger } from './logger'
import { MessageQueue, Task, TaskResponsePayload } from './queue'
import type { TaskError } from './queue'
import { ResultStore } from './resultStore'
import type { IncomingMessage, MessageResponse } from './server'
import { parseBackgroundPrefix, serializeSessionKey } from './session'
import type { SessionProcessor } from './session'

export interface PendingTaskEntry {
  status: 'completed' | 'waiting_approval' | 'failed'
  response?: string
  attachments?: Attachment[]
  error?: TaskError
  model: string
  storedAt: number
  /**
   * True only for the C3 recoverability entry written by the SYNC resolve path
   * (handleResponse's \`if (!this.resolved)\` branch). Such a result was already
   * delivered INLINE to the sync caller's socket; it exists solely so a
   * socket-death duplicate that received \`{status:pending,taskId:priorId}\` can
   * still poll GET /v1/runtime/tasks/:id/result and recover the outcome. It is
   * NOT genuinely-pending work (nobody is waiting to consume it), so it must
   * NOT pin the stateless \`pendingResults\` idle gauge, which would otherwise
   * block drain/suspend forever after any sync turn. Async results
   * (executeAsync) leave this undefined and DO count toward pendingResults.
   */
  deliveredInline?: boolean
  /**
   * Epoch ms of the first successful owner READ of this entry (poll on
   * GET /v1/runtime/tasks/:id/result — main.ts handleTaskResult). Same class
   * as \`deliveredInline\`: a fetched result is recoverability cache — it stays
   * for repeat polls (multi-reader idempotency) but must not pin the stateless
   * \`pendingResults\` idle gauge (see runtime/resultDelivery.ts).
   */
  deliveredAt?: number
  source: {
    channelType: string
    channelId: string
    sender: string
  }
  approval?: {
    requestId: string
    userId: string
    notification: string
    // U5 — connect_required discriminator carried from the tool:approval_needed
    // event so a REST poll rebuilds the connect suspension. Absent reason ⇒
    // generic approval (back-compat).
    reason?: 'approval_required' | 'connect_required'
    mcpServerName?: string
  }
}

/**
 * U5 — pick the connect_required discriminator off a `tool:approval_needed`
 * event payload for the polling `approval` projections (both the inline
 * MessageResponse and the stored PendingTaskEntry). Returns an empty object for
 * a generic approval so the legacy wire shape stays byte-identical
 * (back-compat). Pure field propagation — the decision already happened
 * upstream (buildConnectRequiredApproval); nothing is inferred here.
 */
function connectApprovalFields(data: Record<string, unknown>): {
  reason?: 'approval_required' | 'connect_required'
  mcpServerName?: string
} {
  const reason = data.reason as 'approval_required' | 'connect_required' | undefined
  const isConnect = reason === 'connect_required'
  return {
    ...(reason ? { reason } : {}),
    // mcpServerName rides ONLY connect_required, matching the SSE producer.
    // Runtime typeof guard: never pass a non-string through the blind cast even if
    // an upstream contract ever regressed.
    ...(isConnect && typeof data.mcpServerName === 'string'
      ? { mcpServerName: data.mcpServerName }
      : {}),
  }
}

export interface MessageHandlerDeps {
  messageQueue: MessageQueue
  agent: AgentStateMachine | null
  pendingTaskResults: ResultStore<PendingTaskEntry>
  getModel: () => string
  sanitizeAttachments: (raw: Attachment[] | undefined) => Attachment[] | undefined
  timeoutMs?: number
  sessionProcessor?: SessionProcessor
  taskLifecycle: TaskLifecycle
}

/**
 * Handles a single incoming message lifecycle: enqueue, listen for
 * completion/approval, clean up listeners.
 *
 * Replaces the nested-closure pattern in main.ts handleIncomingMessage().
 */
export class IncomingMessageHandler {
  private resolved = false
  private task: Task
  private resolve!: (response: MessageResponse) => void
  private timeout: ReturnType<typeof setTimeout> | null = null
  private backgroundSessionKey: string | null = null

  // Bound listener refs for cleanup
  private onApprovalNeeded: ((event: { data: unknown }) => void) | null = null
  private onApprovalConsumed: ((event: { data: unknown }) => void) | null = null
  private onComplete: ((event: { task?: Task }) => void) | null = null
  private onFail: ((event: { task?: Task }) => void) | null = null

  constructor(
    private readonly message: IncomingMessage,
    private readonly deps: MessageHandlerDeps
  ) {
    const { isBackground, content } = parseBackgroundPrefix(message.content)
    const effectiveMessage = isBackground ? { ...message, content } : message

    const responseCallback = this.handleResponse.bind(this)
    this.task = deps.messageQueue.createTaskFromMessage(
      effectiveMessage,
      responseCallback,
      'normal'
    )

    // Reporter creation lives in TaskExecutor.buildLoopConfig — single
    // construction site so SSE redaction wiring can't drift between callers.
    // SSE clients connecting before the agent dequeues block in
    // progressReporterRegistry.waitFor(taskId, 180s) until taskExecutor registers.

    if (isBackground && deps.sessionProcessor) {
      this.backgroundSessionKey = `bg-${this.task.id}`
    }
  }

  execute(): Promise<MessageResponse> {
    // Background tasks always return immediately
    if (this.backgroundSessionKey) {
      return Promise.resolve(this.executeAsync())
    }

    return new Promise<MessageResponse>(resolve => {
      this.resolve = resolve

      // Single admission sink. MessageQueue.admit registers the lifecycle
      // record (Invariant I12: registration precedes sessionProcessor.enqueue
      // below) and rejects duplicates — a suppressed duplicate resolves
      // immediately and NEVER starts a second execution.
      const admission = this.deps.messageQueue.admit(this.task)
      if (!admission.admitted && admission.reason !== 'queue_full') {
        this.resolved = true
        resolve(this.duplicateSuppressedResponse(admission))
        return
      }
      if (!admission.admitted) {
        // queue_full keeps the legacy contract: the task still executes via
        // SessionProcessor, so register here to satisfy Invariant I12.
        this.deps.taskLifecycle.register(this.task)
      }
      logger.info({ taskId: this.task.id }, 'Message queued')

      // Dispatch to SessionProcessor if available
      if (this.deps.sessionProcessor) {
        const msg = this.message
        const sessionKey =
          this.backgroundSessionKey ??
          serializeSessionKey({
            userId: msg.sender,
            channelType: msg.channelType,
            channelId: msg.channelId || 'default',
            threadId: msg.threadId,
          })
        this.deps.sessionProcessor.enqueue(sessionKey, this.task)
      }

      this.attachListeners()
      this.startTimeout()
    })
  }

  /**
   * Enqueue the task and return immediately with a pending status.
   * Results and approval notifications are stored in pendingTaskResults
   * for polling via GET /v1/runtime/tasks/:taskId/result.
   */
  executeAsync(): MessageResponse {
    // Single admission sink — runs before any listener wiring so a suppressed
    // duplicate attaches nothing, leaks nothing, and NEVER executes again.
    const admission = this.deps.messageQueue.admit(this.task)
    if (!admission.admitted && admission.reason !== 'queue_full') {
      return this.duplicateSuppressedResponse(admission)
    }
    if (!admission.admitted) {
      // queue_full keeps the legacy contract: the task still executes via
      // SessionProcessor, so register here to satisfy Invariant I12.
      this.deps.taskLifecycle.register(this.task)
    }

    // Override responseCallback to store results in pendingTaskResults
    this.task.responseCallback = async payload => {
      const attachments = this.deps.sanitizeAttachments(payload.attachments)
      const status: PendingTaskEntry['status'] = payload.error ? 'failed' : 'completed'
      logger.info(
        { taskId: this.task.id, status, attachmentCount: attachments?.length ?? 0 },
        'Storing async result'
      )
      this.deps.pendingTaskResults.set(this.task.id, {
        status,
        response: payload.response,
        attachments,
        error: payload.error,
        model: this.deps.getModel(),
        storedAt: Date.now(),
        source: {
          channelType: this.message.channelType,
          channelId: this.message.channelId,
          sender: this.message.sender,
        },
      })
    }

    // Attach approval listener that stores to pendingTaskResults
    if (this.deps.agent) {
      const onApproval = (event: { data: unknown }) => {
        const data = event.data as Record<string, unknown>
        if (data.taskId !== this.task.id) return
        logger.info({ taskId: this.task.id }, 'Storing async approval notification')
        this.deps.pendingTaskResults.set(this.task.id, {
          status: 'waiting_approval',
          model: this.deps.getModel(),
          storedAt: Date.now(),
          source: {
            channelType: this.message.channelType,
            channelId: this.message.channelId,
            sender: this.message.sender,
          },
          approval: {
            requestId: data.requestId as string,
            userId: data.userId as string,
            notification: data.notification as string,
            ...connectApprovalFields(data),
          },
        })
      }
      this.deps.agent.on('tool:approval_needed', onApproval)

      // Listener that clears the stored waiting_approval entry once the
      // approval is granted or denied. Without this, GET /tasks/:id/result
      // keeps returning the obsolete notification (with the now-consumed
      // requestId) until either a *new* approval fires (overwriting the entry)
      // or the task fully completes — which makes channel-reader's poll loop
      // believe the same approval is still needed and re-prompt the user.
      const onApprovalConsumed = (event: { data: unknown }) => {
        const data = event.data as Record<string, unknown>
        if (data.taskId !== this.task.id) return
        logger.info(
          { taskId: this.task.id, requestId: data.requestId },
          'Clearing stale approval entry'
        )
        this.deps.pendingTaskResults.delete(this.task.id)
      }
      this.deps.agent.on('tool:approval_granted', onApprovalConsumed)
      this.deps.agent.on('tool:approval_denied', onApprovalConsumed)

      // Clean up listeners when task completes or fails
      const cleanup = (event: { task?: { id: string; error?: TaskError } }) => {
        if (event.task?.id !== this.task.id) return
        this.deps.agent?.removeListener('tool:approval_needed', onApproval)
        this.deps.agent?.removeListener('tool:approval_granted', onApprovalConsumed)
        this.deps.agent?.removeListener('tool:approval_denied', onApprovalConsumed)
        this.deps.messageQueue.off('task:completed', cleanup)
        this.deps.messageQueue.off('task:failed', onFinalFailure)
      }
      // Delivery is handled by the overridden responseCallback above — this handler
      // only cleans up the event listeners so we don't leak them after the task ends.
      const onFinalFailure = (event: { task?: { id: string } }) => {
        if (event.task?.id !== this.task.id) return
        cleanup(event)
      }
      this.deps.messageQueue.on('task:completed', cleanup)
      this.deps.messageQueue.on('task:failed', onFinalFailure)
    }

    logger.info({ taskId: this.task.id }, 'Message queued asynchronously')

    // Dispatch to SessionProcessor if available
    if (this.deps.sessionProcessor) {
      const msg = this.message
      const sessionKey =
        this.backgroundSessionKey ??
        serializeSessionKey({
          userId: msg.sender,
          channelType: msg.channelType,
          channelId: msg.channelId,
          threadId: msg.threadId,
        })
      this.deps.sessionProcessor.enqueue(sessionKey, this.task)
    }

    return {
      success: true,
      taskId: this.task.id,
      status: 'pending',
    }
  }

  /**
   * Build the response for a suppressed duplicate delivery. The duplicate is
   * NEVER executed:
   *   - prior task completed → replay the recorded outcome (duplicate-suppressed success)
   *   - prior task failed/cancelled → replay the recorded terminal error
   *   - prior task still in flight → attach/no-op: the in-flight execution
   *     answers; callers can poll GET /v1/runtime/tasks/:taskId/result with
   *     the prior task id.
   * Fail-loud: every suppression is logged with task id + prior state.
   */
  private duplicateSuppressedResponse(admission: {
    reason: 'duplicate_task_id' | 'duplicate_delivery'
    priorTaskId: string
    prior: TaskRecord
  }): MessageResponse {
    const { reason, priorTaskId, prior } = admission
    const priorStatus = prior.status
    logger.warn(
      { taskId: this.task.id, reason, priorTaskId, priorStatus },
      'Duplicate delivery suppressed'
    )
    // #666 — a success names the attachments and file references the first
    // delivery admitted, as the first response did; without them a client reads
    // a Host that dropped them. `prior` is the record admission read, so a TTL
    // eviction since then cannot remove them.
    const accepted: Pick<MessageResponse, 'acceptedAttachmentIds' | 'acceptedFileReferenceIds'> = {
      ...(prior.acceptedAttachmentIds.length
        ? { acceptedAttachmentIds: [...prior.acceptedAttachmentIds] }
        : {}),
      ...(prior.acceptedFileReferenceIds.length
        ? { acceptedFileReferenceIds: [...prior.acceptedFileReferenceIds] }
        : {}),
    }
    if (priorStatus === 'completed') {
      return {
        success: true,
        status: 'completed',
        response: prior.response,
        attachments: this.deps.sanitizeAttachments(prior.attachments),
        model: this.deps.getModel(),
        taskId: priorTaskId,
        ...accepted,
      }
    }
    if (priorStatus === 'failed' || priorStatus === 'cancelled') {
      const error: TaskError = prior.error ?? {
        code: 'TASK_ALREADY_TERMINAL',
        message: `Duplicate delivery suppressed — original task ${priorTaskId} is ${priorStatus}`,
        retryable: false,
        provider: 'unknown',
      }
      return {
        success: false,
        error,
        model: this.deps.getModel(),
        taskId: priorTaskId,
      }
    }
    // pending / processing / waiting_approval — in flight; the first delivery
    // owns result delivery. Attach by returning the prior task id as pending.
    return {
      success: true,
      status: 'pending',
      taskId: priorTaskId,
      ...accepted,
    }
  }

  private async handleResponse(payload: TaskResponsePayload): Promise<void> {
    const attachments = this.deps.sanitizeAttachments(payload.attachments)

    if (!this.resolved) {
      this.resolved = true
      // C3: the SYNC path binds the outcome to THIS request's socket via the
      // resolver below. If rpc-proxy's wake-and-hold retries the same messageId
      // after a socket death, the admission sink suppresses the duplicate and
      // returns { status: 'pending', taskId: priorId } — but without persisting
      // here, GET /v1/runtime/tasks/:priorId/result would find nothing and the
      // turn's result would be LOST (never re-executed, but unpollable). Persist
      // the terminal outcome keyed by task id (mirrors executeAsync's store) so
      // the suppressed duplicate's returned taskId is actually recoverable.
      // Tag it deliveredInline: the sync caller already got this answer on its
      // own socket via the resolve() below, so this entry is recoverability
      // cache only and must be EXCLUDED from the pendingResults idle gauge
      // (otherwise every sync turn would pin the gauge and block suspend).
      this.deps.pendingTaskResults.set(this.task.id, {
        status: payload.error ? 'failed' : 'completed',
        response: payload.response,
        attachments,
        error: payload.error,
        model: this.deps.getModel(),
        storedAt: Date.now(),
        deliveredInline: true,
        source: {
          channelType: this.message.channelType,
          channelId: this.message.channelId,
          sender: this.message.sender,
        },
      })
      if (payload.error) {
        this.resolve({
          success: false,
          error: payload.error,
          ...(attachments?.length ? { attachments } : {}),
          model: this.deps.getModel(),
        })
      } else {
        this.resolve({
          success: true,
          status: 'completed',
          response: payload.response,
          attachments,
          model: this.deps.getModel(),
        })
      }
    } else {
      logger.info({ taskId: this.task.id }, 'Storing post-approval result')
      // Compute the terminal status from the payload (parity with the sync
      // branch above and executeAsync): a post-approval turn can FAIL, and
      // hardcoding 'completed' here made the poll report success:false with
      // a contradictory status:'completed'.
      this.deps.pendingTaskResults.set(this.task.id, {
        status: payload.error ? 'failed' : 'completed',
        response: payload.response,
        attachments,
        error: payload.error,
        model: this.deps.getModel(),
        storedAt: Date.now(),
        source: {
          channelType: this.message.channelType,
          channelId: this.message.channelId,
          sender: this.message.sender,
        },
      })
    }
  }

  private attachListeners(): void {
    // Approval listener
    this.onApprovalNeeded = (event: { data: unknown }) => {
      const data = event.data as Record<string, unknown>
      if (data.taskId !== this.task.id) return

      if (!this.resolved) {
        this.resolved = true
        logger.info({ taskId: this.task.id }, 'Returning approval notification')
        this.resolve({
          success: true,
          status: 'waiting_approval',
          approval: {
            taskId: this.task.id,
            requestId: data.requestId as string,
            userId: data.userId as string,
            notification: data.notification as string,
            ...connectApprovalFields(data),
          },
          model: this.deps.getModel(),
        })
      } else {
        logger.info(
          { taskId: this.task.id, requestId: data.requestId },
          'Storing approval notification'
        )
        this.deps.pendingTaskResults.set(this.task.id, {
          status: 'waiting_approval',
          model: this.deps.getModel(),
          storedAt: Date.now(),
          source: {
            channelType: this.message.channelType,
            channelId: this.message.channelId,
            sender: this.message.sender,
          },
          approval: {
            requestId: data.requestId as string,
            userId: data.userId as string,
            notification: data.notification as string,
            ...connectApprovalFields(data),
          },
        })
      }
    }

    if (this.deps.agent) {
      this.deps.agent.on('tool:approval_needed', this.onApprovalNeeded)
    }

    // See executeAsync's onApprovalConsumed for the rationale — same staleness
    // bug applies to subsequent-approval entries stored after the first resolve.
    this.onApprovalConsumed = (event: { data: unknown }) => {
      const data = event.data as Record<string, unknown>
      if (data.taskId !== this.task.id) return
      logger.info(
        { taskId: this.task.id, requestId: data.requestId },
        'Clearing stale approval entry'
      )
      this.deps.pendingTaskResults.delete(this.task.id)
    }
    if (this.deps.agent) {
      this.deps.agent.on('tool:approval_granted', this.onApprovalConsumed)
      this.deps.agent.on('tool:approval_denied', this.onApprovalConsumed)
    }

    // Completion listener
    this.onComplete = (event: { task?: Task }) => {
      if (event.task?.id === this.task.id) {
        this.dispose()
      }
    }

    // Failure listener
    this.onFail = (event: { task?: Task }) => {
      if (event.task?.id !== this.task.id) return
      this.dispose()
      if (!this.resolved) {
        this.resolved = true
        const err: TaskError = event.task?.error ?? {
          code: LlmErrorCode.ApiCallFailed,
          message: 'Task failed',
          retryable: true,
          provider: 'unknown',
        }
        this.resolve({ success: false, error: err })
      }
    }

    this.deps.messageQueue.on('task:completed', this.onComplete)
    this.deps.messageQueue.on('task:failed', this.onFail)
  }

  private startTimeout(): void {
    const ms = this.deps.timeoutMs ?? 5 * 60 * 1000
    this.timeout = setTimeout(() => {
      if (!this.resolved) {
        this.resolved = true
        this.dispose()
        this.resolve({
          success: false,
          error: {
            code: LlmErrorCode.ApiCallFailed,
            message: 'Task processing timeout',
            retryable: true,
            provider: 'unknown',
          },
        })
      }
    }, ms)
  }

  private dispose(): void {
    if (this.timeout) {
      clearTimeout(this.timeout)
      this.timeout = null
    }
    if (this.onApprovalNeeded) {
      this.deps.agent?.removeListener('tool:approval_needed', this.onApprovalNeeded)
      this.onApprovalNeeded = null
    }
    if (this.onApprovalConsumed) {
      this.deps.agent?.removeListener('tool:approval_granted', this.onApprovalConsumed)
      this.deps.agent?.removeListener('tool:approval_denied', this.onApprovalConsumed)
      this.onApprovalConsumed = null
    }
    if (this.onComplete) {
      this.deps.messageQueue.off('task:completed', this.onComplete)
      this.onComplete = null
    }
    if (this.onFail) {
      this.deps.messageQueue.off('task:failed', this.onFail)
      this.onFail = null
    }
  }
}