Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.9 KB
Raw Blame
/**
 * Step-level approval gating: posts an approval request and polls until a
 * terminal status. On 401 it refreshes; on persistent refresh 401 it falls
 * back to the caller-supplied `reIssueTokens` callback for a one-shot recovery.
 */
import { createHash } from 'node:crypto'
import client, { Counter } from 'prom-client'
import { refreshRuntimeAuthFromPersistedState } from './mcpHostJwtState'
import { getJwtRuntimeBinding } from './mcpHostRuntimeJwt'
import {
  recordRuntimeAuthRecoveryFailure,
  recordRuntimeAuthRecoverySuccess,
} from './runtimeAuthHealth'

export interface ApprovalTarget {
  userId?: string
  teamId?: string
}

export interface WorkflowTriggerRunIntentPayload {
  inputs?: Record<string, unknown> | null
  intermediateParameters?: Record<string, unknown> | null
  outputOverrides?: Record<string, unknown> | null
}

export interface ApprovalGateParams {
  stepId: string
  executionId?: string
  runBindingProof?: string
  idempotencyKeyOverride?: string
  runtimeMcpHostRef?: string
  approvalRecipe?: {
    recipeNamespace: string
    recipeName: string
  }
  target: ApprovalTarget
  message: string
  payloadMetadata?: unknown
  workflowTriggerRunIntent?: WorkflowTriggerRunIntentPayload
  /** Broker-only retry opt-in for already-consumed approvals. */
  allowConsumedTerminal?: boolean
  timeoutSeconds?: number
}

export interface ReIssuedTokenPair {
  accessToken: string
  refreshToken: string
  mcpHostControlToken?: string
}

/**
 * Auth credential the mcp-host pod uses for ALL outbound calls to
 * control-api (workflow approvals, LLM usage reporting, anything else
 * added later). Holds the live access/refresh JWT pair and rotates it in
 * place on 401, so a single instance is shared across consumers in the
 * pod and refresh-on-401 mutations propagate.
 */
export interface McpHostRuntimeAuth {
  accessToken: string
  refreshToken: string
  baseUrl: string
  /** Canonical bearer identity from the runtime access JWT hostRefs[0]. */
  hostRef: string
  /** Sent in request bodies and kept aligned with the runtime access JWT claims. */
  recipeNamespace: string
  recipeName: string
  /** Live workflow-control JWT used by workflow_list/status/health/trigger broker calls. */
  mcpHostControlToken?: string
  persistRotatedTokens?: (tokens: {
    accessToken: string
    refreshToken: string
    mcpHostControlToken?: string
  }) => Promise<void>
  /** Optional re-issue callback invoked after persistent refresh 401s. */
  reIssueTokens?: () => Promise<ReIssuedTokenPair>
}

export interface ApprovalGateResult {
  approvalRequestId?: string
  status: string
  decidedBy?: unknown
  note?: string
}

type PollStatusResult =
  | { ok: true; status: string; decisionMaker?: unknown; note?: string }
  | { ok: false; reason: 'unauthorized' }

type RefreshResult = { ok: true } | { ok: false; status: number }

const POLL_INTERVAL_MS = 5_000
const POLL_JITTER_MS = 2_000
const REQUEST_RETRY_BASE_MS = 750
const REQUEST_RETRY_JITTER_MS = 250
const REQUEST_MAX_ATTEMPTS = 3
const POLL_MAX_ATTEMPTS = 3
const TERMINAL_STATUSES = new Set(['approved', 'denied', 'expired', 'cancelled', 'consumed'])

const REFRESH_MAX_ATTEMPTS = 3
const REFRESH_BACKOFF_MS = [1_000, 2_000, 4_000]
/** Consecutive 401s on refresh that triggers re-issue. */
const REFRESH_CONSECUTIVE_401_THRESHOLD = 2

function getOrCreateCounter<Label extends string>(options: {
  name: string
  help: string
  labelNames: readonly Label[]
}): Counter<Label> {
  const existing = client.register.getSingleMetric(options.name)
  if (existing) return existing as Counter<Label>
  return new Counter<Label>({
    name: options.name,
    help: options.help,
    labelNames: options.labelNames as Label[],
  })
}

export const refreshFailureCounter = getOrCreateCounter({
  name: 'workflow_auth_refresh_failures_total',
  help: 'Total refresh attempts that returned a non-2xx status',
  labelNames: ['status'] as const,
})

export const reIssueCounter = getOrCreateCounter({
  name: 'workflow_auth_reissue_attempts_total',
  help: 'Total re-issue attempts triggered by persistent refresh 401s',
  labelNames: ['result'] as const,
})

export const workflowTokenRefreshCounter = getOrCreateCounter({
  name: 'workflow_token_refresh_total',
  help: 'Total workflow runtime token refresh attempts',
  labelNames: ['recipe', 'outcome'] as const,
})

async function sleep(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms))
}

function isRetryableStatus(status: number): boolean {
  return status === 502 || status === 503 || status === 504 || (status >= 500 && status < 600)
}

function retryDelayMs(attempt: number): number {
  return REQUEST_RETRY_BASE_MS * attempt + Math.random() * REQUEST_RETRY_JITTER_MS
}

async function approvalErrorCodeFromResponse(res: Response): Promise<string | undefined> {
  const json = (res as { json?: () => Promise<unknown> }).json
  if (typeof json !== 'function') return undefined
  try {
    const data = await json.call(res)
    if (!data || typeof data !== 'object') return undefined
    const record = data as Record<string, unknown>
    if (typeof record.error === 'string') return record.error
    if (typeof record.code === 'string') return record.code
  } catch {
    return undefined
  }
  return undefined
}

function recipeLabel(auth: McpHostRuntimeAuth): string {
  return auth.recipeName ?? '<unknown>'
}

function reloadPersistedAuthIfAvailable(auth: McpHostRuntimeAuth): boolean {
  if (!refreshRuntimeAuthFromPersistedState(auth)) {
    return false
  }
  workflowTokenRefreshCounter.inc({
    recipe: auth.recipeName ?? '<unknown>',
    outcome: 'loaded_from_state',
  })
  return true
}

/** Mutex (per auth identity) so concurrent callers share one re-issue. */
const inflightReIssues = new WeakMap<McpHostRuntimeAuth, Promise<void>>()

function approvalPayloadMetadata(params: ApprovalGateParams): unknown {
  if (!params.runtimeMcpHostRef) return params.payloadMetadata
  const existing =
    params.payloadMetadata &&
    typeof params.payloadMetadata === 'object' &&
    !Array.isArray(params.payloadMetadata)
      ? (params.payloadMetadata as Record<string, unknown>)
      : {}
  return {
    ...existing,
    runtimeMcpHostRef: params.runtimeMcpHostRef,
  }
}

/** Mutates `auth` on success; returns `{ ok: false, status }` without throwing. */
async function attemptRefresh(auth: McpHostRuntimeAuth): Promise<RefreshResult> {
  const url = `${auth.baseUrl}/api/v1/workflow-auth/refresh`
  let res: Response
  try {
    res = await fetch(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${auth.refreshToken}`,
      },
    })
  } catch (err) {
    // Synthetic 0 status so callers treat network errors as transient.
    workflowTokenRefreshCounter.inc({ recipe: auth.recipeName ?? '<unknown>', outcome: 'failed' })
    return { ok: false, status: 0 }
  }

  if (!res.ok) {
    refreshFailureCounter.inc({ status: String(res.status) })
    workflowTokenRefreshCounter.inc({ recipe: auth.recipeName ?? '<unknown>', outcome: 'failed' })
    return { ok: false, status: res.status }
  }

  const data = (await res.json()) as {
    accessToken?: string
    refreshToken?: string
    mcpHostControlToken?: string
    expiresInSeconds?: number
  }
  if (!data.accessToken) {
    throw new Error('Token refresh response missing accessToken')
  }
  if (!data.refreshToken) {
    throw new Error('Token refresh response missing refreshToken')
  }

  auth.accessToken = data.accessToken
  auth.refreshToken = data.refreshToken
  if (typeof data.mcpHostControlToken === 'string' && data.mcpHostControlToken.trim()) {
    auth.mcpHostControlToken = data.mcpHostControlToken
  }
  const binding = getJwtRuntimeBinding(data.accessToken)
  if (!binding) {
    throw new Error('Refreshed runtime access token missing canonical mcp-host binding claims')
  }
  Object.assign(auth, binding)
  if (auth.persistRotatedTokens) {
    try {
      await auth.persistRotatedTokens({
        accessToken: data.accessToken,
        refreshToken: data.refreshToken,
        ...(auth.mcpHostControlToken ? { mcpHostControlToken: auth.mcpHostControlToken } : {}),
      })
    } catch (err) {
      console.warn(
        '[UserApprovalRequester] Failed to persist rotated approval tokens:',
        err instanceof Error ? err.message : String(err)
      )
    }
  }

  workflowTokenRefreshCounter.inc({ recipe: auth.recipeName ?? '<unknown>', outcome: 'succeeded' })
  return { ok: true }
}

/** Recovery path. Mutex'd so concurrent refresh failures share one /issue call. */
async function recoverTokenPair(auth: McpHostRuntimeAuth): Promise<void> {
  const inflight = inflightReIssues.get(auth)
  if (inflight) {
    await inflight
    return
  }

  if (!auth.reIssueTokens) {
    throw new Error(
      'Approval refresh persistently returned 401 and no reIssueTokens recovery callback is configured'
    )
  }

  const reIssuer = auth.reIssueTokens

  const job = (async () => {
    try {
      console.info(
        `[UserApprovalRequester] Token pair re-issue requested (recipe=${recipeLabel(auth)})`
      )
      const newPair = await reIssuer()
      if (!newPair?.accessToken || !newPair?.refreshToken) {
        throw new Error('reIssueTokens returned an empty or invalid token pair')
      }
      auth.accessToken = newPair.accessToken
      auth.refreshToken = newPair.refreshToken
      if (typeof newPair.mcpHostControlToken === 'string' && newPair.mcpHostControlToken.trim()) {
        auth.mcpHostControlToken = newPair.mcpHostControlToken
      }
      const binding = getJwtRuntimeBinding(newPair.accessToken)
      if (!binding) {
        throw new Error('Re-issued runtime access token missing canonical mcp-host binding claims')
      }
      Object.assign(auth, binding)
      if (auth.persistRotatedTokens) {
        try {
          await auth.persistRotatedTokens({
            accessToken: newPair.accessToken,
            refreshToken: newPair.refreshToken,
            ...(auth.mcpHostControlToken ? { mcpHostControlToken: auth.mcpHostControlToken } : {}),
          })
        } catch (err) {
          console.warn(
            '[UserApprovalRequester] Failed to persist re-issued approval tokens:',
            err instanceof Error ? err.message : String(err)
          )
        }
      }
      reIssueCounter.inc({ result: 'success' })
      console.info(
        `[UserApprovalRequester] Token pair re-issued successfully (recipe=${recipeLabel(auth)})`
      )
    } catch (err) {
      reIssueCounter.inc({ result: 'failed' })
      const message = err instanceof Error ? err.message : String(err)
      console.error(
        `[UserApprovalRequester] Workflow auth re-issue failed (recipe=${recipeLabel(auth)}): ${message}`
      )
      throw err
    }
  })()

  inflightReIssues.set(auth, job)
  try {
    await job
  } finally {
    if (inflightReIssues.get(auth) === job) {
      inflightReIssues.delete(auth)
    }
  }
}

async function refreshWithRecoveryInner(auth: McpHostRuntimeAuth): Promise<void> {
  if (reloadPersistedAuthIfAvailable(auth)) {
    return
  }

  let consecutive401 = 0
  let lastStatus: number | undefined

  for (let attempt = 1; attempt <= REFRESH_MAX_ATTEMPTS; attempt++) {
    const result = await attemptRefresh(auth)
    if (result.ok) {
      return
    }

    lastStatus = result.status

    if (result.status === 401) {
      if (reloadPersistedAuthIfAvailable(auth)) {
        return
      }
      consecutive401 += 1
      console.warn(
        `[UserApprovalRequester] Refresh failed with 401; attempting workflow auth re-issue (recipe=${recipeLabel(auth)}, attempt=${attempt}, consecutive401=${consecutive401})`
      )
      if (consecutive401 >= REFRESH_CONSECUTIVE_401_THRESHOLD) {
        if (reloadPersistedAuthIfAvailable(auth)) {
          return
        }
        await recoverTokenPair(auth)
        return
      }
    } else {
      // Reset on non-401 so transient 5xx doesn't trigger recovery.
      consecutive401 = 0
    }

    if (attempt < REFRESH_MAX_ATTEMPTS) {
      const delay =
        REFRESH_BACKOFF_MS[attempt - 1] ?? REFRESH_BACKOFF_MS[REFRESH_BACKOFF_MS.length - 1]!
      await sleep(delay)
    }
  }

  throw new Error(
    `Token refresh failed after ${REFRESH_MAX_ATTEMPTS} attempts (last status=${lastStatus ?? 'unknown'})`
  )
}

/** Refresh with retry; escalates to recoverTokenPair on persistent 401s. */
export async function refreshWithRecovery(auth: McpHostRuntimeAuth): Promise<void> {
  try {
    await refreshWithRecoveryInner(auth)
    recordRuntimeAuthRecoverySuccess()
  } catch (err) {
    recordRuntimeAuthRecoveryFailure('refresh_recovery_failed')
    throw err
  }
}

async function requestApproval(
  params: ApprovalGateParams,
  auth: McpHostRuntimeAuth
): Promise<{ id: string }> {
  const url = `${auth.baseUrl}/api/v1/workflow-approvals/request`
  // Idempotency key collapses retried POSTs within one execution.
  const idempotencyKey =
    params.idempotencyKeyOverride?.trim() ||
    createHash('sha256')
      .update(
        `${params.executionId ?? ''}:${params.stepId}:${params.target.userId ?? ''}:${params.target.teamId ?? ''}:${params.message}`
      )
      .digest('hex')

  // At most one post-recovery retry; otherwise propagate to avoid 401 loops.
  let refreshedAfterUnauthorized = false

  for (let attempt = 1; attempt <= REQUEST_MAX_ATTEMPTS; attempt++) {
    const res = await fetch(url, {
      method: 'POST',
      signal: AbortSignal.timeout(15_000),
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${auth.accessToken}`,
        'Idempotency-Key': idempotencyKey,
      },
      body: JSON.stringify({
        // Default to the token recipe binding. Trigger-bound workflow approvals
        // may override this with the target recipe; control-api still validates
        // the typed workflowTrigger intent and caller binding before accepting it.
        recipeNamespace: params.approvalRecipe?.recipeNamespace ?? auth.recipeNamespace,
        recipeName: params.approvalRecipe?.recipeName ?? auth.recipeName,
        target: params.target,
        payload: {
          message: params.message,
          ...(params.payloadMetadata !== undefined || params.runtimeMcpHostRef
            ? { metadata: approvalPayloadMetadata(params) }
            : {}),
        },
        ...(params.timeoutSeconds ? { ttlSeconds: params.timeoutSeconds } : {}),
        ...(params.workflowTriggerRunIntent
          ? { workflowTriggerRunIntent: params.workflowTriggerRunIntent }
          : {}),
        ...(params.executionId
          ? { correlation: { taskId: params.executionId, stepId: params.stepId } }
          : { correlation: { stepId: params.stepId } }),
        ...(params.runBindingProof ? { workflowRunBindingProof: params.runBindingProof } : {}),
      }),
    })

    if (res.status === 401 && !refreshedAfterUnauthorized) {
      await refreshWithRecovery(auth)
      refreshedAfterUnauthorized = true
      continue
    }

    if (res.status === 409) {
      const data = (await res.json().catch(() => ({}))) as {
        approvalRequestId?: string
        status?: string
        error?: string
      }
      if (
        params.runBindingProof &&
        data.error === 'workflow_approval_run_binding_invalid' &&
        attempt < REQUEST_MAX_ATTEMPTS
      ) {
        await sleep(retryDelayMs(attempt))
        continue
      }
      if (!data.approvalRequestId) {
        throw new Error(
          data.error === 'workflow_approval_run_binding_invalid'
            ? 'Approval request failed (409)'
            : 'Approval request conflict response missing approvalRequestId'
        )
      }
      return { id: data.approvalRequestId }
    }

    if (!res.ok) {
      if (isRetryableStatus(res.status) && attempt < REQUEST_MAX_ATTEMPTS) {
        await sleep(retryDelayMs(attempt))
        continue
      }
      const errorCode = await approvalErrorCodeFromResponse(res)
      if (res.status === 422 && errorCode === 'idempotency_key_payload_mismatch') {
        throw new Error(
          'Workflow approval request conflicted with an earlier trigger attempt. Send the workflow request again as a new channel message, or change the workflow inputs.'
        )
      }
      throw new Error(`Approval request failed (${res.status})`)
    }

    const data = (await res.json()) as {
      approvalRequestId?: string
      status?: string
      expiresAt?: string
    }
    if (!data.approvalRequestId) {
      throw new Error('Approval request response missing approvalRequestId')
    }
    return { id: data.approvalRequestId }
  }

  throw new Error('Approval request failed after retry budget was exhausted')
}

async function pollStatus(
  approvalId: string,
  accessToken: string,
  baseUrl: string
): Promise<PollStatusResult> {
  const url = `${baseUrl}/api/v1/workflow-approvals/${approvalId}/status`
  for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) {
    const res = await fetch(url, {
      method: 'GET',
      signal: AbortSignal.timeout(15_000),
      headers: {
        Authorization: `Bearer ${accessToken}`,
      },
    })

    if (res.status === 401) {
      return { ok: false, reason: 'unauthorized' }
    }

    if (!res.ok) {
      if (isRetryableStatus(res.status) && attempt < POLL_MAX_ATTEMPTS) {
        await sleep(retryDelayMs(attempt))
        continue
      }
      throw new Error(`Approval status poll failed (${res.status})`)
    }

    const data = (await res.json()) as {
      status: string
      expiresAt?: string
      decisionMaker?: unknown
      note?: string
    }
    return {
      ok: true,
      status: data.status,
      decisionMaker: data.decisionMaker,
      note: data.note,
    }
  }

  throw new Error(
    `Approval status poll failed after retry budget was exhausted for request ${approvalId}`
  )
}

/** Best-effort approval cancel on local timeout. Never throws. */
async function cancelApprovalSafely(approvalId: string, auth: McpHostRuntimeAuth): Promise<void> {
  try {
    const url = `${auth.baseUrl}/api/v1/workflow-approvals/${approvalId}/cancel`
    const res = await fetch(url, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${auth.accessToken}`,
      },
      signal: AbortSignal.timeout(5_000),
    })
    if (!res.ok && res.status !== 404 && res.status !== 409) {
      console.warn(
        `[UserApprovalRequester] Timeout cancel returned non-OK status ${res.status} for ${approvalId}`
      )
    }
  } catch (err) {
    console.warn(
      `[UserApprovalRequester] Timeout cancel failed for ${approvalId}:`,
      err instanceof Error ? err.message : String(err)
    )
  }
}

/** Polls until terminal. Throws if denied, expired, or cancelled. */
export async function gateStep(
  params: ApprovalGateParams,
  auth: McpHostRuntimeAuth
): Promise<ApprovalGateResult> {
  const deadline =
    params.timeoutSeconds != null ? Date.now() + params.timeoutSeconds * 1000 : Infinity

  const { id: approvalId } = await requestApproval(params, auth)

  // At most one recovery per gate to prevent loops on revoked auth.
  let recoveredDuringPoll = false
  while (Date.now() < deadline) {
    const result = await pollStatus(approvalId, auth.accessToken, auth.baseUrl)

    if (!result.ok) {
      if (recoveredDuringPoll) {
        throw new Error(
          `Approval poll returned 401 after token recovery (request ${approvalId}); upstream auth appears broken`
        )
      }
      await refreshWithRecovery(auth)
      recoveredDuringPoll = true
      continue
    }

    if (TERMINAL_STATUSES.has(result.status)) {
      if (
        result.status === 'approved' ||
        (result.status === 'consumed' && params.allowConsumedTerminal)
      ) {
        return {
          approvalRequestId: approvalId,
          status: result.status,
          decidedBy: result.decisionMaker,
          note: result.note,
        }
      }
      throw new Error(
        `Approval ${result.status} for step (request ${approvalId})` +
          (result.note ? `: ${result.note}` : '')
      )
    }

    await sleep(POLL_INTERVAL_MS + Math.random() * POLL_JITTER_MS)
  }

  await cancelApprovalSafely(approvalId, auth)
  throw new Error(
    `Approval polling timed out after ${params.timeoutSeconds}s for request ${approvalId}`
  )
}