Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.4 KB
Raw Blame
/**
 * Tests for the refresh-rotation recovery path in userApprovalRequester:
 * persistent refresh 401 → caller-supplied `reIssueTokens` callback →
 * retry the original operation once.
 */
import { type Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'node:fs/promises'
import * as os from 'node:os'
import * as path from 'node:path'
import { persistRuntimeAuthTokens } from '../mcpHostJwtState'
import { resetRuntimeAuthHealthForTests, runtimeAuthHealthSnapshot } from '../runtimeAuthHealth'
import {
  type ApprovalGateParams,
  type McpHostRuntimeAuth,
  type ReIssuedTokenPair,
  gateStep,
  reIssueCounter,
  refreshFailureCounter,
  workflowTokenRefreshCounter,
} from '../userApprovalRequester'

const mockFetch = vi.fn() as Mock
vi.stubGlobal('fetch', mockFetch)
const tempDirs: string[] = []

function makeAuth(
  overrides: Partial<McpHostRuntimeAuth> = {}
): McpHostRuntimeAuth & { reIssueTokens: Mock<() => Promise<ReIssuedTokenPair>> } {
  const reIssueTokens = vi.fn<() => Promise<ReIssuedTokenPair>>()
  return {
    accessToken: RECIPE_TEST_ACCESS_TOKEN,
    refreshToken: 'initial-refresh',
    baseUrl: 'http://gateway:8092',
    hostRef: 'sandbox-recipes/recipe-test',
    recipeNamespace: 'sandbox-recipes',
    recipeName: 'recipe-test',
    reIssueTokens,
    ...overrides,
  } as McpHostRuntimeAuth & { reIssueTokens: Mock<() => Promise<ReIssuedTokenPair>> }
}

function makeRuntimeJwt(
  binding: {
    hostRefs: string[]
    recipeNamespace: string
    recipeName: string
  } & Record<string, unknown>
): string {
  const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url')
  const payload = Buffer.from(JSON.stringify(binding)).toString('base64url')
  return `${header}.${payload}.sig`
}

const RECIPE_TEST_ACCESS_TOKEN = makeRuntimeJwt({
  hostRefs: ['sandbox-recipes/recipe-test'],
  recipeNamespace: 'sandbox-recipes',
  recipeName: 'recipe-test',
})

const REISSUED_ACCESS_TOKEN = makeRuntimeJwt({
  hostRefs: ['sandbox-recipes/reissued-recipe'],
  recipeNamespace: 'sandbox-recipes',
  recipeName: 'reissued-recipe',
})

function jsonResponse(body: unknown, init?: { status?: number; ok?: boolean }): unknown {
  const status = init?.status ?? 200
  return {
    ok: init?.ok ?? (status >= 200 && status < 300),
    status,
    json: async () => body,
    text: async () => JSON.stringify(body),
  }
}

function textResponse(status: number, body: string): unknown {
  return {
    ok: status >= 200 && status < 300,
    status,
    json: async () => {
      throw new Error('not json')
    },
    text: async () => body,
  }
}

const PARAMS: ApprovalGateParams = {
  stepId: 'step-recovery',
  executionId: 'exec-recovery',
  target: { userId: 'user-1' },
  message: 'approve this',
}

describe('userApprovalRequester — refresh rotation crash recovery', () => {
  beforeEach(() => {
    vi.clearAllMocks()
    mockFetch.mockReset()
    // Reset prom-client counters between tests so assertions are isolated.
    refreshFailureCounter.reset()
    reIssueCounter.reset()
    workflowTokenRefreshCounter.reset()
    resetRuntimeAuthHealthForTests()
    vi.useFakeTimers()
  })

  afterEach(async () => {
    vi.useRealTimers()
    vi.unstubAllEnvs()
    await Promise.all(tempDirs.splice(0).map(dir => fs.rm(dir, { recursive: true, force: true })))
  })

  it('does not invoke re-issue when refresh is not needed', async () => {
    const auth = makeAuth()

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-ok',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'approved' }, { status: 200 }))

    const promise = gateStep(PARAMS, auth)
    await vi.advanceTimersByTimeAsync(1_000)
    const result = await promise

    expect(result.status).toBe('approved')
    expect(auth.reIssueTokens).not.toHaveBeenCalled()
    expect(
      mockFetch.mock.calls.some(
        ([url]) => typeof url === 'string' && url.includes('/workflow-auth/refresh')
      )
    ).toBe(false)
  })

  it('rejects consumed approvals by default because they are already spent', async () => {
    const auth = makeAuth()

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-consumed',
        status: 'consumed',
      })
    )
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'consumed' }, { status: 200 }))

    const assertion = expect(gateStep(PARAMS, auth)).rejects.toThrow('Approval consumed')
    await vi.advanceTimersByTimeAsync(1_000)
    await assertion
  })

  it('treats consumed as terminal only when workflow trigger retries opt in', async () => {
    const auth = makeAuth()

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-consumed',
        status: 'consumed',
      })
    )
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'consumed' }, { status: 200 }))

    const promise = gateStep({ ...PARAMS, allowConsumedTerminal: true }, auth)
    await vi.advanceTimersByTimeAsync(1_000)
    const result = await promise

    expect(result).toMatchObject({
      approvalRequestId: 'approval-consumed',
      status: 'consumed',
    })
  })

  // ── Test 2: single refresh 401 then 200 — no re-issue ────────────────

  it('recovers from a single refresh 401 followed by 200 without re-issuing', async () => {
    const auth = makeAuth()

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-single-401',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    mockFetch.mockResolvedValueOnce(textResponse(401, 'refresh token expired'))
    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        accessToken: RECIPE_TEST_ACCESS_TOKEN,
        refreshToken: 'rotated-refresh',
        expiresInSeconds: 600,
      })
    )
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'approved' }, { status: 200 }))

    const promise = gateStep(PARAMS, auth)
    // 1s backoff between refresh attempts → 2s window covers it.
    await vi.advanceTimersByTimeAsync(2_000)
    const result = await promise

    expect(result.status).toBe('approved')
    expect(auth.accessToken).toBe(RECIPE_TEST_ACCESS_TOKEN)
    expect(auth.refreshToken).toBe('rotated-refresh')
    expect(auth.reIssueTokens).not.toHaveBeenCalled()
    expect(runtimeAuthHealthSnapshot()).toMatchObject({
      state: 'ok',
      consecutiveFailures: 0,
      lastFailureReason: null,
    })
    const metric = await workflowTokenRefreshCounter.get()
    expect(metric.values).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          value: 1,
          labels: { recipe: 'recipe-test', outcome: 'failed' },
        }),
        expect.objectContaining({
          value: 1,
          labels: { recipe: 'recipe-test', outcome: 'succeeded' },
        }),
      ])
    )
  })

  it('labels persisted auth reloads separately from network refresh success', async () => {
    const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), 'workflow-auth-state-'))
    tempDirs.push(stateDir)
    vi.stubEnv('MCP_HOST_RUNTIME_AUTH_STATE_DIR', stateDir)
    const nowSecs = Math.floor(Date.now() / 1000)
    const persistedAccessToken = makeRuntimeJwt({
      hostRefs: ['sandbox-recipes/recipe-test'],
      recipeNamespace: 'sandbox-recipes',
      recipeName: 'recipe-test',
      exp: nowSecs + 600,
    })
    const persistedRefreshToken = makeRuntimeJwt({
      hostRefs: ['sandbox-recipes/recipe-test'],
      recipeNamespace: 'sandbox-recipes',
      recipeName: 'recipe-test',
      exp: nowSecs + 1200,
    })
    await persistRuntimeAuthTokens(
      {
        accessToken: persistedAccessToken,
        refreshToken: persistedRefreshToken,
      },
      stateDir
    )
    const auth = makeAuth({ refreshToken: 'stale-refresh' })

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-state-reload',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'approved' }, { status: 200 }))

    const promise = gateStep(PARAMS, auth)
    await vi.advanceTimersByTimeAsync(1_000)
    const result = await promise

    expect(result.status).toBe('approved')
    expect(auth.accessToken).toBe(persistedAccessToken)
    expect(auth.refreshToken).toBe(persistedRefreshToken)
    expect(
      mockFetch.mock.calls.some(
        ([url]) => typeof url === 'string' && url.includes('/workflow-auth/refresh')
      )
    ).toBe(false)
    const metric = await workflowTokenRefreshCounter.get()
    expect(metric.values).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          value: 1,
          labels: { recipe: 'recipe-test', outcome: 'loaded_from_state' },
        }),
      ])
    )
    expect(metric.values).not.toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          labels: { recipe: 'recipe-test', outcome: 'succeeded' },
        }),
      ])
    )
  })

  it('reloads persisted auth after a refresh 401 race before re-issuing', async () => {
    const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), 'workflow-auth-state-'))
    tempDirs.push(stateDir)
    vi.stubEnv('MCP_HOST_RUNTIME_AUTH_STATE_DIR', stateDir)
    const nowSecs = Math.floor(Date.now() / 1000)
    const racedAccessToken = makeRuntimeJwt({
      hostRefs: ['sandbox-recipes/recipe-test'],
      recipeNamespace: 'sandbox-recipes',
      recipeName: 'recipe-test',
      iat: nowSecs + 1,
      exp: nowSecs + 600,
    })
    const racedRefreshToken = makeRuntimeJwt({
      hostRefs: ['sandbox-recipes/recipe-test'],
      recipeNamespace: 'sandbox-recipes',
      recipeName: 'recipe-test',
      iat: nowSecs + 1,
      exp: nowSecs + 1200,
    })
    const auth = makeAuth({ refreshToken: 'stale-refresh' })

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-refresh-race',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    mockFetch.mockImplementationOnce(async () => {
      await persistRuntimeAuthTokens(
        {
          accessToken: racedAccessToken,
          refreshToken: racedRefreshToken,
        },
        stateDir
      )
      return textResponse(401, 'refresh token already rotated')
    })
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'approved' }, { status: 200 }))

    const promise = gateStep(PARAMS, auth)
    await vi.advanceTimersByTimeAsync(1_000)
    const result = await promise

    expect(result.status).toBe('approved')
    expect(auth.accessToken).toBe(racedAccessToken)
    expect(auth.refreshToken).toBe(racedRefreshToken)
    expect(auth.reIssueTokens).not.toHaveBeenCalled()
    const refreshCalls = mockFetch.mock.calls.filter(
      ([url]) => typeof url === 'string' && url.includes('/workflow-auth/refresh')
    )
    expect(refreshCalls).toHaveLength(1)
    const metric = await workflowTokenRefreshCounter.get()
    expect(metric.values).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          value: 1,
          labels: { recipe: 'recipe-test', outcome: 'loaded_from_state' },
        }),
      ])
    )
  })

  it('invokes re-issue after 2 consecutive refresh 401s, then retries the original op', async () => {
    const auth = makeAuth()
    auth.reIssueTokens.mockResolvedValueOnce({
      accessToken: REISSUED_ACCESS_TOKEN,
      refreshToken: 'reissued-refresh',
    })

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-reissue',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
    // Second 401 reaches the threshold → recovery via reIssueTokens.
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
    mockFetch.mockResolvedValueOnce(jsonResponse({ status: 'approved' }, { status: 200 }))

    const promise = gateStep(PARAMS, auth)
    await vi.advanceTimersByTimeAsync(3_000)
    const result = await promise

    expect(result.status).toBe('approved')
    expect(auth.reIssueTokens).toHaveBeenCalledTimes(1)
    expect(auth.accessToken).toContain('.')
    expect(auth.refreshToken).toBe('reissued-refresh')
    expect(auth.hostRef).toBe('sandbox-recipes/reissued-recipe')
    expect(auth.recipeNamespace).toBe('sandbox-recipes')
    expect(auth.recipeName).toBe('reissued-recipe')
  })

  it('propagates the error when re-issue itself fails', async () => {
    vi.stubEnv('MCP_HOST_RUNTIME_AUTH_DEGRADED_AFTER_FAILURES', '1')
    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    try {
      const auth = makeAuth()
      auth.reIssueTokens.mockRejectedValueOnce(
        new Error('WRC /auth/mcp-host/:ns/:name/tokens returned HTTP 500')
      )

      mockFetch.mockResolvedValueOnce(
        jsonResponse({
          approvalRequestId: 'approval-fail',
          status: 'pending',
          expiresAt: '2099-01-01T00:00:00Z',
        })
      )
      mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
      mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
      mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))

      const promise = gateStep(PARAMS, auth)
      const assertion = expect(promise).rejects.toThrow(
        /WRC \/auth\/mcp-host\/:ns\/:name\/tokens returned HTTP 500/
      )
      await vi.advanceTimersByTimeAsync(3_000)
      await assertion

      expect(auth.reIssueTokens).toHaveBeenCalledTimes(1)
      expect(runtimeAuthHealthSnapshot()).toMatchObject({
        state: 'degraded',
        consecutiveFailures: 1,
        lastFailureReason: 'refresh_recovery_failed',
      })
      expect(consoleError).toHaveBeenCalledWith(
        expect.stringContaining('Workflow auth re-issue failed')
      )
    } finally {
      consoleError.mockRestore()
    }
  })

  it('does NOT re-issue again if the original operation fails with 401 after recovery', async () => {
    const auth = makeAuth()
    auth.reIssueTokens.mockResolvedValueOnce({
      accessToken: REISSUED_ACCESS_TOKEN,
      refreshToken: 'reissued-refresh',
    })

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-loop-guard',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    // First poll → 401
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    // Refresh attempts both 401 → triggers re-issue
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
    // Re-issued tokens are also rejected (e.g. admin revocation persists).
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })

    const promise = gateStep(PARAMS, auth)
    const assertion = expect(promise).rejects.toThrow(/upstream auth appears broken/)
    await vi.advanceTimersByTimeAsync(3_000)
    await assertion

    expect(auth.reIssueTokens).toHaveBeenCalledTimes(1)
  })

  it('serialises concurrent re-issue attempts on the same auth object', async () => {
    // Real timers: the mutex contract is wall-clock driven — both gateStep
    // invocations must reach `recoverTokenPair` on the same auth object
    // before the first one finishes.
    vi.useRealTimers()

    const auth = makeAuth()
    // Barrier: first reIssue blocks until released so the second caller
    // observes the in-flight promise in the mutex map.
    let releaseReIssue: (v: ReIssuedTokenPair) => void
    const reIssueGate = new Promise<ReIssuedTokenPair>(resolve => {
      releaseReIssue = resolve
    })
    auth.reIssueTokens.mockImplementation(() => reIssueGate)

    let requestCount = 0
    let refreshCount = 0
    mockFetch.mockImplementation(async (url: string, init?: RequestInit) => {
      if (url.endsWith('/workflow-approvals/request') && init?.method === 'POST') {
        requestCount += 1
        return jsonResponse({
          approvalRequestId: `approval-concurrent-${requestCount}`,
          status: 'pending',
          expiresAt: '2099-01-01T00:00:00Z',
        })
      }
      if (url.includes('/workflow-approvals/') && url.endsWith('/status')) {
        const isApproved = auth.accessToken === REISSUED_ACCESS_TOKEN
        if (isApproved) {
          return jsonResponse({ status: 'approved' }, { status: 200 })
        }
        return { ok: false, status: 401 } as unknown
      }
      if (url.endsWith('/workflow-auth/refresh')) {
        refreshCount += 1
        return textResponse(401, 'revoked')
      }
      throw new Error(`Unexpected fetch: ${init?.method ?? 'GET'} ${url}`)
    })

    const p1 = gateStep(PARAMS, auth)
    const p2 = gateStep(
      { ...PARAMS, stepId: 'step-recovery-b', executionId: 'exec-recovery-b' },
      auth
    )

    // Wait for the first reIssueTokens invocation; refresh backoff is 1s
    // between attempts so ≥1.5s gives both gates time to reach recovery.
    const started = Date.now()
    while (auth.reIssueTokens.mock.calls.length === 0 && Date.now() - started < 5_000) {
      await new Promise(resolve => setTimeout(resolve, 20))
    }
    await new Promise(resolve => setTimeout(resolve, 1_500))

    releaseReIssue!({ accessToken: REISSUED_ACCESS_TOKEN, refreshToken: 'reissued-refresh' })

    const [r1, r2] = await Promise.all([p1, p2])
    expect(r1.status).toBe('approved')
    expect(r2.status).toBe('approved')
    expect(auth.reIssueTokens).toHaveBeenCalledTimes(1)
    expect(refreshCount).toBeGreaterThanOrEqual(2)
  }, 15_000)

  it('throws a descriptive error when persistent 401 has no reIssueTokens callback', async () => {
    const auth = makeAuth()
    delete (auth as { reIssueTokens?: unknown }).reIssueTokens

    mockFetch.mockResolvedValueOnce(
      jsonResponse({
        approvalRequestId: 'approval-no-recovery',
        status: 'pending',
        expiresAt: '2099-01-01T00:00:00Z',
      })
    )
    mockFetch.mockResolvedValueOnce({ ok: false, status: 401 })
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))
    mockFetch.mockResolvedValueOnce(textResponse(401, 'revoked'))

    const promise = gateStep(PARAMS, auth)
    const assertion = expect(promise).rejects.toThrow(
      /no reIssueTokens recovery callback is configured/
    )
    await vi.advanceTimersByTimeAsync(3_000)
    await assertion
  })
})