Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
11.5 KB
Raw Blame
/**
 * Integration: WorkflowRecipe full lifecycle with coordinator + mcp_host Pods
 *
 * Validates the two-Pod model (coordinator + mcp_host) for agentic workflow execution:
 *   1. WorkflowRecipe with spec.steps[] created via control-api admin endpoint
 *   2. WRC reconciler creates coordinator Pod + mcp_host Pod in sandbox-recipes
 *   3. Coordinator drives step execution via mcp_host
 *   4. Recipe reaches completed or failed phase (not hanging)
 *   5. Status is reflected in WorkflowRecipe CRD status
 *
 * Requires:
 *   - minikube cluster (clerum-test) running
 *   - control-api port-forwarded on :8090  (make minikube-pf-control-ui)
 *   - WRC running with CLERUM_COORDINATOR_IMAGE=clerum/workflow-coordinator:test
 *   - CLERUM_MCP_HOST_IMAGE=clerum/mcp-host:test
 *   - clerum-model-secret-mapping ConfigMap in mcp-host ns with a `zai` key (R1: keyed by provider)
 *   - chatllm-api-keys Secret in mcp-host ns with zai-api-key populated (single source of truth post-refactor)
 *
 * Model: zai / glm-4.7 (default for all steps)
 */
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { execSync } from 'node:child_process'
import {
  CONTROL_API_URL,
  bearer,
  deleteJson,
  fetchJson,
  isServiceUp,
  postJson,
} from './helpers.integration.js'

// ─── Config ─────────────────────────────────────────────────────────────────

/** Max wait time for workflow to reach terminal phase (completed/failed) */
const WORKFLOW_TIMEOUT_MS = 5 * 60 * 1000 // 5 minutes
/** How often to poll workflow status */
const POLL_INTERVAL_MS = 10_000
/** Test recipe name — prefix with wf- to identify as workflow (not workload) */
const SMOKE_RECIPE = 'wf-integration-smoke'
/** Namespace where coordinator + mcp_host Pods are created */
const SANDBOX_NS = 'sandbox-recipes'

// ─── State ──────────────────────────────────────────────────────────────────

let controlApiUp = false
let adminToken = ''

// ─── Helpers ─────────────────────────────────────────────────────────────────

/**
 * Execute kubectl and return stdout. Returns null on error.
 * Safe for read-only checks in tests (not destructive).
 */
function kubectl(args: string): string | null {
  try {
    return execSync(`kubectl ${args} --context clerum-test`, {
      encoding: 'utf-8',
      timeout: 10_000,
      stdio: ['pipe', 'pipe', 'pipe'],
    }).trim()
  } catch {
    return null
  }
}

/** Poll workflow phase until terminal or timeout. Returns final phase. */
async function waitForWorkflowPhase(
  recipeName: string,
  token: string,
  terminalPhases: string[],
  timeoutMs: number
): Promise<string> {
  const deadline = Date.now() + timeoutMs
  let lastPhase = 'unknown'

  while (Date.now() < deadline) {
    // control-api /status returns resource.status directly (not wrapped in { status: ... })
    const { status, data } = await fetchJson<{
      workflowExecution?: { phase?: string }
    }>(`${CONTROL_API_URL}/api/v1/admin/recipes/${recipeName}/status`, { headers: bearer(token) })

    if (status === 200 && data) {
      const phase = (data as { workflowExecution?: { phase?: string } }).workflowExecution?.phase
      if (phase) {
        lastPhase = phase
        if (terminalPhases.includes(lastPhase)) {
          return lastPhase
        }
      }
    }

    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS))
  }

  return lastPhase
}

/** Wait for a Pod matching labelSelector to appear in a namespace. */
async function waitForPod(
  namespace: string,
  labelSelector: string,
  timeoutMs = 60_000
): Promise<boolean> {
  const deadline = Date.now() + timeoutMs
  while (Date.now() < deadline) {
    const out = kubectl(`get pods -n ${namespace} -l ${labelSelector} --no-headers 2>/dev/null`)
    if (out && out.trim().length > 0) return true
    await new Promise(r => setTimeout(r, 3_000))
  }
  return false
}

/** Wait for kubectl jsonpath output to become available. */
async function waitForKubectlOutput(args: string, timeoutMs = 5_000): Promise<string | null> {
  const deadline = Date.now() + timeoutMs
  while (Date.now() < deadline) {
    const out = kubectl(args)
    if (out && out.trim().length > 0) return out
    await new Promise(r => setTimeout(r, 500))
  }
  return null
}

// ─── Setup / Teardown ────────────────────────────────────────────────────────

beforeAll(async () => {
  controlApiUp = await isServiceUp(CONTROL_API_URL)
  if (!controlApiUp) {
    console.log('[workflow-lifecycle] control-api not available — tests will be skipped')
    return
  }

  const adminUsername = process.env.TEST_ADMIN_USERNAME ?? 'admin'
  const adminPassword = process.env.TEST_ADMIN_PASSWORD ?? 'changeme123!'
  const { status, data } = await postJson<{ token?: string }>(
    `${CONTROL_API_URL}/api/v1/admin/auth/login`,
    { username: adminUsername, password: adminPassword }
  )

  if (status === 200 && data.token) {
    adminToken = data.token
  } else {
    console.log('[workflow-lifecycle] Admin login failed — tests will be skipped')
  }

  // Pre-cleanup: remove leftover recipe from a previous crashed run
  if (adminToken) {
    await deleteJson(`${CONTROL_API_URL}/api/v1/admin/recipes/${SMOKE_RECIPE}`, bearer(adminToken))
    // Remove any leftover Pods — ignore errors
    kubectl(
      `delete pods -n ${SANDBOX_NS} -l clerum.io/recipe=${SMOKE_RECIPE} --ignore-not-found=true`
    )
  }
})

afterAll(async () => {
  if (!adminToken) return
  // Best-effort cleanup
  await deleteJson(`${CONTROL_API_URL}/api/v1/admin/recipes/${SMOKE_RECIPE}`, bearer(adminToken))
  kubectl(
    `delete pods -n ${SANDBOX_NS} -l clerum.io/recipe=${SMOKE_RECIPE} --ignore-not-found=true`
  )
})

// ─── Tests ───────────────────────────────────────────────────────────────────

describe('WorkflowRecipe lifecycle — coordinator + mcp_host two-Pod model', () => {
  it('control-api is reachable', async () => {
    if (!controlApiUp) return
    const { status } = await fetchJson(`${CONTROL_API_URL}/health`)
    expect(status).toBe(200)
  })

  it('admin login succeeds', async () => {
    if (!controlApiUp) return
    expect(adminToken).toBeTruthy()
    expect(adminToken.length).toBeGreaterThan(10)
  })

  it('WRC is running in control-plane', () => {
    if (!controlApiUp) return
    const out = kubectl('get deployment workflow-recipes -n control-plane --no-headers')
    expect(out).toBeTruthy()
    expect(out).toContain('1/1')
  })

  it('creates WorkflowRecipe with two steps — default model zai/glm-4.7', async () => {
    if (!controlApiUp || !adminToken) return

    const recipeBody = {
      metadata: { name: SMOKE_RECIPE },
      spec: {
        // Pure workflow mode — no workloads[] (coordinator + mcp_host only)
        steps: [
          {
            id: 'step-hello',
            instruction: 'Respond with exactly: HELLO_WORKFLOW_OK',
          },
          {
            id: 'step-confirm',
            instruction: 'Respond with exactly: CONFIRM_OK',
            dependsOn: ['step-hello'],
          },
        ],
        // Provider + model applied to ALL steps (no per-step override)
        agent: {
          provider: 'zai',
          model: 'glm-4.7',
        },
      },
    }

    const { status, data } = await postJson<{ metadata?: { name: string } }>(
      `${CONTROL_API_URL}/api/v1/admin/recipes`,
      recipeBody,
      bearer(adminToken)
    )

    expect(status).toBe(201)
    expect(data.metadata?.name).toBe(SMOKE_RECIPE)
  }, 15_000)

  it('WRC creates coordinator Pod in sandbox-recipes within 60s', async () => {
    if (!controlApiUp || !adminToken) return

    const found = await waitForPod(
      SANDBOX_NS,
      `clerum.io/recipe=${SMOKE_RECIPE},clerum.io/component=workflow-coordinator`,
      60_000
    )

    expect(found).toBe(true)
  }, 70_000)

  it('WRC creates mcp_host Pod in sandbox-recipes within 60s', async () => {
    if (!controlApiUp || !adminToken) return

    const found = await waitForPod(
      SANDBOX_NS,
      `clerum.io/recipe=${SMOKE_RECIPE},clerum.io/component=workflow-mcp-host`,
      60_000
    )

    expect(found).toBe(true)
  }, 70_000)

  it('mcp_host Pod env has CLERUM_WORKFLOW_ENABLED=true', async () => {
    if (!controlApiUp || !adminToken) return

    const out = await waitForKubectlOutput(
      `get pods -n ${SANDBOX_NS} -l clerum.io/recipe=${SMOKE_RECIPE},clerum.io/component=workflow-mcp-host -o jsonpath='{.items[0].spec.containers[0].env}'`
    )
    // mcp_host should have CLERUM_WORKFLOW_ENABLED in its env
    // Soft assertion: pod may still be in ContainerCreating, skip check if not ready yet
    if (out && out.trim().length > 0) {
      expect(out).toContain('CLERUM_WORKFLOW_ENABLED')
    } else {
      console.log('[workflow-lifecycle] mcp_host pod env not yet available — skipping env check')
    }
  }, 15_000)

  it(
    'WorkflowRecipe reaches terminal phase (completed or failed) within 5 minutes',
    async () => {
      if (!controlApiUp || !adminToken) return

      const terminalPhases = ['completed', 'failed', 'timed_out']
      const finalPhase = await waitForWorkflowPhase(
        SMOKE_RECIPE,
        adminToken,
        terminalPhases,
        WORKFLOW_TIMEOUT_MS
      )

      // Both completed and failed are valid terminal states — the important
      // thing is that the workflow DID run and didn't hang forever.
      expect(terminalPhases).toContain(finalPhase)
      console.log(`[workflow-lifecycle] Final phase: ${finalPhase}`)
    },
    WORKFLOW_TIMEOUT_MS + 30_000
  )

  it('coordinator Pod reached terminal state (Succeeded or Failed)', async () => {
    if (!controlApiUp || !adminToken) return

    const out = kubectl(
      `get pods -n ${SANDBOX_NS} -l clerum.io/recipe=${SMOKE_RECIPE},clerum.io/component=workflow-coordinator --no-headers`
    )
    // Pod should be in Succeeded or Failed phase after workflow completes
    if (out && out.trim().length > 0) {
      const inTerminalState =
        out.includes('Completed') ||
        out.includes('Error') ||
        out.includes('OOMKilled') ||
        out.includes('CrashLoopBackOff') ||
        out.includes('0/1')
      // If the pod exists, it should not be in Running state indefinitely
      // (Completed = exit 0, Error/OOMKilled = exit 1 — both are expected)
      expect(inTerminalState || out.includes('Running')).toBe(true)
    }
    // If pod was cleaned up, that's also fine
  }, 15_000)

  it('WorkflowRecipe CRD has workflowExecution.phase after workflow runs', async () => {
    if (!controlApiUp || !adminToken) return

    const { status, data } = await fetchJson<Record<string, unknown>>(
      `${CONTROL_API_URL}/api/v1/admin/recipes/${SMOKE_RECIPE}/status`,
      { headers: bearer(adminToken) }
    )

    expect([200, 404]).toContain(status)
    if (status === 200 && data) {
      // control-api /status returns resource.status → workflowExecution.phase
      const phase = (data as { workflowExecution?: { phase?: string } }).workflowExecution?.phase
      if (phase !== undefined) {
        expect(typeof phase).toBe('string')
        expect(String(phase).length).toBeGreaterThan(0)
      }
    }
  }, 15_000)
})