Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
7.2 KB
Raw Blame
import { expect } from '@playwright/test'
import {
  UUID_RE,
  firstDataLine,
  runControlPostgresSql,
  sqlLiteral,
} from '../../../tests/e2e/gfsUiFixtures'
import {
  GFS_PLUGIN_NAMESPACE,
  GFS_PLUGIN_RECIPE,
  gfsWorkflowRunsAfter,
} from './workflowAgentChatGfsRuntime'
import { kubectl } from './workflowAgentChatTools'

type WorkflowRecipeMetadata = {
  labels?: Record<string, string>
  name?: string
  namespace?: string
  ownerReferences?: Array<{
    apiVersion?: string
    controller?: boolean
    kind?: string
    name?: string
    uid?: string
  }>
  uid?: string
}

type WorkflowRecipeResource = { metadata?: WorkflowRecipeMetadata }

// Covers one 25s Kubernetes delete, late-child observation, and the required
// 10s quiet window within the enclosing 900s Desktop journey budget.
const CLEANUP_TIMEOUT_MS = 75_000
const LATE_CHILD_QUIET_WINDOW_MS = 10_000

function workflowRecipeOrUndefined(name: string): WorkflowRecipeResource | undefined {
  const raw = kubectl(
    [
      '-n',
      GFS_PLUGIN_NAMESPACE,
      'get',
      'workflowrecipe',
      name,
      '--ignore-not-found=true',
      '-o',
      'json',
    ],
    undefined,
    20_000
  ).trim()
  return raw ? (JSON.parse(raw) as WorkflowRecipeResource) : undefined
}

function isVerifiedRunChild(
  child: WorkflowRecipeResource,
  input: { childName: string; parentUid: string; runId: string }
): boolean {
  const owners = child.metadata?.ownerReferences ?? []
  const owner = owners[0]
  return (
    child.metadata?.name === input.childName &&
    child.metadata?.namespace === GFS_PLUGIN_NAMESPACE &&
    child.metadata?.labels?.['clerum.io/workflow-run-id'] === input.runId &&
    child.metadata?.labels?.['clerum.io/parent-recipe'] === GFS_PLUGIN_RECIPE &&
    owners.length === 1 &&
    owner?.controller === true &&
    owner.apiVersion === 'clerum.io/v1alpha1' &&
    owner.kind === 'WorkflowRecipe' &&
    owner.name === GFS_PLUGIN_RECIPE &&
    owner.uid === input.parentUid
  )
}

function namesByLabel(namespace: string, resources: string, selector: string): string[] {
  return kubectl(
    ['-n', namespace, 'get', resources, '-l', selector, '-o', 'name'],
    undefined,
    20_000
  )
    .split('\n')
    .map(value => value.trim())
    .filter(Boolean)
    .sort()
}

function runChildNames(runId: string): string[] {
  return namesByLabel(GFS_PLUGIN_NAMESPACE, 'workflowrecipe', `clerum.io/workflow-run-id=${runId}`)
}

function deleteVerifiedRunChildren(
  runId: string,
  candidateNames: string[]
): { deleted: number; names: string[] } {
  const parentUid = workflowRecipeOrUndefined(GFS_PLUGIN_RECIPE)?.metadata?.uid ?? ''
  const discoveredNames = runChildNames(runId).map(name => name.slice(name.indexOf('/') + 1))
  const names = [...new Set([...candidateNames, ...discoveredNames])]
  let deleted = 0

  for (const childName of names) {
    if (childName === GFS_PLUGIN_RECIPE) {
      throw new Error(`Refusing to clean parent WorkflowRecipe for run ${runId}`)
    }
    const child = workflowRecipeOrUndefined(childName)
    if (!child) continue
    if (!parentUid || !isVerifiedRunChild(child, { childName, parentUid, runId })) {
      throw new Error(
        `Refusing to delete unverified workflow child ${GFS_PLUGIN_NAMESPACE}/${childName} for run ${runId}`
      )
    }
    kubectl(
      [
        '-n',
        GFS_PLUGIN_NAMESPACE,
        'delete',
        'workflowrecipe',
        childName,
        '--wait=true',
        '--timeout=25s',
      ],
      undefined,
      CLEANUP_TIMEOUT_MS
    )
    deleted += 1
  }
  return { deleted, names }
}

function leakedRuntimeResources(candidateNames: string[], runId: string) {
  const childNames = runChildNames(runId).map(name => name.slice(name.indexOf('/') + 1))
  const names = [...new Set([...candidateNames, ...childNames])]
  const sandboxResources = names.flatMap(name =>
    namesByLabel(
      GFS_PLUGIN_NAMESPACE,
      'pod,service,secret,networkpolicy,configmap,persistentvolumeclaim',
      `clerum.io/recipe=${name}`
    )
  )
  const transportResources = names.flatMap(name =>
    namesByLabel(
      'mcp-server',
      'mcpserver,context,service,deployment,secret',
      `clerum.io/recipe=${name}`
    )
  )
  return {
    childRecipes: runChildNames(runId),
    sandboxResources: [...new Set(sandboxResources)].sort(),
    transportResources: [...new Set(transportResources)].sort(),
  }
}

async function cleanupGfsWorkflowRun(
  runId: string,
  observedChildName: string | undefined
): Promise<void> {
  if (!UUID_RE.test(runId)) return

  const persistedChildName = firstDataLine(
    runControlPostgresSql(`
      UPDATE workflow_runs
         SET phase = 'Canceled', completed_at = COALESCE(completed_at, now()), updated_at = now()
       WHERE run_id = ${sqlLiteral(runId)}::uuid
         AND recipe_namespace = ${sqlLiteral(GFS_PLUGIN_NAMESPACE)}
         AND recipe_name = ${sqlLiteral(GFS_PLUGIN_RECIPE)}
      RETURNING child_recipe_name;
    `)
  )
  let candidates = [...new Set([observedChildName, persistedChildName].filter(Boolean))] as string[]

  let lastChildActivityAt = Date.now()
  await expect
    .poll(
      () => {
        const cleanup = deleteVerifiedRunChildren(runId, candidates)
        candidates = cleanup.names
        if (cleanup.deleted > 0) lastChildActivityAt = Date.now()
        return {
          ...leakedRuntimeResources(candidates, runId),
          quietWindowElapsed: Date.now() - lastChildActivityAt >= LATE_CHILD_QUIET_WINDOW_MS,
        }
      },
      {
        timeout: CLEANUP_TIMEOUT_MS,
        intervals: [500, 1_000, 2_000],
        message: `workflow run ${runId} should stop creating children after cancellation`,
      }
    )
    .toEqual({
      childRecipes: [],
      sandboxResources: [],
      transportResources: [],
      quietWindowElapsed: true,
    })

  runControlPostgresSql(`
    DELETE FROM workflow_approval_requests
     WHERE bound_workflow_run_id = ${sqlLiteral(runId)}::uuid
       AND recipe_namespace = ${sqlLiteral(GFS_PLUGIN_NAMESPACE)}
       AND recipe_name = ${sqlLiteral(GFS_PLUGIN_RECIPE)};
    DELETE FROM workflow_runs
     WHERE run_id = ${sqlLiteral(runId)}::uuid
       AND recipe_namespace = ${sqlLiteral(GFS_PLUGIN_NAMESPACE)}
       AND recipe_name = ${sqlLiteral(GFS_PLUGIN_RECIPE)};
  `)

  expect(leakedRuntimeResources(candidates, runId)).toEqual({
    childRecipes: [],
    sandboxResources: [],
    transportResources: [],
  })

  expect(
    firstDataLine(
      runControlPostgresSql(`
        SELECT
          (SELECT COUNT(*) FROM workflow_approval_requests
            WHERE bound_workflow_run_id = ${sqlLiteral(runId)}::uuid)::text || '|' ||
          (SELECT COUNT(*) FROM workflow_run_steps
            WHERE run_id = ${sqlLiteral(runId)}::uuid)::text || '|' ||
          (SELECT COUNT(*) FROM workflow_runs
            WHERE run_id = ${sqlLiteral(runId)}::uuid)::text;
      `)
    )
  ).toBe('0|0|0')
}

export async function cleanupGfsWorkflowRunsAfter(baselineIds: ReadonlySet<string>): Promise<void> {
  const errors: unknown[] = []
  for (const run of gfsWorkflowRunsAfter(baselineIds)) {
    try {
      await cleanupGfsWorkflowRun(run.runId, run.childName || undefined)
    } catch (error) {
      errors.push(error)
    }
  }
  if (errors.length > 0) {
    throw new AggregateError(errors, 'failed to clean every new GFS workflow run')
  }
}