Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Crash recovery for workflow Pods.
*
* Detects Pod failures, creates replacement Pods, manages attempt counters,
* and transitions workflow phase to `recovering`.
*
* Max 3 attempts, 120s recovery timeout.
* */
import * as k8s from '@kubernetes/client-node'
import { getErrorCode } from '../reconciler/k8sErrors'
import { WorkflowExecutionStatus, WorkflowPhase } from './types'
const MAX_ATTEMPTS = 3
const RECOVERABLE_WAITING_REASONS = new Set([
'CreateContainerConfigError',
'CreateContainerError',
'ImagePullBackOff',
'ErrImagePull',
'InvalidImageName',
'CrashLoopBackOff',
])
export interface CrashRecoveryResult {
action: 'none' | 'replace' | 'fail'
message: string
newPhase?: WorkflowPhase
newAttempt?: number
}
export interface PodReadiness {
phase?: string
ready: boolean
waitingReason?: string
schedulingReason?: string
/** Pod UID — changes on recreate, lets callers detect a restarted pod. */
uid?: string
}
export function evaluateCompletedRuntimePodRecovery(
currentExecution: WorkflowExecutionStatus | undefined,
componentName: string
): CrashRecoveryResult {
const attempt = currentExecution?.attempt ?? 0
if (attempt >= MAX_ATTEMPTS) {
return {
action: 'fail',
message: `${componentName} pod completed before workflow became terminal after ${MAX_ATTEMPTS} recovery attempts`,
newPhase: 'failed',
}
}
return {
action: 'replace',
message: `${componentName} pod completed before workflow became terminal; creating replacement (attempt ${attempt + 1}/${MAX_ATTEMPTS})`,
newPhase: 'recovering',
newAttempt: attempt + 1,
}
}
export function isRecoverableContainerWaitingReason(
podPhase: string | undefined,
containerWaitingReason: string | undefined
): boolean {
if (!containerWaitingReason || !RECOVERABLE_WAITING_REASONS.has(containerWaitingReason)) {
return false
}
return (
podPhase === 'Pending' ||
(podPhase === 'Running' && containerWaitingReason === 'CrashLoopBackOff')
)
}
export function evaluateCrashRecovery(
podPhase: string | undefined,
currentExecution: WorkflowExecutionStatus | undefined,
containerWaitingReason?: string
): CrashRecoveryResult {
// Pending image/config errors and Running CrashLoopBackOff pods will not
// become ready without replacing the pod, so keep them bounded.
if (isRecoverableContainerWaitingReason(podPhase, containerWaitingReason)) {
const attempt = currentExecution?.attempt ?? 0
if (attempt >= MAX_ATTEMPTS) {
return {
action: 'fail',
message: `Pod in ${podPhase} with ${containerWaitingReason} after ${MAX_ATTEMPTS} recovery attempts`,
newPhase: 'failed',
}
}
return {
action: 'replace',
message: `Pod in ${podPhase} with ${containerWaitingReason}; creating replacement (attempt ${attempt + 1}/${MAX_ATTEMPTS})`,
newPhase: 'recovering',
newAttempt: attempt + 1,
}
}
if (!podPhase || podPhase === 'Running' || podPhase === 'Pending') {
return { action: 'none', message: 'Pod is healthy' }
}
const attempt = currentExecution?.attempt ?? 0
if (podPhase === 'Succeeded') {
return { action: 'none', message: 'Pod completed successfully' }
}
// Pod is Failed or Unknown
if (attempt >= MAX_ATTEMPTS) {
return {
action: 'fail',
message: `Pod failed after ${MAX_ATTEMPTS} recovery attempts`,
newPhase: 'failed',
}
}
return {
action: 'replace',
message: `Pod in phase '${podPhase}' — creating replacement (attempt ${attempt + 1}/${MAX_ATTEMPTS})`,
newPhase: 'recovering',
newAttempt: attempt + 1,
}
}
export async function deletePodIfExists(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<void> {
try {
await coreApi.deleteNamespacedPod({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) !== 404) throw error
// 404 = already gone, safe to proceed
}
}
const DEFAULT_POD_DELETE_WAIT_MS = 30_000
const POD_DELETE_POLL_MS = 250
/** Poll until the pod is gone (404) or timeout. Returns true when deleted. */
export async function waitForPodDeletion(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string,
options: { timeoutMs?: number; pollIntervalMs?: number } = {}
): Promise<boolean> {
const timeoutMs = options.timeoutMs ?? DEFAULT_POD_DELETE_WAIT_MS
const pollIntervalMs = options.pollIntervalMs ?? POD_DELETE_POLL_MS
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if ((await getPodPresence(coreApi, name, namespace)).kind === 'absent') return true
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
}
return (await getPodPresence(coreApi, name, namespace)).kind === 'absent'
}
/**
* Discriminated GET result. `unread` is only for callers that skipped the GET;
* `getPodPresence` never returns it. Skip DELETE only on `absent` — a live
* object with an empty `status.phase` is `present`.
*/
export type PodPresence =
| { kind: 'absent' }
| { kind: 'present'; phase?: string }
| { kind: 'unread' }
export type ObservedPodPresence = Exclude<PodPresence, { kind: 'unread' }>
export async function getPodPresence(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<ObservedPodPresence> {
try {
const pod = await coreApi.readNamespacedPod({ name, namespace })
return { kind: 'present', phase: pod.status?.phase }
} catch (error: unknown) {
if (getErrorCode(error) === 404) return { kind: 'absent' }
throw error
}
}
export async function getPodPhase(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<string | undefined> {
const presence = await getPodPresence(coreApi, name, namespace)
switch (presence.kind) {
case 'absent':
return undefined
case 'present':
return presence.phase
default: {
const _exhaustive: never = presence
return _exhaustive
}
}
}
export async function getPodReadiness(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<PodReadiness> {
try {
const pod = await coreApi.readNamespacedPod({ name, namespace })
return inspectPodReadiness(pod)
} catch (error: unknown) {
if (getErrorCode(error) === 404) return { ready: false }
throw error
}
}
export function inspectPodReadiness(pod: k8s.V1Pod): PodReadiness {
const phase = pod.status?.phase
const readyCondition = pod.status?.conditions?.find(condition => condition.type === 'Ready')
const scheduledCondition = pod.status?.conditions?.find(
condition => condition.type === 'PodScheduled' && condition.status !== 'True'
)
const waitingReason = firstContainerWaitingReason(pod)
return {
phase,
ready: phase === 'Running' && readyCondition?.status === 'True',
...(waitingReason && { waitingReason }),
...(scheduledCondition?.reason && { schedulingReason: scheduledCondition.reason }),
...(pod.metadata?.uid && { uid: pod.metadata.uid }),
}
}
/**
* Returns the first container waiting reason that should influence recovery.
* Pending image/config errors and Running CrashLoopBackOff are actionable.
*/
export async function getContainerWaitingReason(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<string | undefined> {
try {
const pod = await coreApi.readNamespacedPod({ name, namespace })
const reason = firstContainerWaitingReason(pod)
if (pod.status?.phase === 'Pending') return reason
if (pod.status?.phase === 'Running' && reason === 'CrashLoopBackOff') return reason
return undefined
} catch (error: unknown) {
if (getErrorCode(error) === 404) return undefined
throw error
}
}
function firstContainerWaitingReason(pod: k8s.V1Pod): string | undefined {
const containerStatuses = pod.status?.containerStatuses ?? []
for (const cs of containerStatuses) {
const reason = (cs.state as { waiting?: { reason?: string } })?.waiting?.reason
if (reason) return reason
}
return undefined
}