Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
23.2 KB
Raw Blame
/**
 * DB-first WorkflowRun processor — Fase 4 of the DB-first refactor
 * (plan §serene-sauteeing-jellyfish.md §Fase 4).
 *
 * Replaces `WorkflowRunReconciler`'s CRD watch loop with a Postgres
 * LISTEN/NOTIFY consumer. Runs the non-owner state transitions:
 *
 *   Pending → (create child recipe) → Running   [processPending]
 *   Running → Succeeded|Failed|Canceled         [syncFromRecipeExecution]
 *   Running → Failed (timeout)                  [checkStuckRuns]
 *
 * Every UPDATE carries `WHERE owner_instance_id = $self AND phase = $expected`
 * as defense-in-depth: if another replica grabbed the row, we silently no-op
 * (rowCount=0) instead of colliding.
 *
 * The LISTEN session is isolated on a dedicated `pool.connect()` client —
 * the Postgres session pool must stay sticky for notifications to flow. A
 * 30-second poll tick is kept as a fallback for notifications that get
 * dropped across reconnects.
 */
import type { Pool, PoolClient } from 'pg'
import { type Logger, createLogger } from '../observability/logger.js'
import type { WorkflowRecipeCRD } from '../types.js'

// ─── Types ─────────────────────────────────────────────────────────────

export type WorkflowRunPhase = 'Pending' | 'Running' | 'Succeeded' | 'Failed' | 'Canceled'

/** Subset of workflow_runs we load when processing a notification. */
export interface DbRunRow {
  run_id: string
  recipe_namespace: string
  recipe_name: string
  phase: WorkflowRunPhase
  team_id: string | null
  usage_team_id: string | null
  actor_type: string
  actor_id: string | null
  inputs: Record<string, unknown> | null
  intermediate_parameters: Record<string, unknown> | null
  output_overrides: Record<string, unknown> | null
  trigger_source: string
  owner_instance_id: string | null
  max_duration_seconds: number | null
  started_at: string | null
  child_recipe_name: string | null
  child_recipe_namespace: string | null
}

export interface ChildRecipeRef {
  name: string
  namespace: string
}

export type ChildRecipeExistenceChecker = (child: ChildRecipeRef) => Promise<boolean>

/**
 * Caller-provided hook that builds + creates a child WorkflowRecipe for a
 * given run row. Lives in the caller so this module stays K8s-unaware (and
 * fully testable with a fake). Returns the `{ name, namespace }` of the
 * created child resource.
 */
export type ChildRecipeCreator = (run: DbRunRow) => Promise<ChildRecipeRef>

export interface DbRunProcessorOptions {
  /** Stamped onto `workflow_runs.owner_instance_id` at acquisition time. */
  instanceId: string
  /** Pool used for transactional work (SELECT FOR UPDATE, UPDATEs). */
  pool: Pool
  /** Fallback poll interval (ms) for runs that slipped past LISTEN. */
  runPollMs: number
  /** Builds + creates the child WorkflowRecipe for a run (injected). */
  createChildRecipe: ChildRecipeCreator
  /** Checks whether a previously created child WorkflowRecipe still exists. */
  childRecipeExists?: ChildRecipeExistenceChecker
  /** Called after the DB commit that claims a run as Running. */
  onRunStarted?: (runId: string) => void
  /** Called once the child recipe's status sync lands a terminal phase. */
  onRunTerminal?: (runId: string, phase: WorkflowRunPhase) => void
  /** Injection seams for tests. */
  connect?: () => Promise<PoolClient>
  logger?: Logger
}

export interface DbRunProcessor {
  start(): Promise<void>
  stop(): Promise<void>
  processPending(runId: string): Promise<void>
  syncFromRecipeExecution(recipe: WorkflowRecipeCRD): Promise<void>
  checkStuckRuns(): Promise<number>
}

// ─── SQL literal helpers ───────────────────────────────────────────────

const SELECT_RUN_FOR_UPDATE = `
  SELECT run_id, recipe_namespace, recipe_name, phase,
         team_id, usage_team_id, actor_type, actor_id, inputs, intermediate_parameters,
         output_overrides, trigger_source, owner_instance_id,
         max_duration_seconds, started_at,
         child_recipe_name, child_recipe_namespace
    FROM workflow_runs
   WHERE run_id = $1
   FOR UPDATE`

const CLAIM_PENDING = `
  UPDATE workflow_runs
     SET phase = 'Running',
         owner_instance_id = $1,
         child_recipe_name = $2,
         child_recipe_namespace = $3,
         started_at = now(),
         last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $4
     AND phase = 'Pending'
     AND (owner_instance_id IS NULL OR owner_instance_id = $1)`

const UPDATE_RUN_TERMINAL = `
  UPDATE workflow_runs
     SET phase = $1,
         completed_at = COALESCE($2::timestamptz, now()),
         last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $3
     AND (
       phase IN ('Pending', 'Running')
       OR (phase = 'Failed' AND $1 = 'Succeeded')
     )`

const UPDATE_RUN_PROGRESS = `
  UPDATE workflow_runs
     SET last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $1
     AND phase IN ('Pending', 'Running')`

const UPSERT_STEP = `
  INSERT INTO workflow_run_steps
    (run_id, step_id, phase, started_at, completed_at, output, tools_called, error, approval_binding_sha256)
  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
  ON CONFLICT (run_id, step_id) DO UPDATE
    SET phase = EXCLUDED.phase,
        started_at = COALESCE(workflow_run_steps.started_at, EXCLUDED.started_at),
        completed_at = COALESCE(EXCLUDED.completed_at, workflow_run_steps.completed_at),
        output = COALESCE(EXCLUDED.output, workflow_run_steps.output),
        tools_called = COALESCE(EXCLUDED.tools_called, workflow_run_steps.tools_called),
        error = COALESCE(EXCLUDED.error, workflow_run_steps.error),
        approval_binding_sha256 = COALESCE(
          workflow_run_steps.approval_binding_sha256,
          EXCLUDED.approval_binding_sha256
        )`

const FIND_STUCK_RUNS = `
  SELECT run_id, started_at, max_duration_seconds
    FROM workflow_runs
   WHERE phase = 'Running'
     AND owner_instance_id = $1
     AND max_duration_seconds IS NOT NULL
     AND started_at IS NOT NULL
     AND started_at < now() - (max_duration_seconds || ' seconds')::interval
   FOR UPDATE SKIP LOCKED
   LIMIT 50`

const FAIL_STUCK_RUN = `
  UPDATE workflow_runs
     SET phase = 'Failed',
         completed_at = now(),
         last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $1
     AND phase = 'Running'
     AND owner_instance_id = $2`

const FIND_ORPHAN_RUNS = `
  SELECT run_id, phase
    FROM workflow_runs
   WHERE phase IN ('Pending', 'Running')
     AND (owner_instance_id IS NULL
          OR last_reconciled_at IS NULL
          OR last_reconciled_at < now() - interval '1 minute')
   LIMIT 50`

const CLAIM_ORPHAN_RUNNING = `
  UPDATE workflow_runs
     SET owner_instance_id = $1,
         last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $2
     AND phase = 'Running'
     AND (owner_instance_id IS NULL
          OR last_reconciled_at IS NULL
          OR last_reconciled_at < now() - interval '1 minute')
  RETURNING child_recipe_name, child_recipe_namespace`

const FAIL_RECLAIMED_MISSING_CHILD = `
  UPDATE workflow_runs
     SET phase = 'Failed',
         completed_at = now(),
         last_reconciled_at = now(),
         updated_at = now()
   WHERE run_id = $1
     AND phase = 'Running'
     AND owner_instance_id = $2`

// ─── Factory ───────────────────────────────────────────────────────────

export function createDbRunProcessor(opts: DbRunProcessorOptions): DbRunProcessor {
  const log = opts.logger ?? createLogger('wrc', 'db-run-processor:' + opts.instanceId)
  const connect = opts.connect ?? (() => opts.pool.connect())

  let listenClient: PoolClient | null = null
  let pollTimer: ReturnType<typeof setInterval> | null = null
  let stopped = true

  async function attachListener(): Promise<void> {
    try {
      const c = await connect()
      c.on('notification', msg => {
        if (msg.channel !== 'workflow_run_update' || !msg.payload) return
        void processPending(msg.payload).catch(err => {
          log.error('notification dispatch failed', {
            run_id: msg.payload,
            err: err instanceof Error ? err.message : String(err),
          })
        })
      })
      c.on('error', (err: Error) => {
        log.warn('listen session error — will reconnect', { err: err.message })
        try {
          c.release(err)
        } catch {
          /* idempotent */
        }
        listenClient = null
        if (!stopped) void reconnectLater()
      })
      await c.query('LISTEN workflow_run_update')
      listenClient = c
      log.info('LISTEN workflow_run_update attached')
    } catch (err) {
      log.warn('LISTEN attach failed — will retry', {
        err: err instanceof Error ? err.message : String(err),
      })
      if (!stopped) void reconnectLater()
    }
  }

  async function reconnectLater(): Promise<void> {
    if (stopped) return
    await new Promise<void>(resolve => setTimeout(resolve, 1_000))
    if (!stopped) await attachListener()
  }

  async function failReclaimedRunMissingChild(runId: string): Promise<boolean> {
    const client = await connect()
    let committed = false
    try {
      await client.query('BEGIN')
      const res = await client.query(FAIL_RECLAIMED_MISSING_CHILD, [runId, opts.instanceId])
      await client.query('COMMIT')
      committed = true
      return Boolean(res.rowCount)
    } catch (err) {
      if (!committed) {
        try {
          await client.query('ROLLBACK')
        } catch {
          /* swallow */
        }
      }
      throw err
    } finally {
      client.release()
    }
  }

  async function verifyReclaimedChild(
    runId: string,
    childRecipeName: string | null,
    childRecipeNamespace: string | null
  ): Promise<void> {
    if (!opts.childRecipeExists) return

    if (!childRecipeName || !childRecipeNamespace) {
      const failed = await failReclaimedRunMissingChild(runId)
      if (failed) {
        log.warn('reclaimed running run missing child recipe reference; failing run', {
          run_id: runId,
          owner_instance_id: opts.instanceId,
        })
        notifyRunTerminal(runId, 'Failed')
      }
      return
    }

    let childExists = false
    try {
      childExists = await opts.childRecipeExists({
        name: childRecipeName,
        namespace: childRecipeNamespace,
      })
    } catch (err) {
      log.warn('child existence check failed during running-run reclaim', {
        run_id: runId,
        child_name: childRecipeName,
        child_namespace: childRecipeNamespace,
        err: err instanceof Error ? err.message : String(err),
      })
      return
    }

    if (childExists) return

    const failed = await failReclaimedRunMissingChild(runId)
    if (failed) {
      log.warn('reclaimed running run child recipe missing; failing run', {
        run_id: runId,
        child_name: childRecipeName,
        child_namespace: childRecipeNamespace,
        owner_instance_id: opts.instanceId,
      })
      notifyRunTerminal(runId, 'Failed')
    }
  }

  function notifyRunStarted(runId: string): void {
    try {
      opts.onRunStarted?.(runId)
    } catch (err) {
      log.warn('run-start lifecycle callback failed after committed transition', {
        run_id: runId,
        err: err instanceof Error ? err.message : String(err),
      })
    }
  }

  function notifyRunTerminal(runId: string, phase: WorkflowRunPhase): void {
    try {
      opts.onRunTerminal?.(runId, phase)
    } catch (err) {
      log.warn('run-terminal lifecycle callback failed after committed transition', {
        run_id: runId,
        phase,
        err: err instanceof Error ? err.message : String(err),
      })
    }
  }

  async function reclaimRunning(runId: string): Promise<void> {
    const client = await connect()
    let committed = false
    try {
      await client.query('BEGIN')
      await client.query("SELECT pg_advisory_xact_lock(hashtext('wrc-run-' || $1::text))", [runId])
      const claimRes = await client.query<{
        child_recipe_name: string | null
        child_recipe_namespace: string | null
      }>(CLAIM_ORPHAN_RUNNING, [opts.instanceId, runId])
      await client.query('COMMIT')
      committed = true
      if (claimRes.rowCount && claimRes.rowCount > 0) {
        log.info('orphaned running run reclaimed', {
          run_id: runId,
          owner_instance_id: opts.instanceId,
        })
        // The row was committed before this notification. Re-emitting the
        // stable run_start source occurrence lets reporter recovery converge.
        notifyRunStarted(runId)
        const claimed = claimRes.rows[0]
        await verifyReclaimedChild(
          runId,
          claimed?.child_recipe_name ?? null,
          claimed?.child_recipe_namespace ?? null
        )
      }
    } catch (err) {
      if (!committed) {
        try {
          await client.query('ROLLBACK')
        } catch {
          /* swallow */
        }
      }
      throw err
    } finally {
      client.release()
    }
  }

  async function pollOrphans(): Promise<void> {
    if (stopped) return
    try {
      const res = await opts.pool.query<{ run_id: string; phase: WorkflowRunPhase }>(
        FIND_ORPHAN_RUNS
      )
      for (const row of res.rows) {
        const dispatch =
          row.phase === 'Pending' ? processPending(row.run_id) : reclaimRunning(row.run_id)
        await dispatch.catch(err => {
          log.warn('orphan dispatch failed', {
            run_id: row.run_id,
            err: err instanceof Error ? err.message : String(err),
          })
        })
      }
    } catch (err) {
      log.warn('poll orphans failed', {
        err: err instanceof Error ? err.message : String(err),
      })
    }
  }

  // ─── Public: processPending ──────────────────────────────────────────
  //
  // Invariant: only called when we hold the WRC leader lock. Per-run
  // concurrency is further bounded by `pg_advisory_xact_lock(hash(run_id))`
  // so two replicas cannot process the same run concurrently, even during a
  // brief leader handover.
  async function processPending(runId: string): Promise<void> {
    const client = await connect()
    let committed = false
    try {
      await client.query('BEGIN')
      // Per-run lock — released at COMMIT/ROLLBACK. hashtextextended(text,
      // bigint) returns bigint, which `pg_advisory_xact_lock(bigint)`
      // accepts. Fall back to hashtext when the extended form is unavailable.
      await client.query("SELECT pg_advisory_xact_lock(hashtext('wrc-run-' || $1::text))", [runId])
      const selectRes = await client.query<DbRunRow>(SELECT_RUN_FOR_UPDATE, [runId])
      const run = selectRes.rows[0]
      if (!run) {
        await client.query('ROLLBACK')
        committed = true
        return
      }

      if (run.phase !== 'Pending') {
        // Someone else already transitioned us past Pending — nothing to do
        // here. `syncFromRecipeExecution` drives Running→terminal.
        await client.query('ROLLBACK')
        committed = true
        return
      }

      const child = await opts.createChildRecipe(run)
      const claimRes = await client.query(CLAIM_PENDING, [
        opts.instanceId,
        child.name,
        child.namespace,
        run.run_id,
      ])
      if (claimRes.rowCount === 0) {
        // Another replica claimed the row between our SELECT and UPDATE —
        // rollback and let the winner handle it. Child creation is idempotent
        // on the run_id-derived name, so a retry re-attaches to the same
        // child instead of materializing a duplicate execution.
        log.warn('pending claim lost race', { run_id: run.run_id })
        await client.query('ROLLBACK')
        committed = true
        return
      }
      await client.query('COMMIT')
      committed = true
      log.info('run transitioned to Running', {
        run_id: run.run_id,
        child_name: child.name,
        child_namespace: child.namespace,
      })
      // Business state is durable before any best-effort tracing notification.
      notifyRunStarted(run.run_id)
    } catch (err) {
      if (!committed) {
        try {
          await client.query('ROLLBACK')
        } catch {
          /* swallow */
        }
      }
      log.error('processPending failed', {
        run_id: runId,
        err: err instanceof Error ? err.message : String(err),
      })
      throw err
    } finally {
      client.release()
    }
  }

  // ─── Public: syncFromRecipeExecution ─────────────────────────────────
  //
  // Pure SQL translation of the legacy CRD merge-patch. Mapped phase rules
  // come from the reconciler's `mapWorkflowExecutionPhase` (identical
  // semantics — see below). Steps are UPSERTed so repeated child events are
  // idempotent.
  async function syncFromRecipeExecution(recipe: WorkflowRecipeCRD): Promise<void> {
    // run_id lives on metadata.labels so `kubectl delete -l ...` works from
    // the control-api archive cron. See k8sClient.ts for the write side.
    const runId = recipe.metadata.labels?.['clerum.io/workflow-run-id']
    if (!runId) return

    const execution = recipe.status?.workflowExecution
    let mappedPhase = mapWorkflowExecutionPhase(execution?.phase)
    if (mappedPhase === 'Succeeded' && !hasCompletedAllDeclaredSteps(recipe)) {
      log.warn('workflow completion ignored because declared steps are not terminal', {
        run_id: runId,
        recipe_name: recipe.metadata.name,
      })
      mappedPhase = null
    }
    const client = await opts.pool.connect()
    let terminalNotification: { runId: string; phase: WorkflowRunPhase } | null = null
    try {
      await client.query('BEGIN')
      for (const step of recipe.status?.steps ?? []) {
        await client.query(UPSERT_STEP, [
          runId,
          step.id,
          mapStepPhase(step.phase),
          (step as { startedAt?: string }).startedAt ?? null,
          (step as { completedAt?: string }).completedAt ?? null,
          (step as { output?: string }).output ?? null,
          // jsonb params must be JSON text here; pg serializes JS arrays as SQL arrays.
          Array.isArray((step as { toolsCalled?: unknown[] }).toolsCalled)
            ? JSON.stringify((step as { toolsCalled?: unknown[] }).toolsCalled)
            : null,
          (step as { error?: string }).error ?? null,
          (step as { approvalBindingSha256?: string }).approvalBindingSha256 ?? null,
        ])
      }

      if (mappedPhase && TERMINAL_PHASES.has(mappedPhase)) {
        const res = await client.query(UPDATE_RUN_TERMINAL, [
          mappedPhase,
          execution?.completedAt ?? null,
          runId,
        ])
        if (res.rowCount && res.rowCount > 0) {
          terminalNotification = { runId, phase: mappedPhase }
        }
      } else {
        // Non-terminal sync — just bump heartbeat. Do NOT overwrite `phase`
        // here: the legacy reconciler preserved Running when no execution
        // phase was present (see "does not overwrite running phase" test).
        await client.query(UPDATE_RUN_PROGRESS, [runId])
      }
      await client.query('COMMIT')
      if (terminalNotification) {
        notifyRunTerminal(terminalNotification.runId, terminalNotification.phase)
      }
    } catch (err) {
      try {
        await client.query('ROLLBACK')
      } catch {
        /* swallow */
      }
      log.error('syncFromRecipeExecution failed', {
        run_id: runId,
        err: err instanceof Error ? err.message : String(err),
      })
      throw err
    } finally {
      client.release()
    }
  }

  // ─── Public: checkStuckRuns (timeout enforcement) ────────────────────
  async function checkStuckRuns(): Promise<number> {
    const client = await opts.pool.connect()
    let failed = 0
    const terminalNotifications: Array<{ runId: string; phase: WorkflowRunPhase }> = []
    try {
      await client.query('BEGIN')
      const res = await client.query<{ run_id: string }>(FIND_STUCK_RUNS, [opts.instanceId])
      for (const row of res.rows) {
        const upd = await client.query(FAIL_STUCK_RUN, [row.run_id, opts.instanceId])
        if (upd.rowCount && upd.rowCount > 0) {
          failed += 1
          log.warn('run force-failed (max duration exceeded)', { run_id: row.run_id })
          terminalNotifications.push({ runId: row.run_id, phase: 'Failed' })
        }
      }
      await client.query('COMMIT')
      for (const notification of terminalNotifications) {
        notifyRunTerminal(notification.runId, notification.phase)
      }
    } catch (err) {
      try {
        await client.query('ROLLBACK')
      } catch {
        /* swallow */
      }
      log.error('checkStuckRuns failed', {
        err: err instanceof Error ? err.message : String(err),
      })
    } finally {
      client.release()
    }
    return failed
  }

  return {
    async start() {
      if (!stopped) return
      stopped = false
      await attachListener()
      pollTimer = setInterval(() => {
        void pollOrphans()
        void checkStuckRuns().catch(() => {
          /* logged inside */
        })
      }, opts.runPollMs)
      if (typeof pollTimer.unref === 'function') pollTimer.unref()
    },

    async stop() {
      stopped = true
      if (pollTimer) {
        clearInterval(pollTimer)
        pollTimer = null
      }
      if (listenClient) {
        try {
          await listenClient.query('UNLISTEN workflow_run_update')
        } catch {
          /* likely already disconnected */
        }
        try {
          listenClient.release()
        } catch {
          /* idempotent */
        }
        listenClient = null
      }
    },

    processPending,
    syncFromRecipeExecution,
    checkStuckRuns,
  }
}

// ─── Pure helpers (exported for reuse) ─────────────────────────────────

const TERMINAL_PHASES = new Set<WorkflowRunPhase>(['Succeeded', 'Failed', 'Canceled'])

export function mapWorkflowExecutionPhase(phase?: string): WorkflowRunPhase | null {
  switch (phase) {
    case 'initializing':
    case 'queued':
      return 'Pending'
    case 'running':
      return 'Running'
    case 'completed':
      return 'Succeeded'
    case 'failed':
      return 'Failed'
    case 'cancelled':
      return 'Canceled'
    default:
      return null
  }
}

export function mapStepPhase(phase: string): string {
  switch (phase) {
    case 'completed':
      return 'Succeeded'
    case 'failed':
      return 'Failed'
    case 'skipped':
      return 'Skipped'
    case 'cancelled':
      return 'Canceled'
    case 'running':
      return 'Running'
    default:
      return 'Pending'
  }
}

export function hasCompletedAllDeclaredSteps(recipe: WorkflowRecipeCRD): boolean {
  const declaredStepIds = (recipe.spec.steps ?? [])
    .map(step => step.id)
    .filter((id): id is string => typeof id === 'string' && id.length > 0)

  if (declaredStepIds.length === 0) return true

  const statusById = new Map((recipe.status?.steps ?? []).map(step => [step.id, step.phase]))
  return declaredStepIds.every(id => {
    const phase = statusById.get(id)
    return phase === 'completed' || phase === 'skipped'
  })
}