Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { Pool, PoolClient } from 'pg'
import type { WorkflowRecipeCRD } from '../types.js'
import {
type ChildRecipeCreator,
type DbRunProcessor,
type DbRunRow,
createDbRunProcessor,
hasCompletedAllDeclaredSteps,
mapStepPhase,
mapWorkflowExecutionPhase,
} from './dbRunProcessor.js'
/**
* Tests for `createDbRunProcessor` — the DB-first replacement for
* `WorkflowRunReconciler` (plan §serene-sauteeing-jellyfish.md §Fase 4).
*
* Scope:
* - Seven cases migrated from `workflowRunReconciler.test.ts` translated from
* CRD merge-patch assertions to SQL state assertions.
* - Three additional cases specific to the DB-first architecture (LISTEN
* reconnect, advisory-lock split-brain, orphan poll fallback).
*
* Design notes:
* - We drive a programmable fake `Pool`/`PoolClient` that matches SQL by
* regex. No real Postgres.
* - We intentionally do NOT mock the K8s client — the module consumes a
* `ChildRecipeCreator` callback so tests can spy on child creation without
* touching kubernetes.
* - `process.nextTick` is used to flush microtasks because `vi.useFakeTimers`
* fakes `setImmediate`/`queueMicrotask` but not `process.nextTick`.
*/
type QueryHandler = (
sql: string,
params?: unknown[]
) => Promise<{ rows: unknown[]; rowCount: number }>
interface FakeClient {
query: ReturnType<typeof vi.fn>
release: ReturnType<typeof vi.fn>
on: ReturnType<typeof vi.fn>
listeners: Record<string, Array<(...args: unknown[]) => void>>
emit: (event: string, ...args: unknown[]) => void
calls: Array<{ sql: string; params?: unknown[] }>
}
function makeClient(handler: QueryHandler): FakeClient {
const listeners: Record<string, Array<(...args: unknown[]) => void>> = {}
const calls: Array<{ sql: string; params?: unknown[] }> = []
const client: FakeClient = {
query: vi.fn(async (sql: unknown, params?: unknown[]) => {
const text = String(sql)
calls.push({ sql: text, params })
return handler(text, params)
}),
release: vi.fn(),
on: vi.fn((event: string, listener: (...args: unknown[]) => void) => {
listeners[event] = listeners[event] ?? []
listeners[event].push(listener)
}),
listeners,
emit(event: string, ...args: unknown[]) {
for (const l of listeners[event] ?? []) l(...args)
},
calls,
}
return client
}
function silentLogger() {
return {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
withStep: () => silentLogger(),
}
}
const flush = () => new Promise<void>(resolve => process.nextTick(resolve))
function baseRun(overrides: Partial<DbRunRow> = {}): DbRunRow {
return {
run_id: '00000000-0000-0000-0000-000000000001',
recipe_namespace: 'demo',
recipe_name: 'echo',
phase: 'Pending',
team_id: null,
usage_team_id: null,
actor_type: 'user',
actor_id: null,
inputs: null,
intermediate_parameters: null,
output_overrides: null,
trigger_source: 'onDemand',
owner_instance_id: null,
max_duration_seconds: 600,
// Pending runs carry a NULL started_at until CLAIM_PENDING sets it.
// Overrides can still populate this for Running/terminal-phase fixtures.
started_at: null,
child_recipe_name: null,
child_recipe_namespace: null,
...overrides,
}
}
function makeRecipeWithAnnotation(
runId: string,
executionPhase?: string,
steps: WorkflowRecipeCRD['status'] extends infer S ? S : never = undefined as never
): WorkflowRecipeCRD {
const statusSteps = (steps ?? undefined) as WorkflowRecipeCRD['status']
const rec: WorkflowRecipeCRD = {
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: 'child-xyz',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/workflow-run-id': runId },
},
spec: {},
status: {
phase: 'active',
workflowExecution: executionPhase ? { phase: executionPhase } : undefined,
...(statusSteps ?? {}),
},
}
return rec
}
// ─── Suite ───────────────────────────────────────────────────────────────
describe('createDbRunProcessor', () => {
let instances: DbRunProcessor[] = []
beforeEach(() => {
instances = []
})
afterEach(async () => {
for (const inst of instances) {
try {
await inst.stop()
} catch {
/* best-effort */
}
}
vi.useRealTimers()
vi.restoreAllMocks()
})
function spawn(opts: Parameters<typeof createDbRunProcessor>[0]): DbRunProcessor {
const p = createDbRunProcessor(opts)
instances.push(p)
return p
}
// ─── 1) processPending happy path: creates child + claims Pending → Running
it('processPending creates child and transitions Pending → Running atomically', async () => {
const run = baseRun()
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/FROM workflow_runs[\s\S]*FOR UPDATE/i.test(sql)) {
return { rows: [run], rowCount: 1 }
}
if (/UPDATE workflow_runs[\s\S]*SET phase = 'Running'/i.test(sql)) {
return { rows: [], rowCount: 1 }
}
if (/^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const pool = { connect: vi.fn(async () => client as unknown as PoolClient) } as unknown as Pool
const createChildRecipe: ChildRecipeCreator = vi.fn(async () => ({
name: 'child-run-1',
namespace: 'sandbox-recipes',
}))
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe,
logger: silentLogger(),
})
await proc.processPending(run.run_id)
expect(createChildRecipe).toHaveBeenCalledWith(run)
expect(client.release).toHaveBeenCalledTimes(1)
const sqlTrail = client.calls.map(c => c.sql.trim().slice(0, 40))
expect(sqlTrail[0]).toBe('BEGIN')
expect(sqlTrail.some(s => /pg_advisory_xact_lock/.test(s))).toBe(true)
expect(sqlTrail[sqlTrail.length - 1]).toBe('COMMIT')
const claim = client.calls.find(c => /SET phase = 'Running'/.test(c.sql))
expect(claim).toBeDefined()
expect(claim?.sql).toContain('started_at = now()')
expect(claim?.params).toEqual(['wrc-1', 'child-run-1', 'sandbox-recipes', run.run_id])
})
it('does not fail processPending when the post-commit run-start callback throws', async () => {
const run = baseRun({ run_id: 'run-callback-start' })
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/FOR UPDATE/.test(sql)) return { rows: [run], rowCount: 1 }
if (/SET phase = 'Running'/.test(sql)) return { rows: [], rowCount: 1 }
if (/^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/^ROLLBACK$/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const pool = { connect: vi.fn(async () => client as unknown as PoolClient) } as unknown as Pool
const logger = silentLogger()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(async () => ({
name: 'child-callback-start',
namespace: 'sandbox-recipes',
})),
logger,
onRunStarted: vi.fn(() => {
throw new Error('trace callback failed')
}),
})
await expect(proc.processPending(run.run_id)).resolves.toBeUndefined()
expect(client.calls.some(c => /^COMMIT$/i.test(c.sql))).toBe(true)
expect(client.calls.some(c => /^ROLLBACK$/i.test(c.sql))).toBe(false)
expect(logger.warn).toHaveBeenCalledWith(
'run-start lifecycle callback failed after committed transition',
expect.objectContaining({ run_id: run.run_id, err: 'trace callback failed' })
)
})
// ─── 2) idempotency: if the row is already Running, skip child creation
it('processPending is a no-op when the row is no longer Pending', async () => {
const run = baseRun({ phase: 'Running', owner_instance_id: 'wrc-other' })
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/FOR UPDATE/.test(sql)) return { rows: [run], rowCount: 1 }
if (/^ROLLBACK$/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const pool = { connect: vi.fn(async () => client as unknown as PoolClient) } as unknown as Pool
const createChildRecipe = vi.fn(async () => ({ name: 'x', namespace: 'y' }))
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe,
logger: silentLogger(),
})
await proc.processPending(run.run_id)
expect(createChildRecipe).not.toHaveBeenCalled()
const hasRollback = client.calls.some(c => /^ROLLBACK$/i.test(c.sql))
expect(hasRollback).toBe(true)
})
// ─── 3) child recipe name+namespace land on claim UPDATE params (labels source)
it('stamps child name/namespace into workflow_runs and preserves owner_instance_id', async () => {
const run = baseRun({ run_id: 'aaaaaaaa-0000-0000-0000-000000000001' })
const client = makeClient(async sql => {
if (/FOR UPDATE/.test(sql)) return { rows: [run], rowCount: 1 }
if (/SET phase = 'Running'/.test(sql)) return { rows: [], rowCount: 1 }
return { rows: [], rowCount: 0 }
})
const pool = { connect: vi.fn(async () => client as unknown as PoolClient) } as unknown as Pool
const createChildRecipe: ChildRecipeCreator = vi.fn(async () => ({
name: 'child-labelled',
namespace: 'sandbox-recipes',
}))
const proc = spawn({
instanceId: 'wrc-owner-xyz',
pool,
runPollMs: 30_000,
createChildRecipe,
logger: silentLogger(),
})
await proc.processPending(run.run_id)
const claim = client.calls.find(c => /SET phase = 'Running'/.test(c.sql))
expect(claim?.sql).toContain('started_at = now()')
expect(claim?.params).toEqual([
'wrc-owner-xyz',
'child-labelled',
'sandbox-recipes',
run.run_id,
])
})
// ─── 4) syncFromRecipeExecution writes terminal phase + fires onRunTerminal
it('syncFromRecipeExecution transitions Running → Succeeded and upserts steps', async () => {
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/UPDATE workflow_runs[\s\S]*completed_at/i.test(sql)) {
return { rows: [], rowCount: 1 }
}
if (/INSERT INTO workflow_run_steps/i.test(sql)) {
return { rows: [], rowCount: 1 }
}
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
onRunTerminal,
})
const recipe: WorkflowRecipeCRD = {
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: 'child-xyz',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/workflow-run-id': 'run-1' },
},
spec: {},
status: {
phase: 'active',
workflowExecution: {
phase: 'completed',
completedAt: '2026-04-20T12:05:00Z',
},
steps: [
{
id: 'step-1',
phase: 'completed',
output: 'hello',
toolsCalled: [{ serverName: 'clerum', toolName: 'generate_docx' }],
approvalBindingSha256: 'b'.repeat(64),
},
{ id: 'step-2', phase: 'running' },
],
},
}
await proc.syncFromRecipeExecution(recipe)
const terminalUpdate = client.calls.find(c =>
/UPDATE workflow_runs[\s\S]*completed_at/i.test(c.sql)
)
expect(terminalUpdate?.params).toEqual(['Succeeded', '2026-04-20T12:05:00Z', 'run-1'])
const upserts = client.calls.filter(c => /INSERT INTO workflow_run_steps/i.test(c.sql))
expect(upserts).toHaveLength(2)
expect(upserts[0]?.params?.[1]).toBe('step-1')
expect(upserts[0]?.params?.[2]).toBe('Succeeded')
expect(upserts[0]?.params?.[6]).toBe(
JSON.stringify([{ serverName: 'clerum', toolName: 'generate_docx' }])
)
expect(upserts[0]?.params?.[8]).toBe('b'.repeat(64))
expect(upserts[1]?.params?.[2]).toBe('Running')
expect(upserts[1]?.params?.[6]).toBeNull()
expect(upserts[1]?.params?.[8]).toBeNull()
expect(client.calls.indexOf(upserts[0]!)).toBeLessThan(client.calls.indexOf(terminalUpdate!))
expect(onRunTerminal).toHaveBeenCalledWith('run-1', 'Succeeded')
})
it('does not fail syncFromRecipeExecution when the post-commit terminal callback throws', async () => {
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/UPDATE workflow_runs[\s\S]*completed_at/i.test(sql)) return { rows: [], rowCount: 1 }
if (/^ROLLBACK$/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const logger = silentLogger()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger,
onRunTerminal: vi.fn(() => {
throw new Error('terminal trace callback failed')
}),
})
await expect(
proc.syncFromRecipeExecution({
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: 'child-callback-terminal',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/workflow-run-id': 'run-callback-terminal' },
},
spec: {},
status: {
phase: 'active',
workflowExecution: {
phase: 'completed',
completedAt: '2026-04-20T12:05:00Z',
},
},
})
).resolves.toBeUndefined()
expect(client.calls.some(c => /^COMMIT$/i.test(c.sql))).toBe(true)
expect(client.calls.some(c => /^ROLLBACK$/i.test(c.sql))).toBe(false)
expect(logger.warn).toHaveBeenCalledWith(
'run-terminal lifecycle callback failed after committed transition',
expect.objectContaining({
run_id: 'run-callback-terminal',
phase: 'Succeeded',
err: 'terminal trace callback failed',
})
)
})
it('syncFromRecipeExecution lets a recovered child recipe correct Failed → Succeeded', async () => {
let sawRecoveredTerminalPredicate = false
const client = makeClient(async (sql, params) => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/UPDATE workflow_runs[\s\S]*completed_at/i.test(sql)) {
sawRecoveredTerminalPredicate =
/phase = 'Failed' AND \$1 = 'Succeeded'/.test(sql) &&
params?.[0] === 'Succeeded' &&
params?.[2] === 'run-recovered'
return { rows: [], rowCount: sawRecoveredTerminalPredicate ? 1 : 0 }
}
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
onRunTerminal,
})
await proc.syncFromRecipeExecution({
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: 'child-recovered',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/workflow-run-id': 'run-recovered' },
},
spec: {},
status: {
phase: 'active',
workflowExecution: {
phase: 'completed',
completedAt: '2026-04-20T12:09:00Z',
},
},
})
expect(sawRecoveredTerminalPredicate).toBe(true)
expect(onRunTerminal).toHaveBeenCalledWith('run-recovered', 'Succeeded')
})
it('does not mark a run Succeeded when workflowExecution is completed but declared steps are open', async () => {
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/SET last_reconciled_at = now\(\)/i.test(sql)) return { rows: [], rowCount: 1 }
if (/INSERT INTO workflow_run_steps/i.test(sql)) return { rows: [], rowCount: 1 }
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
onRunTerminal,
})
const recipe: WorkflowRecipeCRD = {
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: 'child-xyz',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/workflow-run-id': 'run-inconsistent' },
},
spec: { steps: [{ id: 'prepare' }, { id: 'emit' }] },
status: {
phase: 'active',
workflowExecution: {
phase: 'completed',
completedAt: '2026-04-20T12:05:00Z',
},
steps: [
{ id: 'prepare', phase: 'completed' },
{ id: 'emit', phase: 'running' },
],
},
}
expect(hasCompletedAllDeclaredSteps(recipe)).toBe(false)
await proc.syncFromRecipeExecution(recipe)
const terminalUpdate = client.calls.find(c =>
/UPDATE workflow_runs[\s\S]*completed_at/i.test(c.sql)
)
expect(terminalUpdate).toBeUndefined()
expect(client.calls.find(c => /SET last_reconciled_at = now\(\)/i.test(c.sql))).toBeDefined()
expect(onRunTerminal).not.toHaveBeenCalled()
})
// ─── 5) heartbeat-only path (no execution phase) — DO NOT overwrite phase
it('syncFromRecipeExecution preserves Running when execution phase is missing', async () => {
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/SET last_reconciled_at = now\(\)/i.test(sql)) return { rows: [], rowCount: 1 }
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
onRunTerminal,
})
const recipe = makeRecipeWithAnnotation('run-heartbeat')
await proc.syncFromRecipeExecution(recipe)
const terminalCall = client.calls.find(c => /completed_at/i.test(c.sql))
expect(terminalCall).toBeUndefined()
const heartbeat = client.calls.find(c => /SET last_reconciled_at = now\(\)/i.test(c.sql))
expect(heartbeat).toBeDefined()
expect(heartbeat?.params).toEqual(['run-heartbeat'])
expect(onRunTerminal).not.toHaveBeenCalled()
})
// ─── 6) checkStuckRuns no-op when no rows match the timeout predicate
it('checkStuckRuns is a no-op when no run exceeds max_duration_seconds', async () => {
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/FROM workflow_runs[\s\S]*FOR UPDATE SKIP LOCKED/i.test(sql)) {
return { rows: [], rowCount: 0 }
}
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
})
const failed = await proc.checkStuckRuns()
expect(failed).toBe(0)
const anyFail = client.calls.find(c =>
/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(c.sql)
)
expect(anyFail).toBeUndefined()
})
// ─── 7) checkStuckRuns flips to Failed when started_at + max_duration elapsed
it('checkStuckRuns force-fails stuck runs and notifies onRunTerminal', async () => {
const stuckRows = [{ run_id: 'run-stuck-1' }]
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/FROM workflow_runs[\s\S]*FOR UPDATE SKIP LOCKED/i.test(sql)) {
return { rows: stuckRows, rowCount: stuckRows.length }
}
if (/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(sql)) {
return { rows: [], rowCount: 1 }
}
return { rows: [], rowCount: 0 }
})
const pool = {
connect: vi.fn(async () => client as unknown as PoolClient),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
onRunTerminal,
})
const failed = await proc.checkStuckRuns()
expect(failed).toBe(1)
expect(onRunTerminal).toHaveBeenCalledWith('run-stuck-1', 'Failed')
const failUpdate = client.calls.find(c =>
/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(c.sql)
)
expect(failUpdate?.params).toEqual(['run-stuck-1', 'wrc-1'])
})
// ─── 8) LISTEN reconnect: on session error, a new LISTEN session is
// attempted after the backoff.
it('re-attaches LISTEN after the listener session errors', async () => {
vi.useFakeTimers()
let sessionIdx = 0
const sessions: FakeClient[] = []
const pool = {
connect: vi.fn(async () => {
const s = makeClient(async sql => {
if (/^LISTEN workflow_run_update$/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
sessions.push(s)
sessionIdx += 1
return s as unknown as PoolClient
}),
query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
} as unknown as Pool
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 60_000,
createChildRecipe: vi.fn(),
logger: silentLogger(),
})
await proc.start()
await flush()
expect(sessionIdx).toBe(1)
// Emit a session error on the listener — the module should release and
// schedule a reconnect in 1 second.
sessions[0]!.emit('error', new Error('connection terminated'))
expect(sessions[0]!.release).toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1_000)
await flush()
await flush()
expect(sessionIdx).toBeGreaterThanOrEqual(2)
})
// ─── 9) Advisory-lock split-brain: if another replica claims during SELECT
// FOR UPDATE→UPDATE window, the lost claim ends in ROLLBACK and does NOT
// stamp owner_instance_id.
it('rolls back when another replica wins the claim (rowCount = 0 on CLAIM_PENDING)', async () => {
const run = baseRun()
let beginCount = 0
let rollbackCount = 0
let commitCount = 0
const client = makeClient(async sql => {
if (/^BEGIN$/i.test(sql)) {
beginCount += 1
return { rows: [], rowCount: 0 }
}
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/FOR UPDATE/.test(sql) && !/SKIP LOCKED/.test(sql)) {
return { rows: [run], rowCount: 1 }
}
if (/SET phase = 'Running'/.test(sql)) {
// Simulate a winning concurrent UPDATE that already flipped phase away
// from Pending, so our WHERE clause matches zero rows.
return { rows: [], rowCount: 0 }
}
if (/^ROLLBACK$/i.test(sql)) {
rollbackCount += 1
return { rows: [], rowCount: 0 }
}
if (/^COMMIT$/i.test(sql)) {
commitCount += 1
return { rows: [], rowCount: 0 }
}
return { rows: [], rowCount: 0 }
})
const pool = { connect: vi.fn(async () => client as unknown as PoolClient) } as unknown as Pool
const proc = spawn({
instanceId: 'wrc-loser',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(async () => ({ name: 'c', namespace: 'sandbox-recipes' })),
logger: silentLogger(),
})
await proc.processPending(run.run_id)
expect(beginCount).toBe(1)
expect(rollbackCount).toBe(1)
expect(commitCount).toBe(0)
})
// ─── 10) Poll fallback picks up runs that missed LISTEN
it('poll loop invokes processPending for orphan runs found via FIND_ORPHAN_RUNS', async () => {
vi.useFakeTimers()
const orphanRow = baseRun({ run_id: 'run-orphan-1' })
const pool = {
connect: vi.fn(async () => {
const c = makeClient(async (sql, params) => {
if (/^LISTEN/i.test(sql)) return { rows: [], rowCount: 0 }
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql) || /^ROLLBACK$/i.test(sql)) {
return { rows: [], rowCount: 0 }
}
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/FOR UPDATE\s*$/i.test(sql.trim())) {
// SELECT_RUN_FOR_UPDATE — gate by param so only the orphan matches.
expect(params).toEqual(['run-orphan-1'])
return { rows: [orphanRow], rowCount: 1 }
}
if (/SET phase = 'Running'/.test(sql)) return { rows: [], rowCount: 1 }
return { rows: [], rowCount: 0 }
})
return c as unknown as PoolClient
}),
query: vi.fn(async (sql: string) => {
if (/FROM workflow_runs[\s\S]*LIMIT 50/i.test(sql)) {
return { rows: [{ run_id: 'run-orphan-1', phase: 'Pending' }], rowCount: 1 }
}
return { rows: [], rowCount: 0 }
}),
} as unknown as Pool
const childSpy = vi.fn(async (_run: DbRunRow) => ({
name: 'child-orphan',
namespace: 'sandbox-recipes',
}))
const createChildRecipe: ChildRecipeCreator = childSpy
const proc = spawn({
instanceId: 'wrc-1',
pool,
runPollMs: 30_000,
createChildRecipe,
logger: silentLogger(),
})
await proc.start()
await flush()
// Advance to the next poll tick.
await vi.advanceTimersByTimeAsync(30_000)
await flush()
await flush()
await flush()
expect(childSpy).toHaveBeenCalledTimes(1)
expect(childSpy.mock.calls[0]?.[0]?.run_id).toBe('run-orphan-1')
})
it('poll loop reclaims orphaned Running runs without recreating the child', async () => {
vi.useFakeTimers()
const createChildRecipe = vi.fn(async () => ({
name: 'should-not-run',
namespace: 'sandbox-recipes',
}))
const reclaimClient = makeClient(async (sql, params) => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/RETURNING child_recipe_name,\s*child_recipe_namespace/i.test(sql)) {
expect(params).toEqual(['wrc-2', 'run-orphan-running-1'])
return {
rows: [{ child_recipe_name: 'child-orphan', child_recipe_namespace: 'sandbox-recipes' }],
rowCount: 1,
}
}
return { rows: [], rowCount: 0 }
})
const listenClient = makeClient(async sql => {
if (/^LISTEN/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
let connectCount = 0
const pool = {
connect: vi.fn(async () => {
connectCount += 1
return (connectCount === 1 ? listenClient : reclaimClient) as unknown as PoolClient
}),
query: vi.fn(async (sql: string) => {
if (/FROM workflow_runs[\s\S]*LIMIT 50/i.test(sql)) {
return {
rows: [{ run_id: 'run-orphan-running-1', phase: 'Running' }],
rowCount: 1,
}
}
return { rows: [], rowCount: 0 }
}),
} as unknown as Pool
const proc = spawn({
instanceId: 'wrc-2',
pool,
runPollMs: 30_000,
createChildRecipe,
childRecipeExists: vi.fn(async () => true),
logger: silentLogger(),
})
await proc.start()
await flush()
await vi.advanceTimersByTimeAsync(30_000)
await flush()
await flush()
expect(createChildRecipe).not.toHaveBeenCalled()
const reclaim = reclaimClient.calls.find(c => /owner_instance_id = \$1/i.test(c.sql))
expect(reclaim).toBeDefined()
})
it('reclaimed Running runs become eligible for timeout enforcement on the new owner', async () => {
vi.useFakeTimers()
const listenClient = makeClient(async sql => {
if (/^LISTEN/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const reclaimClient = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/RETURNING child_recipe_name,\s*child_recipe_namespace/i.test(sql)) {
return {
rows: [{ child_recipe_name: 'child-orphan', child_recipe_namespace: 'sandbox-recipes' }],
rowCount: 1,
}
}
return { rows: [], rowCount: 0 }
})
const stuckClient = makeClient(async sql => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/FROM workflow_runs[\s\S]*FOR UPDATE SKIP LOCKED/i.test(sql)) {
return { rows: [{ run_id: 'run-orphan-running-2' }], rowCount: 1 }
}
if (/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(sql)) {
return { rows: [], rowCount: 1 }
}
return { rows: [], rowCount: 0 }
})
let connectCount = 0
const pool = {
connect: vi.fn(async () => {
connectCount += 1
if (connectCount === 1) return listenClient as unknown as PoolClient
if (connectCount === 2) return reclaimClient as unknown as PoolClient
return stuckClient as unknown as PoolClient
}),
query: vi.fn(async (sql: string) => {
if (/FROM workflow_runs[\s\S]*LIMIT 50/i.test(sql)) {
return {
rows: [{ run_id: 'run-orphan-running-2', phase: 'Running' }],
rowCount: 1,
}
}
return { rows: [], rowCount: 0 }
}),
} as unknown as Pool
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-3',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
childRecipeExists: vi.fn(async () => true),
logger: silentLogger(),
onRunTerminal,
})
await proc.start()
await flush()
await vi.advanceTimersByTimeAsync(30_000)
await flush()
await flush()
const failed = await proc.checkStuckRuns()
expect(failed).toBe(1)
expect(onRunTerminal).toHaveBeenCalledWith('run-orphan-running-2', 'Failed')
const failUpdate = stuckClient.calls.find(c =>
/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(c.sql)
)
expect(failUpdate?.params).toEqual(['run-orphan-running-2', 'wrc-3'])
})
it('fails a reclaimed Running run when its child recipe no longer exists', async () => {
vi.useFakeTimers()
const listenClient = makeClient(async sql => {
if (/^LISTEN/i.test(sql)) return { rows: [], rowCount: 0 }
return { rows: [], rowCount: 0 }
})
const workerClient = makeClient(async (sql, params) => {
if (/^BEGIN$/i.test(sql) || /^COMMIT$/i.test(sql)) return { rows: [], rowCount: 0 }
if (/pg_advisory_xact_lock/.test(sql)) return { rows: [], rowCount: 0 }
if (/RETURNING child_recipe_name,\s*child_recipe_namespace/i.test(sql)) {
return {
rows: [{ child_recipe_name: 'child-missing', child_recipe_namespace: 'sandbox-recipes' }],
rowCount: 1,
}
}
if (/SET phase = 'Failed'/.test(sql)) {
expect(params).toEqual(['run-orphan-running-3', 'wrc-4'])
return { rows: [], rowCount: 1 }
}
if (/FROM workflow_runs[\s\S]*FOR UPDATE SKIP LOCKED/i.test(sql)) {
return { rows: [], rowCount: 0 }
}
return { rows: [], rowCount: 0 }
})
let connectCount = 0
const pool = {
connect: vi.fn(async () => {
connectCount += 1
if (connectCount === 1) return listenClient as unknown as PoolClient
return workerClient as unknown as PoolClient
}),
query: vi.fn(async (sql: string) => {
if (/FROM workflow_runs[\s\S]*LIMIT 50/i.test(sql)) {
return {
rows: [{ run_id: 'run-orphan-running-3', phase: 'Running' }],
rowCount: 1,
}
}
return { rows: [], rowCount: 0 }
}),
} as unknown as Pool
const childRecipeExists = vi.fn(async () => false)
const onRunTerminal = vi.fn()
const proc = spawn({
instanceId: 'wrc-4',
pool,
runPollMs: 30_000,
createChildRecipe: vi.fn(),
childRecipeExists,
logger: silentLogger(),
onRunTerminal,
})
await proc.start()
await flush()
await vi.advanceTimersByTimeAsync(30_000)
await flush()
await flush()
await flush()
expect(childRecipeExists).toHaveBeenCalledWith({
name: 'child-missing',
namespace: 'sandbox-recipes',
})
const failUpdate = workerClient.calls.find(c =>
/UPDATE workflow_runs[\s\S]*SET phase = 'Failed'/i.test(c.sql)
)
expect(failUpdate).toBeDefined()
expect(onRunTerminal).toHaveBeenCalledWith('run-orphan-running-3', 'Failed')
})
})
// ─── Pure helper tests ─────────────────────────────────────────────────────
describe('mapWorkflowExecutionPhase', () => {
it.each([
['initializing', 'Pending'],
['queued', 'Pending'],
['running', 'Running'],
['completed', 'Succeeded'],
['failed', 'Failed'],
['cancelled', 'Canceled'],
] as const)('maps %s → %s', (input, expected) => {
expect(mapWorkflowExecutionPhase(input)).toBe(expected)
})
it('returns null for unknown phases', () => {
expect(mapWorkflowExecutionPhase(undefined)).toBeNull()
expect(mapWorkflowExecutionPhase('bogus')).toBeNull()
})
})
describe('mapStepPhase', () => {
it.each([
['completed', 'Succeeded'],
['failed', 'Failed'],
['skipped', 'Skipped'],
['cancelled', 'Canceled'],
['running', 'Running'],
['anything-else', 'Pending'],
])('maps %s → %s', (input, expected) => {
expect(mapStepPhase(input)).toBe(expected)
})
})