Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { K8sGateway } from '../src/k8s.js'
// Import AFTER vi.mock() so the service picks up the mocked modules.
import { archiveTerminalRuns } from '../src/services/workflowRunsArchiveService.js'
/**
* Tests for `workflowRunsArchiveService.archiveTerminalRuns`.
*
* The service acquires a session-scoped `pg_try_advisory_lock` on a client
* obtained via `pool.connect()`, then drives batch transactions on that same
* client (so the lock stays alive for the whole sweep). We mock `pool.connect`
* to return a programmable fake client and route queries by SQL regex.
*
* Post-commit, the service best-effort deletes the child WorkflowRecipe via a
* `K8sGateway.deleteResource(...)` — we assert the call count + status-code
* handling (404 = not_found, other = error) without throwing.
*/
type LockQueryResult = { rows: Array<{ acquired: boolean }>; rowCount: number }
type AnyQueryResult = { rows: unknown[]; rowCount: number | null }
type RunRow = {
run_id: string
child_recipe_name: string | null
child_recipe_namespace: string | null
}
// ─── Mock state ────────────────────────────────────────────────────────────
const eligibleRuns: RunRow[] = []
let lockAcquired = true // flip to false to simulate contention from another replica
let batchErrorMode: 'none' | 'throw-on-insert-audit' = 'none'
const clientQuery = vi.fn(
async (sql: unknown, params?: unknown[]): Promise<AnyQueryResult | LockQueryResult> => {
const text = typeof sql === 'string' ? sql : ''
// Advisory lock acquisition
if (/pg_try_advisory_lock/i.test(text)) {
return { rows: [{ acquired: lockAcquired }], rowCount: 1 }
}
if (/pg_advisory_unlock/i.test(text)) {
return { rows: [], rowCount: 1 }
}
// Transaction boundary statements are no-ops in the mock
if (/^BEGIN$|^COMMIT$|^ROLLBACK$/i.test(text.trim())) {
return { rows: [], rowCount: null }
}
// SELECT eligible terminal runs (drain the shared array in batches)
if (
/SELECT run_id, child_recipe_name, child_recipe_namespace\s+FROM workflow_runs/i.test(text)
) {
const limit = Number(params?.[1] ?? 0)
const batch = eligibleRuns.splice(0, limit)
return { rows: batch, rowCount: batch.length }
}
// Copy INSERT — optionally throw to exercise the ROLLBACK path
if (/INSERT INTO workflow_runs_audit/i.test(text)) {
if (batchErrorMode === 'throw-on-insert-audit') {
throw new Error('simulated audit insert failure')
}
const ids = (params?.[0] as string[]) ?? []
return { rows: [], rowCount: ids.length }
}
if (/INSERT INTO workflow_run_step_audit/i.test(text)) {
return { rows: [], rowCount: 0 }
}
// Live DELETE
if (/DELETE FROM workflow_runs WHERE run_id = ANY/i.test(text)) {
const ids = (params?.[0] as string[]) ?? []
return { rows: [], rowCount: ids.length }
}
return { rows: [], rowCount: null }
}
)
const clientRelease = vi.fn()
const mockConnect = vi.fn(async () => ({
query: clientQuery,
release: clientRelease,
}))
vi.mock('../src/db.js', () => ({
pool: {
query: vi.fn(),
connect: () => mockConnect(),
},
withTransaction: vi.fn(),
}))
// Silence logger — observability is covered by integration, not unit tests.
vi.mock('../src/observability/logger.js', () => ({
rootLogger: {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
}))
// ─── Fake gateway ──────────────────────────────────────────────────────────
type DeleteError = Error & { statusCode?: number }
function makeGateway(
deleteImpl: (plural: string, name: string, ns?: string) => Promise<unknown>
): K8sGateway {
return {
deleteResource: vi.fn(deleteImpl),
} as unknown as K8sGateway
}
// ─── Tests ─────────────────────────────────────────────────────────────────
describe('archiveTerminalRuns', () => {
beforeEach(() => {
eligibleRuns.length = 0
lockAcquired = true
batchErrorMode = 'none'
clientQuery.mockClear()
clientRelease.mockClear()
mockConnect.mockClear()
})
it('archives eligible terminal runs and deletes each child recipe post-commit', async () => {
eligibleRuns.push(
{
run_id: 'uuid-1',
child_recipe_name: 'recipe-1',
child_recipe_namespace: 'sandbox-recipes',
},
{ run_id: 'uuid-2', child_recipe_name: 'recipe-2', child_recipe_namespace: 'sandbox-recipes' }
)
const deleteResource = vi.fn(async () => ({}))
const gateway = makeGateway(deleteResource)
const total = await archiveTerminalRuns(gateway)
expect(total).toBe(2)
// Transaction shape: BEGIN → SELECT → INSERT audit → INSERT step audit → DELETE → COMMIT,
// then one more drain SELECT returning 0 rows.
const selectCalls = clientQuery.mock.calls.filter(([sql]) =>
/SELECT run_id, child_recipe_name/i.test(String(sql))
)
expect(selectCalls.length).toBe(2) // drain batch + empty-tail
expect(String(selectCalls[0]?.[0])).toContain('COALESCE(ttl_seconds_after_finished, $1)')
const insertAuditCalls = clientQuery.mock.calls.filter(([sql]) =>
/INSERT INTO workflow_runs_audit/i.test(String(sql))
)
expect(insertAuditCalls.length).toBe(1)
expect(String(insertAuditCalls[0]?.[0])).toContain('wr.started_at')
expect(String(insertAuditCalls[0]?.[0])).toContain('wr.completed_at - wr.started_at')
expect(String(insertAuditCalls[0]?.[0])).not.toContain('wr.completed_at - wr.created_at')
const deleteCalls = clientQuery.mock.calls.filter(([sql]) =>
/DELETE FROM workflow_runs/i.test(String(sql))
)
expect(deleteCalls.length).toBe(1)
// Child recipe deletes happen AFTER the SQL commit, once per archived row.
expect(deleteResource).toHaveBeenCalledTimes(2)
expect(deleteResource).toHaveBeenCalledWith('workflowrecipes', 'recipe-1', 'sandbox-recipes')
expect(deleteResource).toHaveBeenCalledWith('workflowrecipes', 'recipe-2', 'sandbox-recipes')
// Lock hygiene: acquired, released, and client returned to the pool.
const lockCalls = clientQuery.mock.calls.filter(([sql]) =>
/pg_try_advisory_lock|pg_advisory_unlock/i.test(String(sql))
)
expect(lockCalls.length).toBe(2)
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('is a no-op when another replica holds the advisory lock', async () => {
lockAcquired = false
eligibleRuns.push({
run_id: 'uuid-contended',
child_recipe_name: 'recipe-contended',
child_recipe_namespace: 'sandbox-recipes',
})
const deleteResource = vi.fn(async () => ({}))
const gateway = makeGateway(deleteResource)
const total = await archiveTerminalRuns(gateway)
expect(total).toBe(0)
// Only the lock probe ran — no SELECT/INSERT/DELETE against workflow_runs.
const nonLockQueries = clientQuery.mock.calls.filter(
([sql]) => !/pg_try_advisory_lock|pg_advisory_unlock/i.test(String(sql))
)
expect(nonLockQueries.length).toBe(0)
// The unreleased lock must NOT be unlocked (we never acquired it).
const unlockCalls = clientQuery.mock.calls.filter(([sql]) =>
/pg_advisory_unlock/i.test(String(sql))
)
expect(unlockCalls.length).toBe(0)
expect(deleteResource).not.toHaveBeenCalled()
// Client is ALWAYS released so it returns to the pool.
expect(clientRelease).toHaveBeenCalledTimes(1)
// Eligible row is still waiting for the next sweep.
expect(eligibleRuns.length).toBe(1)
})
it('treats 404 NotFound on child delete as success (not_found), not as a sweep failure', async () => {
eligibleRuns.push({
run_id: 'uuid-orphan',
child_recipe_name: 'recipe-gone',
child_recipe_namespace: 'sandbox-recipes',
})
const notFoundErr: DeleteError = Object.assign(new Error('not found'), { statusCode: 404 })
const deleteResource = vi.fn(async () => {
throw notFoundErr
})
const gateway = makeGateway(deleteResource)
const total = await archiveTerminalRuns(gateway)
expect(total).toBe(1)
expect(deleteResource).toHaveBeenCalledTimes(1)
// Lock released + client returned despite K8s 404.
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('continues the sweep when one child delete fails with a non-404 error', async () => {
eligibleRuns.push(
{ run_id: 'uuid-a', child_recipe_name: 'rec-a', child_recipe_namespace: 'sandbox-recipes' },
{ run_id: 'uuid-b', child_recipe_name: 'rec-b', child_recipe_namespace: 'sandbox-recipes' }
)
const serverErr: DeleteError = Object.assign(new Error('500 internal'), { statusCode: 500 })
const deleteResource = vi.fn(async (_plural: string, name: string) => {
if (name === 'rec-a') throw serverErr
return {}
})
const gateway = makeGateway(deleteResource)
// Must NOT throw — child delete is best-effort and the SQL is already committed.
const total = await archiveTerminalRuns(gateway)
expect(total).toBe(2)
expect(deleteResource).toHaveBeenCalledTimes(2)
})
it('rolls back the batch transaction and surfaces the error when INSERT audit fails', async () => {
eligibleRuns.push({
run_id: 'uuid-fail',
child_recipe_name: 'rec-fail',
child_recipe_namespace: 'sandbox-recipes',
})
batchErrorMode = 'throw-on-insert-audit'
const deleteResource = vi.fn(async () => ({}))
const gateway = makeGateway(deleteResource)
await expect(archiveTerminalRuns(gateway)).rejects.toThrow('simulated audit insert failure')
// Rollback must fire; commit must NOT.
const rollbackCalls = clientQuery.mock.calls.filter(([sql]) =>
/^ROLLBACK$/i.test(String(sql).trim())
)
const commitCalls = clientQuery.mock.calls.filter(([sql]) =>
/^COMMIT$/i.test(String(sql).trim())
)
expect(rollbackCalls.length).toBe(1)
expect(commitCalls.length).toBe(0)
// No child delete fired (SQL did not commit → no archived children to clean up).
expect(deleteResource).not.toHaveBeenCalled()
// Lock + client are still released in the finally block.
const unlockCalls = clientQuery.mock.calls.filter(([sql]) =>
/pg_advisory_unlock/i.test(String(sql))
)
expect(unlockCalls.length).toBe(1)
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('preserves FK safety guard and trigger_source preservation in the audit INSERT SQL', async () => {
// This is a contract test: if someone removes the `EXISTS (SELECT 1 FROM users ...)`
// guard, user-session runs with a stale/missing actor_id will violate the FK on
// workflow_runs_audit.triggerer_user_id → users(id). Likewise, if the
// `trigger_source IN ('schedule','autonomous')` CASE is removed, autonomous runs
// will be mis-attributed as 'onDemand' in the audit log.
eligibleRuns.push({
run_id: 'uuid-contract',
child_recipe_name: 'rec-contract',
child_recipe_namespace: 'sandbox-recipes',
})
const gateway = makeGateway(async () => ({}))
await archiveTerminalRuns(gateway)
const auditInsert = clientQuery.mock.calls.find(([sql]) =>
/INSERT INTO workflow_runs_audit/i.test(String(sql))
)
expect(auditInsert).toBeDefined()
const sql = String(auditInsert![0])
// FK safety: only keep actor_id when the user row still exists.
expect(sql).toMatch(
/EXISTS\s*\(\s*SELECT\s+1\s+FROM\s+users\s+WHERE\s+id\s*=\s*wr\.actor_id\s*\)/i
)
expect(sql).toMatch(/wr\.actor_type\s*=\s*'user'/i)
// Trigger-source preservation: autonomous + schedule must pass through.
expect(sql).toMatch(/wr\.trigger_source\s+IN\s*\(\s*'schedule'\s*,\s*'autonomous'\s*\)/i)
expect(sql).toMatch(/ELSE\s+'onDemand'/i)
})
it('respects maxBatches as a safety rail (100 × batchSize per sweep)', async () => {
// 250 eligible rows, batch 5, maxBatches 3 → we should process exactly 15.
for (let i = 0; i < 250; i++) {
eligibleRuns.push({
run_id: `uuid-${i}`,
child_recipe_name: null,
child_recipe_namespace: null,
})
}
const gateway = makeGateway(async () => ({}))
const total = await archiveTerminalRuns(gateway, { batchSize: 5, maxBatches: 3 })
expect(total).toBe(15)
// 235 rows still queued for the next sweep.
expect(eligibleRuns.length).toBe(235)
})
})