Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { rootLogger } from '../src/observability/logger.js'
import {
workflowScheduleWorkerConcurrencyForbiddenSustainedTotal,
workflowScheduleWorkerFiresTotal,
} from '../src/observability/metrics.js'
// Imports AFTER vi.mock() so the service picks up the fakes.
import {
FORBID_PENDING_ACTIVE_BOUND_SECONDS,
FORBID_RUNNING_GRACE_SECONDS,
FORBID_RUNNING_MAX_DURATION_CAP_SECONDS,
FORBID_SKIP_STREAK_CAPACITY,
FORBID_SUSTAINED_SKIP_THRESHOLD,
computeNextFire,
concurrencyForbiddenStreakCountForTests,
processMaturedSchedules,
resetConcurrencyForbiddenStreaksForTests,
} from '../src/services/workflowScheduleWorkerService.js'
/**
* Tests for `workflowScheduleWorkerService.processMaturedSchedules`.
*
* The worker acquires a session-scoped `pg_try_advisory_lock` on a client
* obtained via `pool.connect()`, drains matured rows from `workflow_schedules`,
* and for each row calls `createRun()` + advances `next_fire_at`. We mock:
* - `pool.connect` → programmable fake client
* - `./workflowRunService.createRun` → spy that returns a deterministic row
* - `../observability/logger.js` → silent
* - `../observability/metrics.js` → no-op counters
*/
type LockQueryResult = { rows: Array<{ acquired: boolean }>; rowCount: number }
type AnyQueryResult = { rows: unknown[]; rowCount: number | null }
type ScheduleRow = {
schedule_id: string
recipe_namespace: string
recipe_name: string
team_id: string | null
cron_expression: string
timezone: string
next_fire_at: Date
input_template: Record<string, unknown> | null
allowed_actors?: Array<'user' | 'autonomous' | 'scheduled'> | null
max_duration_seconds?: number | null
ttl_seconds_after_finished?: number | null
concurrency_policy?: 'Forbid' | 'Replace' | 'Allow' | null
}
// ─── Mock state ────────────────────────────────────────────────────────────
const maturedRows: ScheduleRow[] = []
let lockAcquired = true
const updatedSchedules: Array<{ schedule_id: string; next_fire_at?: Date; enabled?: boolean }> = []
const txEvents: Array<'BEGIN' | 'COMMIT' | 'ROLLBACK'> = []
/** Test-configurable hook that can inject a failure at SELECT time. */
let selectQueryError: Error | null = null
/** Non-terminal workflow_runs keyed by `${recipe_namespace}/${recipe_name}`. */
type FakeRun = {
run_id: string
phase: 'Pending' | 'Running'
created_at: Date
started_at: Date | null
max_duration_seconds: number | null
}
const activeRuns = new Map<string, FakeRun>()
/** Test-configurable hook that can inject a failure into the active-run lookup. */
let activeRunQueryError: Error | null = null
let nextFireUpdateError: Error | null = null
const clientQuery = vi.fn(
async (sql: unknown, params?: unknown[]): Promise<AnyQueryResult | LockQueryResult> => {
const text = typeof sql === 'string' ? sql : ''
if (/^\s*BEGIN\s*$/i.test(text)) {
txEvents.push('BEGIN')
return { rows: [], rowCount: null }
}
if (/^\s*COMMIT\s*$/i.test(text)) {
txEvents.push('COMMIT')
return { rows: [], rowCount: null }
}
if (/^\s*ROLLBACK\s*$/i.test(text)) {
txEvents.push('ROLLBACK')
return { rows: [], rowCount: null }
}
if (/pg_try_advisory_lock/i.test(text)) {
return { rows: [{ acquired: lockAcquired }], rowCount: 1 }
}
if (/pg_advisory_unlock/i.test(text)) {
return { rows: [], rowCount: 1 }
}
if (
/SELECT schedule_id, recipe_namespace, recipe_name/i.test(text) &&
/FROM workflow_schedules/i.test(text)
) {
if (selectQueryError) throw selectQueryError
// Contract assertions: the worker MUST read the denormalized policy fields
// written by the WRC schedule sync so it can preserve runtime semantics
// from the WorkflowRecipe CRD.
if (!/allowed_actors/i.test(text)) {
throw new Error('SELECT must include allowed_actors column')
}
if (!/max_duration_seconds/i.test(text)) {
throw new Error('SELECT must include max_duration_seconds column')
}
if (!/ttl_seconds_after_finished/i.test(text)) {
throw new Error('SELECT must include ttl_seconds_after_finished column')
}
if (!/team_id/i.test(text)) {
throw new Error('SELECT must include team_id column')
}
if (!/concurrency_policy/i.test(text)) {
throw new Error('SELECT must include concurrency_policy column')
}
const limit = Number(params?.[0] ?? 0)
const batch = maturedRows.splice(0, limit)
return { rows: batch, rowCount: batch.length }
}
if (/UPDATE workflow_schedules/i.test(text) && /enabled = FALSE/i.test(text)) {
const scheduleId = String(params?.[0] ?? '')
updatedSchedules.push({ schedule_id: scheduleId, enabled: false })
return { rows: [], rowCount: 1 }
}
if (/UPDATE workflow_schedules/i.test(text) && /next_fire_at/i.test(text)) {
if (nextFireUpdateError) throw nextFireUpdateError
const nextFire = params?.[0] as Date
const scheduleId = String(params?.[1] ?? '')
updatedSchedules.push({ schedule_id: scheduleId, next_fire_at: nextFire })
return { rows: [], rowCount: 1 }
}
if (/FROM workflow_runs/i.test(text)) {
if (activeRunQueryError) throw activeRunQueryError
if (!/phase = 'Pending'/.test(text) || !/phase = 'Running'/.test(text)) {
throw new Error('active-run lookup must filter on non-terminal phases')
}
const run = activeRuns.get(`${String(params?.[0])}/${String(params?.[1])}`)
if (!run) return { rows: [], rowCount: 0 }
// Minimal emulation of the stale-run bounds ($3 Pending bound, $4 cap,
// $5 grace) so the worker's wiring is exercised. The SQL semantics are
// proven against real Postgres in
// services.workflowScheduleForbid.realPostgres.integration.test.ts.
const [pendingBound, cap, grace] = [
Number(params?.[2]),
Number(params?.[3]),
Number(params?.[4]),
]
const now = Date.now()
const live =
run.phase === 'Pending'
? run.created_at.getTime() > now - pendingBound * 1000
: (run.started_at ?? run.created_at).getTime() >
now - (Math.min(run.max_duration_seconds ?? cap, cap) + grace) * 1000
if (!live) return { rows: [], rowCount: 0 }
const { run_id, phase, created_at, started_at } = run
return { rows: [{ run_id, phase, created_at, started_at }], rowCount: 1 }
}
return { rows: [], rowCount: null }
}
)
const clientRelease = vi.fn()
const mockConnect = vi.fn(async () => ({
query: clientQuery,
release: clientRelease,
}))
vi.mock('../src/db.js', () => ({
pool: {
query: vi.fn(),
connect: () => mockConnect(),
},
withTransaction: vi.fn(),
}))
const createRunMock = vi.fn(async (input: Record<string, unknown>, _client?: unknown) => ({
row: {
run_id: `run-${(input.idempotency_key as string) ?? 'missing'}`,
},
}))
vi.mock('../src/services/workflowRunService.js', () => ({
createRun: (input: Record<string, unknown>, client?: unknown) => createRunMock(input, client),
}))
vi.mock('../src/observability/logger.js', () => ({
rootLogger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
}))
vi.mock('../src/observability/metrics.js', () => ({
workflowScheduleWorkerRunsTotal: { inc: vi.fn() },
workflowScheduleWorkerFiresTotal: { inc: vi.fn() },
workflowScheduleWorkerConcurrencyForbiddenSustainedTotal: { inc: vi.fn() },
workflowScheduleWorkerDurationSeconds: { observe: vi.fn() },
}))
// ─── Tests ─────────────────────────────────────────────────────────────────
describe('processMaturedSchedules', () => {
beforeEach(() => {
maturedRows.length = 0
updatedSchedules.length = 0
txEvents.length = 0
lockAcquired = true
selectQueryError = null
activeRuns.clear()
activeRunQueryError = null
nextFireUpdateError = null
clientQuery.mockClear()
clientRelease.mockClear()
mockConnect.mockClear()
createRunMock.mockClear()
})
it('fires a single matured schedule, inserts a run, and advances next_fire_at', async () => {
const fireTime = new Date('2026-04-20T09:00:00Z')
maturedRows.push({
schedule_id: 's-1',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'daily-report',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 9 * * *',
timezone: 'UTC',
next_fire_at: fireTime,
input_template: { topic: 'hello' },
max_duration_seconds: 900,
ttl_seconds_after_finished: 1800,
})
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(result.actorNotAllowed).toBe(0)
expect(result.errors).toBe(0)
expect(result.skippedLock).toBe(false)
expect(createRunMock).toHaveBeenCalledTimes(1)
const createRunArg = createRunMock.mock.calls[0][0]
expect(createRunArg.recipe_namespace).toBe('sandbox-recipes')
expect(createRunArg.recipe_name).toBe('daily-report')
expect(createRunArg.actor_type).toBe('scheduled')
expect(createRunArg.team_id).toBe('11111111-1111-4111-8111-111111111111')
expect(createRunArg.trigger_source).toBe('schedule')
expect(createRunArg.max_duration_seconds).toBe(900)
expect(createRunArg.ttl_seconds_after_finished).toBe(1800)
// Idempotency key is deterministic — same (schedule_id, fire time) → same key.
expect(createRunArg.idempotency_key).toBe(`schedule/s-1/${fireTime.toISOString()}`)
expect(createRunArg.inputs).toEqual({ topic: 'hello' })
// next_fire_at was advanced exactly one cron period forward (not to "now").
expect(updatedSchedules).toHaveLength(1)
expect(updatedSchedules[0].schedule_id).toBe('s-1')
expect(updatedSchedules[0].next_fire_at?.toISOString()).toBe('2026-04-21T09:00:00.000Z')
// Lock hygiene.
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('fires a batch of matured schedules in a single sweep', async () => {
const baseTime = new Date('2026-04-20T10:00:00Z')
for (let i = 0; i < 3; i++) {
maturedRows.push({
schedule_id: `s-${i}`,
recipe_namespace: 'sandbox-recipes',
recipe_name: `recipe-${i}`,
team_id: `11111111-1111-4111-8111-11111111111${i}`,
cron_expression: '*/5 * * * *',
timezone: 'UTC',
next_fire_at: baseTime,
input_template: null,
})
}
const result = await processMaturedSchedules({ batchSize: 10 })
expect(result.fired).toBe(3)
expect(createRunMock).toHaveBeenCalledTimes(3)
// All 3 schedules should be advanced by the same +5min window.
expect(updatedSchedules).toHaveLength(3)
for (const u of updatedSchedules) {
expect(u.next_fire_at?.toISOString()).toBe('2026-04-20T10:05:00.000Z')
}
})
it('skips entirely when another replica holds the advisory lock', async () => {
lockAcquired = false
maturedRows.push({
schedule_id: 's-contended',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'contended',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 * * * *',
timezone: 'UTC',
next_fire_at: new Date('2026-04-20T09:00:00Z'),
input_template: null,
})
const result = await processMaturedSchedules()
expect(result.skippedLock).toBe(true)
expect(result.fired).toBe(0)
expect(result.actorNotAllowed).toBe(0)
expect(createRunMock).not.toHaveBeenCalled()
// No SELECT against workflow_schedules either.
const selectCalls = clientQuery.mock.calls.filter(([sql]) =>
/FROM workflow_schedules/i.test(String(sql))
)
expect(selectCalls.length).toBe(0)
// Contended row is preserved for the next sweep.
expect(maturedRows.length).toBe(1)
// Client released so the pool recovers even on contention.
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('disables schedules with invalid cron expressions (no fire, no run)', async () => {
maturedRows.push({
schedule_id: 's-bad',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'broken',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: 'not-a-real-cron',
timezone: 'UTC',
next_fire_at: new Date('2026-04-20T09:00:00Z'),
input_template: null,
})
const result = await processMaturedSchedules()
expect(result.fired).toBe(0)
expect(result.actorNotAllowed).toBe(0)
expect(result.errors).toBe(1)
expect(createRunMock).not.toHaveBeenCalled()
// Schedule was disabled via UPDATE … SET enabled = FALSE.
const disabled = updatedSchedules.find(u => u.enabled === false)
expect(disabled?.schedule_id).toBe('s-bad')
})
it('keeps next_fire_at unchanged when createRun fails so the same window can retry', async () => {
createRunMock.mockRejectedValueOnce(new Error('db write failed'))
const fireTime = new Date('2026-04-20T09:00:00Z')
maturedRows.push({
schedule_id: 's-fail',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'fails',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 9 * * *',
timezone: 'UTC',
next_fire_at: fireTime,
input_template: null,
})
const result = await processMaturedSchedules()
expect(result.fired).toBe(0)
expect(result.actorNotAllowed).toBe(0)
expect(result.errors).toBe(1)
expect(createRunMock).toHaveBeenCalledTimes(1)
// Preserve the current slot for retry; dropping the window would lose a run.
expect(updatedSchedules).toHaveLength(0)
})
it("skips fire when allowed_actors is set and does not include 'scheduled'", async () => {
// Recipe sets spec.triggers.onDemand.allowedActors = ['user'], denormalized
// into workflow_schedules.allowed_actors. The worker MUST refuse to create
// a run but SHOULD still advance next_fire_at so the same window does not
// re-queue forever.
const fireTime = new Date('2026-04-20T09:00:00Z')
maturedRows.push({
schedule_id: 's-gated',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'user-only',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 9 * * *',
timezone: 'UTC',
next_fire_at: fireTime,
input_template: null,
allowed_actors: ['user'],
})
const result = await processMaturedSchedules()
expect(result.fired).toBe(0)
expect(result.actorNotAllowed).toBe(1)
// Policy-driven skip is NOT an error — dashboards would otherwise page
// on intentional configuration.
expect(result.errors).toBe(0)
expect(createRunMock).not.toHaveBeenCalled()
// next_fire_at was advanced so the policy-blocked window doesn't keep
// re-matching the SELECT predicate on every sweep.
expect(updatedSchedules).toHaveLength(1)
expect(updatedSchedules[0].schedule_id).toBe('s-gated')
expect(updatedSchedules[0].next_fire_at?.toISOString()).toBe('2026-04-21T09:00:00.000Z')
})
it("fires normally when allowed_actors explicitly includes 'scheduled'", async () => {
const fireTime = new Date('2026-04-20T09:00:00Z')
maturedRows.push({
schedule_id: 's-scheduled-ok',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'cron-allowed',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 9 * * *',
timezone: 'UTC',
next_fire_at: fireTime,
input_template: null,
allowed_actors: ['user', 'scheduled'],
})
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(result.actorNotAllowed).toBe(0)
expect(createRunMock).toHaveBeenCalledTimes(1)
})
it('rolls back the whole batch transaction when the SELECT query fails', async () => {
// "All-or-nothing" batch semantic: if the transaction-level query throws
// (lost connection, deadlock, etc.), the worker must issue ROLLBACK and
// propagate the error so the outer sweep can record it and release the
// advisory lock. No partial side-effects may leak.
selectQueryError = new Error('connection reset by peer')
// Push rows that WOULD be fired if the SELECT had succeeded; they must
// NOT be processed on this sweep.
maturedRows.push({
schedule_id: 's-lost-1',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'rolled-back',
team_id: '11111111-1111-4111-8111-111111111111',
cron_expression: '0 9 * * *',
timezone: 'UTC',
next_fire_at: new Date('2026-04-20T09:00:00Z'),
input_template: null,
})
await expect(processMaturedSchedules()).rejects.toThrow('connection reset by peer')
// Contract: BEGIN must precede the failed SELECT, ROLLBACK must follow it,
// COMMIT must NOT appear between them.
expect(txEvents[0]).toBe('BEGIN')
expect(txEvents).toContain('ROLLBACK')
expect(txEvents).not.toContain('COMMIT')
expect(createRunMock).not.toHaveBeenCalled()
expect(updatedSchedules).toHaveLength(0)
// Lock is still released even though the batch threw — otherwise the next
// sweep would skip forever.
expect(clientRelease).toHaveBeenCalledTimes(1)
})
describe('concurrencyPolicy', () => {
const teamId = '11111111-1111-4111-8111-111111111111'
const NOW = new Date('2026-04-20T09:00:05Z')
const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000)
function scheduleRow(overrides: Partial<ScheduleRow> = {}): ScheduleRow {
return {
schedule_id: 's-forbid',
recipe_namespace: 'sandbox-recipes',
recipe_name: 'pr-review',
team_id: teamId,
cron_expression: '*/5 * * * *',
timezone: 'UTC',
next_fire_at: new Date('2026-04-20T09:00:00Z'),
input_template: null,
concurrency_policy: 'Forbid',
...overrides,
}
}
function setActiveRun(overrides: Partial<FakeRun> = {}, key = 'sandbox-recipes/pr-review') {
activeRuns.set(key, {
run_id: 'run-active',
phase: 'Running',
created_at: minutesAgo(3),
started_at: minutesAgo(2),
max_duration_seconds: null,
...overrides,
})
}
function activeRunLookups(): unknown[][] {
return clientQuery.mock.calls
.filter(([sql]) => /FROM workflow_runs/i.test(String(sql)))
.map(([, params]) => (params ?? []) as unknown[])
}
beforeEach(() => {
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(NOW)
resetConcurrencyForbiddenStreaksForTests()
vi.mocked(rootLogger.info).mockClear()
vi.mocked(rootLogger.warn).mockClear()
vi.mocked(workflowScheduleWorkerFiresTotal.inc).mockClear()
vi.mocked(workflowScheduleWorkerConcurrencyForbiddenSustainedTotal.inc).mockClear()
})
afterEach(() => {
vi.useRealTimers()
})
it('skips the tick under Forbid while a Running run of the recipe exists, records the reason, and advances', async () => {
setActiveRun()
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.fired).toBe(0)
expect(result.concurrencyForbidden).toBe(1)
expect(result.actorNotAllowed).toBe(0)
expect(result.errors).toBe(0)
expect(createRunMock).not.toHaveBeenCalled()
expect(updatedSchedules).toEqual([
{ schedule_id: 's-forbid', next_fire_at: new Date('2026-04-20T09:05:00.000Z') },
])
expect(workflowScheduleWorkerFiresTotal.inc).toHaveBeenCalledWith(
{ result: 'concurrency_forbidden' },
1
)
expect(rootLogger.info).toHaveBeenCalledWith(
expect.objectContaining({
event: 'workflow_schedule_worker_concurrency_forbidden',
scheduleId: 's-forbid',
recipe: 'sandbox-recipes/pr-review',
activeRunId: 'run-active',
activeRunPhase: 'Running',
activeRunAgeSeconds: 120,
consecutiveSkips: 1,
skippedFireAt: '2026-04-20T09:00:00.000Z',
nextFireAt: '2026-04-20T09:05:00.000Z',
}),
expect.any(String)
)
// The skip is not a fire: last_fire_at must not be stamped.
const skipUpdate = clientQuery.mock.calls.find(([sql]) =>
/UPDATE workflow_schedules/i.test(String(sql))
)
expect(String(skipUpdate?.[0])).not.toMatch(/last_fire_at/)
expect(txEvents).toEqual(['BEGIN', 'COMMIT'])
})
it('emits the sweep summary log for a skip-only sweep', async () => {
setActiveRun()
maturedRows.push(scheduleRow())
await processMaturedSchedules()
expect(rootLogger.info).toHaveBeenCalledWith(
expect.objectContaining({
event: 'workflow_schedule_worker_run',
fired: 0,
errors: 0,
concurrencyForbidden: 1,
}),
'schedule worker sweep complete'
)
})
it('treats a fresh Pending run as active', async () => {
setActiveRun({
run_id: 'run-pending',
phase: 'Pending',
created_at: minutesAgo(1),
started_at: null,
})
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.concurrencyForbidden).toBe(1)
expect(createRunMock).not.toHaveBeenCalled()
expect(rootLogger.info).toHaveBeenCalledWith(
expect.objectContaining({ activeRunId: 'run-pending', activeRunAgeSeconds: 60 }),
expect.any(String)
)
})
it('stops counting a Pending run as active once it is older than the Pending bound', async () => {
// Nothing terminalises a Pending row whose child CR never got created.
setActiveRun({ phase: 'Pending', created_at: minutesAgo(16), started_at: null })
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(result.concurrencyForbidden).toBe(0)
expect(createRunMock).toHaveBeenCalledTimes(1)
})
it('blocks on a Running run still inside max_duration_seconds + grace', async () => {
// 600 s deadline passed 2 min ago, but the 5 min reaper grace has not.
setActiveRun({ max_duration_seconds: 600, started_at: minutesAgo(12) })
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.concurrencyForbidden).toBe(1)
expect(createRunMock).not.toHaveBeenCalled()
})
it('stops counting a Running run as active past max_duration_seconds + grace', async () => {
setActiveRun({ max_duration_seconds: 600, started_at: minutesAgo(16) })
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(createRunMock).toHaveBeenCalledTimes(1)
})
it('bounds a Running run with no or an oversized max_duration_seconds by the 24 h cap', async () => {
setActiveRun({ max_duration_seconds: null, started_at: minutesAgo(23 * 60) })
maturedRows.push(scheduleRow())
expect((await processMaturedSchedules()).concurrencyForbidden).toBe(1)
// The CRD default (604800) exceeds the controller ceiling: clamp to the cap.
setActiveRun({ max_duration_seconds: 604_800, started_at: minutesAgo(24 * 60 + 6) })
maturedRows.push(scheduleRow({ next_fire_at: new Date('2026-04-20T09:05:00Z') }))
vi.setSystemTime(new Date('2026-04-20T09:05:05Z'))
expect((await processMaturedSchedules()).fired).toBe(1)
})
it('binds the named stale-run bounds into the lookup', async () => {
maturedRows.push(scheduleRow())
await processMaturedSchedules()
expect(FORBID_PENDING_ACTIVE_BOUND_SECONDS).toBe(900)
expect(FORBID_RUNNING_MAX_DURATION_CAP_SECONDS).toBe(86_400)
expect(FORBID_RUNNING_GRACE_SECONDS).toBe(300)
expect(activeRunLookups()).toEqual([['sandbox-recipes', 'pr-review', 900, 86_400, 300]])
const lookupSql = String(
clientQuery.mock.calls.find(([sql]) => /FROM workflow_runs/i.test(String(sql)))?.[0]
)
expect(lookupSql).toMatch(/created_at > now\(\) - make_interval\(secs => \$3\)/)
expect(lookupSql).toMatch(
/COALESCE\(started_at, created_at\) > now\(\) - make_interval\(secs => LEAST\(COALESCE\(max_duration_seconds, \$4\), \$4\) \+ \$5\)/
)
})
it('advances past a backlog to the next future window instead of replaying it', async () => {
// The worker was down for an hour: next_fire_at is 12 windows behind.
setActiveRun()
maturedRows.push(scheduleRow({ next_fire_at: new Date('2026-04-20T08:00:00Z') }))
await processMaturedSchedules()
// 09:05 (first window after now), not 08:05 (one step after the stale slot).
expect(updatedSchedules[0]?.next_fire_at?.toISOString()).toBe('2026-04-20T09:05:00.000Z')
})
it('fires the first tick after the active run finishes', async () => {
setActiveRun()
maturedRows.push(scheduleRow())
const skipped = await processMaturedSchedules()
expect(skipped.concurrencyForbidden).toBe(1)
const advancedTo = updatedSchedules[0]?.next_fire_at
expect(advancedTo?.toISOString()).toBe('2026-04-20T09:05:00.000Z')
// The run reaches a terminal phase; the schedule matures at its new slot.
activeRuns.clear()
updatedSchedules.length = 0
vi.setSystemTime(new Date('2026-04-20T09:05:02Z'))
maturedRows.push(scheduleRow({ next_fire_at: advancedTo! }))
const fired = await processMaturedSchedules()
expect(fired.fired).toBe(1)
expect(fired.concurrencyForbidden).toBe(0)
expect(createRunMock).toHaveBeenCalledTimes(1)
expect(createRunMock.mock.calls[0]?.[0].idempotency_key).toBe(
'schedule/s-forbid/2026-04-20T09:05:00.000Z'
)
expect(updatedSchedules[0]?.next_fire_at?.toISOString()).toBe('2026-04-20T09:10:00.000Z')
})
it('warns once after 6 consecutive skips and restarts the streak after a fire', async () => {
setActiveRun({ started_at: minutesAgo(1) })
let slot = new Date('2026-04-20T09:00:00Z')
const tick = async () => {
vi.setSystemTime(new Date(slot.getTime() + 5_000))
updatedSchedules.length = 0
maturedRows.push(scheduleRow({ next_fire_at: slot }))
const result = await processMaturedSchedules()
slot = updatedSchedules[0]!.next_fire_at!
return result
}
for (let i = 1; i < FORBID_SUSTAINED_SKIP_THRESHOLD; i += 1) await tick()
expect(rootLogger.warn).not.toHaveBeenCalled()
await tick() // 6th consecutive skip
expect(rootLogger.warn).toHaveBeenCalledTimes(1)
expect(rootLogger.warn).toHaveBeenCalledWith(
expect.objectContaining({
event: 'workflow_schedule_worker_concurrency_forbidden_sustained',
scheduleId: 's-forbid',
activeRunId: 'run-active',
consecutiveSkips: FORBID_SUSTAINED_SKIP_THRESHOLD,
}),
expect.any(String)
)
expect(workflowScheduleWorkerConcurrencyForbiddenSustainedTotal.inc).toHaveBeenCalledTimes(1)
await tick() // 7th: still skipping, no second warning for the same streak
expect(rootLogger.warn).toHaveBeenCalledTimes(1)
activeRuns.clear()
expect((await tick()).fired).toBe(1)
setActiveRun()
vi.mocked(rootLogger.info).mockClear()
await tick()
expect(rootLogger.info).toHaveBeenCalledWith(
expect.objectContaining({ consecutiveSkips: 1 }),
expect.any(String)
)
})
it('restarts the skip streak when the row presents a slot this replica did not write', async () => {
setActiveRun()
maturedRows.push(scheduleRow())
await processMaturedSchedules() // skips 09:00, this replica expects 09:05 next
// Failover/rollback: the row now carries 09:10, not the 09:05 slot written above.
vi.setSystemTime(new Date('2026-04-20T09:10:05Z'))
maturedRows.push(scheduleRow({ next_fire_at: new Date('2026-04-20T09:10:00Z') }))
await processMaturedSchedules()
const forbiddenSkips = vi
.mocked(rootLogger.info)
.mock.calls.map(([fields]) => fields as Record<string, unknown>)
.filter(fields => fields?.event === 'workflow_schedule_worker_concurrency_forbidden')
.map(fields => fields.consecutiveSkips)
expect(forbiddenSkips).toEqual([1, 1])
})
it('caps the skip-streak map and evicts the least recently skipped schedule first', async () => {
setActiveRun()
const total = FORBID_SKIP_STREAK_CAPACITY + 1
for (let i = 0; i < total; i += 1) maturedRows.push(scheduleRow({ schedule_id: `s-${i}` }))
await processMaturedSchedules({ batchSize: total })
expect(concurrencyForbiddenStreakCountForTests()).toBe(FORBID_SKIP_STREAK_CAPACITY)
// s-0 was evicted, so its next consecutive skip restarts at 1; s-1 was kept and counts 2.
vi.setSystemTime(new Date('2026-04-20T09:05:05Z'))
vi.mocked(rootLogger.info).mockClear()
const slot = new Date('2026-04-20T09:05:00Z')
// s-1 goes first: re-adding s-0 evicts the then-oldest entry, which would be s-1.
maturedRows.push(scheduleRow({ schedule_id: 's-1', next_fire_at: slot }))
maturedRows.push(scheduleRow({ schedule_id: 's-0', next_fire_at: slot }))
await processMaturedSchedules()
const streaks = Object.fromEntries(
vi
.mocked(rootLogger.info)
.mock.calls.map(([fields]) => fields as Record<string, unknown>)
.filter(fields => fields?.event === 'workflow_schedule_worker_concurrency_forbidden')
.map(fields => [fields.scheduleId, fields.consecutiveSkips])
)
expect(streaks).toEqual({ 's-0': 1, 's-1': 2 })
expect(concurrencyForbiddenStreakCountForTests()).toBe(FORBID_SKIP_STREAK_CAPACITY)
})
it('defaults a null concurrency_policy to Forbid', async () => {
setActiveRun()
maturedRows.push(scheduleRow({ concurrency_policy: null }))
const result = await processMaturedSchedules()
expect(result.concurrencyForbidden).toBe(1)
expect(createRunMock).not.toHaveBeenCalled()
})
it('still overlaps under Allow and never looks up active runs', async () => {
setActiveRun()
maturedRows.push(scheduleRow({ concurrency_policy: 'Allow' }))
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(result.concurrencyForbidden).toBe(0)
expect(createRunMock).toHaveBeenCalledTimes(1)
expect(activeRunLookups()).toHaveLength(0)
})
it('treats Replace exactly like Allow (Replace is not implemented)', async () => {
setActiveRun()
maturedRows.push(scheduleRow({ concurrency_policy: 'Replace' }))
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(result.concurrencyForbidden).toBe(0)
expect(createRunMock).toHaveBeenCalledTimes(1)
expect(activeRunLookups()).toHaveLength(0)
})
it('does not block on an active run of a different recipe', async () => {
setActiveRun({ run_id: 'run-other' }, 'sandbox-recipes/other-recipe')
setActiveRun({ run_id: 'run-other-ns' }, 'other-namespace/pr-review')
maturedRows.push(scheduleRow())
const result = await processMaturedSchedules()
expect(result.fired).toBe(1)
expect(activeRunLookups().map(params => params.slice(0, 2))).toEqual([
['sandbox-recipes', 'pr-review'],
])
const lookupSql = String(
clientQuery.mock.calls.find(([sql]) => /FROM workflow_runs/i.test(String(sql)))?.[0]
)
expect(lookupSql).toMatch(/recipe_namespace = \$1/)
expect(lookupSql).toMatch(/AND recipe_name = \$2/)
expect(lookupSql).not.toMatch(/child_recipe_name/)
})
it('runs the active-run check inside the locked batch transaction, before createRun, on the same client', async () => {
maturedRows.push(scheduleRow())
await processMaturedSchedules()
const calls = clientQuery.mock.calls.map(([sql]) => String(sql))
const order = clientQuery.mock.invocationCallOrder
const lockAt = calls.findIndex(sql => /pg_try_advisory_lock/i.test(sql))
const beginAt = calls.findIndex(sql => /^\s*BEGIN\s*$/i.test(sql))
const selectAt = calls.findIndex(sql => /FOR UPDATE SKIP LOCKED/i.test(sql))
const lookupAt = calls.findIndex(sql => /FROM workflow_runs/i.test(sql))
const commitAt = calls.findIndex(sql => /^\s*COMMIT\s*$/i.test(sql))
expect(lockAt).toBeGreaterThanOrEqual(0)
expect(beginAt).toBeGreaterThan(lockAt)
expect(selectAt).toBeGreaterThan(beginAt)
expect(lookupAt).toBeGreaterThan(selectAt)
expect(commitAt).toBeGreaterThan(lookupAt)
expect(createRunMock).toHaveBeenCalledTimes(1)
const createRunOrder = createRunMock.mock.invocationCallOrder[0]!
expect(createRunOrder).toBeGreaterThan(order[lookupAt]!)
expect(createRunOrder).toBeLessThan(order[commitAt]!)
// createRun inserts through the same transaction-scoped client.
const runClient = createRunMock.mock.calls[0]?.[1] as { query: unknown }
expect(runClient.query).toBe(clientQuery)
})
it('rolls back without firing when the active-run lookup fails', async () => {
activeRunQueryError = new Error('canceling statement due to statement timeout')
maturedRows.push(scheduleRow())
await expect(processMaturedSchedules()).rejects.toThrow('statement timeout')
expect(createRunMock).not.toHaveBeenCalled()
expect(updatedSchedules).toHaveLength(0)
expect(txEvents).toEqual(['BEGIN', 'ROLLBACK'])
expect(clientRelease).toHaveBeenCalledTimes(1)
})
it('rolls back without firing when the skip write fails', async () => {
setActiveRun()
nextFireUpdateError = new Error('could not serialize access')
maturedRows.push(scheduleRow())
await expect(processMaturedSchedules()).rejects.toThrow('could not serialize access')
expect(createRunMock).not.toHaveBeenCalled()
expect(activeRunLookups()).toHaveLength(1)
expect(txEvents).toEqual(['BEGIN', 'ROLLBACK'])
expect(clientRelease).toHaveBeenCalledTimes(1)
expect(concurrencyForbiddenStreakCountForTests()).toBe(0)
})
})
})
describe('computeNextFire', () => {
it('advances by one cron period relative to the from timestamp, not to now()', () => {
const fromBehind = new Date('2026-04-20T08:59:59Z')
// "0 9 * * *" UTC should fire at 09:00 UTC.
const next = computeNextFire('0 9 * * *', 'UTC', fromBehind)
expect(next.toISOString()).toBe('2026-04-20T09:00:00.000Z')
})
it('honors IANA timezone for DST-sensitive schedules', () => {
// 09:00 America/New_York during EDT (UTC-4) = 13:00 UTC.
const from = new Date('2026-04-20T08:00:00Z')
const next = computeNextFire('0 9 * * *', 'America/New_York', from)
expect(next.toISOString()).toBe('2026-04-20T13:00:00.000Z')
})
it('defaults to UTC when timezone is an empty string', () => {
const from = new Date('2026-04-20T10:15:00Z')
const next = computeNextFire('30 10 * * *', '', from)
expect(next.toISOString()).toBe('2026-04-20T10:30:00.000Z')
})
})