Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): r1-m3 complete shutdown cleanup

Josué9981057b6180
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 5866a0ac6..b18e36f6c 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.638",
+ "version": "0.1.639",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.638",
+ "version": "0.1.639",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 2783e9829..770c65c01 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.638",
+ "version": "0.1.639",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/main.ts b/control-api/src/main.ts
index c2bd65307..1fe8ba9ba 100644
--- a/control-api/src/main.ts
+++ b/control-api/src/main.ts
@@ -2,6 +2,7 @@ import { config } from './config.js'
import { assertDbReady, pool, rateLimitPool } from './db.js'
import { K8sGateway } from './k8s.js'
import { reconcileAllowedModelsConfigMapOnBoot } from './llmAllowedModelsBootReconcile.js'
+import { rootLogger } from './observability/logger.js'
import { logRegistryConnectionState } from './registryBootGuard.js'
import { ControlApiServer } from './server.js'
import {
@@ -27,12 +28,14 @@ import { startRateLimiterCleanup, stopRateLimiterCleanup } from './services/rate
import {
reconcileRegistryPullSecret,
startRegistryPullSecretReconcileCron,
+ stopRegistryPullSecretReconcileCron,
} from './services/registryPullSecretReconcileCron.js'
import {
reconcileSubscriptionCatalogsFromEnv,
startSubscriptionCatalogSyncCron,
stopSubscriptionCatalogSyncCron,
} from './services/subscriptionCatalogSyncCron.js'
+import { closeTracingPools } from './services/tracing/pools.js'
import {
startWorkflowApprovalTraceProjector,
stopWorkflowApprovalTraceProjector,
@@ -53,29 +56,94 @@ import {
startWorkflowScheduleWorker,
stopWorkflowScheduleWorker,
} from './services/workflowScheduleWorkerCron.js'
+import {
+ createControlApiShutdownSteps,
+ createShutdownHandler,
+ runShutdownSteps,
+} from './shutdown.js'
import { validateStartupGuards } from './startupGuards.js'
+const CONTROL_API_SHUTDOWN_TIMEOUT_MS = 25_000
+const logger = rootLogger.child({ module: 'control-api-shutdown' })
+let activeServer: ControlApiServer | null = null
+let requestedExitCode = 0
+let shutdownRequested = false
+
+const shutdown = createShutdownHandler(async () => {
+ const result = await runShutdownSteps(
+ createControlApiShutdownSteps({
+ 'http-server-and-streams': () => activeServer?.stop(),
+ 'approval-expiry-cron': stopExpiryCron,
+ 'plugin-sdk-maintenance-cron': stopPluginWorkloadSdkMaintenanceCron,
+ 'rate-limit-cleanup': stopRateLimiterCleanup,
+ 'revoked-token-cleanup': stopAdminRevokedTokenCleanup,
+ 'usage-rollup-cron': stopUsageRollupCron,
+ 'usage-retention-cron': stopUsageRetentionCron,
+ 'budget-reservation-sweep': stopBudgetReservationSweepCron,
+ 'approval-archive-cron': stopArchiveCron,
+ 'llm-catalog-sync-cron': stopLlmCatalogSyncCron,
+ 'subscription-catalog-sync-cron': stopSubscriptionCatalogSyncCron,
+ 'registry-pull-secret-cron': stopRegistryPullSecretReconcileCron,
+ 'workflow-runs-archive-cron': stopWorkflowRunsArchiveCron,
+ 'workflow-schedule-worker': stopWorkflowScheduleWorker,
+ 'workflow-approval-notification-worker': stopWorkflowApprovalNotificationDeliveryWorker,
+ 'workflow-approval-trace-projector': stopWorkflowApprovalTraceProjector,
+ 'entity-change-dispatcher': stopEntityChangeDispatcher,
+ 'core-database-pool': () => pool.end(),
+ 'rate-limit-database-pool': () => rateLimitPool.end(),
+ 'trace-database-pools': closeTracingPools,
+ }),
+ CONTROL_API_SHUTDOWN_TIMEOUT_MS
+ )
+ for (const failure of result.errors) {
+ logger.error(
+ { event: 'control_api_shutdown_step_failed', step: failure.name, err: failure.error },
+ 'Control API shutdown step failed'
+ )
+ }
+ if (result.timedOut) {
+ logger.error(
+ { event: 'control_api_shutdown_timed_out', timeoutMs: CONTROL_API_SHUTDOWN_TIMEOUT_MS },
+ 'Control API shutdown exceeded its deadline'
+ )
+ }
+ process.exit(requestedExitCode || result.errors.length > 0 || result.timedOut ? 1 : 0)
+})
+
+const requestShutdown = () => {
+ shutdownRequested = true
+ void shutdown()
+}
+process.once('SIGTERM', requestShutdown)
+process.once('SIGINT', requestShutdown)
+
+function ensureStartupActive(): void {
+ if (shutdownRequested) throw new Error('Control API startup interrupted by shutdown')
+}
+
async function main(): Promise<void> {
- console.log('[ControlAPI] Starting')
- console.log(`[ControlAPI] Namespace: ${config.namespace}`)
- console.log(`[ControlAPI] Port: ${config.port}`)
+ logger.info({ event: 'control_api_starting', namespace: config.namespace, port: config.port })
validateStartupGuards(config)
- console.log(
- `[ControlAPI] Allowed issuance namespaces: ${config.allowedIssuanceNamespaces.join(',')}`
+ logger.info(
+ { event: 'control_api_issuance_namespaces', namespaces: config.allowedIssuanceNamespaces },
+ 'Allowed issuance namespaces'
)
await assertDbReady()
- console.log('[ControlAPI] Database schema ready')
+ ensureStartupActive()
+ logger.info({ event: 'control_api_database_ready' }, 'Database schema ready')
startEntityChangeDispatcher()
// Observability only (never fatal): report whether this self-hosted deployment
// holds a registry identity. Auth is derived from credential presence, so a
// missing row simply means auth is inactive until the connect flow runs.
await logRegistryConnectionState()
+ ensureStartupActive()
// Anti-drift (spec §3-R3.4 / V7): re-materialize the LLM allowlist ConfigMap
// from Postgres. Non-fatal — logs + metric on failure, never aborts boot.
await reconcileAllowedModelsConfigMapOnBoot()
+ ensureStartupActive()
startExpiryCron(config.userApprovalRequestExpiryIntervalMs)
startPluginWorkloadSdkMaintenanceCron()
@@ -95,11 +163,19 @@ async function main(): Promise<void> {
retentionDays: config.approvalRetentionDays,
batchSize: config.userApprovalRequestArchiveBatchSize,
})
- console.log(
- `[ControlAPI] Approval archive cron enabled (retention=${config.approvalRetentionDays}d, batch=${config.userApprovalRequestArchiveBatchSize})`
+ logger.info(
+ {
+ event: 'control_api_approval_archive_enabled',
+ retentionDays: config.approvalRetentionDays,
+ batchSize: config.userApprovalRequestArchiveBatchSize,
+ },
+ 'Approval archive cron enabled'
)
} else {
- console.log('[ControlAPI] Approval archive cron disabled (APPROVAL_ARCHIVE_CRON_ENABLED=false)')
+ logger.info(
+ { event: 'control_api_approval_archive_disabled' },
+ 'Approval archive cron disabled'
+ )
}
const gateway = new K8sGateway(config.namespace)
@@ -117,12 +193,17 @@ async function main(): Promise<void> {
{ sync: () => syncDiscoveredModels({ materializer: gateway.llmAllowedModelsConfigMap() }) },
config.llmCatalogSyncIntervalMs
)
- console.log(
- `[ControlAPI] LLM catalog sync cron enabled (interval=${config.llmCatalogSyncIntervalMs}ms)`
+ logger.info(
+ {
+ event: 'control_api_llm_catalog_sync_enabled',
+ intervalMs: config.llmCatalogSyncIntervalMs,
+ },
+ 'LLM catalog sync cron enabled'
)
} else {
- console.log(
- '[ControlAPI] LLM catalog sync cron disabled (LLM_CATALOG_SYNC_CRON_ENABLED not "true")'
+ logger.info(
+ { event: 'control_api_llm_catalog_sync_disabled' },
+ 'LLM catalog sync cron disabled'
)
}
@@ -141,12 +222,17 @@ async function main(): Promise<void> {
{ sync: () => reconcileSubscriptionCatalogsFromEnv(gateway.llmAllowedModelsConfigMap()) },
config.subscriptionCatalogSyncIntervalMs
)
- console.log(
- `[ControlAPI] Subscription catalog sync cron enabled (interval=${config.subscriptionCatalogSyncIntervalMs}ms)`
+ logger.info(
+ {
+ event: 'control_api_subscription_catalog_sync_enabled',
+ intervalMs: config.subscriptionCatalogSyncIntervalMs,
+ },
+ 'Subscription catalog sync cron enabled'
)
} else {
- console.log(
- '[ControlAPI] Subscription catalog sync cron disabled (SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED not "true")'
+ logger.info(
+ { event: 'control_api_subscription_catalog_sync_disabled' },
+ 'Subscription catalog sync cron disabled'
)
}
@@ -164,12 +250,19 @@ async function main(): Promise<void> {
graceMs: config.workflowRunsArchiveGraceMs,
batchSize: config.workflowRunsArchiveBatchSize,
})
- console.log(
- `[ControlAPI] Workflow-runs archive cron enabled (interval=${config.workflowRunsArchiveIntervalMs}ms, grace=${config.workflowRunsArchiveGraceMs}ms, batch=${config.workflowRunsArchiveBatchSize})`
+ logger.info(
+ {
+ event: 'control_api_workflow_runs_archive_enabled',
+ intervalMs: config.workflowRunsArchiveIntervalMs,
+ graceMs: config.workflowRunsArchiveGraceMs,
+ batchSize: config.workflowRunsArchiveBatchSize,
+ },
+ 'Workflow-runs archive cron enabled'
)
} else {
- console.log(
- '[ControlAPI] Workflow-runs archive cron disabled (WORKFLOW_RUNS_ARCHIVE_CRON_ENABLED=false)'
+ logger.info(
+ { event: 'control_api_workflow_runs_archive_disabled' },
+ 'Workflow-runs archive cron disabled'
)
}
@@ -178,12 +271,18 @@ async function main(): Promise<void> {
intervalMs: config.workflowScheduleWorkerIntervalMs,
batchSize: config.workflowScheduleWorkerBatchSize,
})
- console.log(
- `[ControlAPI] Workflow schedule worker enabled (interval=${config.workflowScheduleWorkerIntervalMs}ms, batch=${config.workflowScheduleWorkerBatchSize})`
+ logger.info(
+ {
+ event: 'control_api_workflow_schedule_worker_enabled',
+ intervalMs: config.workflowScheduleWorkerIntervalMs,
+ batchSize: config.workflowScheduleWorkerBatchSize,
+ },
+ 'Workflow schedule worker enabled'
)
} else {
- console.log(
- '[ControlAPI] Workflow schedule worker disabled (WORKFLOW_SCHEDULE_WORKER_ENABLED=false)'
+ logger.info(
+ { event: 'control_api_workflow_schedule_worker_disabled' },
+ 'Workflow schedule worker disabled'
)
}
@@ -191,30 +290,27 @@ async function main(): Promise<void> {
// Pass the K8sGateway so the worker resolves each delivery's per-channel bot
// from its CommunicationChannel Secret (Figure D multi-bot).
startWorkflowApprovalNotificationDeliveryWorker(undefined, gateway)
- console.log(
- `[ControlAPI] Workflow approval notification delivery enabled (interval=${config.workflowApprovalNotificationDeliveryIntervalMs}ms, batch=${config.workflowApprovalNotificationDeliveryBatchSize})`
+ logger.info(
+ {
+ event: 'control_api_workflow_approval_delivery_enabled',
+ intervalMs: config.workflowApprovalNotificationDeliveryIntervalMs,
+ batchSize: config.workflowApprovalNotificationDeliveryBatchSize,
+ },
+ 'Workflow approval notification delivery enabled'
)
} else {
- console.log(
- '[ControlAPI] Workflow approval notification delivery disabled (WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_ENABLED=false)'
+ logger.info(
+ { event: 'control_api_workflow_approval_delivery_disabled' },
+ 'Workflow approval delivery disabled'
)
}
const server = new ControlApiServer(gateway, config.port)
+ activeServer = server
await server.start()
- console.log('[ControlAPI] Running')
-
- let shuttingDown = false
- const shutdown = () => {
- if (shuttingDown) return
- shuttingDown = true
- void server.stop().finally(() => {
- void pool.end().finally(() => process.exit(0))
- })
- }
- process.once('SIGTERM', shutdown)
- process.once('SIGINT', shutdown)
+ ensureStartupActive()
+ logger.info({ event: 'control_api_running' }, 'Control API running')
// Hosted member-registration self-enrollment (spec §8.4): degrade, never
// block. Fire-and-forget, and only AFTER the listener is up — the liveness
@@ -228,23 +324,8 @@ async function main(): Promise<void> {
}
main().catch(error => {
- console.error('[ControlAPI] Fatal error:', error)
- stopExpiryCron()
- stopPluginWorkloadSdkMaintenanceCron()
- stopArchiveCron()
- stopWorkflowRunsArchiveCron()
- stopWorkflowScheduleWorker()
- stopWorkflowApprovalNotificationDeliveryWorker()
- stopEntityChangeDispatcher()
- stopRateLimiterCleanup()
- stopAdminRevokedTokenCleanup()
- stopUsageRollupCron()
- stopUsageRetentionCron()
- stopBudgetReservationSweepCron()
- stopLlmCatalogSyncCron()
- stopSubscriptionCatalogSyncCron()
- stopWorkflowApprovalTraceProjector()
- void pool.end()
- void rateLimitPool.end()
- process.exit(1)
+ if (shutdownRequested) return
+ requestedExitCode = 1
+ logger.fatal({ event: 'control_api_startup_failed', err: error }, 'Control API startup failed')
+ void shutdown()
})
diff --git a/control-api/src/server.ts b/control-api/src/server.ts
index 8817e3820..3ce00920d 100644
--- a/control-api/src/server.ts
+++ b/control-api/src/server.ts
@@ -18,6 +18,7 @@ import { startExpiryCron, stopExpiryCron } from './services/userApprovalRequestE
export class ControlApiServer {
private httpServer: HttpServer | null = null
+ private stopPromise: Promise<void> | null = null
constructor(
private readonly gateway: K8sGateway,
@@ -45,6 +46,12 @@ export class ControlApiServer {
}
async stop(): Promise<void> {
+ if (this.stopPromise) return this.stopPromise
+ this.stopPromise = this.stopOnce()
+ return this.stopPromise
+ }
+
+ private async stopOnce(): Promise<void> {
stopExpiryCron()
stopUsageRollupCron()
stopUsageRetentionCron()
diff --git a/control-api/src/services/tracing/pools.ts b/control-api/src/services/tracing/pools.ts
index e1112277d..ad6b691f6 100644
--- a/control-api/src/services/tracing/pools.ts
+++ b/control-api/src/services/tracing/pools.ts
@@ -127,6 +127,30 @@ export async function closeTraceMaintenancePool(): Promise<void> {
if (maintenancePool === pool) maintenancePool = undefined
}
+let closeTracingPoolsPromise: Promise<void> | null = null
+
+/** Close the lazily-created request tracing pools during Control API shutdown. */
+export function closeTracingPools(): Promise<void> {
+ if (closeTracingPoolsPromise) return closeTracingPoolsPromise
+ const active = pools
+ if (!active) return Promise.resolve()
+ pools = undefined
+ closeTracingPoolsPromise = Promise.allSettled([
+ active.traceIngestPool.end(),
+ active.traceReadPool.end(),
+ ]).then(results => {
+ closeTracingPoolsPromise = null
+ const failures = results.filter(result => result.status === 'rejected')
+ if (failures.length > 0) {
+ throw new AggregateError(
+ failures.map(result => (result as PromiseRejectedResult).reason),
+ 'One or more tracing pools failed to close'
+ )
+ }
+ })
+ return closeTracingPoolsPromise
+}
+
async function releaseAfterRollback(client: PoolClient): Promise<void> {
try {
await client.query('ROLLBACK')
diff --git a/control-api/src/shutdown.ts b/control-api/src/shutdown.ts
new file mode 100644
index 000000000..ce8f3a4d1
--- /dev/null
+++ b/control-api/src/shutdown.ts
@@ -0,0 +1,76 @@
+export type ShutdownStep = {
+ name: string
+ run: () => void | Promise<void>
+}
+
+export type ShutdownResult = {
+ errors: Array<{ name: string; error: unknown }>
+ timedOut: boolean
+}
+
+export const CONTROL_API_SHUTDOWN_STEP_NAMES = [
+ 'http-server-and-streams',
+ 'approval-expiry-cron',
+ 'plugin-sdk-maintenance-cron',
+ 'rate-limit-cleanup',
+ 'revoked-token-cleanup',
+ 'usage-rollup-cron',
+ 'usage-retention-cron',
+ 'budget-reservation-sweep',
+ 'approval-archive-cron',
+ 'llm-catalog-sync-cron',
+ 'subscription-catalog-sync-cron',
+ 'registry-pull-secret-cron',
+ 'workflow-runs-archive-cron',
+ 'workflow-schedule-worker',
+ 'workflow-approval-notification-worker',
+ 'workflow-approval-trace-projector',
+ 'entity-change-dispatcher',
+ 'core-database-pool',
+ 'rate-limit-database-pool',
+ 'trace-database-pools',
+] as const
+
+export type ControlApiShutdownStepName = (typeof CONTROL_API_SHUTDOWN_STEP_NAMES)[number]
+
+export function createControlApiShutdownSteps(
+ actions: Record<ControlApiShutdownStepName, () => void | Promise<void>>
+): ShutdownStep[] {
+ return CONTROL_API_SHUTDOWN_STEP_NAMES.map(name => ({ name, run: actions[name] }))
+}
+
+/** Run every cleanup step in order under one shared deadline. */
+export async function runShutdownSteps(
+ steps: readonly ShutdownStep[],
+ timeoutMs: number
+): Promise<ShutdownResult> {
+ const errors: ShutdownResult['errors'] = []
+ let timeout: NodeJS.Timeout | undefined
+ const cleanup = (async () => {
+ for (const step of steps) {
+ try {
+ await step.run()
+ } catch (error) {
+ errors.push({ name: step.name, error })
+ }
+ }
+ })()
+ const completed = await Promise.race([
+ cleanup.then(() => true),
+ new Promise<false>(resolve => {
+ timeout = setTimeout(() => resolve(false), timeoutMs)
+ timeout.unref?.()
+ }),
+ ])
+ if (timeout) clearTimeout(timeout)
+ return { errors, timedOut: !completed }
+}
+
+/** Coalesce repeated signals/fatal paths onto one in-flight shutdown. */
+export function createShutdownHandler(run: () => Promise<void>): () => Promise<void> {
+ let shutdown: Promise<void> | null = null
+ return () => {
+ shutdown ??= run()
+ return shutdown
+ }
+}
diff --git a/control-api/test/server.shutdown.test.ts b/control-api/test/server.shutdown.test.ts
new file mode 100644
index 000000000..07a7d6cd1
--- /dev/null
+++ b/control-api/test/server.shutdown.test.ts
@@ -0,0 +1,30 @@
+import { describe, expect, it, vi } from 'vitest'
+import { ControlApiServer } from '../src/server.js'
+
+describe('ControlApiServer shutdown idempotency', () => {
+ it('makes concurrent stop callers await the same HTTP close operation', async () => {
+ let finishClose!: () => void
+ const httpServer = {
+ close: vi.fn((callback: () => void) => {
+ finishClose = callback
+ }),
+ closeAllConnections: vi.fn(),
+ }
+ const server = new ControlApiServer({} as never, 0) as any
+ server.httpServer = httpServer
+
+ const first = server.stop()
+ const second = server.stop()
+ let secondFinished = false
+ void second.then(() => {
+ secondFinished = true
+ })
+ await Promise.resolve()
+
+ expect(httpServer.close).toHaveBeenCalledOnce()
+ expect(secondFinished).toBe(false)
+ finishClose()
+ await Promise.all([first, second])
+ expect(secondFinished).toBe(true)
+ })
+})
diff --git a/control-api/test/services.tracingPools.test.ts b/control-api/test/services.tracingPools.test.ts
index 17603bce9..ede018ac4 100644
--- a/control-api/test/services.tracingPools.test.ts
+++ b/control-api/test/services.tracingPools.test.ts
@@ -120,6 +120,18 @@ describe('tracing pool isolation', () => {
expect(fakePools).toHaveLength(2)
})
+ it('closes both lazily-created request pools once during API shutdown', async () => {
+ const { closeTracingPools, getTracingPools } = await import('../src/services/tracing/pools.js')
+ const first = getTracingPools()
+
+ await Promise.all([closeTracingPools(), closeTracingPools()])
+
+ expect(first.traceIngestPool.end).toHaveBeenCalledOnce()
+ expect(first.traceReadPool.end).toHaveBeenCalledOnce()
+ expect(getTracingPools()).not.toBe(first)
+ expect(fakePools).toHaveLength(4)
+ })
+
it('requires an explicit maintenance connection string', async () => {
const { createTraceMaintenancePool, getTraceMaintenancePool } =
await import('../src/services/tracing/pools.js')
diff --git a/control-api/test/shutdown.test.ts b/control-api/test/shutdown.test.ts
new file mode 100644
index 000000000..660946c23
--- /dev/null
+++ b/control-api/test/shutdown.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ CONTROL_API_SHUTDOWN_STEP_NAMES,
+ createControlApiShutdownSteps,
+ createShutdownHandler,
+ runShutdownSteps,
+} from '../src/shutdown.js'
+
+describe('Control API shutdown', () => {
+ it('closes every registered resource in order and continues after failures', async () => {
+ const completed: string[] = []
+ const actions = Object.fromEntries(
+ CONTROL_API_SHUTDOWN_STEP_NAMES.map(name => [
+ name,
+ vi.fn(async () => {
+ completed.push(name)
+ if (name === 'rate-limit-cleanup') throw new Error('fixture failure')
+ }),
+ ])
+ ) as Record<(typeof CONTROL_API_SHUTDOWN_STEP_NAMES)[number], () => Promise<void>>
+
+ const result = await runShutdownSteps(createControlApiShutdownSteps(actions), 1000)
+
+ expect(completed).toEqual(CONTROL_API_SHUTDOWN_STEP_NAMES)
+ expect(Object.keys(actions).sort()).toEqual([...CONTROL_API_SHUTDOWN_STEP_NAMES].sort())
+ expect(result.errors.map(error => error.name)).toEqual(['rate-limit-cleanup'])
+ expect(result.timedOut).toBe(false)
+ })
+
+ it('shares one in-flight cleanup operation across repeated shutdown signals', async () => {
+ let finish!: () => void
+ const run = vi.fn(() => new Promise<void>(resolve => (finish = resolve)))
+ const shutdown = createShutdownHandler(run)
+
+ const first = shutdown()
+ const second = shutdown()
+
+ expect(second).toBe(first)
+ expect(run).toHaveBeenCalledOnce()
+ finish()
+ await first
+ })
+
+ it('reports a shared-deadline timeout without skipping later cleanup continuation', async () => {
+ vi.useFakeTimers()
+ let finishSlowStep!: () => void
+ const steps = [
+ { name: 'slow', run: () => new Promise<void>(resolve => (finishSlowStep = resolve)) },
+ { name: 'later', run: vi.fn() },
+ ]
+ const pending = runShutdownSteps(steps, 25)
+ await vi.advanceTimersByTimeAsync(25)
+ await expect(pending).resolves.toMatchObject({ timedOut: true, errors: [] })
+ finishSlowStep()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(steps[1]?.run).toHaveBeenCalledOnce()
+ vi.useRealTimers()
+ })
+})