Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): r1-m3 complete shutdown cleanup
Josué9981057b6180
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 5866a0ac6..b18e36f6c 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.638",+ "version": "0.1.639","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.638",+ "version": "0.1.639","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 2783e9829..770c65c01 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.638",+ "version": "0.1.639","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/main.ts b/control-api/src/main.tsindex c2bd65307..1fe8ba9ba 100644--- a/control-api/src/main.ts+++ b/control-api/src/main.ts@@ -2,6 +2,7 @@ import { config } from './config.js'import { assertDbReady, pool, rateLimitPool } from './db.js'import { K8sGateway } from './k8s.js'import { reconcileAllowedModelsConfigMapOnBoot } from './llmAllowedModelsBootReconcile.js'+import { rootLogger } from './observability/logger.js'import { logRegistryConnectionState } from './registryBootGuard.js'import { ControlApiServer } from './server.js'import {@@ -27,12 +28,14 @@ import { startRateLimiterCleanup, stopRateLimiterCleanup } from './services/rateimport {reconcileRegistryPullSecret,startRegistryPullSecretReconcileCron,+ stopRegistryPullSecretReconcileCron,} from './services/registryPullSecretReconcileCron.js'import {reconcileSubscriptionCatalogsFromEnv,startSubscriptionCatalogSyncCron,stopSubscriptionCatalogSyncCron,} from './services/subscriptionCatalogSyncCron.js'+import { closeTracingPools } from './services/tracing/pools.js'import {startWorkflowApprovalTraceProjector,stopWorkflowApprovalTraceProjector,@@ -53,29 +56,94 @@ import {startWorkflowScheduleWorker,stopWorkflowScheduleWorker,} from './services/workflowScheduleWorkerCron.js'+import {+ createControlApiShutdownSteps,+ createShutdownHandler,+ runShutdownSteps,+} from './shutdown.js'import { validateStartupGuards } from './startupGuards.js'+const CONTROL_API_SHUTDOWN_TIMEOUT_MS = 25_000+const logger = rootLogger.child({ module: 'control-api-shutdown' })+let activeServer: ControlApiServer | null = null+let requestedExitCode = 0+let shutdownRequested = false++const shutdown = createShutdownHandler(async () => {+ const result = await runShutdownSteps(+ createControlApiShutdownSteps({+ 'http-server-and-streams': () => activeServer?.stop(),+ 'approval-expiry-cron': stopExpiryCron,+ 'plugin-sdk-maintenance-cron': stopPluginWorkloadSdkMaintenanceCron,+ 'rate-limit-cleanup': stopRateLimiterCleanup,+ 'revoked-token-cleanup': stopAdminRevokedTokenCleanup,+ 'usage-rollup-cron': stopUsageRollupCron,+ 'usage-retention-cron': stopUsageRetentionCron,+ 'budget-reservation-sweep': stopBudgetReservationSweepCron,+ 'approval-archive-cron': stopArchiveCron,+ 'llm-catalog-sync-cron': stopLlmCatalogSyncCron,+ 'subscription-catalog-sync-cron': stopSubscriptionCatalogSyncCron,+ 'registry-pull-secret-cron': stopRegistryPullSecretReconcileCron,+ 'workflow-runs-archive-cron': stopWorkflowRunsArchiveCron,+ 'workflow-schedule-worker': stopWorkflowScheduleWorker,+ 'workflow-approval-notification-worker': stopWorkflowApprovalNotificationDeliveryWorker,+ 'workflow-approval-trace-projector': stopWorkflowApprovalTraceProjector,+ 'entity-change-dispatcher': stopEntityChangeDispatcher,+ 'core-database-pool': () => pool.end(),+ 'rate-limit-database-pool': () => rateLimitPool.end(),+ 'trace-database-pools': closeTracingPools,+ }),+ CONTROL_API_SHUTDOWN_TIMEOUT_MS+ )+ for (const failure of result.errors) {+ logger.error(+ { event: 'control_api_shutdown_step_failed', step: failure.name, err: failure.error },+ 'Control API shutdown step failed'+ )+ }+ if (result.timedOut) {+ logger.error(+ { event: 'control_api_shutdown_timed_out', timeoutMs: CONTROL_API_SHUTDOWN_TIMEOUT_MS },+ 'Control API shutdown exceeded its deadline'+ )+ }+ process.exit(requestedExitCode || result.errors.length > 0 || result.timedOut ? 1 : 0)+})++const requestShutdown = () => {+ shutdownRequested = true+ void shutdown()+}+process.once('SIGTERM', requestShutdown)+process.once('SIGINT', requestShutdown)++function ensureStartupActive(): void {+ if (shutdownRequested) throw new Error('Control API startup interrupted by shutdown')+}+async function main(): Promise<void> {- console.log('[ControlAPI] Starting')- console.log(`[ControlAPI] Namespace: ${config.namespace}`)- console.log(`[ControlAPI] Port: ${config.port}`)+ logger.info({ event: 'control_api_starting', namespace: config.namespace, port: config.port })validateStartupGuards(config)- console.log(- `[ControlAPI] Allowed issuance namespaces: ${config.allowedIssuanceNamespaces.join(',')}`+ logger.info(+ { event: 'control_api_issuance_namespaces', namespaces: config.allowedIssuanceNamespaces },+ 'Allowed issuance namespaces')await assertDbReady()- console.log('[ControlAPI] Database schema ready')+ ensureStartupActive()+ logger.info({ event: 'control_api_database_ready' }, 'Database schema ready')startEntityChangeDispatcher()// Observability only (never fatal): report whether this self-hosted deployment// holds a registry identity. Auth is derived from credential presence, so a// missing row simply means auth is inactive until the connect flow runs.await logRegistryConnectionState()+ ensureStartupActive()// Anti-drift (spec §3-R3.4 / V7): re-materialize the LLM allowlist ConfigMap// from Postgres. Non-fatal — logs + metric on failure, never aborts boot.await reconcileAllowedModelsConfigMapOnBoot()+ ensureStartupActive()startExpiryCron(config.userApprovalRequestExpiryIntervalMs)startPluginWorkloadSdkMaintenanceCron()@@ -95,11 +163,19 @@ async function main(): Promise<void> {retentionDays: config.approvalRetentionDays,batchSize: config.userApprovalRequestArchiveBatchSize,})- console.log(- `[ControlAPI] Approval archive cron enabled (retention=${config.approvalRetentionDays}d, batch=${config.userApprovalRequestArchiveBatchSize})`+ logger.info(+ {+ event: 'control_api_approval_archive_enabled',+ retentionDays: config.approvalRetentionDays,+ batchSize: config.userApprovalRequestArchiveBatchSize,+ },+ 'Approval archive cron enabled')} else {- console.log('[ControlAPI] Approval archive cron disabled (APPROVAL_ARCHIVE_CRON_ENABLED=false)')+ logger.info(+ { event: 'control_api_approval_archive_disabled' },+ 'Approval archive cron disabled'+ )}const gateway = new K8sGateway(config.namespace)@@ -117,12 +193,17 @@ async function main(): Promise<void> {{ sync: () => syncDiscoveredModels({ materializer: gateway.llmAllowedModelsConfigMap() }) },config.llmCatalogSyncIntervalMs)- console.log(- `[ControlAPI] LLM catalog sync cron enabled (interval=${config.llmCatalogSyncIntervalMs}ms)`+ logger.info(+ {+ event: 'control_api_llm_catalog_sync_enabled',+ intervalMs: config.llmCatalogSyncIntervalMs,+ },+ 'LLM catalog sync cron enabled')} else {- console.log(- '[ControlAPI] LLM catalog sync cron disabled (LLM_CATALOG_SYNC_CRON_ENABLED not "true")'+ logger.info(+ { event: 'control_api_llm_catalog_sync_disabled' },+ 'LLM catalog sync cron disabled')}@@ -141,12 +222,17 @@ async function main(): Promise<void> {{ sync: () => reconcileSubscriptionCatalogsFromEnv(gateway.llmAllowedModelsConfigMap()) },config.subscriptionCatalogSyncIntervalMs)- console.log(- `[ControlAPI] Subscription catalog sync cron enabled (interval=${config.subscriptionCatalogSyncIntervalMs}ms)`+ logger.info(+ {+ event: 'control_api_subscription_catalog_sync_enabled',+ intervalMs: config.subscriptionCatalogSyncIntervalMs,+ },+ 'Subscription catalog sync cron enabled')} else {- console.log(- '[ControlAPI] Subscription catalog sync cron disabled (SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED not "true")'+ logger.info(+ { event: 'control_api_subscription_catalog_sync_disabled' },+ 'Subscription catalog sync cron disabled')}@@ -164,12 +250,19 @@ async function main(): Promise<void> {graceMs: config.workflowRunsArchiveGraceMs,batchSize: config.workflowRunsArchiveBatchSize,})- console.log(- `[ControlAPI] Workflow-runs archive cron enabled (interval=${config.workflowRunsArchiveIntervalMs}ms, grace=${config.workflowRunsArchiveGraceMs}ms, batch=${config.workflowRunsArchiveBatchSize})`+ logger.info(+ {+ event: 'control_api_workflow_runs_archive_enabled',+ intervalMs: config.workflowRunsArchiveIntervalMs,+ graceMs: config.workflowRunsArchiveGraceMs,+ batchSize: config.workflowRunsArchiveBatchSize,+ },+ 'Workflow-runs archive cron enabled')} else {- console.log(- '[ControlAPI] Workflow-runs archive cron disabled (WORKFLOW_RUNS_ARCHIVE_CRON_ENABLED=false)'+ logger.info(+ { event: 'control_api_workflow_runs_archive_disabled' },+ 'Workflow-runs archive cron disabled')}@@ -178,12 +271,18 @@ async function main(): Promise<void> {intervalMs: config.workflowScheduleWorkerIntervalMs,batchSize: config.workflowScheduleWorkerBatchSize,})- console.log(- `[ControlAPI] Workflow schedule worker enabled (interval=${config.workflowScheduleWorkerIntervalMs}ms, batch=${config.workflowScheduleWorkerBatchSize})`+ logger.info(+ {+ event: 'control_api_workflow_schedule_worker_enabled',+ intervalMs: config.workflowScheduleWorkerIntervalMs,+ batchSize: config.workflowScheduleWorkerBatchSize,+ },+ 'Workflow schedule worker enabled')} else {- console.log(- '[ControlAPI] Workflow schedule worker disabled (WORKFLOW_SCHEDULE_WORKER_ENABLED=false)'+ logger.info(+ { event: 'control_api_workflow_schedule_worker_disabled' },+ 'Workflow schedule worker disabled')}@@ -191,30 +290,27 @@ async function main(): Promise<void> {// Pass the K8sGateway so the worker resolves each delivery's per-channel bot// from its CommunicationChannel Secret (Figure D multi-bot).startWorkflowApprovalNotificationDeliveryWorker(undefined, gateway)- console.log(- `[ControlAPI] Workflow approval notification delivery enabled (interval=${config.workflowApprovalNotificationDeliveryIntervalMs}ms, batch=${config.workflowApprovalNotificationDeliveryBatchSize})`+ logger.info(+ {+ event: 'control_api_workflow_approval_delivery_enabled',+ intervalMs: config.workflowApprovalNotificationDeliveryIntervalMs,+ batchSize: config.workflowApprovalNotificationDeliveryBatchSize,+ },+ 'Workflow approval notification delivery enabled')} else {- console.log(- '[ControlAPI] Workflow approval notification delivery disabled (WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_ENABLED=false)'+ logger.info(+ { event: 'control_api_workflow_approval_delivery_disabled' },+ 'Workflow approval delivery disabled')}const server = new ControlApiServer(gateway, config.port)+ activeServer = serverawait server.start()- console.log('[ControlAPI] Running')-- let shuttingDown = false- const shutdown = () => {- if (shuttingDown) return- shuttingDown = true- void server.stop().finally(() => {- void pool.end().finally(() => process.exit(0))- })- }- process.once('SIGTERM', shutdown)- process.once('SIGINT', shutdown)+ ensureStartupActive()+ logger.info({ event: 'control_api_running' }, 'Control API running')// Hosted member-registration self-enrollment (spec §8.4): degrade, never// block. Fire-and-forget, and only AFTER the listener is up — the liveness@@ -228,23 +324,8 @@ async function main(): Promise<void> {}main().catch(error => {- console.error('[ControlAPI] Fatal error:', error)- stopExpiryCron()- stopPluginWorkloadSdkMaintenanceCron()- stopArchiveCron()- stopWorkflowRunsArchiveCron()- stopWorkflowScheduleWorker()- stopWorkflowApprovalNotificationDeliveryWorker()- stopEntityChangeDispatcher()- stopRateLimiterCleanup()- stopAdminRevokedTokenCleanup()- stopUsageRollupCron()- stopUsageRetentionCron()- stopBudgetReservationSweepCron()- stopLlmCatalogSyncCron()- stopSubscriptionCatalogSyncCron()- stopWorkflowApprovalTraceProjector()- void pool.end()- void rateLimitPool.end()- process.exit(1)+ if (shutdownRequested) return+ requestedExitCode = 1+ logger.fatal({ event: 'control_api_startup_failed', err: error }, 'Control API startup failed')+ void shutdown()})diff --git a/control-api/src/server.ts b/control-api/src/server.tsindex 8817e3820..3ce00920d 100644--- a/control-api/src/server.ts+++ b/control-api/src/server.ts@@ -18,6 +18,7 @@ import { startExpiryCron, stopExpiryCron } from './services/userApprovalRequestEexport class ControlApiServer {private httpServer: HttpServer | null = null+ private stopPromise: Promise<void> | null = nullconstructor(private readonly gateway: K8sGateway,@@ -45,6 +46,12 @@ export class ControlApiServer {}async stop(): Promise<void> {+ if (this.stopPromise) return this.stopPromise+ this.stopPromise = this.stopOnce()+ return this.stopPromise+ }++ private async stopOnce(): Promise<void> {stopExpiryCron()stopUsageRollupCron()stopUsageRetentionCron()diff --git a/control-api/src/services/tracing/pools.ts b/control-api/src/services/tracing/pools.tsindex e1112277d..ad6b691f6 100644--- a/control-api/src/services/tracing/pools.ts+++ b/control-api/src/services/tracing/pools.ts@@ -127,6 +127,30 @@ export async function closeTraceMaintenancePool(): Promise<void> {if (maintenancePool === pool) maintenancePool = undefined}+let closeTracingPoolsPromise: Promise<void> | null = null++/** Close the lazily-created request tracing pools during Control API shutdown. */+export function closeTracingPools(): Promise<void> {+ if (closeTracingPoolsPromise) return closeTracingPoolsPromise+ const active = pools+ if (!active) return Promise.resolve()+ pools = undefined+ closeTracingPoolsPromise = Promise.allSettled([+ active.traceIngestPool.end(),+ active.traceReadPool.end(),+ ]).then(results => {+ closeTracingPoolsPromise = null+ const failures = results.filter(result => result.status === 'rejected')+ if (failures.length > 0) {+ throw new AggregateError(+ failures.map(result => (result as PromiseRejectedResult).reason),+ 'One or more tracing pools failed to close'+ )+ }+ })+ return closeTracingPoolsPromise+}+async function releaseAfterRollback(client: PoolClient): Promise<void> {try {await client.query('ROLLBACK')diff --git a/control-api/src/shutdown.ts b/control-api/src/shutdown.tsnew file mode 100644index 000000000..ce8f3a4d1--- /dev/null+++ b/control-api/src/shutdown.ts@@ -0,0 +1,76 @@+export type ShutdownStep = {+ name: string+ run: () => void | Promise<void>+}++export type ShutdownResult = {+ errors: Array<{ name: string; error: unknown }>+ timedOut: boolean+}++export const CONTROL_API_SHUTDOWN_STEP_NAMES = [+ 'http-server-and-streams',+ 'approval-expiry-cron',+ 'plugin-sdk-maintenance-cron',+ 'rate-limit-cleanup',+ 'revoked-token-cleanup',+ 'usage-rollup-cron',+ 'usage-retention-cron',+ 'budget-reservation-sweep',+ 'approval-archive-cron',+ 'llm-catalog-sync-cron',+ 'subscription-catalog-sync-cron',+ 'registry-pull-secret-cron',+ 'workflow-runs-archive-cron',+ 'workflow-schedule-worker',+ 'workflow-approval-notification-worker',+ 'workflow-approval-trace-projector',+ 'entity-change-dispatcher',+ 'core-database-pool',+ 'rate-limit-database-pool',+ 'trace-database-pools',+] as const++export type ControlApiShutdownStepName = (typeof CONTROL_API_SHUTDOWN_STEP_NAMES)[number]++export function createControlApiShutdownSteps(+ actions: Record<ControlApiShutdownStepName, () => void | Promise<void>>+): ShutdownStep[] {+ return CONTROL_API_SHUTDOWN_STEP_NAMES.map(name => ({ name, run: actions[name] }))+}++/** Run every cleanup step in order under one shared deadline. */+export async function runShutdownSteps(+ steps: readonly ShutdownStep[],+ timeoutMs: number+): Promise<ShutdownResult> {+ const errors: ShutdownResult['errors'] = []+ let timeout: NodeJS.Timeout | undefined+ const cleanup = (async () => {+ for (const step of steps) {+ try {+ await step.run()+ } catch (error) {+ errors.push({ name: step.name, error })+ }+ }+ })()+ const completed = await Promise.race([+ cleanup.then(() => true),+ new Promise<false>(resolve => {+ timeout = setTimeout(() => resolve(false), timeoutMs)+ timeout.unref?.()+ }),+ ])+ if (timeout) clearTimeout(timeout)+ return { errors, timedOut: !completed }+}++/** Coalesce repeated signals/fatal paths onto one in-flight shutdown. */+export function createShutdownHandler(run: () => Promise<void>): () => Promise<void> {+ let shutdown: Promise<void> | null = null+ return () => {+ shutdown ??= run()+ return shutdown+ }+}diff --git a/control-api/test/server.shutdown.test.ts b/control-api/test/server.shutdown.test.tsnew file mode 100644index 000000000..07a7d6cd1--- /dev/null+++ b/control-api/test/server.shutdown.test.ts@@ -0,0 +1,30 @@+import { describe, expect, it, vi } from 'vitest'+import { ControlApiServer } from '../src/server.js'++describe('ControlApiServer shutdown idempotency', () => {+ it('makes concurrent stop callers await the same HTTP close operation', async () => {+ let finishClose!: () => void+ const httpServer = {+ close: vi.fn((callback: () => void) => {+ finishClose = callback+ }),+ closeAllConnections: vi.fn(),+ }+ const server = new ControlApiServer({} as never, 0) as any+ server.httpServer = httpServer++ const first = server.stop()+ const second = server.stop()+ let secondFinished = false+ void second.then(() => {+ secondFinished = true+ })+ await Promise.resolve()++ expect(httpServer.close).toHaveBeenCalledOnce()+ expect(secondFinished).toBe(false)+ finishClose()+ await Promise.all([first, second])+ expect(secondFinished).toBe(true)+ })+})diff --git a/control-api/test/services.tracingPools.test.ts b/control-api/test/services.tracingPools.test.tsindex 17603bce9..ede018ac4 100644--- a/control-api/test/services.tracingPools.test.ts+++ b/control-api/test/services.tracingPools.test.ts@@ -120,6 +120,18 @@ describe('tracing pool isolation', () => {expect(fakePools).toHaveLength(2)})+ it('closes both lazily-created request pools once during API shutdown', async () => {+ const { closeTracingPools, getTracingPools } = await import('../src/services/tracing/pools.js')+ const first = getTracingPools()++ await Promise.all([closeTracingPools(), closeTracingPools()])++ expect(first.traceIngestPool.end).toHaveBeenCalledOnce()+ expect(first.traceReadPool.end).toHaveBeenCalledOnce()+ expect(getTracingPools()).not.toBe(first)+ expect(fakePools).toHaveLength(4)+ })+it('requires an explicit maintenance connection string', async () => {const { createTraceMaintenancePool, getTraceMaintenancePool } =await import('../src/services/tracing/pools.js')diff --git a/control-api/test/shutdown.test.ts b/control-api/test/shutdown.test.tsnew file mode 100644index 000000000..660946c23--- /dev/null+++ b/control-api/test/shutdown.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it, vi } from 'vitest'+import {+ CONTROL_API_SHUTDOWN_STEP_NAMES,+ createControlApiShutdownSteps,+ createShutdownHandler,+ runShutdownSteps,+} from '../src/shutdown.js'++describe('Control API shutdown', () => {+ it('closes every registered resource in order and continues after failures', async () => {+ const completed: string[] = []+ const actions = Object.fromEntries(+ CONTROL_API_SHUTDOWN_STEP_NAMES.map(name => [+ name,+ vi.fn(async () => {+ completed.push(name)+ if (name === 'rate-limit-cleanup') throw new Error('fixture failure')+ }),+ ])+ ) as Record<(typeof CONTROL_API_SHUTDOWN_STEP_NAMES)[number], () => Promise<void>>++ const result = await runShutdownSteps(createControlApiShutdownSteps(actions), 1000)++ expect(completed).toEqual(CONTROL_API_SHUTDOWN_STEP_NAMES)+ expect(Object.keys(actions).sort()).toEqual([...CONTROL_API_SHUTDOWN_STEP_NAMES].sort())+ expect(result.errors.map(error => error.name)).toEqual(['rate-limit-cleanup'])+ expect(result.timedOut).toBe(false)+ })++ it('shares one in-flight cleanup operation across repeated shutdown signals', async () => {+ let finish!: () => void+ const run = vi.fn(() => new Promise<void>(resolve => (finish = resolve)))+ const shutdown = createShutdownHandler(run)++ const first = shutdown()+ const second = shutdown()++ expect(second).toBe(first)+ expect(run).toHaveBeenCalledOnce()+ finish()+ await first+ })++ it('reports a shared-deadline timeout without skipping later cleanup continuation', async () => {+ vi.useFakeTimers()+ let finishSlowStep!: () => void+ const steps = [+ { name: 'slow', run: () => new Promise<void>(resolve => (finishSlowStep = resolve)) },+ { name: 'later', run: vi.fn() },+ ]+ const pending = runShutdownSteps(steps, 25)+ await vi.advanceTimersByTimeAsync(25)+ await expect(pending).resolves.toMatchObject({ timedOut: true, errors: [] })+ finishSlowStep()+ await vi.advanceTimersByTimeAsync(0)+ expect(steps[1]?.run).toHaveBeenCalledOnce()+ vi.useRealTimers()+ })+})