Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): r2-m2 drain trace work before shutdown
Josué83afcd3002ae
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex ea36133be..4847ffa76 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.640",+ "version": "0.1.641","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.640",+ "version": "0.1.641","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex f26ce09b6..26cc03598 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.640",+ "version": "0.1.641","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/services/tracing/pools.ts b/control-api/src/services/tracing/pools.tsindex ad6b691f6..4c84b1037 100644--- a/control-api/src/services/tracing/pools.ts+++ b/control-api/src/services/tracing/pools.ts@@ -207,7 +207,8 @@ async function acquireTraceClient(pool: Pool, label: TracePoolMetricLabel): Promexport async function withTraceIngestTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {- const client = await acquireTraceClient(getTracingPools().traceIngestPool, 'ingest')+ const pool = getTracingPools().traceIngestPool+ const client = await acquireTraceClient(pool, 'ingest')let released = falsetry {await client.query('BEGIN')@@ -221,12 +222,13 @@ export async function withTraceIngestTransaction<T>(throw error} finally {if (!released) client.release()- observePoolConnections(getTracingPools().traceIngestPool, 'ingest')+ observePoolConnections(pool, 'ingest')}}export async function withTraceReadTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {- const client = await acquireTraceClient(getTracingPools().traceReadPool, 'read')+ const pool = getTracingPools().traceReadPool+ const client = await acquireTraceClient(pool, 'read')let released = falsetry {await client.query('BEGIN READ ONLY')@@ -243,7 +245,7 @@ export async function withTraceReadTransaction<T>(work: (db: DbClient) => Promisthrow error} finally {if (!released) client.release()- observePoolConnections(getTracingPools().traceReadPool, 'read')+ observePoolConnections(pool, 'read')}}diff --git a/control-api/src/services/tracing/workflowApprovalTraceProjector.ts b/control-api/src/services/tracing/workflowApprovalTraceProjector.tsindex 84bc30e4a..bb4460b44 100644--- a/control-api/src/services/tracing/workflowApprovalTraceProjector.ts+++ b/control-api/src/services/tracing/workflowApprovalTraceProjector.ts@@ -243,6 +243,7 @@ export function projectWorkflowApprovalTrace(requestId: string): Promise<number>let started = falselet running = false+let activeWake: Promise<void> | null = nulllet timer: ReturnType<typeof setInterval> | undefinedlet scanAfterId = ''const queuedIds = new Set<string>()@@ -254,7 +255,16 @@ export function enqueueWorkflowApprovalTraceProjection(requestId: string): voidreturn}queuedIds.add(requestId)- queueMicrotask(() => void wake())+ queueMicrotask(scheduleWake)+}++function scheduleWake(): void {+ if (!started || running) return+ const wakePromise = wake()+ activeWake = wakePromise+ void wakePromise.finally(() => {+ if (activeWake === wakePromise) activeWake = null+ })}async function scanMissingApprovalIds(limit: number): Promise<string[]> {@@ -333,16 +343,17 @@ async function wake(): Promise<void> {export function startWorkflowApprovalTraceProjector(intervalMs = DEFAULT_INTERVAL_MS): void {if (started) returnstarted = true- timer = setInterval(() => void wake(), intervalMs)+ timer = setInterval(scheduleWake, intervalMs)timer.unref?.()- void wake()+ scheduleWake()}-export function stopWorkflowApprovalTraceProjector(): void {+export async function stopWorkflowApprovalTraceProjector(): Promise<void> {started = false+ if (timer) clearInterval(timer)+ timer = undefined+ await activeWakerunning = falsescanAfterId = ''queuedIds.clear()- if (timer) clearInterval(timer)- timer = undefined}diff --git a/control-api/test/services.tracingPools.test.ts b/control-api/test/services.tracingPools.test.tsindex ede018ac4..7e773977b 100644--- a/control-api/test/services.tracingPools.test.ts+++ b/control-api/test/services.tracingPools.test.ts@@ -132,6 +132,33 @@ describe('tracing pool isolation', () => {expect(fakePools).toHaveLength(4)})+ it('does not recreate a request pool when an in-flight trace transaction finishes during shutdown', async () => {+ const { closeTracingPools, getTracingPools, withTraceIngestTransaction } =+ await import('../src/services/tracing/pools.js')+ const { traceIngestPool } = getTracingPools()+ const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }+ vi.mocked(traceIngestPool.connect).mockResolvedValue(client as never)+ let finishWork!: () => void+ let enteredWork!: () => void+ const entered = new Promise<void>(resolve => (enteredWork = resolve))+ const transaction = withTraceIngestTransaction(+ () =>+ new Promise<void>(resolve => {+ finishWork = resolve+ enteredWork()+ })+ )++ await entered+ await closeTracingPools()+ finishWork()+ await transaction++ expect(traceIngestPool.end).toHaveBeenCalledOnce()+ expect(client.release).toHaveBeenCalledOnce()+ expect(fakePools).toHaveLength(2)+ })+it('requires an explicit maintenance connection string', async () => {const { createTraceMaintenancePool, getTraceMaintenancePool } =await import('../src/services/tracing/pools.js')diff --git a/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.ts b/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.tsnew file mode 100644index 000000000..5d3be0118--- /dev/null+++ b/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.ts@@ -0,0 +1,48 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'++const traceTransactions = vi.hoisted(() => ({ withTraceIngestTransaction: vi.fn() }))++vi.mock('../src/services/tracing/pools.js', () => ({+ withTraceIngestTransaction: traceTransactions.withTraceIngestTransaction,+}))++describe('workflow approval trace projector shutdown', () => {+ beforeEach(() => {+ vi.resetModules()+ traceTransactions.withTraceIngestTransaction.mockReset()+ })++ it('waits for the active producer-backed scan before allowing pool shutdown', async () => {+ let releaseScan!: () => void+ let markScanStarted!: () => void+ const scanStarted = new Promise<void>(resolve => (markScanStarted = resolve))+ const heldScan = new Promise<void>(resolve => (releaseScan = resolve))+ traceTransactions.withTraceIngestTransaction.mockImplementation(async work => {+ const db = {+ query: vi.fn(async () => {+ markScanStarted()+ await heldScan+ return { rows: [] }+ }),+ }+ return work(db as never)+ })++ const { startWorkflowApprovalTraceProjector, stopWorkflowApprovalTraceProjector } =+ await import('../src/services/tracing/workflowApprovalTraceProjector.js')+ startWorkflowApprovalTraceProjector()+ await scanStarted++ let stopFinished = false+ const stopping = Promise.resolve(stopWorkflowApprovalTraceProjector()).then(() => {+ stopFinished = true+ })+ await Promise.resolve()+ expect(stopFinished).toBe(false)++ releaseScan()+ await stopping+ expect(stopFinished).toBe(true)+ expect(traceTransactions.withTraceIngestTransaction).toHaveBeenCalledOnce()+ })+})