Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): r2-m2 drain trace work before shutdown

Josué83afcd3002ae
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index ea36133be..4847ffa76 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.640",
+ "version": "0.1.641",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.640",
+ "version": "0.1.641",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index f26ce09b6..26cc03598 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.640",
+ "version": "0.1.641",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/tracing/pools.ts b/control-api/src/services/tracing/pools.ts
index ad6b691f6..4c84b1037 100644
--- a/control-api/src/services/tracing/pools.ts
+++ b/control-api/src/services/tracing/pools.ts
@@ -207,7 +207,8 @@ async function acquireTraceClient(pool: Pool, label: TracePoolMetricLabel): Prom
export async function withTraceIngestTransaction<T>(
work: (db: DbClient) => Promise<T>
): Promise<T> {
- const client = await acquireTraceClient(getTracingPools().traceIngestPool, 'ingest')
+ const pool = getTracingPools().traceIngestPool
+ const client = await acquireTraceClient(pool, 'ingest')
let released = false
try {
await client.query('BEGIN')
@@ -221,12 +222,13 @@ export async function withTraceIngestTransaction<T>(
throw error
} finally {
if (!released) client.release()
- observePoolConnections(getTracingPools().traceIngestPool, 'ingest')
+ observePoolConnections(pool, 'ingest')
}
}
export async function withTraceReadTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {
- const client = await acquireTraceClient(getTracingPools().traceReadPool, 'read')
+ const pool = getTracingPools().traceReadPool
+ const client = await acquireTraceClient(pool, 'read')
let released = false
try {
await client.query('BEGIN READ ONLY')
@@ -243,7 +245,7 @@ export async function withTraceReadTransaction<T>(work: (db: DbClient) => Promis
throw error
} finally {
if (!released) client.release()
- observePoolConnections(getTracingPools().traceReadPool, 'read')
+ observePoolConnections(pool, 'read')
}
}
diff --git a/control-api/src/services/tracing/workflowApprovalTraceProjector.ts b/control-api/src/services/tracing/workflowApprovalTraceProjector.ts
index 84bc30e4a..bb4460b44 100644
--- a/control-api/src/services/tracing/workflowApprovalTraceProjector.ts
+++ b/control-api/src/services/tracing/workflowApprovalTraceProjector.ts
@@ -243,6 +243,7 @@ export function projectWorkflowApprovalTrace(requestId: string): Promise<number>
let started = false
let running = false
+let activeWake: Promise<void> | null = null
let timer: ReturnType<typeof setInterval> | undefined
let scanAfterId = ''
const queuedIds = new Set<string>()
@@ -254,7 +255,16 @@ export function enqueueWorkflowApprovalTraceProjection(requestId: string): void
return
}
queuedIds.add(requestId)
- queueMicrotask(() => void wake())
+ queueMicrotask(scheduleWake)
+}
+
+function scheduleWake(): void {
+ if (!started || running) return
+ const wakePromise = wake()
+ activeWake = wakePromise
+ void wakePromise.finally(() => {
+ if (activeWake === wakePromise) activeWake = null
+ })
}
async function scanMissingApprovalIds(limit: number): Promise<string[]> {
@@ -333,16 +343,17 @@ async function wake(): Promise<void> {
export function startWorkflowApprovalTraceProjector(intervalMs = DEFAULT_INTERVAL_MS): void {
if (started) return
started = true
- timer = setInterval(() => void wake(), intervalMs)
+ timer = setInterval(scheduleWake, intervalMs)
timer.unref?.()
- void wake()
+ scheduleWake()
}
-export function stopWorkflowApprovalTraceProjector(): void {
+export async function stopWorkflowApprovalTraceProjector(): Promise<void> {
started = false
+ if (timer) clearInterval(timer)
+ timer = undefined
+ await activeWake
running = false
scanAfterId = ''
queuedIds.clear()
- if (timer) clearInterval(timer)
- timer = undefined
}
diff --git a/control-api/test/services.tracingPools.test.ts b/control-api/test/services.tracingPools.test.ts
index ede018ac4..7e773977b 100644
--- a/control-api/test/services.tracingPools.test.ts
+++ b/control-api/test/services.tracingPools.test.ts
@@ -132,6 +132,33 @@ describe('tracing pool isolation', () => {
expect(fakePools).toHaveLength(4)
})
+ it('does not recreate a request pool when an in-flight trace transaction finishes during shutdown', async () => {
+ const { closeTracingPools, getTracingPools, withTraceIngestTransaction } =
+ await import('../src/services/tracing/pools.js')
+ const { traceIngestPool } = getTracingPools()
+ const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
+ vi.mocked(traceIngestPool.connect).mockResolvedValue(client as never)
+ let finishWork!: () => void
+ let enteredWork!: () => void
+ const entered = new Promise<void>(resolve => (enteredWork = resolve))
+ const transaction = withTraceIngestTransaction(
+ () =>
+ new Promise<void>(resolve => {
+ finishWork = resolve
+ enteredWork()
+ })
+ )
+
+ await entered
+ await closeTracingPools()
+ finishWork()
+ await transaction
+
+ expect(traceIngestPool.end).toHaveBeenCalledOnce()
+ expect(client.release).toHaveBeenCalledOnce()
+ expect(fakePools).toHaveLength(2)
+ })
+
it('requires an explicit maintenance connection string', async () => {
const { createTraceMaintenancePool, getTraceMaintenancePool } =
await import('../src/services/tracing/pools.js')
diff --git a/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.ts b/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.ts
new file mode 100644
index 000000000..5d3be0118
--- /dev/null
+++ b/control-api/test/services.workflowApprovalTraceProjector.shutdown.test.ts
@@ -0,0 +1,48 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const traceTransactions = vi.hoisted(() => ({ withTraceIngestTransaction: vi.fn() }))
+
+vi.mock('../src/services/tracing/pools.js', () => ({
+ withTraceIngestTransaction: traceTransactions.withTraceIngestTransaction,
+}))
+
+describe('workflow approval trace projector shutdown', () => {
+ beforeEach(() => {
+ vi.resetModules()
+ traceTransactions.withTraceIngestTransaction.mockReset()
+ })
+
+ it('waits for the active producer-backed scan before allowing pool shutdown', async () => {
+ let releaseScan!: () => void
+ let markScanStarted!: () => void
+ const scanStarted = new Promise<void>(resolve => (markScanStarted = resolve))
+ const heldScan = new Promise<void>(resolve => (releaseScan = resolve))
+ traceTransactions.withTraceIngestTransaction.mockImplementation(async work => {
+ const db = {
+ query: vi.fn(async () => {
+ markScanStarted()
+ await heldScan
+ return { rows: [] }
+ }),
+ }
+ return work(db as never)
+ })
+
+ const { startWorkflowApprovalTraceProjector, stopWorkflowApprovalTraceProjector } =
+ await import('../src/services/tracing/workflowApprovalTraceProjector.js')
+ startWorkflowApprovalTraceProjector()
+ await scanStarted
+
+ let stopFinished = false
+ const stopping = Promise.resolve(stopWorkflowApprovalTraceProjector()).then(() => {
+ stopFinished = true
+ })
+ await Promise.resolve()
+ expect(stopFinished).toBe(false)
+
+ releaseScan()
+ await stopping
+ expect(stopFinished).toBe(true)
+ expect(traceTransactions.withTraceIngestTransaction).toHaveBeenCalledOnce()
+ })
+})