Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): r2-h1 single-flight dispatcher listener

Josuéfb8b83f66290
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 4847ffa76..dac8a3f72 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.641",
+ "version": "0.1.642",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.641",
+ "version": "0.1.642",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 26cc03598..48ff97068 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.641",
+ "version": "0.1.642",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/entityChangeService.ts b/control-api/src/services/entityChangeService.ts
index abd2f1bc6..bf3ef6b1b 100644
--- a/control-api/src/services/entityChangeService.ts
+++ b/control-api/src/services/entityChangeService.ts
@@ -61,8 +61,10 @@ export async function dispatchEntityChangeOutbox(): Promise<number> {
let dispatcherTimer: NodeJS.Timeout | null = null
let wakeupListener: PoolClient | null = null
+let wakeupConnect: Promise<void> | null = null
let listenerRetryTimer: NodeJS.Timeout | null = null
let dispatcherRunning = false
+let dispatcherGeneration = 0
async function runDispatcherTick(): Promise<void> {
if (dispatcherRunning) return
@@ -85,63 +87,73 @@ async function runDispatcherTick(): Promise<void> {
}
async function connectWakeupListener(): Promise<void> {
- if (wakeupListener || !dispatcherTimer) return
- let client: PoolClient | null = null
- try {
- client = await pool.connect()
- if (!dispatcherTimer) {
- client.release()
- client = null
- return
- }
- await client.query('LISTEN entity_change_outbox')
- if (!dispatcherTimer) {
- client.release(true)
+ if (wakeupListener || wakeupConnect || listenerRetryTimer || !dispatcherTimer) return
+ const generation = dispatcherGeneration
+ const connecting = (async () => {
+ let client: PoolClient | null = null
+ try {
+ client = await pool.connect()
+ if (!dispatcherTimer || generation !== dispatcherGeneration) {
+ client.release()
+ client = null
+ return
+ }
+ await client.query('LISTEN entity_change_outbox')
+ if (!dispatcherTimer || generation !== dispatcherGeneration) {
+ client.release(true)
+ client = null
+ return
+ }
+ const activeClient = client
+ wakeupListener = activeClient
client = null
- return
- }
- const activeClient = client
- wakeupListener = activeClient
- client = null
- activeClient.on('notification', () => void runDispatcherTick())
- const disconnected = (err?: Error) => {
- if (wakeupListener !== activeClient) return
- wakeupListener = null
- activeClient.removeAllListeners('notification')
- activeClient.release(true)
- if (err) {
- logger.warn(
- { event: 'entity_change_dispatch_listener_lost', err },
- 'dispatcher wake-up degraded'
- )
+ activeClient.on('notification', () => void runDispatcherTick())
+ const disconnected = (err?: Error) => {
+ if (wakeupListener !== activeClient) return
+ wakeupListener = null
+ activeClient.removeAllListeners('notification')
+ activeClient.release(true)
+ if (err) {
+ logger.warn(
+ { event: 'entity_change_dispatch_listener_lost', err },
+ 'dispatcher wake-up degraded'
+ )
+ }
+ if (dispatcherTimer && !listenerRetryTimer) {
+ listenerRetryTimer = setTimeout(() => {
+ listenerRetryTimer = null
+ void connectWakeupListener()
+ }, 5000)
+ }
}
- if (dispatcherTimer && !listenerRetryTimer) {
+ activeClient.on('error', disconnected)
+ activeClient.on('end', () => disconnected())
+ } catch (err) {
+ client?.release(true)
+ logger.warn(
+ { event: 'entity_change_dispatch_listener_unavailable', err },
+ 'dispatcher wake-up unavailable'
+ )
+ if (dispatcherTimer && generation === dispatcherGeneration && !listenerRetryTimer) {
listenerRetryTimer = setTimeout(() => {
listenerRetryTimer = null
void connectWakeupListener()
}, 5000)
}
}
- activeClient.on('error', disconnected)
- activeClient.on('end', () => disconnected())
- } catch (err) {
- client?.release(true)
- logger.warn(
- { event: 'entity_change_dispatch_listener_unavailable', err },
- 'dispatcher wake-up unavailable'
- )
- if (dispatcherTimer && !listenerRetryTimer) {
- listenerRetryTimer = setTimeout(() => {
- listenerRetryTimer = null
- void connectWakeupListener()
- }, 5000)
- }
+ })()
+ wakeupConnect = connecting
+ try {
+ await connecting
+ } finally {
+ if (wakeupConnect === connecting) wakeupConnect = null
}
}
/** Start a per-instance dispatcher; the database advisory lock elects one active owner. */
export function startEntityChangeDispatcher(): void {
if (dispatcherTimer) return
+ dispatcherGeneration += 1
dispatcherTimer = setInterval(() => {
void runDispatcherTick()
if (!wakeupListener) void connectWakeupListener()
@@ -151,6 +163,7 @@ export function startEntityChangeDispatcher(): void {
}
export function stopEntityChangeDispatcher(): void {
+ dispatcherGeneration += 1
if (dispatcherTimer) clearInterval(dispatcherTimer)
if (listenerRetryTimer) clearTimeout(listenerRetryTimer)
dispatcherTimer = null
diff --git a/control-api/test/services.entityChangeService.test.ts b/control-api/test/services.entityChangeService.test.ts
index ecb063347..186d535c8 100644
--- a/control-api/test/services.entityChangeService.test.ts
+++ b/control-api/test/services.entityChangeService.test.ts
@@ -12,6 +12,7 @@ import {
const dbMock = vi.hoisted(() => ({ query: vi.fn(), connect: vi.fn() }))
const configMock = vi.hoisted(() => ({
entityChangeDispatchBatchSize: 37,
+ entityChangeDispatchIntervalMs: 100,
entityChangeRetentionSeconds: 86_400,
entityChangeMaxRecoveryEvents: 500,
}))
@@ -122,4 +123,65 @@ describe('entityChangeService', () => {
stopEntityChangeDispatcher()
expect(listener.release).toHaveBeenCalledWith(true)
})
+
+ it('single-flights dispatcher LISTEN connection attempts while pool connection is pending', async () => {
+ vi.useFakeTimers()
+ type TestListener = EventEmitter & {
+ query: ReturnType<typeof vi.fn>
+ release: ReturnType<typeof vi.fn>
+ }
+ const pendingConnections: Array<{
+ resolve: (client: TestListener) => void
+ listener: TestListener
+ }> = []
+ dbMock.connect.mockImplementation(
+ () =>
+ new Promise(resolve => {
+ const listener = new EventEmitter() as TestListener
+ listener.query = vi.fn().mockResolvedValue({ rows: [] })
+ listener.release = vi.fn()
+ pendingConnections.push({ resolve, listener })
+ })
+ )
+ dbMock.query.mockResolvedValue({ rows: [] })
+
+ try {
+ startEntityChangeDispatcher()
+ await vi.advanceTimersByTimeAsync(500)
+ expect(dbMock.connect).toHaveBeenCalledOnce()
+ stopEntityChangeDispatcher()
+ pendingConnections[0].resolve(pendingConnections[0].listener)
+ vi.useRealTimers()
+ await new Promise<void>(resolve => setImmediate(resolve))
+ expect(pendingConnections[0].listener.release).toHaveBeenCalledOnce()
+ } finally {
+ stopEntityChangeDispatcher()
+ for (const { resolve, listener } of pendingConnections) {
+ resolve(listener)
+ }
+ vi.useRealTimers()
+ await new Promise<void>(resolve => setImmediate(resolve))
+ }
+ })
+
+ it('does not bypass the bounded dispatcher listener retry delay', async () => {
+ vi.useFakeTimers()
+ dbMock.connect.mockRejectedValue(new Error('database unavailable'))
+ dbMock.query.mockResolvedValue({ rows: [] })
+
+ try {
+ startEntityChangeDispatcher()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(dbMock.connect).toHaveBeenCalledOnce()
+
+ await vi.advanceTimersByTimeAsync(4_999)
+ expect(dbMock.connect).toHaveBeenCalledOnce()
+
+ await vi.advanceTimersByTimeAsync(1)
+ expect(dbMock.connect).toHaveBeenCalledTimes(2)
+ } finally {
+ stopEntityChangeDispatcher()
+ vi.useRealTimers()
+ }
+ })
})