Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): resolve r2-m2 bound live stream admission

Josué71329349d7f7
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 8fa990243..e21fcf1e4 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 6b17fc92c..7e01ada66 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index 8c3d9ef69..49beaa3dd 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -242,6 +242,8 @@ type Config = {
entityChangeStreamPollMs: number
entityChangeStreamHeartbeatMs: number
entityChangeStreamMaxLifetimeMs: number
+ entityChangeStreamMaxConnections: number
+ entityChangeStreamMaxConnectionsPerPrincipal: number
notificationsDesktopFirstEnabled: boolean
notificationDesktopGraceSeconds: number
// Stricter than the normal refresh limiter: a client reaching reissue has
@@ -1107,6 +1109,18 @@ export const config: Config = {
30 * 1000,
60 * 60 * 1000
),
+ entityChangeStreamMaxConnections: boundedIntegerEnv(
+ 'ENTITY_CHANGE_STREAM_MAX_CONNECTIONS',
+ 256,
+ 1,
+ 10000
+ ),
+ entityChangeStreamMaxConnectionsPerPrincipal: boundedIntegerEnv(
+ 'ENTITY_CHANGE_STREAM_MAX_CONNECTIONS_PER_PRINCIPAL',
+ 8,
+ 1,
+ 1000
+ ),
notificationsDesktopFirstEnabled:
(process.env.NOTIFICATIONS_DESKTOP_FIRST_ENABLED ?? 'true') !== 'false',
// Parse safely: a non-empty non-numeric env (e.g. "disabled") yields NaN,
diff --git a/control-api/src/routes/entityChangeStream.ts b/control-api/src/routes/entityChangeStream.ts
index a9a6c92fc..a0f05c9e0 100644
--- a/control-api/src/routes/entityChangeStream.ts
+++ b/control-api/src/routes/entityChangeStream.ts
@@ -41,6 +41,7 @@ type EntityChangeStreamMessage =
}
const activeEntityChangeStreams = new Set<() => void>()
+const activeEntityChangeStreamsByPrincipal = new Map<string, number>()
export function closeActiveEntityChangeStreams(): void {
for (const close of Array.from(activeEntityChangeStreams)) close()
@@ -66,7 +67,8 @@ export function streamEntityChanges(
res: Response,
initialCursor: string | null,
isAuthorized: () => Promise<boolean>,
- principalKind: 'user' | 'operator'
+ principalKind: 'user' | 'operator',
+ principalId: string = principalKind
): void {
void (async () => {
let closed = false
@@ -80,6 +82,7 @@ export function streamEntityChanges(
let heartbeatTimer: NodeJS.Timeout | null = null
let lifetimeTimer: NodeJS.Timeout | null = null
let unsubscribeFeedWake: (() => void) | null = null
+ let admittedPrincipal: string | null = null
let shutdown = () => undefined
const cleanup = () => {
@@ -91,6 +94,12 @@ export function streamEntityChanges(
unsubscribeFeedWake?.()
unsubscribeFeedWake = null
activeEntityChangeStreams.delete(shutdown)
+ if (admittedPrincipal) {
+ const count = activeEntityChangeStreamsByPrincipal.get(admittedPrincipal) ?? 0
+ if (count <= 1) activeEntityChangeStreamsByPrincipal.delete(admittedPrincipal)
+ else activeEntityChangeStreamsByPrincipal.set(admittedPrincipal, count - 1)
+ admittedPrincipal = null
+ }
if (metricsActive) {
metricsActive = false
entityChangeStreamConnectionsActive.dec({ principal_kind: principalKind })
@@ -229,6 +238,19 @@ export function streamEntityChanges(
return
}
if (closed || res.destroyed) return
+ const principalConnections = activeEntityChangeStreamsByPrincipal.get(principalId) ?? 0
+ if (
+ activeEntityChangeStreams.size >= config.entityChangeStreamMaxConnections ||
+ principalConnections >= config.entityChangeStreamMaxConnectionsPerPrincipal
+ ) {
+ res.status(429)
+ res.setHeader('retry-after', '5')
+ res.json({ error: 'Entity change stream capacity reached' })
+ cleanup()
+ return
+ }
+ activeEntityChangeStreamsByPrincipal.set(principalId, principalConnections + 1)
+ admittedPrincipal = principalId
lastAuthorizationCheck = Date.now()
setStreamHeaders(res)
unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())
diff --git a/control-api/src/routes/external/entityChanges.routes.ts b/control-api/src/routes/external/entityChanges.routes.ts
index 2d7f3e63b..15a912760 100644
--- a/control-api/src/routes/external/entityChanges.routes.ts
+++ b/control-api/src/routes/external/entityChanges.routes.ts
@@ -55,7 +55,8 @@ export function createExternalEntityChangesRouter(): Router {
res,
cursor,
() => isEntityChangeExternalSessionCurrent(claims),
- 'user'
+ 'user',
+ `user:${claims.userId}`
)
})
)
diff --git a/control-api/src/routes/gfs/entityChanges.ts b/control-api/src/routes/gfs/entityChanges.ts
index a5cdc9cd9..e0e7ac355 100644
--- a/control-api/src/routes/gfs/entityChanges.ts
+++ b/control-api/src/routes/gfs/entityChanges.ts
@@ -24,7 +24,8 @@ export function registerGfsEntityChangeRoutes(router: Router): void {
const current = await authenticateAdminSession(token)
return Boolean(current && current.sub === adminReq.adminAuth?.sub)
},
- 'operator'
+ 'operator',
+ `operator:${adminReq.adminAuth?.sub ?? 'unknown'}`
)
}
)
diff --git a/control-api/test/routes.entityChangeStream.test.ts b/control-api/test/routes.entityChangeStream.test.ts
index 6588290f9..bdf344c4e 100644
--- a/control-api/test/routes.entityChangeStream.test.ts
+++ b/control-api/test/routes.entityChangeStream.test.ts
@@ -16,6 +16,8 @@ const configMock = vi.hoisted(() => ({
entityChangeStreamHeartbeatMs: 20_000,
entityChangeStreamMaxLifetimeMs: 600_000,
entityChangeStreamPollMs: 250,
+ entityChangeStreamMaxConnections: 100,
+ entityChangeStreamMaxConnectionsPerPrincipal: 8,
}))
vi.mock('../src/config.js', () => ({ config: configMock }))
@@ -63,6 +65,8 @@ const CURSOR = 'd119f895-1ef8-4e73-8f08-f9754919682a'
describe('routes/entityChangeStream', () => {
beforeEach(() => {
+ configMock.entityChangeStreamMaxConnections = 100
+ configMock.entityChangeStreamMaxConnectionsPerPrincipal = 8
serviceMock.isEntityChangeCursor.mockImplementation((value: string) =>
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)
)
@@ -113,6 +117,45 @@ describe('routes/entityChangeStream', () => {
expect(res.headersSent).toBe(false)
})
+ it('bounds admitted streams per authenticated principal and per process', async () => {
+ configMock.entityChangeStreamMaxConnections = 2
+ configMock.entityChangeStreamMaxConnectionsPerPrincipal = 1
+ serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
+ resyncRequired: false,
+ cursor: CURSOR,
+ scopes: [],
+ })
+ const start = (principalId: string) => {
+ const req = new FakeRequest()
+ const res = new FakeResponse()
+ streamEntityChanges(
+ req as unknown as Request,
+ res as unknown as Response,
+ null,
+ async () => true,
+ 'user',
+ principalId
+ )
+ return { req, res }
+ }
+
+ const first = start('user-1')
+ await vi.waitFor(() => expect(first.res.headersSent).toBe(true))
+ const samePrincipal = start('user-1')
+ await vi.waitFor(() => expect(samePrincipal.res.statusCode).toBe(429))
+ expect(samePrincipal.res.headersSent).toBe(false)
+
+ const secondPrincipal = start('user-2')
+ await vi.waitFor(() => expect(secondPrincipal.res.headersSent).toBe(true))
+ const overProcessLimit = start('user-3')
+ await vi.waitFor(() => expect(overProcessLimit.res.statusCode).toBe(429))
+ expect(overProcessLimit.res.headersSent).toBe(false)
+ closeActiveEntityChangeStreams()
+ await vi.waitFor(() =>
+ expect(first.res.writableEnded && secondPrincipal.res.writableEnded).toBe(true)
+ )
+ })
+
it('sends only coarse invalidations and closes cleanly on server shutdown', async () => {
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: true,