Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): r1-h2 isolate user change timing

Josué9d93771ab316
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 404ca1cdf..c0d3f00d5 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.635",
+ "version": "0.1.636",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.635",
+ "version": "0.1.636",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 3199e881b..7bab7053e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.635",
+ "version": "0.1.636",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index b550caafd..4cda70275 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -245,6 +245,7 @@ type Config = {
entityChangeDispatchBatchSize: number
entityChangeMaxRecoveryEvents: number
entityChangeStreamPollMs: number
+ entityChangeUserVisibilityRefreshMs: number
entityChangeStreamHeartbeatMs: number
entityChangeStreamMaxLifetimeMs: number
entityChangeStreamMaxConnections: number
@@ -1188,6 +1189,12 @@ export const config: Config = {
100000
),
entityChangeStreamPollMs: boundedIntegerEnv('ENTITY_CHANGE_STREAM_POLL_MS', 1000, 250, 5000),
+ entityChangeUserVisibilityRefreshMs: boundedIntegerEnv(
+ 'ENTITY_CHANGE_USER_VISIBILITY_REFRESH_MS',
+ 4000,
+ 1000,
+ 5000
+ ),
entityChangeStreamHeartbeatMs: boundedIntegerEnv(
'ENTITY_CHANGE_STREAM_HEARTBEAT_MS',
20 * 1000,
diff --git a/control-api/src/routes/entityChangeStream.ts b/control-api/src/routes/entityChangeStream.ts
index a0f05c9e0..99382c5bc 100644
--- a/control-api/src/routes/entityChangeStream.ts
+++ b/control-api/src/routes/entityChangeStream.ts
@@ -18,6 +18,7 @@ const logger = rootLogger.child({ module: 'entity-change-stream' })
const MAX_BUFFERED_BYTES = 64 * 1024
const BACKPRESSURE_TIMEOUT_MS = 5000
const AUTHORIZATION_RECHECK_MS = 15_000
+const ZERO_CURSOR = '00000000-0000-0000-0000-000000000000'
type EntityChangeStreamMessage =
| {
@@ -75,7 +76,8 @@ export function streamEntityChanges(
let closing = false
let polling = false
let metricsActive = false
- let cursor = initialCursor
+ let cursor = principalKind === 'user' ? ZERO_CURSOR : initialCursor
+ let userInitialResyncPending = principalKind === 'user'
let lastAuthorizationCheck = 0
let lastHeartbeat = Date.now()
let pollTimer: NodeJS.Timeout | null = null
@@ -169,6 +171,33 @@ export function streamEntityChanges(
if (closed || closing || polling) return
polling = true
try {
+ if (principalKind === 'user') {
+ // The durable feed's global cursor and checkpoint scopes reveal when
+ // resources outside this user's visibility changed. User sessions
+ // therefore use a fixed-cadence, generic invalidation and always
+ // refetch current authorized state; their frames never depend on
+ // global feed contents, retention, or hidden mutations.
+ if (!(await isAuthorized())) {
+ await closeForFailure('session_expired', ZERO_CURSOR)
+ return
+ }
+ const type = userInitialResyncPending ? 'resync_required' : 'scope.invalidated'
+ const sent = await write({
+ schemaVersion: 1,
+ type,
+ cursor: ZERO_CURSOR,
+ scopes: ['gfs', 'authorization'],
+ })
+ if (!sent) {
+ await closeForFailure('slow_consumer', ZERO_CURSOR)
+ return
+ }
+ userInitialResyncPending = false
+ if (type === 'resync_required') {
+ entityChangeStreamResyncRequiredTotal.inc({ principal_kind: principalKind })
+ }
+ return
+ }
const checkpoint = await readEntityChangeCheckpoint(cursor)
const hasChange = checkpoint.resyncRequired || checkpoint.scopes.length > 0
if (hasChange || Date.now() - lastAuthorizationCheck >= AUTHORIZATION_RECHECK_MS) {
@@ -253,14 +282,23 @@ export function streamEntityChanges(
admittedPrincipal = principalId
lastAuthorizationCheck = Date.now()
setStreamHeaders(res)
- unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())
+ if (principalKind === 'operator') {
+ unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())
+ }
activeEntityChangeStreams.add(shutdown)
entityChangeStreamConnectionsActive.inc({ principal_kind: principalKind })
metricsActive = true
await poll()
if (closed) return
- pollTimer = setInterval(() => void poll(), config.entityChangeStreamPollMs)
- heartbeatTimer = setInterval(() => void poll(), config.entityChangeStreamHeartbeatMs)
+ pollTimer = setInterval(
+ () => void poll(),
+ principalKind === 'user'
+ ? config.entityChangeUserVisibilityRefreshMs
+ : config.entityChangeStreamPollMs
+ )
+ if (principalKind === 'operator') {
+ heartbeatTimer = setInterval(() => void poll(), config.entityChangeStreamHeartbeatMs)
+ }
lifetimeTimer = setTimeout(
() => void closeForFailure('max_lifetime'),
config.entityChangeStreamMaxLifetimeMs
diff --git a/control-api/test/routes.entityChangeStream.test.ts b/control-api/test/routes.entityChangeStream.test.ts
index bdf344c4e..bec91f875 100644
--- a/control-api/test/routes.entityChangeStream.test.ts
+++ b/control-api/test/routes.entityChangeStream.test.ts
@@ -16,6 +16,7 @@ const configMock = vi.hoisted(() => ({
entityChangeStreamHeartbeatMs: 20_000,
entityChangeStreamMaxLifetimeMs: 600_000,
entityChangeStreamPollMs: 250,
+ entityChangeUserVisibilityRefreshMs: 4_000,
entityChangeStreamMaxConnections: 100,
entityChangeStreamMaxConnectionsPerPrincipal: 8,
}))
@@ -184,4 +185,85 @@ describe('routes/entityChangeStream', () => {
expect(authorizationChecks).toBeGreaterThanOrEqual(2)
expect(serviceMock.subscribeEntityChangeFeedWake).toHaveBeenCalledOnce()
})
+
+ it('keeps user checkpoints independent of hidden mutations and refreshes on a fixed cadence', async () => {
+ vi.useFakeTimers()
+ serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
+ resyncRequired: false,
+ cursor: CURSOR,
+ scopes: ['gfs', 'authorization'],
+ })
+ const req = new FakeRequest()
+ const res = new FakeResponse()
+ const start = Date.now()
+ streamEntityChanges(
+ req as unknown as Request,
+ res as unknown as Response,
+ CURSOR,
+ async () => true,
+ 'user',
+ 'user-1'
+ )
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(res.frames.map(frame => JSON.parse(frame))).toEqual([
+ {
+ schemaVersion: 1,
+ type: 'resync_required',
+ cursor: '00000000-0000-0000-0000-000000000000',
+ scopes: ['gfs', 'authorization'],
+ },
+ ])
+ expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
+ expect(serviceMock.subscribeEntityChangeFeedWake).not.toHaveBeenCalled()
+
+ const expectedPeriodicFrame = {
+ schemaVersion: 1,
+ type: 'scope.invalidated',
+ cursor: '00000000-0000-0000-0000-000000000000',
+ scopes: ['gfs', 'authorization'],
+ }
+ const checkpointScenarios = [
+ { resyncRequired: false, cursor: CURSOR, scopes: ['gfs'] },
+ {
+ resyncRequired: false,
+ cursor: '0f961d2e-4c95-4ea7-a47d-79f8d4b0da07',
+ scopes: ['authorization'],
+ },
+ { resyncRequired: true, cursor: CURSOR, scopes: ['gfs', 'authorization'] },
+ ]
+ for (const checkpoint of checkpointScenarios) {
+ serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
+ ...checkpoint,
+ })
+ await vi.advanceTimersByTimeAsync(configMock.entityChangeUserVisibilityRefreshMs)
+ expect(JSON.parse(res.frames.at(-1)!)).toEqual(expectedPeriodicFrame)
+ }
+ expect(Date.now() - start).toBe(configMock.entityChangeUserVisibilityRefreshMs * 3)
+ expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
+ closeActiveEntityChangeStreams()
+ await vi.advanceTimersByTimeAsync(0)
+
+ const reconnectReq = new FakeRequest()
+ const reconnectRes = new FakeResponse()
+ streamEntityChanges(
+ reconnectReq as unknown as Request,
+ reconnectRes as unknown as Response,
+ CURSOR,
+ async () => true,
+ 'user',
+ 'user-1'
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ expect(JSON.parse(reconnectRes.frames[0]!)).toEqual({
+ schemaVersion: 1,
+ type: 'resync_required',
+ cursor: '00000000-0000-0000-0000-000000000000',
+ scopes: ['gfs', 'authorization'],
+ })
+ expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
+ closeActiveEntityChangeStreams()
+ await vi.advanceTimersByTimeAsync(0)
+ vi.useRealTimers()
+ })
})