Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import type { Request, Response } from 'express'
import { config } from '../config.js'
import { rootLogger } from '../observability/logger.js'
import {
entityChangeStreamConnectionsActive,
entityChangeStreamDisconnectsTotal,
entityChangeStreamFramesSentTotal,
entityChangeStreamResyncRequiredTotal,
} from '../observability/metrics.js'
import {
type EntityChangeScope,
isEntityChangeCursor,
readEntityChangeCheckpoint,
subscribeEntityChangeFeedWake,
} from '../services/entityChangeService.js'
const logger = rootLogger.child({ module: 'entity-change-stream' })
const MAX_BUFFERED_BYTES = 64 * 1024
const BACKPRESSURE_TIMEOUT_MS = 5000
const AUTHORIZATION_RECHECK_MS = 15_000
const ZERO_CURSOR = '00000000-0000-0000-0000-000000000000'
type EntityChangeStreamMessage =
| {
schemaVersion: 1
type: 'resync_required'
cursor: string
scopes: EntityChangeScope[]
}
| {
schemaVersion: 1
type: 'scope.invalidated'
cursor: string
scopes: EntityChangeScope[]
}
| { schemaVersion: 1; type: 'heartbeat'; cursor: string; observedAt: string }
| {
schemaVersion: 1
type: 'stream.closing'
cursor: string
reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer'
}
const activeEntityChangeStreams = new Set<() => void>()
const activeEntityChangeStreamsByPrincipal = new Map<string, number>()
export function closeActiveEntityChangeStreams(): void {
for (const close of Array.from(activeEntityChangeStreams)) close()
}
export function parseRequestedEntityChangeCursor(req: Request): string | null | false {
const raw = String(req.query?.cursor || '').trim()
if (!raw) return null
return isEntityChangeCursor(raw) ? raw : false
}
function setStreamHeaders(res: Response): void {
res.status(200)
res.setHeader('content-type', 'application/x-ndjson; charset=utf-8')
res.setHeader('cache-control', 'no-cache, no-transform')
res.setHeader('connection', 'keep-alive')
res.setHeader('x-accel-buffering', 'no')
res.flushHeaders?.()
}
export function streamEntityChanges(
req: Request,
res: Response,
initialCursor: string | null,
isAuthorized: () => Promise<boolean>,
principalKind: 'user' | 'operator',
principalId: string = principalKind
): void {
void (async () => {
let closed = false
let closing = false
let polling = false
let metricsActive = false
let cursor = principalKind === 'user' ? ZERO_CURSOR : initialCursor
let userInitialResyncPending = principalKind === 'user'
let lastAuthorizationCheck = 0
let authorizationRetryAfter = 0
let lastHeartbeat = Date.now()
let pollTimer: NodeJS.Timeout | null = null
let heartbeatTimer: NodeJS.Timeout | null = null
let lifetimeTimer: NodeJS.Timeout | null = null
let unsubscribeFeedWake: (() => void) | null = null
let admittedPrincipal: string | null = null
let shutdown = () => undefined
const cleanup = () => {
if (closed) return
closed = true
if (pollTimer) clearInterval(pollTimer)
if (heartbeatTimer) clearInterval(heartbeatTimer)
if (lifetimeTimer) clearTimeout(lifetimeTimer)
unsubscribeFeedWake?.()
unsubscribeFeedWake = null
activeEntityChangeStreams.delete(shutdown)
if (admittedPrincipal) {
const count = activeEntityChangeStreamsByPrincipal.get(admittedPrincipal) ?? 0
if (count <= 1) activeEntityChangeStreamsByPrincipal.delete(admittedPrincipal)
else activeEntityChangeStreamsByPrincipal.set(admittedPrincipal, count - 1)
admittedPrincipal = null
}
if (metricsActive) {
metricsActive = false
entityChangeStreamConnectionsActive.dec({ principal_kind: principalKind })
if (!closing) {
entityChangeStreamDisconnectsTotal.inc({
principal_kind: principalKind,
reason: 'client_disconnect',
})
}
}
}
req.on('aborted', cleanup)
res.on('close', cleanup)
req.setTimeout(0)
const write = async (message: EntityChangeStreamMessage): Promise<boolean> => {
if (closed || res.destroyed) return false
const frame = `${JSON.stringify(message)}\n`
if (res.writableLength + Buffer.byteLength(frame) > MAX_BUFFERED_BYTES) return false
if (res.write(frame)) {
entityChangeStreamFramesSentTotal.inc({
principal_kind: principalKind,
frame_type: message.type,
})
return true
}
const drained = await new Promise<boolean>(resolve => {
let settled = false
const finish = (writable: boolean) => {
if (settled) return
settled = true
clearTimeout(timeout)
res.off('drain', onDrain)
res.off('close', onClose)
resolve(writable)
}
const onDrain = () => finish(true)
const onClose = () => finish(false)
const timeout = setTimeout(() => finish(false), BACKPRESSURE_TIMEOUT_MS)
res.once('drain', onDrain)
res.once('close', onClose)
})
if (drained) {
entityChangeStreamFramesSentTotal.inc({
principal_kind: principalKind,
frame_type: message.type,
})
}
return drained
}
const closeForFailure = async (
reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer',
currentCursor = cursor || '00000000-0000-0000-0000-000000000000'
) => {
if (closed || closing) return
closing = true
entityChangeStreamDisconnectsTotal.inc({ principal_kind: principalKind, reason })
await write({ schemaVersion: 1, type: 'stream.closing', cursor: currentCursor, reason })
res.end()
cleanup()
}
shutdown = () => void closeForFailure('server_shutdown')
const poll = async () => {
if (closed || closing || polling) return
polling = true
try {
if (principalKind === 'user') {
// The durable feed's global cursor and checkpoint scopes reveal when
// resources outside this user's visibility changed. User sessions
// therefore use a fixed-cadence, generic invalidation and always
// refetch current authorized state; their frames never depend on
// global feed contents, retention, or hidden mutations.
if (!(await isAuthorized())) {
await closeForFailure('session_expired', ZERO_CURSOR)
return
}
const type = userInitialResyncPending ? 'resync_required' : 'scope.invalidated'
const sent = await write({
schemaVersion: 1,
type,
cursor: ZERO_CURSOR,
scopes: ['gfs', 'authorization'],
})
if (!sent) {
await closeForFailure('slow_consumer', ZERO_CURSOR)
return
}
userInitialResyncPending = false
if (type === 'resync_required') {
entityChangeStreamResyncRequiredTotal.inc({ principal_kind: principalKind })
}
return
}
const checkpoint = await readEntityChangeCheckpoint(cursor)
const hasChange = checkpoint.resyncRequired || checkpoint.scopes.length > 0
if (hasChange || Date.now() - lastAuthorizationCheck >= AUTHORIZATION_RECHECK_MS) {
if (Date.now() < authorizationRetryAfter) return
try {
if (!(await isAuthorized())) {
await closeForFailure('session_expired', checkpoint.cursor)
return
}
lastAuthorizationCheck = Date.now()
authorizationRetryAfter = 0
} catch (err) {
// Feed wakes may arrive for every committed change while the auth
// service is unavailable. Keep the stream fail-closed and retry
// authorization at a bounded cadence instead of amplifying load.
authorizationRetryAfter = Date.now() + config.entityChangeStreamPollMs
throw err
}
if (closed) {
return
}
}
if (checkpoint.resyncRequired) {
entityChangeStreamResyncRequiredTotal.inc({ principal_kind: principalKind })
if (
!(await write({
schemaVersion: 1,
type: 'resync_required',
cursor: checkpoint.cursor,
scopes: ['gfs', 'authorization'],
}))
) {
await closeForFailure('slow_consumer', cursor || checkpoint.cursor)
return
}
cursor = checkpoint.cursor
} else if (checkpoint.scopes.length > 0) {
if (
!(await write({
schemaVersion: 1,
type: 'scope.invalidated',
cursor: checkpoint.cursor,
scopes: checkpoint.scopes,
}))
) {
await closeForFailure('slow_consumer', cursor || checkpoint.cursor)
return
}
cursor = checkpoint.cursor
}
if (Date.now() - lastHeartbeat >= config.entityChangeStreamHeartbeatMs) {
if (
!(await write({
schemaVersion: 1,
type: 'heartbeat',
cursor: checkpoint.cursor,
observedAt: new Date().toISOString(),
}))
) {
await closeForFailure('slow_consumer', checkpoint.cursor)
return
}
cursor = checkpoint.cursor
lastHeartbeat = Date.now()
}
} catch (err) {
logger.warn(
{ event: 'entity_change_stream_poll_failed', principalKind, err },
'entity change stream poll failed; polling will retry'
)
} finally {
polling = false
}
}
try {
if (!(await isAuthorized())) {
res.status(401).end()
cleanup()
return
}
if (closed || res.destroyed) return
const principalConnections = activeEntityChangeStreamsByPrincipal.get(principalId) ?? 0
if (
activeEntityChangeStreams.size >= config.entityChangeStreamMaxConnections ||
principalConnections >= config.entityChangeStreamMaxConnectionsPerPrincipal
) {
res.status(429)
res.setHeader('retry-after', '5')
res.json({ error: 'Entity change stream capacity reached' })
cleanup()
return
}
activeEntityChangeStreamsByPrincipal.set(principalId, principalConnections + 1)
admittedPrincipal = principalId
lastAuthorizationCheck = Date.now()
setStreamHeaders(res)
if (principalKind === 'operator') {
unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())
}
activeEntityChangeStreams.add(shutdown)
entityChangeStreamConnectionsActive.inc({ principal_kind: principalKind })
metricsActive = true
await poll()
if (closed) return
pollTimer = setInterval(
() => void poll(),
principalKind === 'user'
? config.entityChangeUserVisibilityRefreshMs
: config.entityChangeStreamPollMs
)
if (principalKind === 'operator') {
heartbeatTimer = setInterval(() => void poll(), config.entityChangeStreamHeartbeatMs)
}
lifetimeTimer = setTimeout(
() => void closeForFailure('max_lifetime'),
config.entityChangeStreamMaxLifetimeMs
)
} catch (err) {
logger.error(
{ event: 'entity_change_stream_start_failed', principalKind, err },
'stream setup failed'
)
if (!res.headersSent) res.status(500).json({ error: 'Entity change stream unavailable' })
else res.end()
cleanup()
}
})()
}