Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import type { Request, Response } from 'express'
import { EventEmitter } from 'node:events'
import { get } from 'node:http'
import type { AddressInfo } from 'node:net'
import {
closeActiveEntityChangeStreams,
parseRequestedEntityChangeCursor,
streamEntityChanges,
} from '../src/routes/entityChangeStream.js'
const serviceMock = vi.hoisted(() => ({
isEntityChangeCursor: vi.fn(),
readEntityChangeCheckpoint: vi.fn(),
subscribeEntityChangeFeedWake: vi.fn(),
}))
const configMock = vi.hoisted(() => ({
entityChangeStreamHeartbeatMs: 20_000,
entityChangeStreamMaxLifetimeMs: 600_000,
entityChangeStreamPollMs: 250,
entityChangeUserVisibilityRefreshMs: 4_000,
entityChangeStreamMaxConnections: 100,
entityChangeStreamMaxConnectionsPerPrincipal: 8,
}))
vi.mock('../src/config.js', () => ({ config: configMock }))
vi.mock('../src/services/entityChangeService.js', () => serviceMock)
vi.mock('../src/observability/logger.js', () => ({
rootLogger: { child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }) },
}))
class FakeRequest extends EventEmitter {
query: Record<string, unknown> = {}
setTimeout = vi.fn()
}
class FakeResponse extends EventEmitter {
statusCode = 200
headersSent = false
destroyed = false
writableEnded = false
writableLength = 0
writeReturns = true
frames: string[] = []
status(code: number): this {
this.statusCode = code
return this
}
setHeader(): this {
return this
}
flushHeaders(): void {
this.headersSent = true
}
write(frame: string): boolean {
this.frames.push(frame)
return this.writeReturns
}
end(): void {
this.writableEnded = true
this.emit('close')
}
json(): this {
return this
}
}
const CURSOR = 'd119f895-1ef8-4e73-8f08-f9754919682a'
describe('routes/entityChangeStream', () => {
beforeEach(() => {
configMock.entityChangeStreamMaxConnections = 100
configMock.entityChangeStreamMaxConnectionsPerPrincipal = 8
serviceMock.isEntityChangeCursor.mockImplementation((value: string) =>
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)
)
serviceMock.readEntityChangeCheckpoint.mockReset()
serviceMock.subscribeEntityChangeFeedWake.mockReset()
serviceMock.subscribeEntityChangeFeedWake.mockReturnValue(vi.fn())
})
it('validates requested UUID cursors and represents an absent cursor as a full resync', () => {
expect(parseRequestedEntityChangeCursor({ query: {} } as Request)).toBeNull()
expect(parseRequestedEntityChangeCursor({ query: { cursor: CURSOR } } as Request)).toBe(CURSOR)
expect(parseRequestedEntityChangeCursor({ query: { cursor: 'bad' } } as Request)).toBe(false)
})
it('authorizes before sending streaming headers', async () => {
const req = new FakeRequest()
const res = new FakeResponse()
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
null,
async () => false,
'user'
)
await vi.waitFor(() => expect(res.statusCode).toBe(401))
expect(res.headersSent).toBe(false)
expect(res.frames).toEqual([])
})
it('does not acquire a feed connection when the client disconnects during authorization', async () => {
let finishAuthorization: (authorized: boolean) => void = () => undefined
const authorization = new Promise<boolean>(resolve => {
finishAuthorization = resolve
})
const req = new FakeRequest()
const res = new FakeResponse()
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
null,
() => authorization,
'user'
)
req.emit('aborted')
finishAuthorization(true)
await new Promise(resolve => setImmediate(resolve))
expect(serviceMock.subscribeEntityChangeFeedWake).not.toHaveBeenCalled()
expect(res.headersSent).toBe(false)
})
it('bounds admitted streams per authenticated principal and per process', async () => {
configMock.entityChangeStreamMaxConnections = 2
configMock.entityChangeStreamMaxConnectionsPerPrincipal = 1
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: [],
})
const start = (principalId: string) => {
const req = new FakeRequest()
const res = new FakeResponse()
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
null,
async () => true,
'user',
principalId
)
return { req, res }
}
const first = start('user-1')
await vi.waitFor(() => expect(first.res.headersSent).toBe(true))
const samePrincipal = start('user-1')
await vi.waitFor(() => expect(samePrincipal.res.statusCode).toBe(429))
expect(samePrincipal.res.headersSent).toBe(false)
const secondPrincipal = start('user-2')
await vi.waitFor(() => expect(secondPrincipal.res.headersSent).toBe(true))
const overProcessLimit = start('user-3')
await vi.waitFor(() => expect(overProcessLimit.res.statusCode).toBe(429))
expect(overProcessLimit.res.headersSent).toBe(false)
closeActiveEntityChangeStreams()
await vi.waitFor(() =>
expect(first.res.writableEnded && secondPrincipal.res.writableEnded).toBe(true)
)
})
it('sends only coarse invalidations and closes cleanly on server shutdown', async () => {
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: true,
cursor: CURSOR,
scopes: ['gfs', 'authorization'],
})
const req = new FakeRequest()
const res = new FakeResponse()
let authorizationChecks = 0
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
null,
async () => {
authorizationChecks += 1
return true
},
'operator'
)
await vi.waitFor(() => expect(res.frames.length).toBeGreaterThan(0))
closeActiveEntityChangeStreams()
await vi.waitFor(() => expect(res.writableEnded).toBe(true))
expect(res.frames.join('')).toContain('"type":"resync_required"')
expect(res.frames.join('')).toContain('"type":"stream.closing"')
expect(res.frames.join('')).not.toContain('resource-id')
expect(authorizationChecks).toBeGreaterThanOrEqual(2)
expect(serviceMock.subscribeEntityChangeFeedWake).toHaveBeenCalledOnce()
})
it('keeps user checkpoints independent of hidden mutations and refreshes on a fixed cadence', async () => {
vi.useFakeTimers()
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs', 'authorization'],
})
const req = new FakeRequest()
const res = new FakeResponse()
const start = Date.now()
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
async () => true,
'user',
'user-1'
)
await vi.advanceTimersByTimeAsync(0)
expect(res.frames.map(frame => JSON.parse(frame))).toEqual([
{
schemaVersion: 1,
type: 'resync_required',
cursor: '00000000-0000-0000-0000-000000000000',
scopes: ['gfs', 'authorization'],
},
])
expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
expect(serviceMock.subscribeEntityChangeFeedWake).not.toHaveBeenCalled()
const expectedPeriodicFrame = {
schemaVersion: 1,
type: 'scope.invalidated',
cursor: '00000000-0000-0000-0000-000000000000',
scopes: ['gfs', 'authorization'],
}
const checkpointScenarios = [
{ resyncRequired: false, cursor: CURSOR, scopes: ['gfs'] },
{
resyncRequired: false,
cursor: '0f961d2e-4c95-4ea7-a47d-79f8d4b0da07',
scopes: ['authorization'],
},
{ resyncRequired: true, cursor: CURSOR, scopes: ['gfs', 'authorization'] },
]
for (const checkpoint of checkpointScenarios) {
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
...checkpoint,
})
await vi.advanceTimersByTimeAsync(configMock.entityChangeUserVisibilityRefreshMs)
expect(JSON.parse(res.frames.at(-1)!)).toEqual(expectedPeriodicFrame)
}
expect(Date.now() - start).toBe(configMock.entityChangeUserVisibilityRefreshMs * 3)
expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
closeActiveEntityChangeStreams()
await vi.advanceTimersByTimeAsync(0)
const reconnectReq = new FakeRequest()
const reconnectRes = new FakeResponse()
streamEntityChanges(
reconnectReq as unknown as Request,
reconnectRes as unknown as Response,
CURSOR,
async () => true,
'user',
'user-1'
)
await vi.advanceTimersByTimeAsync(0)
expect(JSON.parse(reconnectRes.frames[0]!)).toEqual({
schemaVersion: 1,
type: 'resync_required',
cursor: '00000000-0000-0000-0000-000000000000',
scopes: ['gfs', 'authorization'],
})
expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
closeActiveEntityChangeStreams()
await vi.advanceTimersByTimeAsync(0)
vi.useRealTimers()
})
it('closes an established user stream when visibility authorization is revoked', async () => {
vi.useFakeTimers()
const req = new FakeRequest()
const res = new FakeResponse()
const isAuthorized = vi
.fn()
.mockResolvedValueOnce(true)
.mockResolvedValueOnce(true)
.mockResolvedValueOnce(false)
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
isAuthorized,
'user',
'user-1'
)
await vi.advanceTimersByTimeAsync(0)
expect(res.frames.map(frame => JSON.parse(frame))).toEqual([
{
schemaVersion: 1,
type: 'resync_required',
cursor: '00000000-0000-0000-0000-000000000000',
scopes: ['gfs', 'authorization'],
},
])
await vi.advanceTimersByTimeAsync(configMock.entityChangeUserVisibilityRefreshMs)
expect(res.frames.map(frame => JSON.parse(frame))).toEqual([
{
schemaVersion: 1,
type: 'resync_required',
cursor: '00000000-0000-0000-0000-000000000000',
scopes: ['gfs', 'authorization'],
},
{
schemaVersion: 1,
type: 'stream.closing',
cursor: '00000000-0000-0000-0000-000000000000',
reason: 'session_expired',
},
])
expect(res.writableEnded).toBe(true)
expect(isAuthorized).toHaveBeenCalledTimes(3)
expect(serviceMock.readEntityChangeCheckpoint).not.toHaveBeenCalled()
vi.useRealTimers()
})
it('rechecks authorization before delivering an established operator invalidation', async () => {
vi.useFakeTimers()
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs'],
})
const req = new FakeRequest()
const res = new FakeResponse()
const isAuthorized = vi.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false)
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
isAuthorized,
'operator'
)
await vi.advanceTimersByTimeAsync(0)
expect(res.frames.map(frame => JSON.parse(frame))).toEqual([
{
schemaVersion: 1,
type: 'stream.closing',
cursor: CURSOR,
reason: 'session_expired',
},
])
expect(res.writableEnded).toBe(true)
expect(res.frames.join('')).not.toContain('scope.invalidated')
expect(serviceMock.subscribeEntityChangeFeedWake).toHaveBeenCalledOnce()
vi.useRealTimers()
})
it('does not deliver a checkpoint when authorization throws and retries after recovery', async () => {
vi.useFakeTimers()
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs'],
})
const req = new FakeRequest()
const res = new FakeResponse()
let wakeFeed: (() => void) | undefined
serviceMock.subscribeEntityChangeFeedWake.mockImplementation((wake: () => void) => {
wakeFeed = wake
return vi.fn()
})
const isAuthorized = vi
.fn()
.mockResolvedValueOnce(true)
.mockRejectedValueOnce(new Error('authorization backend unavailable'))
.mockResolvedValue(true)
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
isAuthorized,
'operator'
)
await vi.advanceTimersByTimeAsync(0)
expect(res.frames).toEqual([])
for (let attempt = 0; attempt < 3; attempt += 1) {
wakeFeed?.()
await vi.advanceTimersByTimeAsync(0)
}
expect(isAuthorized).toHaveBeenCalledTimes(2)
expect(res.frames).toEqual([])
await vi.advanceTimersByTimeAsync(configMock.entityChangeStreamPollMs)
expect(isAuthorized).toHaveBeenCalledTimes(3)
expect(res.frames.map(frame => JSON.parse(frame))).toContainEqual({
schemaVersion: 1,
type: 'scope.invalidated',
cursor: CURSOR,
scopes: ['gfs'],
})
closeActiveEntityChangeStreams()
await vi.advanceTimersByTimeAsync(0)
vi.useRealTimers()
})
it('times out a slow consumer and releases its principal connection slot', async () => {
vi.useFakeTimers()
configMock.entityChangeStreamMaxConnections = 1
configMock.entityChangeStreamMaxConnectionsPerPrincipal = 1
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs'],
})
const start = (writeReturns: boolean) => {
const req = new FakeRequest()
const res = new FakeResponse()
res.writeReturns = writeReturns
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
async () => true,
'operator',
'operator-1'
)
return { req, res }
}
const slow = start(false)
await vi.advanceTimersByTimeAsync(0)
expect(slow.res.frames).toHaveLength(1)
await vi.advanceTimersByTimeAsync(5_000)
expect(slow.res.frames).toHaveLength(2)
const stillOccupied = start(true)
await vi.advanceTimersByTimeAsync(0)
expect(stillOccupied.res.statusCode).toBe(429)
await vi.advanceTimersByTimeAsync(5_000)
expect(slow.res.writableEnded).toBe(true)
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: [],
})
const afterRelease = start(true)
await vi.advanceTimersByTimeAsync(0)
expect(afterRelease.res.headersSent).toBe(true)
closeActiveEntityChangeStreams()
await vi.advanceTimersByTimeAsync(0)
vi.useRealTimers()
})
it('closes before writing when the buffered response is already over the byte limit', async () => {
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs'],
})
const req = new FakeRequest()
const res = new FakeResponse()
res.writableLength = 64 * 1024
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
async () => true,
'operator',
'operator-buffer-limit'
)
try {
await vi.waitFor(() => expect(res.writableEnded).toBe(true))
expect(res.frames).toEqual([])
expect(serviceMock.readEntityChangeCheckpoint).toHaveBeenCalledOnce()
} finally {
closeActiveEntityChangeStreams()
}
})
it('emits an operator heartbeat on the configured cadence without a feed change', async () => {
vi.useFakeTimers()
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: [],
})
const req = new FakeRequest()
const res = new FakeResponse()
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
async () => true,
'operator',
'operator-heartbeat'
)
try {
await vi.advanceTimersByTimeAsync(0)
expect(res.frames).toEqual([])
await vi.advanceTimersByTimeAsync(configMock.entityChangeStreamHeartbeatMs)
expect(res.frames.map(frame => JSON.parse(frame))).toContainEqual(
expect.objectContaining({
schemaVersion: 1,
type: 'heartbeat',
cursor: CURSOR,
observedAt: expect.any(String),
})
)
} finally {
closeActiveEntityChangeStreams()
await vi.advanceTimersByTimeAsync(0)
vi.useRealTimers()
}
})
it('closes the HTTP response after a timed-out slow-consumer write', async () => {
configMock.entityChangeStreamMaxConnections = 1
configMock.entityChangeStreamMaxConnectionsPerPrincipal = 1
serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
resyncRequired: false,
cursor: CURSOR,
scopes: ['gfs'],
})
const app = express()
app.get('/entity-changes', (req, res) => {
const write = res.write.bind(res)
let simulateBackpressure = true
res.write = ((...args: Parameters<typeof res.write>) => {
const accepted = write(...args)
if (simulateBackpressure) {
simulateBackpressure = false
return false
}
return accepted
}) as typeof res.write
streamEntityChanges(
req as unknown as Request,
res as unknown as Response,
CURSOR,
async () => true,
'operator',
'operator-socket'
)
})
const server = app.listen(0, '127.0.0.1')
await new Promise<void>(resolve => server.once('listening', resolve))
const address = server.address() as AddressInfo
try {
const completedResponse = new Promise<{
complete: boolean
lines: string[]
statusCode: number | undefined
}>((resolve, reject) => {
const request = get(
`http://127.0.0.1:${address.port}/entity-changes`,
(response: import('node:http').IncomingMessage) => {
response.setEncoding('utf8')
let body = ''
response.on('data', chunk => {
body += chunk
})
response.on('end', () => {
resolve({
complete: response.complete,
lines: body.trim().split('\n').filter(Boolean),
statusCode: response.statusCode,
})
})
response.on('error', reject)
}
)
request.on('error', reject)
})
const result = await completedResponse
expect(result.statusCode).toBe(200)
expect(result.complete).toBe(true)
expect(result.lines.map(line => JSON.parse(line))).toContainEqual({
schemaVersion: 1,
type: 'stream.closing',
cursor: CURSOR,
reason: 'slow_consumer',
})
} finally {
closeActiveEntityChangeStreams()
await new Promise<void>(resolve => server.close(() => resolve()))
}
})
})