Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(control-api): add durable entity change feed

Josué0266becbbe02
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 211f524b5..aa041be5d 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 64d00e152..adeb0d846 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index fa82b204e..8c3d9ef69 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -1,6 +1,11 @@
import { createPublicKey } from 'node:crypto'
import { DEFAULT_ALLOWED_PLUGIN_IMAGE_PREFIXES } from '@clerum/image-policy'
+function boundedIntegerEnv(name: string, fallback: number, min: number, max: number): number {
+ const parsed = Number(process.env[name])
+ return Number.isSafeInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback
+}
+
type Config = {
port: number
jsonBodyLimit: string
@@ -230,6 +235,13 @@ type Config = {
notificationStreamHeartbeatMs: number
notificationStreamMaxLifetimeMs: number
notificationStreamSnapshotLimit: number
+ entityChangeDispatchIntervalMs: number
+ entityChangeRetentionSeconds: number
+ entityChangeDispatchBatchSize: number
+ entityChangeMaxRecoveryEvents: number
+ entityChangeStreamPollMs: number
+ entityChangeStreamHeartbeatMs: number
+ entityChangeStreamMaxLifetimeMs: number
notificationsDesktopFirstEnabled: boolean
notificationDesktopGraceSeconds: number
// Stricter than the normal refresh limiter: a client reaching reissue has
@@ -1058,6 +1070,43 @@ export const config: Config = {
process.env.NOTIFICATION_STREAM_MAX_LIFETIME_MS || 10 * 60 * 1000
),
notificationStreamSnapshotLimit: Number(process.env.NOTIFICATION_STREAM_SNAPSHOT_LIMIT || 50),
+ entityChangeDispatchIntervalMs: boundedIntegerEnv(
+ 'ENTITY_CHANGE_DISPATCH_INTERVAL_MS',
+ 500,
+ 100,
+ 5000
+ ),
+ entityChangeRetentionSeconds: boundedIntegerEnv(
+ 'ENTITY_CHANGE_RETENTION_SECONDS',
+ 24 * 60 * 60,
+ 60 * 60,
+ 30 * 24 * 60 * 60
+ ),
+ entityChangeDispatchBatchSize: boundedIntegerEnv(
+ 'ENTITY_CHANGE_DISPATCH_BATCH_SIZE',
+ 1000,
+ 1,
+ 10000
+ ),
+ entityChangeMaxRecoveryEvents: boundedIntegerEnv(
+ 'ENTITY_CHANGE_MAX_RECOVERY_EVENTS',
+ 10000,
+ 1,
+ 100000
+ ),
+ entityChangeStreamPollMs: boundedIntegerEnv('ENTITY_CHANGE_STREAM_POLL_MS', 1000, 250, 5000),
+ entityChangeStreamHeartbeatMs: boundedIntegerEnv(
+ 'ENTITY_CHANGE_STREAM_HEARTBEAT_MS',
+ 20 * 1000,
+ 1000,
+ 60 * 1000
+ ),
+ entityChangeStreamMaxLifetimeMs: boundedIntegerEnv(
+ 'ENTITY_CHANGE_STREAM_MAX_LIFETIME_MS',
+ 10 * 60 * 1000,
+ 30 * 1000,
+ 60 * 60 * 1000
+ ),
notificationsDesktopFirstEnabled:
(process.env.NOTIFICATIONS_DESKTOP_FIRST_ENABLED ?? 'true') !== 'false',
// Parse safely: a non-empty non-numeric env (e.g. "disabled") yields NaN,
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index e710d5d98..811982214 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -14,6 +14,7 @@ import {
applyCodexSubscriptionConnectionSchema,
} from './services/codexSubscriptionConnection.js'
import { applyCodexSubscriptionOAuthStateSchema } from './services/codexSubscriptionOAuthState.js'
+import { applyEntityChangeSchema } from './services/entityChangeSchema.js'
import {
applyGfsUploadCleanupSchema,
applyGfsUploadFinalizingSchema,
@@ -6103,6 +6104,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
// freezing exactly the rows a test harness needs to be able to refresh.
},
},
+ {
+ version: '0116_durable_entity_change_feed',
+ apply: applyEntityChangeSchema,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
diff --git a/control-api/src/gfs/resolve.ts b/control-api/src/gfs/resolve.ts
index 5581c6a54..159c90cc5 100644
--- a/control-api/src/gfs/resolve.ts
+++ b/control-api/src/gfs/resolve.ts
@@ -66,6 +66,8 @@ export interface ResolvedResource {
name: string
kind: string
pathCache: string | null
+ bytes: number
+ version: number
updatedAt: string
}
@@ -99,7 +101,7 @@ export interface ResolveDb {
query(text: string, values?: unknown[]): Promise<{ rows: unknown[] }>
}
-const SELECT_COLS = 'resource_id, drive, name, kind, path_cache, updated_at'
+const SELECT_COLS = 'resource_id, drive, name, kind, path_cache, bytes, version, updated_at'
function toIsoTimestamp(value: unknown): string {
if (value instanceof Date) return value.toISOString()
@@ -117,6 +119,8 @@ function rowToResource(row: unknown): ResolvedResource {
name: unknown
kind: unknown
path_cache: unknown
+ bytes: unknown
+ version: unknown
updated_at: unknown
}
return {
@@ -125,6 +129,8 @@ function rowToResource(row: unknown): ResolvedResource {
name: String(r.name),
kind: String(r.kind),
pathCache: r.path_cache == null ? null : String(r.path_cache),
+ bytes: Number(r.bytes),
+ version: Number(r.version),
updatedAt: toIsoTimestamp(r.updated_at),
}
}
diff --git a/control-api/src/main.ts b/control-api/src/main.ts
index d12a0a357..e339b4173 100644
--- a/control-api/src/main.ts
+++ b/control-api/src/main.ts
@@ -12,6 +12,10 @@ import {
startBudgetReservationSweepCron,
stopBudgetReservationSweepCron,
} from './services/budgetReservationSweepCron.js'
+import {
+ startEntityChangeDispatcher,
+ stopEntityChangeDispatcher,
+} from './services/entityChangeService.js'
import { syncDiscoveredModels } from './services/llmCatalogSync.js'
import { startLlmCatalogSyncCron, stopLlmCatalogSyncCron } from './services/llmCatalogSyncCron.js'
import { runBootEnrollment } from './services/memberRegistrationEnrollment.js'
@@ -62,6 +66,7 @@ async function main(): Promise<void> {
await assertDbReady()
console.log('[ControlAPI] Database schema ready')
+ startEntityChangeDispatcher()
// Observability only (never fatal): report whether this self-hosted deployment
// holds a registry identity. Auth is derived from credential presence, so a
@@ -200,6 +205,17 @@ async function main(): Promise<void> {
await server.start()
console.log('[ControlAPI] Running')
+ let shuttingDown = false
+ const shutdown = () => {
+ if (shuttingDown) return
+ shuttingDown = true
+ void server.stop().finally(() => {
+ void pool.end().finally(() => process.exit(0))
+ })
+ }
+ process.once('SIGTERM', shutdown)
+ process.once('SIGINT', shutdown)
+
// Hosted member-registration self-enrollment (spec §8.4): degrade, never
// block. Fire-and-forget, and only AFTER the listener is up — the liveness
// probe has no startupProbe grace (control-api.yaml: initialDelaySeconds=8,
@@ -219,6 +235,7 @@ main().catch(error => {
stopWorkflowRunsArchiveCron()
stopWorkflowScheduleWorker()
stopWorkflowApprovalNotificationDeliveryWorker()
+ stopEntityChangeDispatcher()
stopRateLimiterCleanup()
stopAdminRevokedTokenCleanup()
stopUsageRollupCron()
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 44b326f20..6ab2c427c 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -144,6 +144,37 @@ export const notificationStreamSnapshotSize = getOrCreateHistogram({
buckets: [0, 1, 2, 5, 10, 20, 50],
})
+// ─── Durable entity-change feed/stream metrics ──────────────────────────
+export const entityChangeDispatchBatchesTotal = getOrCreateCounter({
+ name: 'entity_change_dispatch_batches_total',
+ help: 'Entity-change outbox dispatcher ticks by result.',
+ labelNames: ['result'] as const as Array<'result'>,
+})
+
+export const entityChangeStreamConnectionsActive = getOrCreateGauge({
+ name: 'entity_change_stream_connections_active',
+ help: 'Active authenticated entity-change stream connections.',
+ labelNames: ['principal_kind'] as const as Array<'principal_kind'>,
+})
+
+export const entityChangeStreamFramesSentTotal = getOrCreateCounter({
+ name: 'entity_change_stream_frames_sent_total',
+ help: 'Entity-change stream frames successfully handed to HTTP responses.',
+ labelNames: ['principal_kind', 'frame_type'] as const as Array<'principal_kind' | 'frame_type'>,
+})
+
+export const entityChangeStreamDisconnectsTotal = getOrCreateCounter({
+ name: 'entity_change_stream_disconnects_total',
+ help: 'Entity-change stream closures by bounded reason.',
+ labelNames: ['principal_kind', 'reason'] as const as Array<'principal_kind' | 'reason'>,
+})
+
+export const entityChangeStreamResyncRequiredTotal = getOrCreateCounter({
+ name: 'entity_change_stream_resync_required_total',
+ help: 'Entity-change streams that required a full authoritative resync.',
+ labelNames: ['principal_kind'] as const as Array<'principal_kind'>,
+})
+
// ─── Auth token counters ──────────────────────────────────────────────────
export const mcpHostJwtIssueTotal = getOrCreateCounter({
name: 'workflow_auth_issue_total',
diff --git a/control-api/src/routes/entityChangeStream.ts b/control-api/src/routes/entityChangeStream.ts
new file mode 100644
index 000000000..a9a6c92fc
--- /dev/null
+++ b/control-api/src/routes/entityChangeStream.ts
@@ -0,0 +1,256 @@
+import type { Request, Response } from 'express'
+import { config } from '../config.js'
+import { rootLogger } from '../observability/logger.js'
+import {
+ entityChangeStreamConnectionsActive,
+ entityChangeStreamDisconnectsTotal,
+ entityChangeStreamFramesSentTotal,
+ entityChangeStreamResyncRequiredTotal,
+} from '../observability/metrics.js'
+import {
+ type EntityChangeScope,
+ isEntityChangeCursor,
+ readEntityChangeCheckpoint,
+ subscribeEntityChangeFeedWake,
+} from '../services/entityChangeService.js'
+
+const logger = rootLogger.child({ module: 'entity-change-stream' })
+const MAX_BUFFERED_BYTES = 64 * 1024
+const BACKPRESSURE_TIMEOUT_MS = 5000
+const AUTHORIZATION_RECHECK_MS = 15_000
+
+type EntityChangeStreamMessage =
+ | {
+ schemaVersion: 1
+ type: 'resync_required'
+ cursor: string
+ scopes: EntityChangeScope[]
+ }
+ | {
+ schemaVersion: 1
+ type: 'scope.invalidated'
+ cursor: string
+ scopes: EntityChangeScope[]
+ }
+ | { schemaVersion: 1; type: 'heartbeat'; cursor: string; observedAt: string }
+ | {
+ schemaVersion: 1
+ type: 'stream.closing'
+ cursor: string
+ reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer'
+ }
+
+const activeEntityChangeStreams = new Set<() => void>()
+
+export function closeActiveEntityChangeStreams(): void {
+ for (const close of Array.from(activeEntityChangeStreams)) close()
+}
+
+export function parseRequestedEntityChangeCursor(req: Request): string | null | false {
+ const raw = String(req.query?.cursor || '').trim()
+ if (!raw) return null
+ return isEntityChangeCursor(raw) ? raw : false
+}
+
+function setStreamHeaders(res: Response): void {
+ res.status(200)
+ res.setHeader('content-type', 'application/x-ndjson; charset=utf-8')
+ res.setHeader('cache-control', 'no-cache, no-transform')
+ res.setHeader('connection', 'keep-alive')
+ res.setHeader('x-accel-buffering', 'no')
+ res.flushHeaders?.()
+}
+
+export function streamEntityChanges(
+ req: Request,
+ res: Response,
+ initialCursor: string | null,
+ isAuthorized: () => Promise<boolean>,
+ principalKind: 'user' | 'operator'
+): void {
+ void (async () => {
+ let closed = false
+ let closing = false
+ let polling = false
+ let metricsActive = false
+ let cursor = initialCursor
+ let lastAuthorizationCheck = 0
+ let lastHeartbeat = Date.now()
+ let pollTimer: NodeJS.Timeout | null = null
+ let heartbeatTimer: NodeJS.Timeout | null = null
+ let lifetimeTimer: NodeJS.Timeout | null = null
+ let unsubscribeFeedWake: (() => void) | null = null
+ let shutdown = () => undefined
+
+ const cleanup = () => {
+ if (closed) return
+ closed = true
+ if (pollTimer) clearInterval(pollTimer)
+ if (heartbeatTimer) clearInterval(heartbeatTimer)
+ if (lifetimeTimer) clearTimeout(lifetimeTimer)
+ unsubscribeFeedWake?.()
+ unsubscribeFeedWake = null
+ activeEntityChangeStreams.delete(shutdown)
+ if (metricsActive) {
+ metricsActive = false
+ entityChangeStreamConnectionsActive.dec({ principal_kind: principalKind })
+ if (!closing) {
+ entityChangeStreamDisconnectsTotal.inc({
+ principal_kind: principalKind,
+ reason: 'client_disconnect',
+ })
+ }
+ }
+ }
+
+ req.on('aborted', cleanup)
+ res.on('close', cleanup)
+ req.setTimeout(0)
+
+ const write = async (message: EntityChangeStreamMessage): Promise<boolean> => {
+ if (closed || res.destroyed) return false
+ const frame = `${JSON.stringify(message)}\n`
+ if (res.writableLength + Buffer.byteLength(frame) > MAX_BUFFERED_BYTES) return false
+ if (res.write(frame)) {
+ entityChangeStreamFramesSentTotal.inc({
+ principal_kind: principalKind,
+ frame_type: message.type,
+ })
+ return true
+ }
+ const drained = await new Promise<boolean>(resolve => {
+ let settled = false
+ const finish = (writable: boolean) => {
+ if (settled) return
+ settled = true
+ clearTimeout(timeout)
+ res.off('drain', onDrain)
+ res.off('close', onClose)
+ resolve(writable)
+ }
+ const onDrain = () => finish(true)
+ const onClose = () => finish(false)
+ const timeout = setTimeout(() => finish(false), BACKPRESSURE_TIMEOUT_MS)
+ res.once('drain', onDrain)
+ res.once('close', onClose)
+ })
+ if (drained) {
+ entityChangeStreamFramesSentTotal.inc({
+ principal_kind: principalKind,
+ frame_type: message.type,
+ })
+ }
+ return drained
+ }
+
+ const closeForFailure = async (
+ reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer',
+ currentCursor = cursor || '00000000-0000-0000-0000-000000000000'
+ ) => {
+ if (closed || closing) return
+ closing = true
+ entityChangeStreamDisconnectsTotal.inc({ principal_kind: principalKind, reason })
+ await write({ schemaVersion: 1, type: 'stream.closing', cursor: currentCursor, reason })
+ res.end()
+ cleanup()
+ }
+ shutdown = () => void closeForFailure('server_shutdown')
+
+ const poll = async () => {
+ if (closed || closing || polling) return
+ polling = true
+ try {
+ const checkpoint = await readEntityChangeCheckpoint(cursor)
+ const hasChange = checkpoint.resyncRequired || checkpoint.scopes.length > 0
+ if (hasChange || Date.now() - lastAuthorizationCheck >= AUTHORIZATION_RECHECK_MS) {
+ if (!(await isAuthorized())) {
+ await closeForFailure('session_expired', checkpoint.cursor)
+ return
+ }
+ lastAuthorizationCheck = Date.now()
+ }
+ if (checkpoint.resyncRequired) {
+ entityChangeStreamResyncRequiredTotal.inc({ principal_kind: principalKind })
+ if (
+ !(await write({
+ schemaVersion: 1,
+ type: 'resync_required',
+ cursor: checkpoint.cursor,
+ scopes: ['gfs', 'authorization'],
+ }))
+ ) {
+ await closeForFailure('slow_consumer', cursor || checkpoint.cursor)
+ return
+ }
+ cursor = checkpoint.cursor
+ } else if (checkpoint.scopes.length > 0) {
+ if (
+ !(await write({
+ schemaVersion: 1,
+ type: 'scope.invalidated',
+ cursor: checkpoint.cursor,
+ scopes: checkpoint.scopes,
+ }))
+ ) {
+ await closeForFailure('slow_consumer', cursor || checkpoint.cursor)
+ return
+ }
+ cursor = checkpoint.cursor
+ }
+ if (Date.now() - lastHeartbeat >= config.entityChangeStreamHeartbeatMs) {
+ if (
+ !(await write({
+ schemaVersion: 1,
+ type: 'heartbeat',
+ cursor: checkpoint.cursor,
+ observedAt: new Date().toISOString(),
+ }))
+ ) {
+ await closeForFailure('slow_consumer', checkpoint.cursor)
+ return
+ }
+ cursor = checkpoint.cursor
+ lastHeartbeat = Date.now()
+ }
+ } catch (err) {
+ logger.warn(
+ { event: 'entity_change_stream_poll_failed', principalKind, err },
+ 'entity change stream poll failed; polling will retry'
+ )
+ } finally {
+ polling = false
+ }
+ }
+
+ try {
+ if (!(await isAuthorized())) {
+ res.status(401).end()
+ cleanup()
+ return
+ }
+ if (closed || res.destroyed) return
+ lastAuthorizationCheck = Date.now()
+ setStreamHeaders(res)
+ unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())
+ activeEntityChangeStreams.add(shutdown)
+ entityChangeStreamConnectionsActive.inc({ principal_kind: principalKind })
+ metricsActive = true
+ await poll()
+ if (closed) return
+ pollTimer = setInterval(() => void poll(), config.entityChangeStreamPollMs)
+ heartbeatTimer = setInterval(() => void poll(), config.entityChangeStreamHeartbeatMs)
+ lifetimeTimer = setTimeout(
+ () => void closeForFailure('max_lifetime'),
+ config.entityChangeStreamMaxLifetimeMs
+ )
+ } catch (err) {
+ logger.error(
+ { event: 'entity_change_stream_start_failed', principalKind, err },
+ 'stream setup failed'
+ )
+ if (!res.headersSent) res.status(500).json({ error: 'Entity change stream unavailable' })
+ else res.end()
+ cleanup()
+ }
+ })()
+}
diff --git a/control-api/src/routes/external/entityChanges.routes.ts b/control-api/src/routes/external/entityChanges.routes.ts
new file mode 100644
index 000000000..2d7f3e63b
--- /dev/null
+++ b/control-api/src/routes/external/entityChanges.routes.ts
@@ -0,0 +1,63 @@
+import { Request, Response, Router } from 'express'
+import { config } from '../../config.js'
+import { asyncHandler } from '../../http/asyncHandler.js'
+import { createExternalClientRateLimiters } from '../../middleware/externalClientIdentity.js'
+import {
+ type ExternalAuthedRequest,
+ isCurrentExternalSession,
+ requireValidExternalSessionToken,
+} from '../../middleware/externalSessionAuth.js'
+import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'
+import type { AuthClaims } from '../../profileTypes.js'
+import { parseRequestedEntityChangeCursor, streamEntityChanges } from '../entityChangeStream.js'
+
+export function isEntityChangeExternalSessionCurrent(
+ claims: AuthClaims,
+ nowMs = Date.now()
+): Promise<boolean> {
+ if (claims.exp * 1000 <= nowMs) return Promise.resolve(false)
+ return isCurrentExternalSession(claims)
+}
+
+export function createExternalEntityChangesRouter(): Router {
+ const router = Router()
+ const edgeLimits = createExternalClientRateLimiters(
+ 'entity-changes',
+ config.approvalRlExternalClientIpPerMin,
+ config.approvalRlExternalEdgePerMin
+ )
+ router.get(
+ '/external/entity-changes/stream',
+ ...edgeLimits,
+ requireValidExternalSessionToken,
+ rateLimitMiddleware({
+ bucketType: 'external_user',
+ maxPerMinute: config.approvalRlExternalPerMin,
+ getBucketKey: req => {
+ const claims = (req as ExternalAuthedRequest).externalAuth
+ return claims ? `user:${claims.userId}:entity-changes` : null
+ },
+ }),
+ asyncHandler(async (req: Request, res: Response) => {
+ const externalReq = req as ExternalAuthedRequest
+ const claims = externalReq.externalAuth
+ if (!claims) {
+ res.status(401).json({ error: 'Unauthorized' })
+ return
+ }
+ const cursor = parseRequestedEntityChangeCursor(req)
+ if (cursor === false) {
+ res.status(400).json({ error: 'Invalid entity change cursor' })
+ return
+ }
+ streamEntityChanges(
+ req,
+ res,
+ cursor,
+ () => isEntityChangeExternalSessionCurrent(claims),
+ 'user'
+ )
+ })
+ )
+ return router
+}
diff --git a/control-api/src/routes/external/index.ts b/control-api/src/routes/external/index.ts
index 4b6c4e369..76953f79c 100644
--- a/control-api/src/routes/external/index.ts
+++ b/control-api/src/routes/external/index.ts
@@ -3,6 +3,7 @@ import type { K8sGateway } from '../../k8s.js'
import { createExternalAuthRouter } from './auth.js'
import { createExternalUserApprovalDecisionsRouter } from './decisions.routes.js'
import { createExternalDirectoryRouter } from './directory.js'
+import { createExternalEntityChangesRouter } from './entityChanges.routes.js'
import { createExternalGfsRouter } from './gfs.js'
import { createExternalInvitationsRouter } from './invitations.js'
import { createExternalMembersRouter } from './members.js'
@@ -24,6 +25,7 @@ export function createExternalRouter(gateway: K8sGateway): Router {
router.use(createExternalDirectoryRouter())
router.use(createExternalSharedFilesystemsRouter(gateway))
router.use(createExternalGfsRouter())
+ router.use(createExternalEntityChangesRouter())
router.use(createExternalNotificationsRouter())
router.use(createExternalUserApprovalDecisionsRouter())
router.use(createExternalWorkflowApprovalMediumsRouter(gateway))
diff --git a/control-api/src/routes/gfs/entityChanges.ts b/control-api/src/routes/gfs/entityChanges.ts
new file mode 100644
index 000000000..a5cdc9cd9
--- /dev/null
+++ b/control-api/src/routes/gfs/entityChanges.ts
@@ -0,0 +1,31 @@
+import type { Request, Response, Router } from 'express'
+import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'
+import { authenticateAdminSession } from '../../services/adminSessionAuth.js'
+import { CONTROL_UI_ADMIN_SESSION_COOKIE, readCookie } from '../../utils/auth/sessionCookies.js'
+import { parseRequestedEntityChangeCursor, streamEntityChanges } from '../entityChangeStream.js'
+
+export function registerGfsEntityChangeRoutes(router: Router): void {
+ router.get(
+ '/gfs/entity-changes/stream',
+ requireAuthForControlUI,
+ (req: Request, res: Response) => {
+ const adminReq = req as UiAuthedRequest
+ const cursor = parseRequestedEntityChangeCursor(req)
+ if (cursor === false) {
+ res.status(400).json({ error: 'Invalid entity change cursor' })
+ return
+ }
+ const token = readCookie(req, CONTROL_UI_ADMIN_SESSION_COOKIE)
+ streamEntityChanges(
+ req,
+ res,
+ cursor,
+ async () => {
+ const current = await authenticateAdminSession(token)
+ return Boolean(current && current.sub === adminReq.adminAuth?.sub)
+ },
+ 'operator'
+ )
+ }
+ )
+}
diff --git a/control-api/src/routes/gfs/index.ts b/control-api/src/routes/gfs/index.ts
index b82c8b199..68da3f962 100644
--- a/control-api/src/routes/gfs/index.ts
+++ b/control-api/src/routes/gfs/index.ts
@@ -1,4 +1,5 @@
import { Router } from 'express'
+import { registerGfsEntityChangeRoutes } from './entityChanges.js'
import { registerGfsGrantRoutes } from './grants.js'
import { registerLegacyStandaloneGrantReportRoute } from './legacyStandaloneGrants.js'
import { registerGfsProvisionerTokenRoute } from './provisionerToken.js'
@@ -19,6 +20,7 @@ import { registerGfsTreeRoutes } from './tree.js'
export function createGfsRouter(): Router {
const router = Router()
registerGfsTokenRoute(router)
+ registerGfsEntityChangeRoutes(router)
registerGfsProvisionerTokenRoute(router)
registerGfsResolveRoutes(router)
registerGfsTreeRoutes(router)
diff --git a/control-api/src/routes/gfs/resolve.ts b/control-api/src/routes/gfs/resolve.ts
index eb7a44a2d..dd4ea19cd 100644
--- a/control-api/src/routes/gfs/resolve.ts
+++ b/control-api/src/routes/gfs/resolve.ts
@@ -19,6 +19,8 @@ export interface GfsResourceView {
name: string
kind: string
path: string | null
+ bytes: number
+ version: number
updatedAt: string
}
@@ -32,6 +34,8 @@ export function toResolveView(resource: ResolvedResource): GfsResourceView {
name: resource.name,
kind: resource.kind,
path: resource.pathCache,
+ bytes: resource.bytes,
+ version: resource.version,
updatedAt: resource.updatedAt,
}
}
diff --git a/control-api/src/server.ts b/control-api/src/server.ts
index a60e31ec1..8817e3820 100644
--- a/control-api/src/server.ts
+++ b/control-api/src/server.ts
@@ -1,10 +1,13 @@
+import type { Server as HttpServer } from 'node:http'
import { createApp } from './app.js'
import { config } from './config.js'
import { K8sGateway } from './k8s.js'
+import { closeActiveEntityChangeStreams } from './routes/entityChangeStream.js'
import {
startBudgetReservationSweepCron,
stopBudgetReservationSweepCron,
} from './services/budgetReservationSweepCron.js'
+import { stopEntityChangeDispatcher } from './services/entityChangeService.js'
import {
startPluginWorkloadSdkMaintenanceCron,
stopPluginWorkloadSdkMaintenanceCron,
@@ -14,6 +17,8 @@ import { startUsageRollupCron, stopUsageRollupCron } from './services/usageRollu
import { startExpiryCron, stopExpiryCron } from './services/userApprovalRequestExpiryCron.js'
export class ControlApiServer {
+ private httpServer: HttpServer | null = null
+
constructor(
private readonly gateway: K8sGateway,
private readonly port: number
@@ -34,6 +39,7 @@ export class ControlApiServer {
await new Promise<void>((resolve, reject) => {
const server = app.listen(this.port, resolve)
+ this.httpServer = server
server.on('error', reject)
})
}
@@ -44,5 +50,16 @@ export class ControlApiServer {
stopUsageRetentionCron()
stopBudgetReservationSweepCron()
stopPluginWorkloadSdkMaintenanceCron()
+ stopEntityChangeDispatcher()
+ closeActiveEntityChangeStreams()
+ const server = this.httpServer
+ this.httpServer = null
+ if (!server) return
+ const forceCloseTimer = setTimeout(() => server.closeAllConnections(), 5000)
+ forceCloseTimer.unref()
+ await new Promise<void>(resolve => {
+ server.close(() => resolve())
+ })
+ clearTimeout(forceCloseTimer)
}
}
diff --git a/control-api/src/services/entityChangeSchema.ts b/control-api/src/services/entityChangeSchema.ts
new file mode 100644
index 000000000..5a3e9b8df
--- /dev/null
+++ b/control-api/src/services/entityChangeSchema.ts
@@ -0,0 +1,294 @@
+import type { DbClient } from '../db.js'
+
+/**
+ * Durable, generic change-capture storage. Producers only append through
+ * SECURITY DEFINER trigger functions; consumers see coarse scopes, never the
+ * internal resource identity or authorization subject in the outbox.
+ */
+export async function applyEntityChangeSchema(db: DbClient): Promise<void> {
+ await db.query(`
+ CREATE TABLE IF NOT EXISTS entity_change_outbox (
+ id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ scope TEXT NOT NULL CHECK (scope IN ('gfs', 'authorization')),
+ entity_type TEXT NOT NULL,
+ entity_id TEXT NULL,
+ change_kind TEXT NOT NULL,
+ revision BIGINT NULL,
+ facets TEXT[] NOT NULL DEFAULT '{}',
+ created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ );
+ CREATE TABLE IF NOT EXISTS entity_change_watermark (
+ singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton),
+ sequence BIGINT NOT NULL DEFAULT 0 CHECK (sequence >= 0),
+ pruned_through BIGINT NOT NULL DEFAULT 0 CHECK (pruned_through >= 0),
+ current_cursor UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'
+ );
+ ALTER TABLE entity_change_watermark
+ ADD COLUMN IF NOT EXISTS current_cursor UUID NOT NULL
+ DEFAULT '00000000-0000-0000-0000-000000000000';
+ INSERT INTO entity_change_watermark (singleton) VALUES (true)
+ ON CONFLICT (singleton) DO NOTHING;
+
+ CREATE TABLE IF NOT EXISTS entity_change_feed (
+ sequence BIGINT PRIMARY KEY CHECK (sequence > 0),
+ cursor UUID NOT NULL UNIQUE DEFAULT gen_random_uuid(),
+ scope TEXT NOT NULL CHECK (scope IN ('gfs', 'authorization')),
+ created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ );
+ CREATE INDEX IF NOT EXISTS entity_change_feed_created_at_idx
+ ON entity_change_feed (created_at, sequence);
+
+ CREATE OR REPLACE FUNCTION entity_change_capture_resource() RETURNS trigger
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$
+ DECLARE
+ resource_key UUID;
+ event_kind TEXT;
+ event_revision BIGINT;
+ event_facets TEXT[];
+ BEGIN
+ IF TG_OP = 'DELETE' THEN
+ resource_key := OLD.resource_id;
+ event_kind := 'deleted';
+ event_revision := OLD.version;
+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];
+ ELSIF TG_OP = 'INSERT' THEN
+ resource_key := NEW.resource_id;
+ event_kind := 'created';
+ event_revision := NEW.version;
+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];
+ ELSE
+ resource_key := NEW.resource_id;
+ event_revision := NEW.version;
+ IF OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL THEN
+ event_kind := 'deleted';
+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];
+ ELSE
+ event_kind := 'updated';
+ event_facets := ARRAY[]::TEXT[];
+ IF OLD.version IS DISTINCT FROM NEW.version OR OLD.bytes IS DISTINCT FROM NEW.bytes THEN
+ event_facets := array_append(event_facets, 'content');
+ END IF;
+ IF OLD.name IS DISTINCT FROM NEW.name OR
+ OLD.parent_resource_id IS DISTINCT FROM NEW.parent_resource_id OR
+ OLD.path_cache IS DISTINCT FROM NEW.path_cache OR
+ OLD.kind IS DISTINCT FROM NEW.kind THEN
+ event_facets := array_append(event_facets, 'metadata');
+ event_facets := array_append(event_facets, 'hierarchy');
+ END IF;
+ IF OLD.deleted_at IS DISTINCT FROM NEW.deleted_at THEN
+ event_facets := array_append(event_facets, 'visibility');
+ END IF;
+ IF cardinality(event_facets) = 0 THEN
+ event_facets := ARRAY['metadata'];
+ END IF;
+ END IF;
+ END IF;
+
+ INSERT INTO public.entity_change_outbox
+ (scope, entity_type, entity_id, change_kind, revision, facets)
+ VALUES ('gfs', 'gfs.resource', resource_key::TEXT, event_kind, event_revision, event_facets);
+ PERFORM pg_notify('entity_change_outbox', '');
+ IF TG_OP = 'DELETE' THEN
+ RETURN OLD;
+ END IF;
+ RETURN NEW;
+ END;
+ $$;
+
+ CREATE OR REPLACE FUNCTION entity_change_capture_scope() RETURNS trigger
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$
+ DECLARE
+ capture_scope TEXT;
+ capture_type TEXT;
+ BEGIN
+ IF TG_TABLE_NAME IN (
+ 'gfs_grants', 'gfs_shares', 'team_members', 'users',
+ 'control_admin_users', 'gfs_desktop_operator_links'
+ ) THEN
+ capture_scope := 'authorization';
+ capture_type := 'authorization.scope';
+ ELSE
+ capture_scope := 'gfs';
+ capture_type := 'gfs.resource';
+ END IF;
+ INSERT INTO public.entity_change_outbox
+ (scope, entity_type, entity_id, change_kind, facets)
+ VALUES (capture_scope, capture_type, NULL, 'scope_invalidated', ARRAY['visibility', 'hierarchy']);
+ PERFORM pg_notify('entity_change_outbox', '');
+ RETURN NULL;
+ END;
+ $$;
+
+ DROP TRIGGER IF EXISTS entity_change_resource_rows ON gfs_resources;
+ CREATE TRIGGER entity_change_resource_rows
+ AFTER INSERT OR UPDATE OR DELETE ON gfs_resources
+ FOR EACH ROW EXECUTE FUNCTION entity_change_capture_resource();
+ DROP TRIGGER IF EXISTS entity_change_resource_truncate ON gfs_resources;
+ CREATE TRIGGER entity_change_resource_truncate
+ AFTER TRUNCATE ON gfs_resources
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+
+ DROP TRIGGER IF EXISTS entity_change_grant_statements ON gfs_grants;
+ CREATE TRIGGER entity_change_grant_statements
+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_grants
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+ DROP TRIGGER IF EXISTS entity_change_share_statements ON gfs_shares;
+ CREATE TRIGGER entity_change_share_statements
+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_shares
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+
+ DROP TRIGGER IF EXISTS entity_change_team_members_statements ON team_members;
+ CREATE TRIGGER entity_change_team_members_statements
+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON team_members
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+
+ DROP TRIGGER IF EXISTS entity_change_user_lifecycle ON users;
+ CREATE TRIGGER entity_change_user_lifecycle
+ AFTER INSERT OR DELETE OR UPDATE OF lifecycle_state, lifecycle_version ON users
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+ DROP TRIGGER IF EXISTS entity_change_admin_lifecycle ON control_admin_users;
+ CREATE TRIGGER entity_change_admin_lifecycle
+ AFTER INSERT OR DELETE OR UPDATE OF status, session_version ON control_admin_users
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+ DROP TRIGGER IF EXISTS entity_change_desktop_operator_links ON gfs_desktop_operator_links;
+ CREATE TRIGGER entity_change_desktop_operator_links
+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_desktop_operator_links
+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();
+
+ REVOKE ALL ON entity_change_outbox, entity_change_watermark, entity_change_feed
+ FROM PUBLIC, control_api_runtime, gfs_controller, gfs_controller_reader;
+ REVOKE ALL ON SEQUENCE entity_change_outbox_id_seq
+ FROM PUBLIC, control_api_runtime, gfs_controller, gfs_controller_reader;
+ REVOKE ALL ON FUNCTION entity_change_capture_resource() FROM PUBLIC;
+ REVOKE ALL ON FUNCTION entity_change_capture_scope() FROM PUBLIC;
+ `)
+
+ await db.query(`
+ CREATE OR REPLACE FUNCTION entity_change_dispatch_batch(
+ requested_batch_size INTEGER,
+ retention_seconds INTEGER
+ ) RETURNS TABLE (feed_sequence BIGINT, feed_cursor UUID, feed_scope TEXT)
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$
+ DECLARE
+ changed_scopes TEXT[];
+ changed_scope TEXT;
+ next_sequence BIGINT;
+ removed_through BIGINT;
+ BEGIN
+ IF requested_batch_size < 1 OR requested_batch_size > 10000 OR
+ retention_seconds < 3600 OR retention_seconds > 2592000 THEN
+ RAISE EXCEPTION 'entity change dispatcher bounds rejected';
+ END IF;
+ IF NOT pg_try_advisory_xact_lock(1849220361, 1) THEN
+ RETURN;
+ END IF;
+
+ WITH picked AS MATERIALIZED (
+ SELECT id FROM public.entity_change_outbox
+ ORDER BY id
+ LIMIT requested_batch_size
+ FOR UPDATE SKIP LOCKED
+ ), removed AS (
+ DELETE FROM public.entity_change_outbox pending
+ USING picked
+ WHERE pending.id = picked.id
+ RETURNING pending.scope
+ )
+ SELECT array_agg(DISTINCT scope ORDER BY scope) INTO changed_scopes FROM removed;
+
+ IF changed_scopes IS NOT NULL THEN
+ PERFORM 1 FROM public.entity_change_watermark WHERE singleton = true FOR UPDATE;
+ FOREACH changed_scope IN ARRAY changed_scopes LOOP
+ UPDATE public.entity_change_watermark
+ SET sequence = sequence + 1, current_cursor = gen_random_uuid()
+ WHERE singleton = true
+ RETURNING sequence, current_cursor INTO next_sequence, feed_cursor;
+ INSERT INTO public.entity_change_feed (sequence, scope, created_at, cursor)
+ VALUES (next_sequence, changed_scope, clock_timestamp(), feed_cursor);
+ feed_sequence := next_sequence;
+ feed_scope := changed_scope;
+ RETURN NEXT;
+ END LOOP;
+ PERFORM pg_notify('entity_change_feed', '');
+ END IF;
+
+ WITH removed AS (
+ DELETE FROM public.entity_change_feed
+ WHERE created_at < clock_timestamp() - make_interval(secs => retention_seconds)
+ RETURNING sequence
+ ) SELECT max(sequence) INTO removed_through FROM removed;
+ IF removed_through IS NOT NULL THEN
+ UPDATE public.entity_change_watermark
+ SET pruned_through = GREATEST(pruned_through, removed_through)
+ WHERE singleton = true;
+ END IF;
+ RETURN;
+ END;
+ $$;
+
+ DROP FUNCTION IF EXISTS entity_change_read_checkpoint(UUID);
+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(
+ requested_cursor UUID,
+ maximum_recovery_events INTEGER
+ )
+ RETURNS TABLE (
+ needs_resync BOOLEAN,
+ current_cursor UUID,
+ current_sequence BIGINT,
+ invalidated_scopes TEXT[]
+ )
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$
+ DECLARE
+ cursor_sequence BIGINT;
+ current_watermark BIGINT;
+ pruned_watermark BIGINT;
+ BEGIN
+ IF maximum_recovery_events < 1 OR maximum_recovery_events > 100000 THEN
+ RAISE EXCEPTION 'entity change recovery bounds rejected';
+ END IF;
+ SELECT watermark.sequence, watermark.pruned_through, watermark.current_cursor
+ INTO current_watermark, pruned_watermark, current_cursor
+ FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;
+ IF requested_cursor IS NULL THEN
+ needs_resync := true;
+ current_sequence := current_watermark;
+ invalidated_scopes := ARRAY['gfs', 'authorization'];
+ RETURN NEXT;
+ RETURN;
+ END IF;
+
+ IF requested_cursor = '00000000-0000-0000-0000-000000000000' THEN
+ cursor_sequence := 0;
+ ELSE
+ SELECT sequence INTO cursor_sequence FROM public.entity_change_feed
+ WHERE cursor = requested_cursor;
+ END IF;
+ IF cursor_sequence IS NULL AND requested_cursor = current_cursor THEN
+ cursor_sequence := current_watermark;
+ END IF;
+ IF cursor_sequence IS NULL OR cursor_sequence < pruned_watermark OR
+ current_watermark - cursor_sequence > maximum_recovery_events THEN
+ needs_resync := true;
+ current_sequence := current_watermark;
+ invalidated_scopes := ARRAY['gfs', 'authorization'];
+ RETURN NEXT;
+ RETURN;
+ END IF;
+
+ needs_resync := false;
+ SELECT COALESCE(array_agg(DISTINCT scope ORDER BY scope), ARRAY[]::TEXT[]),
+ COALESCE(max(sequence), cursor_sequence)
+ INTO invalidated_scopes, current_sequence
+ FROM public.entity_change_feed WHERE sequence > cursor_sequence;
+ SELECT watermark.current_cursor INTO current_cursor
+ FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;
+ RETURN NEXT;
+ END;
+ $$;
+
+ REVOKE ALL ON FUNCTION entity_change_dispatch_batch(INTEGER, INTEGER) FROM PUBLIC;
+ REVOKE ALL ON FUNCTION entity_change_read_checkpoint(UUID, INTEGER) FROM PUBLIC;
+ GRANT EXECUTE ON FUNCTION entity_change_dispatch_batch(INTEGER, INTEGER) TO control_api_runtime;
+ GRANT EXECUTE ON FUNCTION entity_change_read_checkpoint(UUID, INTEGER) TO control_api_runtime;
+ `)
+}
diff --git a/control-api/src/services/entityChangeService.ts b/control-api/src/services/entityChangeService.ts
new file mode 100644
index 000000000..2d6bc640d
--- /dev/null
+++ b/control-api/src/services/entityChangeService.ts
@@ -0,0 +1,247 @@
+import type { PoolClient } from 'pg'
+import { config } from '../config.js'
+import { pool } from '../db.js'
+import { rootLogger } from '../observability/logger.js'
+import { entityChangeDispatchBatchesTotal } from '../observability/metrics.js'
+
+const logger = rootLogger.child({ module: 'entity-change-feed' })
+const CURSOR_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+const ZERO_CURSOR = '00000000-0000-0000-0000-000000000000'
+
+export type EntityChangeScope = 'gfs' | 'authorization'
+
+export type EntityChangeCheckpoint = {
+ resyncRequired: boolean
+ cursor: string
+ scopes: EntityChangeScope[]
+}
+
+type CheckpointRow = {
+ needs_resync: boolean
+ current_cursor: string | null
+ invalidated_scopes: string[] | null
+}
+
+export function isEntityChangeCursor(value: string): boolean {
+ return CURSOR_RE.test(value)
+}
+
+export async function readEntityChangeCheckpoint(
+ cursor: string | null
+): Promise<EntityChangeCheckpoint> {
+ const result = await pool.query('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ cursor,
+ config.entityChangeMaxRecoveryEvents,
+ ])
+ const row = result.rows[0] as CheckpointRow | undefined
+ if (!row || typeof row.needs_resync !== 'boolean') {
+ throw new Error('Entity change checkpoint query returned an invalid result')
+ }
+ const scopes = Array.isArray(row.invalidated_scopes)
+ ? row.invalidated_scopes.filter(
+ (scope): scope is EntityChangeScope => scope === 'gfs' || scope === 'authorization'
+ )
+ : []
+ return {
+ resyncRequired: row.needs_resync,
+ cursor: isEntityChangeCursor(String(row.current_cursor || ''))
+ ? String(row.current_cursor)
+ : ZERO_CURSOR,
+ scopes,
+ }
+}
+
+export async function dispatchEntityChangeOutbox(): Promise<number> {
+ const result = await pool.query('SELECT * FROM entity_change_dispatch_batch($1, $2)', [
+ config.entityChangeDispatchBatchSize,
+ config.entityChangeRetentionSeconds,
+ ])
+ return result.rows.length
+}
+
+let dispatcherTimer: NodeJS.Timeout | null = null
+let wakeupListener: PoolClient | null = null
+let listenerRetryTimer: NodeJS.Timeout | null = null
+let dispatcherRunning = false
+
+async function runDispatcherTick(): Promise<void> {
+ if (dispatcherRunning) return
+ dispatcherRunning = true
+ try {
+ const count = await dispatchEntityChangeOutbox()
+ entityChangeDispatchBatchesTotal.inc({ result: count > 0 ? 'dispatched' : 'empty' })
+ if (count > 0) {
+ logger.info(
+ { event: 'entity_change_batch_dispatched', feedRows: count },
+ 'entity changes dispatched'
+ )
+ }
+ } catch (err) {
+ entityChangeDispatchBatchesTotal.inc({ result: 'error' })
+ logger.warn({ event: 'entity_change_dispatch_failed', err }, 'entity change dispatch failed')
+ } finally {
+ dispatcherRunning = false
+ }
+}
+
+async function connectWakeupListener(): Promise<void> {
+ if (wakeupListener || !dispatcherTimer) return
+ try {
+ const client = await pool.connect()
+ if (!dispatcherTimer) {
+ client.release()
+ return
+ }
+ await client.query('LISTEN entity_change_outbox')
+ wakeupListener = client
+ client.on('notification', () => void runDispatcherTick())
+ const disconnected = (err?: Error) => {
+ if (wakeupListener !== client) return
+ wakeupListener = null
+ client.removeAllListeners('notification')
+ client.release(true)
+ if (err) {
+ logger.warn(
+ { event: 'entity_change_dispatch_listener_lost', err },
+ 'dispatcher wake-up degraded'
+ )
+ }
+ if (dispatcherTimer && !listenerRetryTimer) {
+ listenerRetryTimer = setTimeout(() => {
+ listenerRetryTimer = null
+ void connectWakeupListener()
+ }, 5000)
+ }
+ }
+ client.on('error', disconnected)
+ client.on('end', () => disconnected())
+ } catch (err) {
+ logger.warn(
+ { event: 'entity_change_dispatch_listener_unavailable', err },
+ 'dispatcher wake-up unavailable'
+ )
+ if (dispatcherTimer && !listenerRetryTimer) {
+ listenerRetryTimer = setTimeout(() => {
+ listenerRetryTimer = null
+ void connectWakeupListener()
+ }, 5000)
+ }
+ }
+}
+
+/** Start a per-instance dispatcher; the database advisory lock elects one active owner. */
+export function startEntityChangeDispatcher(): void {
+ if (dispatcherTimer) return
+ dispatcherTimer = setInterval(() => {
+ void runDispatcherTick()
+ if (!wakeupListener) void connectWakeupListener()
+ }, config.entityChangeDispatchIntervalMs)
+ void connectWakeupListener()
+ void runDispatcherTick()
+}
+
+export function stopEntityChangeDispatcher(): void {
+ if (dispatcherTimer) clearInterval(dispatcherTimer)
+ if (listenerRetryTimer) clearTimeout(listenerRetryTimer)
+ dispatcherTimer = null
+ listenerRetryTimer = null
+ const listener = wakeupListener
+ wakeupListener = null
+ if (listener) {
+ listener.removeAllListeners('notification')
+ listener.release()
+ }
+}
+
+const feedWakeSubscribers = new Set<() => void>()
+let feedWakeListener: PoolClient | null = null
+let feedWakeConnect: Promise<void> | null = null
+let feedWakeRetryTimer: NodeJS.Timeout | null = null
+
+function scheduleFeedWakeReconnect(): void {
+ if (feedWakeSubscribers.size === 0 || feedWakeRetryTimer) return
+ feedWakeRetryTimer = setTimeout(() => {
+ feedWakeRetryTimer = null
+ void connectFeedWakeListener()
+ }, 5000)
+}
+
+async function connectFeedWakeListener(): Promise<void> {
+ if (feedWakeListener || feedWakeConnect || feedWakeSubscribers.size === 0) return
+ const connecting = (async () => {
+ let client: PoolClient | null = null
+ try {
+ client = await pool.connect()
+ if (feedWakeSubscribers.size === 0) {
+ client.release()
+ return
+ }
+ await client.query('LISTEN entity_change_feed')
+ if (feedWakeSubscribers.size === 0) {
+ client.release()
+ return
+ }
+ feedWakeListener = client
+ const connectedClient = client
+ connectedClient.on('notification', () => {
+ for (const subscriber of Array.from(feedWakeSubscribers)) {
+ try {
+ subscriber()
+ } catch {
+ // One stream's wake callback must not interrupt delivery to others.
+ }
+ }
+ })
+ const disconnected = (err?: Error) => {
+ if (feedWakeListener !== connectedClient) return
+ feedWakeListener = null
+ connectedClient.removeAllListeners('notification')
+ connectedClient.removeAllListeners('error')
+ connectedClient.removeAllListeners('end')
+ connectedClient.release(true)
+ if (err) {
+ logger.warn(
+ { event: 'entity_change_feed_listener_lost', err },
+ 'entity change wake-up degraded to polling'
+ )
+ }
+ scheduleFeedWakeReconnect()
+ }
+ connectedClient.on('error', disconnected)
+ connectedClient.on('end', () => disconnected())
+ } catch (err) {
+ if (client) client.release(true)
+ logger.warn(
+ { event: 'entity_change_feed_listener_unavailable', err },
+ 'entity change wake-up unavailable; polling remains active'
+ )
+ scheduleFeedWakeReconnect()
+ }
+ })()
+ feedWakeConnect = connecting
+ try {
+ await connecting
+ } finally {
+ if (feedWakeConnect === connecting) feedWakeConnect = null
+ }
+}
+
+/** Share one process-local LISTEN connection; durable polling remains authoritative. */
+export function subscribeEntityChangeFeedWake(onWake: () => void): () => void {
+ feedWakeSubscribers.add(onWake)
+ void connectFeedWakeListener()
+ return () => {
+ feedWakeSubscribers.delete(onWake)
+ if (feedWakeSubscribers.size > 0) return
+ if (feedWakeRetryTimer) clearTimeout(feedWakeRetryTimer)
+ feedWakeRetryTimer = null
+ const listener = feedWakeListener
+ feedWakeListener = null
+ if (listener) {
+ listener.removeAllListeners('notification')
+ listener.removeAllListeners('error')
+ listener.removeAllListeners('end')
+ listener.release()
+ }
+ }
+}
diff --git a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
index e4759a7f0..001a73a7a 100644
--- a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
+++ b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
@@ -21,7 +21,7 @@ describe('0101 oauth_grants owner generalization migration', () => {
const { CONTROL_API_MIGRATIONS } = await import('../src/db.js')
const versions = CONTROL_API_MIGRATIONS.map(m => m.version)
expect(versions).toContain('0106_oauth_grants_owner_generalization')
- expect(versions.at(-1)).toBe('0115_llm_allowed_models_image_input')
+ expect(versions.at(-1)).toBe('0116_durable_entity_change_feed')
expect(versions.indexOf('0100_seed_minimax_allowed_model')).toBeLessThan(
versions.indexOf('0106_oauth_grants_owner_generalization')
)
diff --git a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
new file mode 100644
index 000000000..fa5ab9884
--- /dev/null
+++ b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
@@ -0,0 +1,384 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { Pool } from 'pg'
+import { initDb } from '../src/db.js'
+import './realPostgres.requirement.ts'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+describeRealPostgres('entity change feed real PostgreSQL contract', () => {
+ const database = `control_api_entity_change_${randomBytes(6).toString('hex')}`
+ const connectionString = databaseUrl(
+ adminUrl ?? 'postgresql://[email protected]/postgres',
+ database
+ )
+ let adminPool: Pool
+ let instancePool: Pool
+ let replicaPool: Pool
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ instancePool = new Pool({ connectionString })
+ replicaPool = new Pool({ connectionString })
+ await initDb({ connect: () => instancePool.connect() })
+ })
+
+ beforeEach(async () => {
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ })
+
+ afterAll(async () => {
+ await instancePool?.end()
+ await replicaPool?.end()
+ if (adminPool) {
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ }
+ })
+
+ it('captures committed create, replace, rename/move, delete, grant, and membership changes', async () => {
+ const client = await instancePool.connect()
+ const drive = `entity-change-${randomUUID()}`
+ const rootId = randomUUID()
+ const parentId = randomUUID()
+ const movedParentId = randomUUID()
+ const fileId = randomUUID()
+ const userId = randomUUID()
+ const teamId = randomUUID()
+ try {
+ await client.query('BEGIN')
+ await client.query('SET ROLE gfs_controller')
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)
+ VALUES ($1, $2, '/', 'directory', '/')`,
+ [rootId, drive]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'parent', 'directory', '/parent')`,
+ [parentId, drive, rootId]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'destination', 'directory', '/destination')`,
+ [movedParentId, drive, rootId]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'draft.md', 'file', '/parent/draft.md')`,
+ [fileId, drive, parentId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(
+ `INSERT INTO gfs_grants (drive, resource_id, subject_type, subject_id, permissions)
+ VALUES ($1, $2, 'team', $3, ARRAY['read'])`,
+ [drive, fileId, teamId]
+ )
+ await client.query(
+ `INSERT INTO gfs_shares (drive, resource_id, subject_type, subject_id, permissions)
+ VALUES ($1, $2, 'user', $3, ARRAY['read'])`,
+ [drive, fileId, userId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ const created = await instancePool.query<{ count: string }>(
+ `SELECT count(*)::text AS count FROM entity_change_outbox WHERE scope = 'gfs'`
+ )
+ expect(Number(created.rows[0]?.count)).toBe(4)
+
+ await client.query('BEGIN')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(
+ `UPDATE gfs_resources SET version = version + 1, bytes = 45 WHERE resource_id = $1`,
+ [fileId]
+ )
+ await client.query(
+ `UPDATE gfs_resources SET parent_resource_id = $2, name = 'renamed.md',
+ path_cache = '/destination/renamed.md' WHERE resource_id = $1`,
+ [fileId, movedParentId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('SET ROLE gfs_controller')
+ await client.query(
+ `UPDATE gfs_resources SET deleted_at = clock_timestamp() WHERE resource_id = $1`,
+ [fileId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ await client.query('BEGIN')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(`INSERT INTO users (id, email) VALUES ($1, $2)`, [
+ userId,
+ `${userId}@entity-change.invalid`,
+ ])
+ await client.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])
+ await client.query(
+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,
+ [teamId, userId]
+ )
+ await client.query('DELETE FROM team_members WHERE team_id = $1 AND user_id = $2', [
+ teamId,
+ userId,
+ ])
+ await client.query('DELETE FROM gfs_grants WHERE drive = $1 AND resource_id = $2', [
+ drive,
+ fileId,
+ ])
+ await client.query('DELETE FROM gfs_shares WHERE drive = $1 AND resource_id = $2', [
+ drive,
+ fileId,
+ ])
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ const outbox = await instancePool.query<{ scope: string; entity_id: string | null }>(
+ 'SELECT scope, entity_id FROM entity_change_outbox ORDER BY id'
+ )
+ expect(outbox.rows.filter(row => row.scope === 'gfs')).toHaveLength(7)
+ expect(outbox.rows.filter(row => row.entity_id === fileId)).toHaveLength(4)
+ expect(outbox.rows.filter(row => row.scope === 'authorization')).toHaveLength(7)
+ // Event evidence contains no file content, path, or authorization subject.
+ expect(JSON.stringify(outbox.rows)).not.toContain('draft.md')
+ expect(JSON.stringify(outbox.rows)).not.toContain(userId)
+ expect(JSON.stringify(outbox.rows)).not.toContain(teamId)
+ } finally {
+ client.release()
+ }
+ })
+
+ it('rolls capture back with the producer transaction and publishes only committed facts', async () => {
+ const client = await instancePool.connect()
+ const drive = `entity-change-rollback-${randomUUID()}`
+ try {
+ const before = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ await client.query('BEGIN')
+ await client.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'rolled-back', 'file')`,
+ [drive]
+ )
+ await client.query('ROLLBACK')
+ const after = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ expect(after.rows[0]?.count).toBe(before.rows[0]?.count)
+
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'committed', 'file')`,
+ [drive]
+ )
+ const firstDispatch = await instancePool.query<{ feed_sequence: string }>(
+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'
+ )
+ const feedAfterFirst = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_feed'
+ )
+ const duplicateDispatch = await instancePool.query<{ feed_sequence: string }>(
+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'
+ )
+ const feedAfterDuplicate = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_feed'
+ )
+ expect(firstDispatch.rows).toHaveLength(1)
+ expect(duplicateDispatch.rows).toHaveLength(0)
+ expect(feedAfterDuplicate.rows[0]?.count).toBe(feedAfterFirst.rows[0]?.count)
+ } finally {
+ client.release()
+ }
+ })
+
+ it('uses dispatcher order rather than identity allocation as public sequence order', async () => {
+ const pending = await instancePool.connect()
+ const earlyDrive = `entity-change-early-${randomUUID()}`
+ try {
+ await pending.query('BEGIN')
+ await pending.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'early', 'file')`,
+ [earlyDrive]
+ )
+ const pendingId = await pending.query<{ id: string }>(
+ `SELECT max(id)::text AS id FROM entity_change_outbox WHERE entity_id IS NOT NULL`
+ )
+ const watermarkBefore = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ const userId = randomUUID()
+ const teamId = randomUUID()
+ await instancePool.query('INSERT INTO users (id, email) VALUES ($1, $2)', [
+ userId,
+ `${userId}@ordering.invalid`,
+ ])
+ await instancePool.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])
+ await instancePool.query(
+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,
+ [teamId, userId]
+ )
+ const committedMax = await instancePool.query<{ id: string }>(
+ 'SELECT max(id)::text AS id FROM entity_change_outbox'
+ )
+ expect(Number(pendingId.rows[0]?.id)).toBeLessThan(Number(committedMax.rows[0]?.id))
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const firstFeed = await instancePool.query<{ sequence: string; scope: string }>(
+ `SELECT sequence::text, scope FROM entity_change_feed
+ WHERE sequence > $1 ORDER BY sequence`,
+ [watermarkBefore.rows[0]?.sequence]
+ )
+ expect(firstFeed.rows.map(row => row.scope)).toEqual(['authorization'])
+ await pending.query('COMMIT')
+ await replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const feed = await instancePool.query<{ sequence: string; scope: string }>(
+ `SELECT sequence::text, scope FROM entity_change_feed
+ WHERE sequence > $1 ORDER BY sequence`,
+ [watermarkBefore.rows[0]?.sequence]
+ )
+ expect(feed.rows.map(row => row.scope)).toEqual(['authorization', 'gfs'])
+ expect(feed.rows.map(row => Number(row.sequence))).toEqual([
+ Number(watermarkBefore.rows[0]?.sequence) + 1,
+ Number(watermarkBefore.rows[0]?.sequence) + 2,
+ ])
+ } finally {
+ await pending.query('ROLLBACK').catch(() => undefined)
+ pending.release()
+ }
+ })
+
+ it('recovers missed wakeups and expired cursors with a coarse authorized checkpoint', async () => {
+ const resource = await instancePool.query<{ resource_id: string }>(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint', 'file')
+ RETURNING resource_id`,
+ [`entity-change-checkpoint-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const lastEvent = await instancePool.query<{ cursor: string; sequence: string }>(
+ 'SELECT cursor::text, sequence::text FROM entity_change_feed ORDER BY sequence DESC LIMIT 1'
+ )
+ // No listener is involved here: the durable query is the fallback after a missed NOTIFY.
+ const recovery = await instancePool.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ lastEvent.rows[0]?.cursor,
+ 10000,
+ ])
+ expect(recovery.rows[0]?.needs_resync).toBe(false)
+
+ await instancePool.query(
+ `UPDATE entity_change_feed SET created_at = clock_timestamp() - interval '2 days'
+ WHERE sequence = $1`,
+ [lastEvent.rows[0]?.sequence]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const expired = await instancePool.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ lastEvent.rows[0]?.cursor,
+ 10000,
+ ])
+ expect(expired.rows[0]?.needs_resync).toBe(true)
+ expect(expired.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])
+ expect(JSON.stringify(expired.rows)).not.toContain(resource.rows[0]?.resource_id)
+ })
+
+ it('coalesces latest-state invalidations and keeps feed storage behind the runtime boundary', async () => {
+ const inserted = await instancePool.query<{ resource_id: string }>(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'many-updates', 'file')
+ RETURNING resource_id`,
+ [`entity-change-coalesce-${randomUUID()}`]
+ )
+ const resourceId = inserted.rows[0]?.resource_id
+ await instancePool.query(
+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',
+ [resourceId]
+ )
+ await instancePool.query(
+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',
+ [resourceId]
+ )
+ const before = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await Promise.all([
+ instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),
+ replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),
+ ])
+ const after = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ const published = await instancePool.query<{ scope: string; count: string }>(
+ `SELECT scope, count(*)::text AS count FROM entity_change_feed
+ WHERE sequence > $1 GROUP BY scope`,
+ [before.rows[0]?.sequence]
+ )
+ expect(Number(after.rows[0]?.sequence) - Number(before.rows[0]?.sequence)).toBe(1)
+ expect(published.rows).toEqual([{ scope: 'gfs', count: '1' }])
+
+ const restricted = await instancePool.connect()
+ try {
+ await restricted.query('SET ROLE control_api_runtime')
+ await expect(restricted.query('SELECT * FROM entity_change_feed')).rejects.toThrow()
+ const checkpoint = await restricted.query<{ invalidated_scopes: string[] }>(
+ 'SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)',
+ ['00000000-0000-0000-0000-000000000000', 10000]
+ )
+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])
+ expect(Object.keys(checkpoint.rows[0] ?? {}).sort()).toEqual([
+ 'current_cursor',
+ 'current_sequence',
+ 'invalidated_scopes',
+ 'needs_resync',
+ ])
+ } finally {
+ await restricted.query('RESET ROLE').catch(() => undefined)
+ restricted.release()
+ }
+ })
+
+ it('resyncs instead of scanning an oversized retained replay window', async () => {
+ const baseline = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind)
+ VALUES ($1, 'bounded-window', 'file')`,
+ [`entity-change-bounded-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind)
+ VALUES ($1, 'bounded-window-2', 'file')`,
+ [`entity-change-bounded-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const bounded = await instancePool.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ baseline.rows[0]?.current_cursor,
+ 1,
+ ])
+ expect(bounded.rows[0]?.needs_resync).toBe(true)
+ expect(bounded.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])
+ })
+})
diff --git a/control-api/test/gfs.resolve.test.ts b/control-api/test/gfs.resolve.test.ts
index fb2951b64..889b28638 100644
--- a/control-api/test/gfs.resolve.test.ts
+++ b/control-api/test/gfs.resolve.test.ts
@@ -46,6 +46,8 @@ function res(partial: Partial<ResolvedResource> & { resourceId: string }): Resol
name: 'n',
kind: 'directory',
pathCache: null,
+ bytes: 0,
+ version: 0,
updatedAt: '2026-01-01T00:00:00.000Z',
...partial,
}
@@ -114,6 +116,8 @@ const row = (resourceId: string, name = 'n', kind = 'directory') => ({
name,
kind,
path_cache: null,
+ bytes: '42',
+ version: 7,
updated_at: new Date('2026-01-01T00:00:00.000Z'),
})
@@ -124,6 +128,8 @@ describe('DbResolveStore', () => {
const out = await new DbResolveStore(db).getByRid('main', RID)
expect(out?.resourceId).toBe('r1')
expect(out?.updatedAt).toBe('2026-01-01T00:00:00.000Z')
+ expect(out?.bytes).toBe(42)
+ expect(out?.version).toBe(7)
expect(db.queries[0].text).toContain('resource_id = $2::uuid')
expect(db.queries[0].text).toContain('deleted_at IS NULL')
expect(db.queries[0].text).toContain('updated_at')
diff --git a/control-api/test/gfs.resolveRoutes.test.ts b/control-api/test/gfs.resolveRoutes.test.ts
index 4377afdb5..25be97d4e 100644
--- a/control-api/test/gfs.resolveRoutes.test.ts
+++ b/control-api/test/gfs.resolveRoutes.test.ts
@@ -11,6 +11,8 @@ function res(partial: Partial<ResolvedResource> & { resourceId: string }): Resol
name: 'n',
kind: 'directory',
pathCache: null,
+ bytes: 0,
+ version: 0,
updatedAt: '2026-01-01T00:00:00.000Z',
...partial,
}
@@ -33,6 +35,8 @@ describe('toResolveView', () => {
expect(view.rid).toBe(RID)
expect(view.gfsUri).toBe(`gfs://main/${RID}`)
expect(view.path).toBe('/org')
+ expect(view.bytes).toBe(0)
+ expect(view.version).toBe(0)
expect(view.updatedAt).toBe('2026-01-01T00:00:00.000Z')
expect(view.updatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/)
})
diff --git a/control-api/test/routes.entityChangeStream.test.ts b/control-api/test/routes.entityChangeStream.test.ts
new file mode 100644
index 000000000..6588290f9
--- /dev/null
+++ b/control-api/test/routes.entityChangeStream.test.ts
@@ -0,0 +1,144 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { Request, Response } from 'express'
+import { EventEmitter } from 'node:events'
+import {
+ closeActiveEntityChangeStreams,
+ parseRequestedEntityChangeCursor,
+ streamEntityChanges,
+} from '../src/routes/entityChangeStream.js'
+
+const serviceMock = vi.hoisted(() => ({
+ isEntityChangeCursor: vi.fn(),
+ readEntityChangeCheckpoint: vi.fn(),
+ subscribeEntityChangeFeedWake: vi.fn(),
+}))
+const configMock = vi.hoisted(() => ({
+ entityChangeStreamHeartbeatMs: 20_000,
+ entityChangeStreamMaxLifetimeMs: 600_000,
+ entityChangeStreamPollMs: 250,
+}))
+
+vi.mock('../src/config.js', () => ({ config: configMock }))
+vi.mock('../src/services/entityChangeService.js', () => serviceMock)
+vi.mock('../src/observability/logger.js', () => ({
+ rootLogger: { child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }) },
+}))
+
+class FakeRequest extends EventEmitter {
+ query: Record<string, unknown> = {}
+ setTimeout = vi.fn()
+}
+
+class FakeResponse extends EventEmitter {
+ statusCode = 200
+ headersSent = false
+ destroyed = false
+ writableEnded = false
+ writableLength = 0
+ frames: string[] = []
+ status(code: number): this {
+ this.statusCode = code
+ return this
+ }
+ setHeader(): this {
+ return this
+ }
+ flushHeaders(): void {
+ this.headersSent = true
+ }
+ write(frame: string): boolean {
+ this.frames.push(frame)
+ return true
+ }
+ end(): void {
+ this.writableEnded = true
+ this.emit('close')
+ }
+ json(): this {
+ return this
+ }
+}
+
+const CURSOR = 'd119f895-1ef8-4e73-8f08-f9754919682a'
+
+describe('routes/entityChangeStream', () => {
+ beforeEach(() => {
+ serviceMock.isEntityChangeCursor.mockImplementation((value: string) =>
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)
+ )
+ serviceMock.readEntityChangeCheckpoint.mockReset()
+ serviceMock.subscribeEntityChangeFeedWake.mockReset()
+ serviceMock.subscribeEntityChangeFeedWake.mockReturnValue(vi.fn())
+ })
+
+ it('validates requested UUID cursors and represents an absent cursor as a full resync', () => {
+ expect(parseRequestedEntityChangeCursor({ query: {} } as Request)).toBeNull()
+ expect(parseRequestedEntityChangeCursor({ query: { cursor: CURSOR } } as Request)).toBe(CURSOR)
+ expect(parseRequestedEntityChangeCursor({ query: { cursor: 'bad' } } as Request)).toBe(false)
+ })
+
+ it('authorizes before sending streaming headers', async () => {
+ const req = new FakeRequest()
+ const res = new FakeResponse()
+ streamEntityChanges(
+ req as unknown as Request,
+ res as unknown as Response,
+ null,
+ async () => false,
+ 'user'
+ )
+ await vi.waitFor(() => expect(res.statusCode).toBe(401))
+ expect(res.headersSent).toBe(false)
+ expect(res.frames).toEqual([])
+ })
+
+ it('does not acquire a feed connection when the client disconnects during authorization', async () => {
+ let finishAuthorization: (authorized: boolean) => void = () => undefined
+ const authorization = new Promise<boolean>(resolve => {
+ finishAuthorization = resolve
+ })
+ const req = new FakeRequest()
+ const res = new FakeResponse()
+ streamEntityChanges(
+ req as unknown as Request,
+ res as unknown as Response,
+ null,
+ () => authorization,
+ 'user'
+ )
+ req.emit('aborted')
+ finishAuthorization(true)
+ await new Promise(resolve => setImmediate(resolve))
+ expect(serviceMock.subscribeEntityChangeFeedWake).not.toHaveBeenCalled()
+ expect(res.headersSent).toBe(false)
+ })
+
+ it('sends only coarse invalidations and closes cleanly on server shutdown', async () => {
+ serviceMock.readEntityChangeCheckpoint.mockResolvedValue({
+ resyncRequired: true,
+ cursor: CURSOR,
+ scopes: ['gfs', 'authorization'],
+ })
+ const req = new FakeRequest()
+ const res = new FakeResponse()
+ let authorizationChecks = 0
+ streamEntityChanges(
+ req as unknown as Request,
+ res as unknown as Response,
+ null,
+ async () => {
+ authorizationChecks += 1
+ return true
+ },
+ 'operator'
+ )
+ await vi.waitFor(() => expect(res.frames.length).toBeGreaterThan(0))
+ closeActiveEntityChangeStreams()
+ await vi.waitFor(() => expect(res.writableEnded).toBe(true))
+ expect(res.frames.join('')).toContain('"type":"resync_required"')
+ expect(res.frames.join('')).toContain('"type":"stream.closing"')
+ expect(res.frames.join('')).not.toContain('resource-id')
+ expect(authorizationChecks).toBeGreaterThanOrEqual(2)
+ expect(serviceMock.subscribeEntityChangeFeedWake).toHaveBeenCalledOnce()
+ })
+})
diff --git a/control-api/test/routes.externalEntityChanges.test.ts b/control-api/test/routes.externalEntityChanges.test.ts
new file mode 100644
index 000000000..def2154ba
--- /dev/null
+++ b/control-api/test/routes.externalEntityChanges.test.ts
@@ -0,0 +1,27 @@
+import { describe, expect, it, vi } from 'vitest'
+import { isEntityChangeExternalSessionCurrent } from '../src/routes/external/entityChanges.routes.js'
+
+const sessionAuth = vi.hoisted(() => ({
+ isCurrentExternalSession: vi.fn(),
+ requireValidExternalSessionToken: vi.fn(),
+}))
+
+vi.mock('../src/middleware/externalSessionAuth.js', () => sessionAuth)
+
+describe('external entity-change stream authorization', () => {
+ it('stops delivery after the authenticated session token expires', async () => {
+ const claims = {
+ userId: 'user-1',
+ email: '[email protected]',
+ teamId: 'team-1',
+ role: 'member' as const,
+ authGeneration: 1,
+ exp: 1_000,
+ }
+ sessionAuth.isCurrentExternalSession.mockReset().mockResolvedValue(true)
+
+ await expect(isEntityChangeExternalSessionCurrent(claims, 999_000)).resolves.toBe(true)
+ await expect(isEntityChangeExternalSessionCurrent(claims, 1_000_000)).resolves.toBe(false)
+ expect(sessionAuth.isCurrentExternalSession).toHaveBeenCalledOnce()
+ })
+})
diff --git a/control-api/test/services.entityChangeService.test.ts b/control-api/test/services.entityChangeService.test.ts
new file mode 100644
index 000000000..63597aef5
--- /dev/null
+++ b/control-api/test/services.entityChangeService.test.ts
@@ -0,0 +1,90 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { EventEmitter } from 'node:events'
+import {
+ dispatchEntityChangeOutbox,
+ isEntityChangeCursor,
+ readEntityChangeCheckpoint,
+ subscribeEntityChangeFeedWake,
+} from '../src/services/entityChangeService.js'
+
+const dbMock = vi.hoisted(() => ({ query: vi.fn(), connect: vi.fn() }))
+const configMock = vi.hoisted(() => ({
+ entityChangeDispatchBatchSize: 37,
+ entityChangeRetentionSeconds: 86_400,
+ entityChangeMaxRecoveryEvents: 500,
+}))
+
+vi.mock('../src/db.js', () => ({ pool: dbMock }))
+vi.mock('../src/config.js', () => ({ config: configMock }))
+vi.mock('../src/observability/logger.js', () => ({
+ rootLogger: { child: () => ({ info: vi.fn(), warn: vi.fn() }) },
+}))
+
+describe('entityChangeService', () => {
+ beforeEach(() => {
+ dbMock.query.mockReset()
+ dbMock.connect.mockReset()
+ })
+
+ it('validates only canonical UUID cursors', () => {
+ expect(isEntityChangeCursor('d119f895-1ef8-4e73-8f08-f9754919682a')).toBe(true)
+ expect(isEntityChangeCursor('d119f895-1ef8-4e73-8f08-f9754919682')).toBe(false)
+ })
+
+ it('exposes only approved coarse scopes from the durable checkpoint', async () => {
+ dbMock.query.mockResolvedValueOnce({
+ rows: [
+ {
+ needs_resync: false,
+ current_cursor: 'd119f895-1ef8-4e73-8f08-f9754919682a',
+ invalidated_scopes: ['gfs', 'authorization', 'private-resource-id'],
+ },
+ ],
+ })
+
+ await expect(
+ readEntityChangeCheckpoint('d119f895-1ef8-4e73-8f08-f9754919682a')
+ ).resolves.toEqual({
+ resyncRequired: false,
+ cursor: 'd119f895-1ef8-4e73-8f08-f9754919682a',
+ scopes: ['gfs', 'authorization'],
+ })
+ expect(dbMock.query).toHaveBeenCalledWith(
+ 'SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)',
+ ['d119f895-1ef8-4e73-8f08-f9754919682a', 500]
+ )
+ })
+
+ it('dispatches with the configured batch and retention bounds', async () => {
+ dbMock.query.mockResolvedValueOnce({ rows: [{ feed_sequence: 1 }] })
+ await expect(dispatchEntityChangeOutbox()).resolves.toBe(1)
+ expect(dbMock.query).toHaveBeenCalledWith(
+ 'SELECT * FROM entity_change_dispatch_batch($1, $2)',
+ [37, 86_400]
+ )
+ })
+
+ it('shares one LISTEN connection among wake subscribers and releases it when empty', async () => {
+ const listener = new EventEmitter() as EventEmitter & {
+ query: ReturnType<typeof vi.fn>
+ release: ReturnType<typeof vi.fn>
+ }
+ listener.query = vi.fn().mockResolvedValue({ rows: [] })
+ listener.release = vi.fn()
+ dbMock.connect.mockResolvedValueOnce(listener)
+ const first = vi.fn()
+ const second = vi.fn()
+ const unsubscribeFirst = subscribeEntityChangeFeedWake(first)
+ const unsubscribeSecond = subscribeEntityChangeFeedWake(second)
+ await vi.waitFor(() => expect(listener.query).toHaveBeenCalledWith('LISTEN entity_change_feed'))
+ expect(dbMock.connect).toHaveBeenCalledOnce()
+
+ listener.emit('notification')
+ expect(first).toHaveBeenCalledOnce()
+ expect(second).toHaveBeenCalledOnce()
+
+ unsubscribeFirst()
+ unsubscribeSecond()
+ expect(listener.release).toHaveBeenCalledOnce()
+ })
+})
diff --git a/deploy/scripts/control-api-runtime-access-profiles.tsv b/deploy/scripts/control-api-runtime-access-profiles.tsv
index 6cbb1641b..0c3d257f1 100644
--- a/deploy/scripts/control-api-runtime-access-profiles.tsv
+++ b/deploy/scripts/control-api-runtime-access-profiles.tsv
@@ -1,6 +1,9 @@
# public relation<TAB>control_api_runtime access profile
administrative_events append
agent_run_events append
+entity_change_feed none
+entity_change_outbox none
+entity_change_watermark none
budget_pending_reservations legacy_dml
codex_subscription_connections upsert
codex_subscription_oauth_states upsert
diff --git a/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv b/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv
index f3ed6db69..a26bbb8c8 100644
--- a/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv
+++ b/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv
@@ -1,6 +1,7 @@
# public sequence<TAB>control_api_runtime access profile
administrative_events_ingest_sequence_seq consume
agent_run_events_ingest_sequence_seq consume
+entity_change_outbox_id_seq none
gfs_audit_sequence_no_seq legacy_rw
governed_event_stream_stream_sequence_seq consume
infrastructure_telemetry_events_ingest_sequence_seq consume
diff --git a/deploy/scripts/run-control-api-db-migration.sh b/deploy/scripts/run-control-api-db-migration.sh
index 2929e85d6..a8019687d 100755
--- a/deploy/scripts/run-control-api-db-migration.sh
+++ b/deploy/scripts/run-control-api-db-migration.sh
@@ -463,7 +463,7 @@ runtime_sequence_access_contract_values() {
/^[[:space:]]*#/ || /^[[:space:]]*$/ { next }
NF != 2 { exit 2 }
$1 !~ /^[a-z][a-z0-9_]*$/ { exit 3 }
- $2 !~ /^(legacy_rw|consume)$/ { exit 4 }
+ $2 !~ /^(legacy_rw|consume|none)$/ { exit 4 }
seen[$1]++ { exit 5 }
{
count++
@@ -630,8 +630,8 @@ verify_runtime_access_contract() {
JOIN actual_sequences actual USING (sequence_name)
CROSS JOIN LATERAL (
VALUES
- ('USAGE', true),
- ('SELECT', true),
+ ('USAGE', expected.access_profile != 'none'),
+ ('SELECT', expected.access_profile != 'none'),
('UPDATE', expected.access_profile = 'legacy_rw')
) required(privilege_name, allowed)
WHERE has_sequence_privilege(
diff --git a/scripts/tests/test-control-api-runtime-access-contract.sh b/scripts/tests/test-control-api-runtime-access-contract.sh
index 89dfb3238..06f28f63f 100755
--- a/scripts/tests/test-control-api-runtime-access-contract.sh
+++ b/scripts/tests/test-control-api-runtime-access-contract.sh
@@ -9,18 +9,21 @@ relation_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { count++ } END { print cou
duplicate_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { seen[$1]++ } END { for (name in seen) if (seen[name] > 1) count++ } END { print count + 0 }' "$PROFILE_FILE")"
invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/) { count++ } END { print count + 0 }' "$PROFILE_FILE")"
-[[ "$relation_count" == "88" ]]
-[[ "$duplicate_count" == "0" ]]
-[[ "$invalid_count" == "0" ]]
-grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$PROFILE_FILE"
-grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$PROFILE_FILE"
-
-grep -Fq '$2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/' "$MIGRATION_SCRIPT"
-grep -Fq "('INSERT', expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle'))" "$MIGRATION_SCRIPT"
-grep -Fq "('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))" "$MIGRATION_SCRIPT"
-grep -Fq "('DELETE', expected.access_profile IN ('legacy_dml'))" "$MIGRATION_SCRIPT"
-grep -Fq "('TRUNCATE', false)" "$MIGRATION_SCRIPT"
-grep -Fq "('REFERENCES', false)" "$MIGRATION_SCRIPT"
-grep -Fq "('TRIGGER', false)" "$MIGRATION_SCRIPT"
+if [[ "$relation_count" != "95" || "$duplicate_count" != "0" || "$invalid_count" != "0" ]] || \
+ ! grep -qx $'entity_change_feed\tnone' "$PROFILE_FILE" || \
+ ! grep -qx $'entity_change_outbox\tnone' "$PROFILE_FILE" || \
+ ! grep -qx $'entity_change_watermark\tnone' "$PROFILE_FILE" || \
+ ! grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$PROFILE_FILE" || \
+ ! grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$PROFILE_FILE" || \
+ ! grep -Fq '$2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/' "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('INSERT', expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle'))" "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))" "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('DELETE', expected.access_profile IN ('legacy_dml'))" "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('TRUNCATE', false)" "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('REFERENCES', false)" "$MIGRATION_SCRIPT" || \
+ ! grep -Fq "('TRIGGER', false)" "$MIGRATION_SCRIPT"; then
+ printf 'FAIL: control-api runtime access contract is incomplete or not exact\n' >&2
+ exit 1
+fi
printf 'PASS: control-api runtime link_lifecycle contract is explicit and least privilege\n'
diff --git a/scripts/tests/test-gcp-scripts.sh b/scripts/tests/test-gcp-scripts.sh
index 2aa971418..a997dac95 100755
--- a/scripts/tests/test-gcp-scripts.sh
+++ b/scripts/tests/test-gcp-scripts.sh
@@ -1464,9 +1464,12 @@ assert_control_api_runtime_access_contract_is_exact() {
invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/) { count++ } END { print count + 0 }' "$profile_file")"
sequence_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { count++ } END { print count + 0 }' "$sequence_profile_file")"
sequence_duplicate_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { seen[$1]++ } END { for (name in seen) if (seen[name] > 1) count++ } END { print count + 0 }' "$sequence_profile_file")"
- sequence_invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_rw|consume)$/) { count++ } END { print count + 0 }' "$sequence_profile_file")"
+ sequence_invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_rw|consume|none)$/) { count++ } END { print count + 0 }' "$sequence_profile_file")"
- if [[ "$relation_count" == "88" && "$duplicate_count" == "0" && "$invalid_count" == "0" ]] && \
+ if [[ "$relation_count" == "95" && "$duplicate_count" == "0" && "$invalid_count" == "0" ]] && \
+ grep -qx $'entity_change_feed\tnone' "$profile_file" && \
+ grep -qx $'entity_change_outbox\tnone' "$profile_file" && \
+ grep -qx $'entity_change_watermark\tnone' "$profile_file" && \
grep -qx $'gfs_upload_parts\tnone' "$profile_file" && \
grep -qx $'gfs_upload_sessions\tnone' "$profile_file" && \
grep -qx $'llm_allowed_models\tlegacy_dml' "$profile_file" && \
@@ -1476,13 +1479,15 @@ assert_control_api_runtime_access_contract_is_exact() {
grep -qx $'plugin_workload_sdk_spend_outcomes\tappend' "$profile_file" && \
grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$profile_file" && \
grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$profile_file" && \
- [[ "$sequence_count" == "7" && "$sequence_duplicate_count" == "0" && "$sequence_invalid_count" == "0" ]] && \
+ [[ "$sequence_count" == "8" && "$sequence_duplicate_count" == "0" && "$sequence_invalid_count" == "0" ]] && \
+ grep -qx $'entity_change_outbox_id_seq\tnone' "$sequence_profile_file" && \
grep -qx $'member_registration_credentials_id_seq\tconsume' "$sequence_profile_file" && \
[[ "$migration_script" != *'RUNTIME_ACCESS_PROFILES_FILE:-'* ]] && \
[[ "$migration_script" == *'FULL OUTER JOIN actual_relations'* ]] && \
[[ "$migration_script" == *'FULL OUTER JOIN actual_sequences'* ]] && \
[[ "$migration_script" == *'IS DISTINCT FROM required.allowed'* ]] && \
[[ "$migration_script" == *'has_sequence_privilege('* ]] && \
+ [[ "$migration_script" == *"('SELECT', expected.access_profile != 'none')"* ]] && \
[[ "$migration_script" == *"expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle')"* ]] && \
[[ "$migration_script" == *"('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))"* ]] && \
[[ "$migration_script" == *"('DELETE', expected.access_profile IN ('legacy_dml'))"* ]] && \