Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(control-api): add durable entity change feed
Josué0266becbbe02
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 211f524b5..aa041be5d 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 64d00e152..adeb0d846 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex fa82b204e..8c3d9ef69 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -1,6 +1,11 @@import { createPublicKey } from 'node:crypto'import { DEFAULT_ALLOWED_PLUGIN_IMAGE_PREFIXES } from '@clerum/image-policy'+function boundedIntegerEnv(name: string, fallback: number, min: number, max: number): number {+ const parsed = Number(process.env[name])+ return Number.isSafeInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback+}+type Config = {port: numberjsonBodyLimit: string@@ -230,6 +235,13 @@ type Config = {notificationStreamHeartbeatMs: numbernotificationStreamMaxLifetimeMs: numbernotificationStreamSnapshotLimit: number+ entityChangeDispatchIntervalMs: number+ entityChangeRetentionSeconds: number+ entityChangeDispatchBatchSize: number+ entityChangeMaxRecoveryEvents: number+ entityChangeStreamPollMs: number+ entityChangeStreamHeartbeatMs: number+ entityChangeStreamMaxLifetimeMs: numbernotificationsDesktopFirstEnabled: booleannotificationDesktopGraceSeconds: number// Stricter than the normal refresh limiter: a client reaching reissue has@@ -1058,6 +1070,43 @@ export const config: Config = {process.env.NOTIFICATION_STREAM_MAX_LIFETIME_MS || 10 * 60 * 1000),notificationStreamSnapshotLimit: Number(process.env.NOTIFICATION_STREAM_SNAPSHOT_LIMIT || 50),+ entityChangeDispatchIntervalMs: boundedIntegerEnv(+ 'ENTITY_CHANGE_DISPATCH_INTERVAL_MS',+ 500,+ 100,+ 5000+ ),+ entityChangeRetentionSeconds: boundedIntegerEnv(+ 'ENTITY_CHANGE_RETENTION_SECONDS',+ 24 * 60 * 60,+ 60 * 60,+ 30 * 24 * 60 * 60+ ),+ entityChangeDispatchBatchSize: boundedIntegerEnv(+ 'ENTITY_CHANGE_DISPATCH_BATCH_SIZE',+ 1000,+ 1,+ 10000+ ),+ entityChangeMaxRecoveryEvents: boundedIntegerEnv(+ 'ENTITY_CHANGE_MAX_RECOVERY_EVENTS',+ 10000,+ 1,+ 100000+ ),+ entityChangeStreamPollMs: boundedIntegerEnv('ENTITY_CHANGE_STREAM_POLL_MS', 1000, 250, 5000),+ entityChangeStreamHeartbeatMs: boundedIntegerEnv(+ 'ENTITY_CHANGE_STREAM_HEARTBEAT_MS',+ 20 * 1000,+ 1000,+ 60 * 1000+ ),+ entityChangeStreamMaxLifetimeMs: boundedIntegerEnv(+ 'ENTITY_CHANGE_STREAM_MAX_LIFETIME_MS',+ 10 * 60 * 1000,+ 30 * 1000,+ 60 * 60 * 1000+ ),notificationsDesktopFirstEnabled:(process.env.NOTIFICATIONS_DESKTOP_FIRST_ENABLED ?? 'true') !== 'false',// Parse safely: a non-empty non-numeric env (e.g. "disabled") yields NaN,diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex e710d5d98..811982214 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -14,6 +14,7 @@ import {applyCodexSubscriptionConnectionSchema,} from './services/codexSubscriptionConnection.js'import { applyCodexSubscriptionOAuthStateSchema } from './services/codexSubscriptionOAuthState.js'+import { applyEntityChangeSchema } from './services/entityChangeSchema.js'import {applyGfsUploadCleanupSchema,applyGfsUploadFinalizingSchema,@@ -6103,6 +6104,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [// freezing exactly the rows a test harness needs to be able to refresh.},},+ {+ version: '0116_durable_entity_change_feed',+ apply: applyEntityChangeSchema,+ },]async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {diff --git a/control-api/src/gfs/resolve.ts b/control-api/src/gfs/resolve.tsindex 5581c6a54..159c90cc5 100644--- a/control-api/src/gfs/resolve.ts+++ b/control-api/src/gfs/resolve.ts@@ -66,6 +66,8 @@ export interface ResolvedResource {name: stringkind: stringpathCache: string | null+ bytes: number+ version: numberupdatedAt: string}@@ -99,7 +101,7 @@ export interface ResolveDb {query(text: string, values?: unknown[]): Promise<{ rows: unknown[] }>}-const SELECT_COLS = 'resource_id, drive, name, kind, path_cache, updated_at'+const SELECT_COLS = 'resource_id, drive, name, kind, path_cache, bytes, version, updated_at'function toIsoTimestamp(value: unknown): string {if (value instanceof Date) return value.toISOString()@@ -117,6 +119,8 @@ function rowToResource(row: unknown): ResolvedResource {name: unknownkind: unknownpath_cache: unknown+ bytes: unknown+ version: unknownupdated_at: unknown}return {@@ -125,6 +129,8 @@ function rowToResource(row: unknown): ResolvedResource {name: String(r.name),kind: String(r.kind),pathCache: r.path_cache == null ? null : String(r.path_cache),+ bytes: Number(r.bytes),+ version: Number(r.version),updatedAt: toIsoTimestamp(r.updated_at),}}diff --git a/control-api/src/main.ts b/control-api/src/main.tsindex d12a0a357..e339b4173 100644--- a/control-api/src/main.ts+++ b/control-api/src/main.ts@@ -12,6 +12,10 @@ import {startBudgetReservationSweepCron,stopBudgetReservationSweepCron,} from './services/budgetReservationSweepCron.js'+import {+ startEntityChangeDispatcher,+ stopEntityChangeDispatcher,+} from './services/entityChangeService.js'import { syncDiscoveredModels } from './services/llmCatalogSync.js'import { startLlmCatalogSyncCron, stopLlmCatalogSyncCron } from './services/llmCatalogSyncCron.js'import { runBootEnrollment } from './services/memberRegistrationEnrollment.js'@@ -62,6 +66,7 @@ async function main(): Promise<void> {await assertDbReady()console.log('[ControlAPI] Database schema ready')+ startEntityChangeDispatcher()// Observability only (never fatal): report whether this self-hosted deployment// holds a registry identity. Auth is derived from credential presence, so a@@ -200,6 +205,17 @@ async function main(): Promise<void> {await server.start()console.log('[ControlAPI] Running')+ let shuttingDown = false+ const shutdown = () => {+ if (shuttingDown) return+ shuttingDown = true+ void server.stop().finally(() => {+ void pool.end().finally(() => process.exit(0))+ })+ }+ process.once('SIGTERM', shutdown)+ process.once('SIGINT', shutdown)+// Hosted member-registration self-enrollment (spec §8.4): degrade, never// block. Fire-and-forget, and only AFTER the listener is up — the liveness// probe has no startupProbe grace (control-api.yaml: initialDelaySeconds=8,@@ -219,6 +235,7 @@ main().catch(error => {stopWorkflowRunsArchiveCron()stopWorkflowScheduleWorker()stopWorkflowApprovalNotificationDeliveryWorker()+ stopEntityChangeDispatcher()stopRateLimiterCleanup()stopAdminRevokedTokenCleanup()stopUsageRollupCron()diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 44b326f20..6ab2c427c 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -144,6 +144,37 @@ export const notificationStreamSnapshotSize = getOrCreateHistogram({buckets: [0, 1, 2, 5, 10, 20, 50],})+// ─── Durable entity-change feed/stream metrics ──────────────────────────+export const entityChangeDispatchBatchesTotal = getOrCreateCounter({+ name: 'entity_change_dispatch_batches_total',+ help: 'Entity-change outbox dispatcher ticks by result.',+ labelNames: ['result'] as const as Array<'result'>,+})++export const entityChangeStreamConnectionsActive = getOrCreateGauge({+ name: 'entity_change_stream_connections_active',+ help: 'Active authenticated entity-change stream connections.',+ labelNames: ['principal_kind'] as const as Array<'principal_kind'>,+})++export const entityChangeStreamFramesSentTotal = getOrCreateCounter({+ name: 'entity_change_stream_frames_sent_total',+ help: 'Entity-change stream frames successfully handed to HTTP responses.',+ labelNames: ['principal_kind', 'frame_type'] as const as Array<'principal_kind' | 'frame_type'>,+})++export const entityChangeStreamDisconnectsTotal = getOrCreateCounter({+ name: 'entity_change_stream_disconnects_total',+ help: 'Entity-change stream closures by bounded reason.',+ labelNames: ['principal_kind', 'reason'] as const as Array<'principal_kind' | 'reason'>,+})++export const entityChangeStreamResyncRequiredTotal = getOrCreateCounter({+ name: 'entity_change_stream_resync_required_total',+ help: 'Entity-change streams that required a full authoritative resync.',+ labelNames: ['principal_kind'] as const as Array<'principal_kind'>,+})+// ─── Auth token counters ──────────────────────────────────────────────────export const mcpHostJwtIssueTotal = getOrCreateCounter({name: 'workflow_auth_issue_total',diff --git a/control-api/src/routes/entityChangeStream.ts b/control-api/src/routes/entityChangeStream.tsnew file mode 100644index 000000000..a9a6c92fc--- /dev/null+++ b/control-api/src/routes/entityChangeStream.ts@@ -0,0 +1,256 @@+import type { Request, Response } from 'express'+import { config } from '../config.js'+import { rootLogger } from '../observability/logger.js'+import {+ entityChangeStreamConnectionsActive,+ entityChangeStreamDisconnectsTotal,+ entityChangeStreamFramesSentTotal,+ entityChangeStreamResyncRequiredTotal,+} from '../observability/metrics.js'+import {+ type EntityChangeScope,+ isEntityChangeCursor,+ readEntityChangeCheckpoint,+ subscribeEntityChangeFeedWake,+} from '../services/entityChangeService.js'++const logger = rootLogger.child({ module: 'entity-change-stream' })+const MAX_BUFFERED_BYTES = 64 * 1024+const BACKPRESSURE_TIMEOUT_MS = 5000+const AUTHORIZATION_RECHECK_MS = 15_000++type EntityChangeStreamMessage =+ | {+ schemaVersion: 1+ type: 'resync_required'+ cursor: string+ scopes: EntityChangeScope[]+ }+ | {+ schemaVersion: 1+ type: 'scope.invalidated'+ cursor: string+ scopes: EntityChangeScope[]+ }+ | { schemaVersion: 1; type: 'heartbeat'; cursor: string; observedAt: string }+ | {+ schemaVersion: 1+ type: 'stream.closing'+ cursor: string+ reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer'+ }++const activeEntityChangeStreams = new Set<() => void>()++export function closeActiveEntityChangeStreams(): void {+ for (const close of Array.from(activeEntityChangeStreams)) close()+}++export function parseRequestedEntityChangeCursor(req: Request): string | null | false {+ const raw = String(req.query?.cursor || '').trim()+ if (!raw) return null+ return isEntityChangeCursor(raw) ? raw : false+}++function setStreamHeaders(res: Response): void {+ res.status(200)+ res.setHeader('content-type', 'application/x-ndjson; charset=utf-8')+ res.setHeader('cache-control', 'no-cache, no-transform')+ res.setHeader('connection', 'keep-alive')+ res.setHeader('x-accel-buffering', 'no')+ res.flushHeaders?.()+}++export function streamEntityChanges(+ req: Request,+ res: Response,+ initialCursor: string | null,+ isAuthorized: () => Promise<boolean>,+ principalKind: 'user' | 'operator'+): void {+ void (async () => {+ let closed = false+ let closing = false+ let polling = false+ let metricsActive = false+ let cursor = initialCursor+ let lastAuthorizationCheck = 0+ let lastHeartbeat = Date.now()+ let pollTimer: NodeJS.Timeout | null = null+ let heartbeatTimer: NodeJS.Timeout | null = null+ let lifetimeTimer: NodeJS.Timeout | null = null+ let unsubscribeFeedWake: (() => void) | null = null+ let shutdown = () => undefined++ const cleanup = () => {+ if (closed) return+ closed = true+ if (pollTimer) clearInterval(pollTimer)+ if (heartbeatTimer) clearInterval(heartbeatTimer)+ if (lifetimeTimer) clearTimeout(lifetimeTimer)+ unsubscribeFeedWake?.()+ unsubscribeFeedWake = null+ activeEntityChangeStreams.delete(shutdown)+ if (metricsActive) {+ metricsActive = false+ entityChangeStreamConnectionsActive.dec({ principal_kind: principalKind })+ if (!closing) {+ entityChangeStreamDisconnectsTotal.inc({+ principal_kind: principalKind,+ reason: 'client_disconnect',+ })+ }+ }+ }++ req.on('aborted', cleanup)+ res.on('close', cleanup)+ req.setTimeout(0)++ const write = async (message: EntityChangeStreamMessage): Promise<boolean> => {+ if (closed || res.destroyed) return false+ const frame = `${JSON.stringify(message)}\n`+ if (res.writableLength + Buffer.byteLength(frame) > MAX_BUFFERED_BYTES) return false+ if (res.write(frame)) {+ entityChangeStreamFramesSentTotal.inc({+ principal_kind: principalKind,+ frame_type: message.type,+ })+ return true+ }+ const drained = await new Promise<boolean>(resolve => {+ let settled = false+ const finish = (writable: boolean) => {+ if (settled) return+ settled = true+ clearTimeout(timeout)+ res.off('drain', onDrain)+ res.off('close', onClose)+ resolve(writable)+ }+ const onDrain = () => finish(true)+ const onClose = () => finish(false)+ const timeout = setTimeout(() => finish(false), BACKPRESSURE_TIMEOUT_MS)+ res.once('drain', onDrain)+ res.once('close', onClose)+ })+ if (drained) {+ entityChangeStreamFramesSentTotal.inc({+ principal_kind: principalKind,+ frame_type: message.type,+ })+ }+ return drained+ }++ const closeForFailure = async (+ reason: 'max_lifetime' | 'session_expired' | 'server_shutdown' | 'slow_consumer',+ currentCursor = cursor || '00000000-0000-0000-0000-000000000000'+ ) => {+ if (closed || closing) return+ closing = true+ entityChangeStreamDisconnectsTotal.inc({ principal_kind: principalKind, reason })+ await write({ schemaVersion: 1, type: 'stream.closing', cursor: currentCursor, reason })+ res.end()+ cleanup()+ }+ shutdown = () => void closeForFailure('server_shutdown')++ const poll = async () => {+ if (closed || closing || polling) return+ polling = true+ try {+ const checkpoint = await readEntityChangeCheckpoint(cursor)+ const hasChange = checkpoint.resyncRequired || checkpoint.scopes.length > 0+ if (hasChange || Date.now() - lastAuthorizationCheck >= AUTHORIZATION_RECHECK_MS) {+ if (!(await isAuthorized())) {+ await closeForFailure('session_expired', checkpoint.cursor)+ return+ }+ lastAuthorizationCheck = Date.now()+ }+ if (checkpoint.resyncRequired) {+ entityChangeStreamResyncRequiredTotal.inc({ principal_kind: principalKind })+ if (+ !(await write({+ schemaVersion: 1,+ type: 'resync_required',+ cursor: checkpoint.cursor,+ scopes: ['gfs', 'authorization'],+ }))+ ) {+ await closeForFailure('slow_consumer', cursor || checkpoint.cursor)+ return+ }+ cursor = checkpoint.cursor+ } else if (checkpoint.scopes.length > 0) {+ if (+ !(await write({+ schemaVersion: 1,+ type: 'scope.invalidated',+ cursor: checkpoint.cursor,+ scopes: checkpoint.scopes,+ }))+ ) {+ await closeForFailure('slow_consumer', cursor || checkpoint.cursor)+ return+ }+ cursor = checkpoint.cursor+ }+ if (Date.now() - lastHeartbeat >= config.entityChangeStreamHeartbeatMs) {+ if (+ !(await write({+ schemaVersion: 1,+ type: 'heartbeat',+ cursor: checkpoint.cursor,+ observedAt: new Date().toISOString(),+ }))+ ) {+ await closeForFailure('slow_consumer', checkpoint.cursor)+ return+ }+ cursor = checkpoint.cursor+ lastHeartbeat = Date.now()+ }+ } catch (err) {+ logger.warn(+ { event: 'entity_change_stream_poll_failed', principalKind, err },+ 'entity change stream poll failed; polling will retry'+ )+ } finally {+ polling = false+ }+ }++ try {+ if (!(await isAuthorized())) {+ res.status(401).end()+ cleanup()+ return+ }+ if (closed || res.destroyed) return+ lastAuthorizationCheck = Date.now()+ setStreamHeaders(res)+ unsubscribeFeedWake = subscribeEntityChangeFeedWake(() => void poll())+ activeEntityChangeStreams.add(shutdown)+ entityChangeStreamConnectionsActive.inc({ principal_kind: principalKind })+ metricsActive = true+ await poll()+ if (closed) return+ pollTimer = setInterval(() => void poll(), config.entityChangeStreamPollMs)+ heartbeatTimer = setInterval(() => void poll(), config.entityChangeStreamHeartbeatMs)+ lifetimeTimer = setTimeout(+ () => void closeForFailure('max_lifetime'),+ config.entityChangeStreamMaxLifetimeMs+ )+ } catch (err) {+ logger.error(+ { event: 'entity_change_stream_start_failed', principalKind, err },+ 'stream setup failed'+ )+ if (!res.headersSent) res.status(500).json({ error: 'Entity change stream unavailable' })+ else res.end()+ cleanup()+ }+ })()+}diff --git a/control-api/src/routes/external/entityChanges.routes.ts b/control-api/src/routes/external/entityChanges.routes.tsnew file mode 100644index 000000000..2d7f3e63b--- /dev/null+++ b/control-api/src/routes/external/entityChanges.routes.ts@@ -0,0 +1,63 @@+import { Request, Response, Router } from 'express'+import { config } from '../../config.js'+import { asyncHandler } from '../../http/asyncHandler.js'+import { createExternalClientRateLimiters } from '../../middleware/externalClientIdentity.js'+import {+ type ExternalAuthedRequest,+ isCurrentExternalSession,+ requireValidExternalSessionToken,+} from '../../middleware/externalSessionAuth.js'+import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'+import type { AuthClaims } from '../../profileTypes.js'+import { parseRequestedEntityChangeCursor, streamEntityChanges } from '../entityChangeStream.js'++export function isEntityChangeExternalSessionCurrent(+ claims: AuthClaims,+ nowMs = Date.now()+): Promise<boolean> {+ if (claims.exp * 1000 <= nowMs) return Promise.resolve(false)+ return isCurrentExternalSession(claims)+}++export function createExternalEntityChangesRouter(): Router {+ const router = Router()+ const edgeLimits = createExternalClientRateLimiters(+ 'entity-changes',+ config.approvalRlExternalClientIpPerMin,+ config.approvalRlExternalEdgePerMin+ )+ router.get(+ '/external/entity-changes/stream',+ ...edgeLimits,+ requireValidExternalSessionToken,+ rateLimitMiddleware({+ bucketType: 'external_user',+ maxPerMinute: config.approvalRlExternalPerMin,+ getBucketKey: req => {+ const claims = (req as ExternalAuthedRequest).externalAuth+ return claims ? `user:${claims.userId}:entity-changes` : null+ },+ }),+ asyncHandler(async (req: Request, res: Response) => {+ const externalReq = req as ExternalAuthedRequest+ const claims = externalReq.externalAuth+ if (!claims) {+ res.status(401).json({ error: 'Unauthorized' })+ return+ }+ const cursor = parseRequestedEntityChangeCursor(req)+ if (cursor === false) {+ res.status(400).json({ error: 'Invalid entity change cursor' })+ return+ }+ streamEntityChanges(+ req,+ res,+ cursor,+ () => isEntityChangeExternalSessionCurrent(claims),+ 'user'+ )+ })+ )+ return router+}diff --git a/control-api/src/routes/external/index.ts b/control-api/src/routes/external/index.tsindex 4b6c4e369..76953f79c 100644--- a/control-api/src/routes/external/index.ts+++ b/control-api/src/routes/external/index.ts@@ -3,6 +3,7 @@ import type { K8sGateway } from '../../k8s.js'import { createExternalAuthRouter } from './auth.js'import { createExternalUserApprovalDecisionsRouter } from './decisions.routes.js'import { createExternalDirectoryRouter } from './directory.js'+import { createExternalEntityChangesRouter } from './entityChanges.routes.js'import { createExternalGfsRouter } from './gfs.js'import { createExternalInvitationsRouter } from './invitations.js'import { createExternalMembersRouter } from './members.js'@@ -24,6 +25,7 @@ export function createExternalRouter(gateway: K8sGateway): Router {router.use(createExternalDirectoryRouter())router.use(createExternalSharedFilesystemsRouter(gateway))router.use(createExternalGfsRouter())+ router.use(createExternalEntityChangesRouter())router.use(createExternalNotificationsRouter())router.use(createExternalUserApprovalDecisionsRouter())router.use(createExternalWorkflowApprovalMediumsRouter(gateway))diff --git a/control-api/src/routes/gfs/entityChanges.ts b/control-api/src/routes/gfs/entityChanges.tsnew file mode 100644index 000000000..a5cdc9cd9--- /dev/null+++ b/control-api/src/routes/gfs/entityChanges.ts@@ -0,0 +1,31 @@+import type { Request, Response, Router } from 'express'+import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'+import { authenticateAdminSession } from '../../services/adminSessionAuth.js'+import { CONTROL_UI_ADMIN_SESSION_COOKIE, readCookie } from '../../utils/auth/sessionCookies.js'+import { parseRequestedEntityChangeCursor, streamEntityChanges } from '../entityChangeStream.js'++export function registerGfsEntityChangeRoutes(router: Router): void {+ router.get(+ '/gfs/entity-changes/stream',+ requireAuthForControlUI,+ (req: Request, res: Response) => {+ const adminReq = req as UiAuthedRequest+ const cursor = parseRequestedEntityChangeCursor(req)+ if (cursor === false) {+ res.status(400).json({ error: 'Invalid entity change cursor' })+ return+ }+ const token = readCookie(req, CONTROL_UI_ADMIN_SESSION_COOKIE)+ streamEntityChanges(+ req,+ res,+ cursor,+ async () => {+ const current = await authenticateAdminSession(token)+ return Boolean(current && current.sub === adminReq.adminAuth?.sub)+ },+ 'operator'+ )+ }+ )+}diff --git a/control-api/src/routes/gfs/index.ts b/control-api/src/routes/gfs/index.tsindex b82c8b199..68da3f962 100644--- a/control-api/src/routes/gfs/index.ts+++ b/control-api/src/routes/gfs/index.ts@@ -1,4 +1,5 @@import { Router } from 'express'+import { registerGfsEntityChangeRoutes } from './entityChanges.js'import { registerGfsGrantRoutes } from './grants.js'import { registerLegacyStandaloneGrantReportRoute } from './legacyStandaloneGrants.js'import { registerGfsProvisionerTokenRoute } from './provisionerToken.js'@@ -19,6 +20,7 @@ import { registerGfsTreeRoutes } from './tree.js'export function createGfsRouter(): Router {const router = Router()registerGfsTokenRoute(router)+ registerGfsEntityChangeRoutes(router)registerGfsProvisionerTokenRoute(router)registerGfsResolveRoutes(router)registerGfsTreeRoutes(router)diff --git a/control-api/src/routes/gfs/resolve.ts b/control-api/src/routes/gfs/resolve.tsindex eb7a44a2d..dd4ea19cd 100644--- a/control-api/src/routes/gfs/resolve.ts+++ b/control-api/src/routes/gfs/resolve.ts@@ -19,6 +19,8 @@ export interface GfsResourceView {name: stringkind: stringpath: string | null+ bytes: number+ version: numberupdatedAt: string}@@ -32,6 +34,8 @@ export function toResolveView(resource: ResolvedResource): GfsResourceView {name: resource.name,kind: resource.kind,path: resource.pathCache,+ bytes: resource.bytes,+ version: resource.version,updatedAt: resource.updatedAt,}}diff --git a/control-api/src/server.ts b/control-api/src/server.tsindex a60e31ec1..8817e3820 100644--- a/control-api/src/server.ts+++ b/control-api/src/server.ts@@ -1,10 +1,13 @@+import type { Server as HttpServer } from 'node:http'import { createApp } from './app.js'import { config } from './config.js'import { K8sGateway } from './k8s.js'+import { closeActiveEntityChangeStreams } from './routes/entityChangeStream.js'import {startBudgetReservationSweepCron,stopBudgetReservationSweepCron,} from './services/budgetReservationSweepCron.js'+import { stopEntityChangeDispatcher } from './services/entityChangeService.js'import {startPluginWorkloadSdkMaintenanceCron,stopPluginWorkloadSdkMaintenanceCron,@@ -14,6 +17,8 @@ import { startUsageRollupCron, stopUsageRollupCron } from './services/usageRolluimport { startExpiryCron, stopExpiryCron } from './services/userApprovalRequestExpiryCron.js'export class ControlApiServer {+ private httpServer: HttpServer | null = null+constructor(private readonly gateway: K8sGateway,private readonly port: number@@ -34,6 +39,7 @@ export class ControlApiServer {await new Promise<void>((resolve, reject) => {const server = app.listen(this.port, resolve)+ this.httpServer = serverserver.on('error', reject)})}@@ -44,5 +50,16 @@ export class ControlApiServer {stopUsageRetentionCron()stopBudgetReservationSweepCron()stopPluginWorkloadSdkMaintenanceCron()+ stopEntityChangeDispatcher()+ closeActiveEntityChangeStreams()+ const server = this.httpServer+ this.httpServer = null+ if (!server) return+ const forceCloseTimer = setTimeout(() => server.closeAllConnections(), 5000)+ forceCloseTimer.unref()+ await new Promise<void>(resolve => {+ server.close(() => resolve())+ })+ clearTimeout(forceCloseTimer)}}diff --git a/control-api/src/services/entityChangeSchema.ts b/control-api/src/services/entityChangeSchema.tsnew file mode 100644index 000000000..5a3e9b8df--- /dev/null+++ b/control-api/src/services/entityChangeSchema.ts@@ -0,0 +1,294 @@+import type { DbClient } from '../db.js'++/**+ * Durable, generic change-capture storage. Producers only append through+ * SECURITY DEFINER trigger functions; consumers see coarse scopes, never the+ * internal resource identity or authorization subject in the outbox.+ */+export async function applyEntityChangeSchema(db: DbClient): Promise<void> {+ await db.query(`+ CREATE TABLE IF NOT EXISTS entity_change_outbox (+ id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,+ scope TEXT NOT NULL CHECK (scope IN ('gfs', 'authorization')),+ entity_type TEXT NOT NULL,+ entity_id TEXT NULL,+ change_kind TEXT NOT NULL,+ revision BIGINT NULL,+ facets TEXT[] NOT NULL DEFAULT '{}',+ created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()+ );+ CREATE TABLE IF NOT EXISTS entity_change_watermark (+ singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton),+ sequence BIGINT NOT NULL DEFAULT 0 CHECK (sequence >= 0),+ pruned_through BIGINT NOT NULL DEFAULT 0 CHECK (pruned_through >= 0),+ current_cursor UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'+ );+ ALTER TABLE entity_change_watermark+ ADD COLUMN IF NOT EXISTS current_cursor UUID NOT NULL+ DEFAULT '00000000-0000-0000-0000-000000000000';+ INSERT INTO entity_change_watermark (singleton) VALUES (true)+ ON CONFLICT (singleton) DO NOTHING;++ CREATE TABLE IF NOT EXISTS entity_change_feed (+ sequence BIGINT PRIMARY KEY CHECK (sequence > 0),+ cursor UUID NOT NULL UNIQUE DEFAULT gen_random_uuid(),+ scope TEXT NOT NULL CHECK (scope IN ('gfs', 'authorization')),+ created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()+ );+ CREATE INDEX IF NOT EXISTS entity_change_feed_created_at_idx+ ON entity_change_feed (created_at, sequence);++ CREATE OR REPLACE FUNCTION entity_change_capture_resource() RETURNS trigger+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$+ DECLARE+ resource_key UUID;+ event_kind TEXT;+ event_revision BIGINT;+ event_facets TEXT[];+ BEGIN+ IF TG_OP = 'DELETE' THEN+ resource_key := OLD.resource_id;+ event_kind := 'deleted';+ event_revision := OLD.version;+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];+ ELSIF TG_OP = 'INSERT' THEN+ resource_key := NEW.resource_id;+ event_kind := 'created';+ event_revision := NEW.version;+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];+ ELSE+ resource_key := NEW.resource_id;+ event_revision := NEW.version;+ IF OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL THEN+ event_kind := 'deleted';+ event_facets := ARRAY['metadata', 'hierarchy', 'content', 'visibility'];+ ELSE+ event_kind := 'updated';+ event_facets := ARRAY[]::TEXT[];+ IF OLD.version IS DISTINCT FROM NEW.version OR OLD.bytes IS DISTINCT FROM NEW.bytes THEN+ event_facets := array_append(event_facets, 'content');+ END IF;+ IF OLD.name IS DISTINCT FROM NEW.name OR+ OLD.parent_resource_id IS DISTINCT FROM NEW.parent_resource_id OR+ OLD.path_cache IS DISTINCT FROM NEW.path_cache OR+ OLD.kind IS DISTINCT FROM NEW.kind THEN+ event_facets := array_append(event_facets, 'metadata');+ event_facets := array_append(event_facets, 'hierarchy');+ END IF;+ IF OLD.deleted_at IS DISTINCT FROM NEW.deleted_at THEN+ event_facets := array_append(event_facets, 'visibility');+ END IF;+ IF cardinality(event_facets) = 0 THEN+ event_facets := ARRAY['metadata'];+ END IF;+ END IF;+ END IF;++ INSERT INTO public.entity_change_outbox+ (scope, entity_type, entity_id, change_kind, revision, facets)+ VALUES ('gfs', 'gfs.resource', resource_key::TEXT, event_kind, event_revision, event_facets);+ PERFORM pg_notify('entity_change_outbox', '');+ IF TG_OP = 'DELETE' THEN+ RETURN OLD;+ END IF;+ RETURN NEW;+ END;+ $$;++ CREATE OR REPLACE FUNCTION entity_change_capture_scope() RETURNS trigger+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$+ DECLARE+ capture_scope TEXT;+ capture_type TEXT;+ BEGIN+ IF TG_TABLE_NAME IN (+ 'gfs_grants', 'gfs_shares', 'team_members', 'users',+ 'control_admin_users', 'gfs_desktop_operator_links'+ ) THEN+ capture_scope := 'authorization';+ capture_type := 'authorization.scope';+ ELSE+ capture_scope := 'gfs';+ capture_type := 'gfs.resource';+ END IF;+ INSERT INTO public.entity_change_outbox+ (scope, entity_type, entity_id, change_kind, facets)+ VALUES (capture_scope, capture_type, NULL, 'scope_invalidated', ARRAY['visibility', 'hierarchy']);+ PERFORM pg_notify('entity_change_outbox', '');+ RETURN NULL;+ END;+ $$;++ DROP TRIGGER IF EXISTS entity_change_resource_rows ON gfs_resources;+ CREATE TRIGGER entity_change_resource_rows+ AFTER INSERT OR UPDATE OR DELETE ON gfs_resources+ FOR EACH ROW EXECUTE FUNCTION entity_change_capture_resource();+ DROP TRIGGER IF EXISTS entity_change_resource_truncate ON gfs_resources;+ CREATE TRIGGER entity_change_resource_truncate+ AFTER TRUNCATE ON gfs_resources+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();++ DROP TRIGGER IF EXISTS entity_change_grant_statements ON gfs_grants;+ CREATE TRIGGER entity_change_grant_statements+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_grants+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();+ DROP TRIGGER IF EXISTS entity_change_share_statements ON gfs_shares;+ CREATE TRIGGER entity_change_share_statements+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_shares+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();++ DROP TRIGGER IF EXISTS entity_change_team_members_statements ON team_members;+ CREATE TRIGGER entity_change_team_members_statements+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON team_members+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();++ DROP TRIGGER IF EXISTS entity_change_user_lifecycle ON users;+ CREATE TRIGGER entity_change_user_lifecycle+ AFTER INSERT OR DELETE OR UPDATE OF lifecycle_state, lifecycle_version ON users+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();+ DROP TRIGGER IF EXISTS entity_change_admin_lifecycle ON control_admin_users;+ CREATE TRIGGER entity_change_admin_lifecycle+ AFTER INSERT OR DELETE OR UPDATE OF status, session_version ON control_admin_users+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();+ DROP TRIGGER IF EXISTS entity_change_desktop_operator_links ON gfs_desktop_operator_links;+ CREATE TRIGGER entity_change_desktop_operator_links+ AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON gfs_desktop_operator_links+ FOR EACH STATEMENT EXECUTE FUNCTION entity_change_capture_scope();++ REVOKE ALL ON entity_change_outbox, entity_change_watermark, entity_change_feed+ FROM PUBLIC, control_api_runtime, gfs_controller, gfs_controller_reader;+ REVOKE ALL ON SEQUENCE entity_change_outbox_id_seq+ FROM PUBLIC, control_api_runtime, gfs_controller, gfs_controller_reader;+ REVOKE ALL ON FUNCTION entity_change_capture_resource() FROM PUBLIC;+ REVOKE ALL ON FUNCTION entity_change_capture_scope() FROM PUBLIC;+ `)++ await db.query(`+ CREATE OR REPLACE FUNCTION entity_change_dispatch_batch(+ requested_batch_size INTEGER,+ retention_seconds INTEGER+ ) RETURNS TABLE (feed_sequence BIGINT, feed_cursor UUID, feed_scope TEXT)+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$+ DECLARE+ changed_scopes TEXT[];+ changed_scope TEXT;+ next_sequence BIGINT;+ removed_through BIGINT;+ BEGIN+ IF requested_batch_size < 1 OR requested_batch_size > 10000 OR+ retention_seconds < 3600 OR retention_seconds > 2592000 THEN+ RAISE EXCEPTION 'entity change dispatcher bounds rejected';+ END IF;+ IF NOT pg_try_advisory_xact_lock(1849220361, 1) THEN+ RETURN;+ END IF;++ WITH picked AS MATERIALIZED (+ SELECT id FROM public.entity_change_outbox+ ORDER BY id+ LIMIT requested_batch_size+ FOR UPDATE SKIP LOCKED+ ), removed AS (+ DELETE FROM public.entity_change_outbox pending+ USING picked+ WHERE pending.id = picked.id+ RETURNING pending.scope+ )+ SELECT array_agg(DISTINCT scope ORDER BY scope) INTO changed_scopes FROM removed;++ IF changed_scopes IS NOT NULL THEN+ PERFORM 1 FROM public.entity_change_watermark WHERE singleton = true FOR UPDATE;+ FOREACH changed_scope IN ARRAY changed_scopes LOOP+ UPDATE public.entity_change_watermark+ SET sequence = sequence + 1, current_cursor = gen_random_uuid()+ WHERE singleton = true+ RETURNING sequence, current_cursor INTO next_sequence, feed_cursor;+ INSERT INTO public.entity_change_feed (sequence, scope, created_at, cursor)+ VALUES (next_sequence, changed_scope, clock_timestamp(), feed_cursor);+ feed_sequence := next_sequence;+ feed_scope := changed_scope;+ RETURN NEXT;+ END LOOP;+ PERFORM pg_notify('entity_change_feed', '');+ END IF;++ WITH removed AS (+ DELETE FROM public.entity_change_feed+ WHERE created_at < clock_timestamp() - make_interval(secs => retention_seconds)+ RETURNING sequence+ ) SELECT max(sequence) INTO removed_through FROM removed;+ IF removed_through IS NOT NULL THEN+ UPDATE public.entity_change_watermark+ SET pruned_through = GREATEST(pruned_through, removed_through)+ WHERE singleton = true;+ END IF;+ RETURN;+ END;+ $$;++ DROP FUNCTION IF EXISTS entity_change_read_checkpoint(UUID);+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(+ requested_cursor UUID,+ maximum_recovery_events INTEGER+ )+ RETURNS TABLE (+ needs_resync BOOLEAN,+ current_cursor UUID,+ current_sequence BIGINT,+ invalidated_scopes TEXT[]+ )+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public AS $$+ DECLARE+ cursor_sequence BIGINT;+ current_watermark BIGINT;+ pruned_watermark BIGINT;+ BEGIN+ IF maximum_recovery_events < 1 OR maximum_recovery_events > 100000 THEN+ RAISE EXCEPTION 'entity change recovery bounds rejected';+ END IF;+ SELECT watermark.sequence, watermark.pruned_through, watermark.current_cursor+ INTO current_watermark, pruned_watermark, current_cursor+ FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;+ IF requested_cursor IS NULL THEN+ needs_resync := true;+ current_sequence := current_watermark;+ invalidated_scopes := ARRAY['gfs', 'authorization'];+ RETURN NEXT;+ RETURN;+ END IF;++ IF requested_cursor = '00000000-0000-0000-0000-000000000000' THEN+ cursor_sequence := 0;+ ELSE+ SELECT sequence INTO cursor_sequence FROM public.entity_change_feed+ WHERE cursor = requested_cursor;+ END IF;+ IF cursor_sequence IS NULL AND requested_cursor = current_cursor THEN+ cursor_sequence := current_watermark;+ END IF;+ IF cursor_sequence IS NULL OR cursor_sequence < pruned_watermark OR+ current_watermark - cursor_sequence > maximum_recovery_events THEN+ needs_resync := true;+ current_sequence := current_watermark;+ invalidated_scopes := ARRAY['gfs', 'authorization'];+ RETURN NEXT;+ RETURN;+ END IF;++ needs_resync := false;+ SELECT COALESCE(array_agg(DISTINCT scope ORDER BY scope), ARRAY[]::TEXT[]),+ COALESCE(max(sequence), cursor_sequence)+ INTO invalidated_scopes, current_sequence+ FROM public.entity_change_feed WHERE sequence > cursor_sequence;+ SELECT watermark.current_cursor INTO current_cursor+ FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;+ RETURN NEXT;+ END;+ $$;++ REVOKE ALL ON FUNCTION entity_change_dispatch_batch(INTEGER, INTEGER) FROM PUBLIC;+ REVOKE ALL ON FUNCTION entity_change_read_checkpoint(UUID, INTEGER) FROM PUBLIC;+ GRANT EXECUTE ON FUNCTION entity_change_dispatch_batch(INTEGER, INTEGER) TO control_api_runtime;+ GRANT EXECUTE ON FUNCTION entity_change_read_checkpoint(UUID, INTEGER) TO control_api_runtime;+ `)+}diff --git a/control-api/src/services/entityChangeService.ts b/control-api/src/services/entityChangeService.tsnew file mode 100644index 000000000..2d6bc640d--- /dev/null+++ b/control-api/src/services/entityChangeService.ts@@ -0,0 +1,247 @@+import type { PoolClient } from 'pg'+import { config } from '../config.js'+import { pool } from '../db.js'+import { rootLogger } from '../observability/logger.js'+import { entityChangeDispatchBatchesTotal } from '../observability/metrics.js'++const logger = rootLogger.child({ module: 'entity-change-feed' })+const CURSOR_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i+const ZERO_CURSOR = '00000000-0000-0000-0000-000000000000'++export type EntityChangeScope = 'gfs' | 'authorization'++export type EntityChangeCheckpoint = {+ resyncRequired: boolean+ cursor: string+ scopes: EntityChangeScope[]+}++type CheckpointRow = {+ needs_resync: boolean+ current_cursor: string | null+ invalidated_scopes: string[] | null+}++export function isEntityChangeCursor(value: string): boolean {+ return CURSOR_RE.test(value)+}++export async function readEntityChangeCheckpoint(+ cursor: string | null+): Promise<EntityChangeCheckpoint> {+ const result = await pool.query('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ cursor,+ config.entityChangeMaxRecoveryEvents,+ ])+ const row = result.rows[0] as CheckpointRow | undefined+ if (!row || typeof row.needs_resync !== 'boolean') {+ throw new Error('Entity change checkpoint query returned an invalid result')+ }+ const scopes = Array.isArray(row.invalidated_scopes)+ ? row.invalidated_scopes.filter(+ (scope): scope is EntityChangeScope => scope === 'gfs' || scope === 'authorization'+ )+ : []+ return {+ resyncRequired: row.needs_resync,+ cursor: isEntityChangeCursor(String(row.current_cursor || ''))+ ? String(row.current_cursor)+ : ZERO_CURSOR,+ scopes,+ }+}++export async function dispatchEntityChangeOutbox(): Promise<number> {+ const result = await pool.query('SELECT * FROM entity_change_dispatch_batch($1, $2)', [+ config.entityChangeDispatchBatchSize,+ config.entityChangeRetentionSeconds,+ ])+ return result.rows.length+}++let dispatcherTimer: NodeJS.Timeout | null = null+let wakeupListener: PoolClient | null = null+let listenerRetryTimer: NodeJS.Timeout | null = null+let dispatcherRunning = false++async function runDispatcherTick(): Promise<void> {+ if (dispatcherRunning) return+ dispatcherRunning = true+ try {+ const count = await dispatchEntityChangeOutbox()+ entityChangeDispatchBatchesTotal.inc({ result: count > 0 ? 'dispatched' : 'empty' })+ if (count > 0) {+ logger.info(+ { event: 'entity_change_batch_dispatched', feedRows: count },+ 'entity changes dispatched'+ )+ }+ } catch (err) {+ entityChangeDispatchBatchesTotal.inc({ result: 'error' })+ logger.warn({ event: 'entity_change_dispatch_failed', err }, 'entity change dispatch failed')+ } finally {+ dispatcherRunning = false+ }+}++async function connectWakeupListener(): Promise<void> {+ if (wakeupListener || !dispatcherTimer) return+ try {+ const client = await pool.connect()+ if (!dispatcherTimer) {+ client.release()+ return+ }+ await client.query('LISTEN entity_change_outbox')+ wakeupListener = client+ client.on('notification', () => void runDispatcherTick())+ const disconnected = (err?: Error) => {+ if (wakeupListener !== client) return+ wakeupListener = null+ client.removeAllListeners('notification')+ client.release(true)+ if (err) {+ logger.warn(+ { event: 'entity_change_dispatch_listener_lost', err },+ 'dispatcher wake-up degraded'+ )+ }+ if (dispatcherTimer && !listenerRetryTimer) {+ listenerRetryTimer = setTimeout(() => {+ listenerRetryTimer = null+ void connectWakeupListener()+ }, 5000)+ }+ }+ client.on('error', disconnected)+ client.on('end', () => disconnected())+ } catch (err) {+ logger.warn(+ { event: 'entity_change_dispatch_listener_unavailable', err },+ 'dispatcher wake-up unavailable'+ )+ if (dispatcherTimer && !listenerRetryTimer) {+ listenerRetryTimer = setTimeout(() => {+ listenerRetryTimer = null+ void connectWakeupListener()+ }, 5000)+ }+ }+}++/** Start a per-instance dispatcher; the database advisory lock elects one active owner. */+export function startEntityChangeDispatcher(): void {+ if (dispatcherTimer) return+ dispatcherTimer = setInterval(() => {+ void runDispatcherTick()+ if (!wakeupListener) void connectWakeupListener()+ }, config.entityChangeDispatchIntervalMs)+ void connectWakeupListener()+ void runDispatcherTick()+}++export function stopEntityChangeDispatcher(): void {+ if (dispatcherTimer) clearInterval(dispatcherTimer)+ if (listenerRetryTimer) clearTimeout(listenerRetryTimer)+ dispatcherTimer = null+ listenerRetryTimer = null+ const listener = wakeupListener+ wakeupListener = null+ if (listener) {+ listener.removeAllListeners('notification')+ listener.release()+ }+}++const feedWakeSubscribers = new Set<() => void>()+let feedWakeListener: PoolClient | null = null+let feedWakeConnect: Promise<void> | null = null+let feedWakeRetryTimer: NodeJS.Timeout | null = null++function scheduleFeedWakeReconnect(): void {+ if (feedWakeSubscribers.size === 0 || feedWakeRetryTimer) return+ feedWakeRetryTimer = setTimeout(() => {+ feedWakeRetryTimer = null+ void connectFeedWakeListener()+ }, 5000)+}++async function connectFeedWakeListener(): Promise<void> {+ if (feedWakeListener || feedWakeConnect || feedWakeSubscribers.size === 0) return+ const connecting = (async () => {+ let client: PoolClient | null = null+ try {+ client = await pool.connect()+ if (feedWakeSubscribers.size === 0) {+ client.release()+ return+ }+ await client.query('LISTEN entity_change_feed')+ if (feedWakeSubscribers.size === 0) {+ client.release()+ return+ }+ feedWakeListener = client+ const connectedClient = client+ connectedClient.on('notification', () => {+ for (const subscriber of Array.from(feedWakeSubscribers)) {+ try {+ subscriber()+ } catch {+ // One stream's wake callback must not interrupt delivery to others.+ }+ }+ })+ const disconnected = (err?: Error) => {+ if (feedWakeListener !== connectedClient) return+ feedWakeListener = null+ connectedClient.removeAllListeners('notification')+ connectedClient.removeAllListeners('error')+ connectedClient.removeAllListeners('end')+ connectedClient.release(true)+ if (err) {+ logger.warn(+ { event: 'entity_change_feed_listener_lost', err },+ 'entity change wake-up degraded to polling'+ )+ }+ scheduleFeedWakeReconnect()+ }+ connectedClient.on('error', disconnected)+ connectedClient.on('end', () => disconnected())+ } catch (err) {+ if (client) client.release(true)+ logger.warn(+ { event: 'entity_change_feed_listener_unavailable', err },+ 'entity change wake-up unavailable; polling remains active'+ )+ scheduleFeedWakeReconnect()+ }+ })()+ feedWakeConnect = connecting+ try {+ await connecting+ } finally {+ if (feedWakeConnect === connecting) feedWakeConnect = null+ }+}++/** Share one process-local LISTEN connection; durable polling remains authoritative. */+export function subscribeEntityChangeFeedWake(onWake: () => void): () => void {+ feedWakeSubscribers.add(onWake)+ void connectFeedWakeListener()+ return () => {+ feedWakeSubscribers.delete(onWake)+ if (feedWakeSubscribers.size > 0) return+ if (feedWakeRetryTimer) clearTimeout(feedWakeRetryTimer)+ feedWakeRetryTimer = null+ const listener = feedWakeListener+ feedWakeListener = null+ if (listener) {+ listener.removeAllListeners('notification')+ listener.removeAllListeners('error')+ listener.removeAllListeners('end')+ listener.release()+ }+ }+}diff --git a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts b/control-api/test/db.oauthGrantsOwnerGeneralization.test.tsindex e4759a7f0..001a73a7a 100644--- a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts+++ b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts@@ -21,7 +21,7 @@ describe('0101 oauth_grants owner generalization migration', () => {const { CONTROL_API_MIGRATIONS } = await import('../src/db.js')const versions = CONTROL_API_MIGRATIONS.map(m => m.version)expect(versions).toContain('0106_oauth_grants_owner_generalization')- expect(versions.at(-1)).toBe('0115_llm_allowed_models_image_input')+ expect(versions.at(-1)).toBe('0116_durable_entity_change_feed')expect(versions.indexOf('0100_seed_minimax_allowed_model')).toBeLessThan(versions.indexOf('0106_oauth_grants_owner_generalization'))diff --git a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts b/control-api/test/entityChangeFeed.realPostgres.integration.test.tsnew file mode 100644index 000000000..fa5ab9884--- /dev/null+++ b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts@@ -0,0 +1,384 @@+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import { initDb } from '../src/db.js'+import './realPostgres.requirement.ts'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++describeRealPostgres('entity change feed real PostgreSQL contract', () => {+ const database = `control_api_entity_change_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ let adminPool: Pool+ let instancePool: Pool+ let replicaPool: Pool++ beforeAll(async () => {+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ instancePool = new Pool({ connectionString })+ replicaPool = new Pool({ connectionString })+ await initDb({ connect: () => instancePool.connect() })+ })++ beforeEach(async () => {+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ })++ afterAll(async () => {+ await instancePool?.end()+ await replicaPool?.end()+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ }+ })++ it('captures committed create, replace, rename/move, delete, grant, and membership changes', async () => {+ const client = await instancePool.connect()+ const drive = `entity-change-${randomUUID()}`+ const rootId = randomUUID()+ const parentId = randomUUID()+ const movedParentId = randomUUID()+ const fileId = randomUUID()+ const userId = randomUUID()+ const teamId = randomUUID()+ try {+ await client.query('BEGIN')+ await client.query('SET ROLE gfs_controller')+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)+ VALUES ($1, $2, '/', 'directory', '/')`,+ [rootId, drive]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'parent', 'directory', '/parent')`,+ [parentId, drive, rootId]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'destination', 'directory', '/destination')`,+ [movedParentId, drive, rootId]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'draft.md', 'file', '/parent/draft.md')`,+ [fileId, drive, parentId]+ )+ await client.query('RESET ROLE')+ await client.query('SET ROLE control_api_runtime')+ await client.query(+ `INSERT INTO gfs_grants (drive, resource_id, subject_type, subject_id, permissions)+ VALUES ($1, $2, 'team', $3, ARRAY['read'])`,+ [drive, fileId, teamId]+ )+ await client.query(+ `INSERT INTO gfs_shares (drive, resource_id, subject_type, subject_id, permissions)+ VALUES ($1, $2, 'user', $3, ARRAY['read'])`,+ [drive, fileId, userId]+ )+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ const created = await instancePool.query<{ count: string }>(+ `SELECT count(*)::text AS count FROM entity_change_outbox WHERE scope = 'gfs'`+ )+ expect(Number(created.rows[0]?.count)).toBe(4)++ await client.query('BEGIN')+ await client.query('SET ROLE control_api_runtime')+ await client.query(+ `UPDATE gfs_resources SET version = version + 1, bytes = 45 WHERE resource_id = $1`,+ [fileId]+ )+ await client.query(+ `UPDATE gfs_resources SET parent_resource_id = $2, name = 'renamed.md',+ path_cache = '/destination/renamed.md' WHERE resource_id = $1`,+ [fileId, movedParentId]+ )+ await client.query('RESET ROLE')+ await client.query('SET ROLE gfs_controller')+ await client.query(+ `UPDATE gfs_resources SET deleted_at = clock_timestamp() WHERE resource_id = $1`,+ [fileId]+ )+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ await client.query('BEGIN')+ await client.query('SET ROLE control_api_runtime')+ await client.query(`INSERT INTO users (id, email) VALUES ($1, $2)`, [+ userId,+ `${userId}@entity-change.invalid`,+ ])+ await client.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])+ await client.query(+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,+ [teamId, userId]+ )+ await client.query('DELETE FROM team_members WHERE team_id = $1 AND user_id = $2', [+ teamId,+ userId,+ ])+ await client.query('DELETE FROM gfs_grants WHERE drive = $1 AND resource_id = $2', [+ drive,+ fileId,+ ])+ await client.query('DELETE FROM gfs_shares WHERE drive = $1 AND resource_id = $2', [+ drive,+ fileId,+ ])+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ const outbox = await instancePool.query<{ scope: string; entity_id: string | null }>(+ 'SELECT scope, entity_id FROM entity_change_outbox ORDER BY id'+ )+ expect(outbox.rows.filter(row => row.scope === 'gfs')).toHaveLength(7)+ expect(outbox.rows.filter(row => row.entity_id === fileId)).toHaveLength(4)+ expect(outbox.rows.filter(row => row.scope === 'authorization')).toHaveLength(7)+ // Event evidence contains no file content, path, or authorization subject.+ expect(JSON.stringify(outbox.rows)).not.toContain('draft.md')+ expect(JSON.stringify(outbox.rows)).not.toContain(userId)+ expect(JSON.stringify(outbox.rows)).not.toContain(teamId)+ } finally {+ client.release()+ }+ })++ it('rolls capture back with the producer transaction and publishes only committed facts', async () => {+ const client = await instancePool.connect()+ const drive = `entity-change-rollback-${randomUUID()}`+ try {+ const before = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ await client.query('BEGIN')+ await client.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'rolled-back', 'file')`,+ [drive]+ )+ await client.query('ROLLBACK')+ const after = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ expect(after.rows[0]?.count).toBe(before.rows[0]?.count)++ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'committed', 'file')`,+ [drive]+ )+ const firstDispatch = await instancePool.query<{ feed_sequence: string }>(+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'+ )+ const feedAfterFirst = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_feed'+ )+ const duplicateDispatch = await instancePool.query<{ feed_sequence: string }>(+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'+ )+ const feedAfterDuplicate = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_feed'+ )+ expect(firstDispatch.rows).toHaveLength(1)+ expect(duplicateDispatch.rows).toHaveLength(0)+ expect(feedAfterDuplicate.rows[0]?.count).toBe(feedAfterFirst.rows[0]?.count)+ } finally {+ client.release()+ }+ })++ it('uses dispatcher order rather than identity allocation as public sequence order', async () => {+ const pending = await instancePool.connect()+ const earlyDrive = `entity-change-early-${randomUUID()}`+ try {+ await pending.query('BEGIN')+ await pending.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'early', 'file')`,+ [earlyDrive]+ )+ const pendingId = await pending.query<{ id: string }>(+ `SELECT max(id)::text AS id FROM entity_change_outbox WHERE entity_id IS NOT NULL`+ )+ const watermarkBefore = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ const userId = randomUUID()+ const teamId = randomUUID()+ await instancePool.query('INSERT INTO users (id, email) VALUES ($1, $2)', [+ userId,+ `${userId}@ordering.invalid`,+ ])+ await instancePool.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])+ await instancePool.query(+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,+ [teamId, userId]+ )+ const committedMax = await instancePool.query<{ id: string }>(+ 'SELECT max(id)::text AS id FROM entity_change_outbox'+ )+ expect(Number(pendingId.rows[0]?.id)).toBeLessThan(Number(committedMax.rows[0]?.id))+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const firstFeed = await instancePool.query<{ sequence: string; scope: string }>(+ `SELECT sequence::text, scope FROM entity_change_feed+ WHERE sequence > $1 ORDER BY sequence`,+ [watermarkBefore.rows[0]?.sequence]+ )+ expect(firstFeed.rows.map(row => row.scope)).toEqual(['authorization'])+ await pending.query('COMMIT')+ await replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const feed = await instancePool.query<{ sequence: string; scope: string }>(+ `SELECT sequence::text, scope FROM entity_change_feed+ WHERE sequence > $1 ORDER BY sequence`,+ [watermarkBefore.rows[0]?.sequence]+ )+ expect(feed.rows.map(row => row.scope)).toEqual(['authorization', 'gfs'])+ expect(feed.rows.map(row => Number(row.sequence))).toEqual([+ Number(watermarkBefore.rows[0]?.sequence) + 1,+ Number(watermarkBefore.rows[0]?.sequence) + 2,+ ])+ } finally {+ await pending.query('ROLLBACK').catch(() => undefined)+ pending.release()+ }+ })++ it('recovers missed wakeups and expired cursors with a coarse authorized checkpoint', async () => {+ const resource = await instancePool.query<{ resource_id: string }>(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint', 'file')+ RETURNING resource_id`,+ [`entity-change-checkpoint-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const lastEvent = await instancePool.query<{ cursor: string; sequence: string }>(+ 'SELECT cursor::text, sequence::text FROM entity_change_feed ORDER BY sequence DESC LIMIT 1'+ )+ // No listener is involved here: the durable query is the fallback after a missed NOTIFY.+ const recovery = await instancePool.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ lastEvent.rows[0]?.cursor,+ 10000,+ ])+ expect(recovery.rows[0]?.needs_resync).toBe(false)++ await instancePool.query(+ `UPDATE entity_change_feed SET created_at = clock_timestamp() - interval '2 days'+ WHERE sequence = $1`,+ [lastEvent.rows[0]?.sequence]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const expired = await instancePool.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ lastEvent.rows[0]?.cursor,+ 10000,+ ])+ expect(expired.rows[0]?.needs_resync).toBe(true)+ expect(expired.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])+ expect(JSON.stringify(expired.rows)).not.toContain(resource.rows[0]?.resource_id)+ })++ it('coalesces latest-state invalidations and keeps feed storage behind the runtime boundary', async () => {+ const inserted = await instancePool.query<{ resource_id: string }>(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'many-updates', 'file')+ RETURNING resource_id`,+ [`entity-change-coalesce-${randomUUID()}`]+ )+ const resourceId = inserted.rows[0]?.resource_id+ await instancePool.query(+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',+ [resourceId]+ )+ await instancePool.query(+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',+ [resourceId]+ )+ const before = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ await Promise.all([+ instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),+ replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),+ ])+ const after = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ const published = await instancePool.query<{ scope: string; count: string }>(+ `SELECT scope, count(*)::text AS count FROM entity_change_feed+ WHERE sequence > $1 GROUP BY scope`,+ [before.rows[0]?.sequence]+ )+ expect(Number(after.rows[0]?.sequence) - Number(before.rows[0]?.sequence)).toBe(1)+ expect(published.rows).toEqual([{ scope: 'gfs', count: '1' }])++ const restricted = await instancePool.connect()+ try {+ await restricted.query('SET ROLE control_api_runtime')+ await expect(restricted.query('SELECT * FROM entity_change_feed')).rejects.toThrow()+ const checkpoint = await restricted.query<{ invalidated_scopes: string[] }>(+ 'SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)',+ ['00000000-0000-0000-0000-000000000000', 10000]+ )+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])+ expect(Object.keys(checkpoint.rows[0] ?? {}).sort()).toEqual([+ 'current_cursor',+ 'current_sequence',+ 'invalidated_scopes',+ 'needs_resync',+ ])+ } finally {+ await restricted.query('RESET ROLE').catch(() => undefined)+ restricted.release()+ }+ })++ it('resyncs instead of scanning an oversized retained replay window', async () => {+ const baseline = await instancePool.query<{ current_cursor: string }>(+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'+ )+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind)+ VALUES ($1, 'bounded-window', 'file')`,+ [`entity-change-bounded-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind)+ VALUES ($1, 'bounded-window-2', 'file')`,+ [`entity-change-bounded-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const bounded = await instancePool.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ baseline.rows[0]?.current_cursor,+ 1,+ ])+ expect(bounded.rows[0]?.needs_resync).toBe(true)+ expect(bounded.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])+ })+})diff --git a/control-api/test/gfs.resolve.test.ts b/control-api/test/gfs.resolve.test.tsindex fb2951b64..889b28638 100644--- a/control-api/test/gfs.resolve.test.ts+++ b/control-api/test/gfs.resolve.test.ts@@ -46,6 +46,8 @@ function res(partial: Partial<ResolvedResource> & { resourceId: string }): Resolname: 'n',kind: 'directory',pathCache: null,+ bytes: 0,+ version: 0,updatedAt: '2026-01-01T00:00:00.000Z',...partial,}@@ -114,6 +116,8 @@ const row = (resourceId: string, name = 'n', kind = 'directory') => ({name,kind,path_cache: null,+ bytes: '42',+ version: 7,updated_at: new Date('2026-01-01T00:00:00.000Z'),})@@ -124,6 +128,8 @@ describe('DbResolveStore', () => {const out = await new DbResolveStore(db).getByRid('main', RID)expect(out?.resourceId).toBe('r1')expect(out?.updatedAt).toBe('2026-01-01T00:00:00.000Z')+ expect(out?.bytes).toBe(42)+ expect(out?.version).toBe(7)expect(db.queries[0].text).toContain('resource_id = $2::uuid')expect(db.queries[0].text).toContain('deleted_at IS NULL')expect(db.queries[0].text).toContain('updated_at')diff --git a/control-api/test/gfs.resolveRoutes.test.ts b/control-api/test/gfs.resolveRoutes.test.tsindex 4377afdb5..25be97d4e 100644--- a/control-api/test/gfs.resolveRoutes.test.ts+++ b/control-api/test/gfs.resolveRoutes.test.ts@@ -11,6 +11,8 @@ function res(partial: Partial<ResolvedResource> & { resourceId: string }): Resolname: 'n',kind: 'directory',pathCache: null,+ bytes: 0,+ version: 0,updatedAt: '2026-01-01T00:00:00.000Z',...partial,}@@ -33,6 +35,8 @@ describe('toResolveView', () => {expect(view.rid).toBe(RID)expect(view.gfsUri).toBe(`gfs://main/${RID}`)expect(view.path).toBe('/org')+ expect(view.bytes).toBe(0)+ expect(view.version).toBe(0)expect(view.updatedAt).toBe('2026-01-01T00:00:00.000Z')expect(view.updatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/)})diff --git a/control-api/test/routes.entityChangeStream.test.ts b/control-api/test/routes.entityChangeStream.test.tsnew file mode 100644index 000000000..6588290f9--- /dev/null+++ b/control-api/test/routes.entityChangeStream.test.ts@@ -0,0 +1,144 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import type { Request, Response } from 'express'+import { EventEmitter } from 'node:events'+import {+ closeActiveEntityChangeStreams,+ parseRequestedEntityChangeCursor,+ streamEntityChanges,+} from '../src/routes/entityChangeStream.js'++const serviceMock = vi.hoisted(() => ({+ isEntityChangeCursor: vi.fn(),+ readEntityChangeCheckpoint: vi.fn(),+ subscribeEntityChangeFeedWake: vi.fn(),+}))+const configMock = vi.hoisted(() => ({+ entityChangeStreamHeartbeatMs: 20_000,+ entityChangeStreamMaxLifetimeMs: 600_000,+ entityChangeStreamPollMs: 250,+}))++vi.mock('../src/config.js', () => ({ config: configMock }))+vi.mock('../src/services/entityChangeService.js', () => serviceMock)+vi.mock('../src/observability/logger.js', () => ({+ rootLogger: { child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }) },+}))++class FakeRequest extends EventEmitter {+ query: Record<string, unknown> = {}+ setTimeout = vi.fn()+}++class FakeResponse extends EventEmitter {+ statusCode = 200+ headersSent = false+ destroyed = false+ writableEnded = false+ writableLength = 0+ frames: string[] = []+ status(code: number): this {+ this.statusCode = code+ return this+ }+ setHeader(): this {+ return this+ }+ flushHeaders(): void {+ this.headersSent = true+ }+ write(frame: string): boolean {+ this.frames.push(frame)+ return true+ }+ end(): void {+ this.writableEnded = true+ this.emit('close')+ }+ json(): this {+ return this+ }+}++const CURSOR = 'd119f895-1ef8-4e73-8f08-f9754919682a'++describe('routes/entityChangeStream', () => {+ beforeEach(() => {+ serviceMock.isEntityChangeCursor.mockImplementation((value: string) =>+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)+ )+ serviceMock.readEntityChangeCheckpoint.mockReset()+ serviceMock.subscribeEntityChangeFeedWake.mockReset()+ serviceMock.subscribeEntityChangeFeedWake.mockReturnValue(vi.fn())+ })++ it('validates requested UUID cursors and represents an absent cursor as a full resync', () => {+ expect(parseRequestedEntityChangeCursor({ query: {} } as Request)).toBeNull()+ expect(parseRequestedEntityChangeCursor({ query: { cursor: CURSOR } } as Request)).toBe(CURSOR)+ expect(parseRequestedEntityChangeCursor({ query: { cursor: 'bad' } } as Request)).toBe(false)+ })++ it('authorizes before sending streaming headers', async () => {+ const req = new FakeRequest()+ const res = new FakeResponse()+ streamEntityChanges(+ req as unknown as Request,+ res as unknown as Response,+ null,+ async () => false,+ 'user'+ )+ await vi.waitFor(() => expect(res.statusCode).toBe(401))+ expect(res.headersSent).toBe(false)+ expect(res.frames).toEqual([])+ })++ it('does not acquire a feed connection when the client disconnects during authorization', async () => {+ let finishAuthorization: (authorized: boolean) => void = () => undefined+ const authorization = new Promise<boolean>(resolve => {+ finishAuthorization = resolve+ })+ const req = new FakeRequest()+ const res = new FakeResponse()+ streamEntityChanges(+ req as unknown as Request,+ res as unknown as Response,+ null,+ () => authorization,+ 'user'+ )+ req.emit('aborted')+ finishAuthorization(true)+ await new Promise(resolve => setImmediate(resolve))+ expect(serviceMock.subscribeEntityChangeFeedWake).not.toHaveBeenCalled()+ expect(res.headersSent).toBe(false)+ })++ it('sends only coarse invalidations and closes cleanly on server shutdown', async () => {+ serviceMock.readEntityChangeCheckpoint.mockResolvedValue({+ resyncRequired: true,+ cursor: CURSOR,+ scopes: ['gfs', 'authorization'],+ })+ const req = new FakeRequest()+ const res = new FakeResponse()+ let authorizationChecks = 0+ streamEntityChanges(+ req as unknown as Request,+ res as unknown as Response,+ null,+ async () => {+ authorizationChecks += 1+ return true+ },+ 'operator'+ )+ await vi.waitFor(() => expect(res.frames.length).toBeGreaterThan(0))+ closeActiveEntityChangeStreams()+ await vi.waitFor(() => expect(res.writableEnded).toBe(true))+ expect(res.frames.join('')).toContain('"type":"resync_required"')+ expect(res.frames.join('')).toContain('"type":"stream.closing"')+ expect(res.frames.join('')).not.toContain('resource-id')+ expect(authorizationChecks).toBeGreaterThanOrEqual(2)+ expect(serviceMock.subscribeEntityChangeFeedWake).toHaveBeenCalledOnce()+ })+})diff --git a/control-api/test/routes.externalEntityChanges.test.ts b/control-api/test/routes.externalEntityChanges.test.tsnew file mode 100644index 000000000..def2154ba--- /dev/null+++ b/control-api/test/routes.externalEntityChanges.test.ts@@ -0,0 +1,27 @@+import { describe, expect, it, vi } from 'vitest'+import { isEntityChangeExternalSessionCurrent } from '../src/routes/external/entityChanges.routes.js'++const sessionAuth = vi.hoisted(() => ({+ isCurrentExternalSession: vi.fn(),+ requireValidExternalSessionToken: vi.fn(),+}))++vi.mock('../src/middleware/externalSessionAuth.js', () => sessionAuth)++describe('external entity-change stream authorization', () => {+ it('stops delivery after the authenticated session token expires', async () => {+ const claims = {+ userId: 'user-1',+ email: '[email protected]',+ teamId: 'team-1',+ role: 'member' as const,+ authGeneration: 1,+ exp: 1_000,+ }+ sessionAuth.isCurrentExternalSession.mockReset().mockResolvedValue(true)++ await expect(isEntityChangeExternalSessionCurrent(claims, 999_000)).resolves.toBe(true)+ await expect(isEntityChangeExternalSessionCurrent(claims, 1_000_000)).resolves.toBe(false)+ expect(sessionAuth.isCurrentExternalSession).toHaveBeenCalledOnce()+ })+})diff --git a/control-api/test/services.entityChangeService.test.ts b/control-api/test/services.entityChangeService.test.tsnew file mode 100644index 000000000..63597aef5--- /dev/null+++ b/control-api/test/services.entityChangeService.test.ts@@ -0,0 +1,90 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import { EventEmitter } from 'node:events'+import {+ dispatchEntityChangeOutbox,+ isEntityChangeCursor,+ readEntityChangeCheckpoint,+ subscribeEntityChangeFeedWake,+} from '../src/services/entityChangeService.js'++const dbMock = vi.hoisted(() => ({ query: vi.fn(), connect: vi.fn() }))+const configMock = vi.hoisted(() => ({+ entityChangeDispatchBatchSize: 37,+ entityChangeRetentionSeconds: 86_400,+ entityChangeMaxRecoveryEvents: 500,+}))++vi.mock('../src/db.js', () => ({ pool: dbMock }))+vi.mock('../src/config.js', () => ({ config: configMock }))+vi.mock('../src/observability/logger.js', () => ({+ rootLogger: { child: () => ({ info: vi.fn(), warn: vi.fn() }) },+}))++describe('entityChangeService', () => {+ beforeEach(() => {+ dbMock.query.mockReset()+ dbMock.connect.mockReset()+ })++ it('validates only canonical UUID cursors', () => {+ expect(isEntityChangeCursor('d119f895-1ef8-4e73-8f08-f9754919682a')).toBe(true)+ expect(isEntityChangeCursor('d119f895-1ef8-4e73-8f08-f9754919682')).toBe(false)+ })++ it('exposes only approved coarse scopes from the durable checkpoint', async () => {+ dbMock.query.mockResolvedValueOnce({+ rows: [+ {+ needs_resync: false,+ current_cursor: 'd119f895-1ef8-4e73-8f08-f9754919682a',+ invalidated_scopes: ['gfs', 'authorization', 'private-resource-id'],+ },+ ],+ })++ await expect(+ readEntityChangeCheckpoint('d119f895-1ef8-4e73-8f08-f9754919682a')+ ).resolves.toEqual({+ resyncRequired: false,+ cursor: 'd119f895-1ef8-4e73-8f08-f9754919682a',+ scopes: ['gfs', 'authorization'],+ })+ expect(dbMock.query).toHaveBeenCalledWith(+ 'SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)',+ ['d119f895-1ef8-4e73-8f08-f9754919682a', 500]+ )+ })++ it('dispatches with the configured batch and retention bounds', async () => {+ dbMock.query.mockResolvedValueOnce({ rows: [{ feed_sequence: 1 }] })+ await expect(dispatchEntityChangeOutbox()).resolves.toBe(1)+ expect(dbMock.query).toHaveBeenCalledWith(+ 'SELECT * FROM entity_change_dispatch_batch($1, $2)',+ [37, 86_400]+ )+ })++ it('shares one LISTEN connection among wake subscribers and releases it when empty', async () => {+ const listener = new EventEmitter() as EventEmitter & {+ query: ReturnType<typeof vi.fn>+ release: ReturnType<typeof vi.fn>+ }+ listener.query = vi.fn().mockResolvedValue({ rows: [] })+ listener.release = vi.fn()+ dbMock.connect.mockResolvedValueOnce(listener)+ const first = vi.fn()+ const second = vi.fn()+ const unsubscribeFirst = subscribeEntityChangeFeedWake(first)+ const unsubscribeSecond = subscribeEntityChangeFeedWake(second)+ await vi.waitFor(() => expect(listener.query).toHaveBeenCalledWith('LISTEN entity_change_feed'))+ expect(dbMock.connect).toHaveBeenCalledOnce()++ listener.emit('notification')+ expect(first).toHaveBeenCalledOnce()+ expect(second).toHaveBeenCalledOnce()++ unsubscribeFirst()+ unsubscribeSecond()+ expect(listener.release).toHaveBeenCalledOnce()+ })+})diff --git a/deploy/scripts/control-api-runtime-access-profiles.tsv b/deploy/scripts/control-api-runtime-access-profiles.tsvindex 6cbb1641b..0c3d257f1 100644--- a/deploy/scripts/control-api-runtime-access-profiles.tsv+++ b/deploy/scripts/control-api-runtime-access-profiles.tsv@@ -1,6 +1,9 @@# public relation<TAB>control_api_runtime access profileadministrative_events appendagent_run_events append+entity_change_feed none+entity_change_outbox none+entity_change_watermark nonebudget_pending_reservations legacy_dmlcodex_subscription_connections upsertcodex_subscription_oauth_states upsertdiff --git a/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv b/deploy/scripts/control-api-runtime-sequence-access-profiles.tsvindex f3ed6db69..a26bbb8c8 100644--- a/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv+++ b/deploy/scripts/control-api-runtime-sequence-access-profiles.tsv@@ -1,6 +1,7 @@# public sequence<TAB>control_api_runtime access profileadministrative_events_ingest_sequence_seq consumeagent_run_events_ingest_sequence_seq consume+entity_change_outbox_id_seq nonegfs_audit_sequence_no_seq legacy_rwgoverned_event_stream_stream_sequence_seq consumeinfrastructure_telemetry_events_ingest_sequence_seq consumediff --git a/deploy/scripts/run-control-api-db-migration.sh b/deploy/scripts/run-control-api-db-migration.shindex 2929e85d6..a8019687d 100755--- a/deploy/scripts/run-control-api-db-migration.sh+++ b/deploy/scripts/run-control-api-db-migration.sh@@ -463,7 +463,7 @@ runtime_sequence_access_contract_values() {/^[[:space:]]*#/ || /^[[:space:]]*$/ { next }NF != 2 { exit 2 }$1 !~ /^[a-z][a-z0-9_]*$/ { exit 3 }- $2 !~ /^(legacy_rw|consume)$/ { exit 4 }+ $2 !~ /^(legacy_rw|consume|none)$/ { exit 4 }seen[$1]++ { exit 5 }{count++@@ -630,8 +630,8 @@ verify_runtime_access_contract() {JOIN actual_sequences actual USING (sequence_name)CROSS JOIN LATERAL (VALUES- ('USAGE', true),- ('SELECT', true),+ ('USAGE', expected.access_profile != 'none'),+ ('SELECT', expected.access_profile != 'none'),('UPDATE', expected.access_profile = 'legacy_rw')) required(privilege_name, allowed)WHERE has_sequence_privilege(diff --git a/scripts/tests/test-control-api-runtime-access-contract.sh b/scripts/tests/test-control-api-runtime-access-contract.shindex 89dfb3238..06f28f63f 100755--- a/scripts/tests/test-control-api-runtime-access-contract.sh+++ b/scripts/tests/test-control-api-runtime-access-contract.sh@@ -9,18 +9,21 @@ relation_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { count++ } END { print couduplicate_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { seen[$1]++ } END { for (name in seen) if (seen[name] > 1) count++ } END { print count + 0 }' "$PROFILE_FILE")"invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/) { count++ } END { print count + 0 }' "$PROFILE_FILE")"-[[ "$relation_count" == "88" ]]-[[ "$duplicate_count" == "0" ]]-[[ "$invalid_count" == "0" ]]-grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$PROFILE_FILE"-grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$PROFILE_FILE"--grep -Fq '$2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/' "$MIGRATION_SCRIPT"-grep -Fq "('INSERT', expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle'))" "$MIGRATION_SCRIPT"-grep -Fq "('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))" "$MIGRATION_SCRIPT"-grep -Fq "('DELETE', expected.access_profile IN ('legacy_dml'))" "$MIGRATION_SCRIPT"-grep -Fq "('TRUNCATE', false)" "$MIGRATION_SCRIPT"-grep -Fq "('REFERENCES', false)" "$MIGRATION_SCRIPT"-grep -Fq "('TRIGGER', false)" "$MIGRATION_SCRIPT"+if [[ "$relation_count" != "95" || "$duplicate_count" != "0" || "$invalid_count" != "0" ]] || \+ ! grep -qx $'entity_change_feed\tnone' "$PROFILE_FILE" || \+ ! grep -qx $'entity_change_outbox\tnone' "$PROFILE_FILE" || \+ ! grep -qx $'entity_change_watermark\tnone' "$PROFILE_FILE" || \+ ! grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$PROFILE_FILE" || \+ ! grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$PROFILE_FILE" || \+ ! grep -Fq '$2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/' "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('INSERT', expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle'))" "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))" "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('DELETE', expected.access_profile IN ('legacy_dml'))" "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('TRUNCATE', false)" "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('REFERENCES', false)" "$MIGRATION_SCRIPT" || \+ ! grep -Fq "('TRIGGER', false)" "$MIGRATION_SCRIPT"; then+ printf 'FAIL: control-api runtime access contract is incomplete or not exact\n' >&2+ exit 1+fiprintf 'PASS: control-api runtime link_lifecycle contract is explicit and least privilege\n'diff --git a/scripts/tests/test-gcp-scripts.sh b/scripts/tests/test-gcp-scripts.shindex 2aa971418..a997dac95 100755--- a/scripts/tests/test-gcp-scripts.sh+++ b/scripts/tests/test-gcp-scripts.sh@@ -1464,9 +1464,12 @@ assert_control_api_runtime_access_contract_is_exact() {invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_dml|upsert|append|read|link_lifecycle|none)$/) { count++ } END { print count + 0 }' "$profile_file")"sequence_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { count++ } END { print count + 0 }' "$sequence_profile_file")"sequence_duplicate_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ { seen[$1]++ } END { for (name in seen) if (seen[name] > 1) count++ } END { print count + 0 }' "$sequence_profile_file")"- sequence_invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_rw|consume)$/) { count++ } END { print count + 0 }' "$sequence_profile_file")"+ sequence_invalid_count="$(awk -F '\t' '!/^[[:space:]]*(#|$)/ && (NF != 2 || $1 !~ /^[a-z][a-z0-9_]*$/ || $2 !~ /^(legacy_rw|consume|none)$/) { count++ } END { print count + 0 }' "$sequence_profile_file")"- if [[ "$relation_count" == "88" && "$duplicate_count" == "0" && "$invalid_count" == "0" ]] && \+ if [[ "$relation_count" == "95" && "$duplicate_count" == "0" && "$invalid_count" == "0" ]] && \+ grep -qx $'entity_change_feed\tnone' "$profile_file" && \+ grep -qx $'entity_change_outbox\tnone' "$profile_file" && \+ grep -qx $'entity_change_watermark\tnone' "$profile_file" && \grep -qx $'gfs_upload_parts\tnone' "$profile_file" && \grep -qx $'gfs_upload_sessions\tnone' "$profile_file" && \grep -qx $'llm_allowed_models\tlegacy_dml' "$profile_file" && \@@ -1476,13 +1479,15 @@ assert_control_api_runtime_access_contract_is_exact() {grep -qx $'plugin_workload_sdk_spend_outcomes\tappend' "$profile_file" && \grep -qx $'gfs_desktop_operator_links\tlink_lifecycle' "$profile_file" && \grep -qx $'desktop_user_retirement_operations\tlink_lifecycle' "$profile_file" && \- [[ "$sequence_count" == "7" && "$sequence_duplicate_count" == "0" && "$sequence_invalid_count" == "0" ]] && \+ [[ "$sequence_count" == "8" && "$sequence_duplicate_count" == "0" && "$sequence_invalid_count" == "0" ]] && \+ grep -qx $'entity_change_outbox_id_seq\tnone' "$sequence_profile_file" && \grep -qx $'member_registration_credentials_id_seq\tconsume' "$sequence_profile_file" && \[[ "$migration_script" != *'RUNTIME_ACCESS_PROFILES_FILE:-'* ]] && \[[ "$migration_script" == *'FULL OUTER JOIN actual_relations'* ]] && \[[ "$migration_script" == *'FULL OUTER JOIN actual_sequences'* ]] && \[[ "$migration_script" == *'IS DISTINCT FROM required.allowed'* ]] && \[[ "$migration_script" == *'has_sequence_privilege('* ]] && \+ [[ "$migration_script" == *"('SELECT', expected.access_profile != 'none')"* ]] && \[[ "$migration_script" == *"expected.access_profile IN ('legacy_dml', 'upsert', 'append', 'link_lifecycle')"* ]] && \[[ "$migration_script" == *"('UPDATE', expected.access_profile IN ('legacy_dml', 'upsert', 'link_lifecycle'))"* ]] && \[[ "$migration_script" == *"('DELETE', expected.access_profile IN ('legacy_dml'))"* ]] && \