Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore: integrate current dev with PR 855 regression fixes
Alfredo Lopez82143e59d6dc
diff --cc control-api/package-lock.jsonindex c57d40107,fe6e679b9..4e19aa7aa--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@@ -1,12 -1,12 +1,12 @@@{"name": "clerum-control-api",- "version": "0.1.692",- "version": "0.1.699",++ "version": "0.1.700","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.692",- "version": "0.1.699",++ "version": "0.1.700","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --cc control-api/package.jsonindex 8acf89846,be18ec391..2d2de50fa--- a/control-api/package.json+++ b/control-api/package.json@@@ -1,6 -1,6 +1,6 @@@{"name": "clerum-control-api",- "version": "0.1.692",- "version": "0.1.699",++ "version": "0.1.700","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --cc control-api/test/db.realPostgresMigration.integration.test.tsindex 7c02d431d,25ebd9eb9..a54a7ebd6--- a/control-api/test/db.realPostgresMigration.integration.test.ts+++ b/control-api/test/db.realPostgresMigration.integration.test.ts@@@ -30,7 -30,7 +30,8 @@@ import { PostgresGovernedSessionReplayRimport { projectAcceptedUsageEvents } from '../src/services/tracing/usageProjection.js'import { ingestUsageEventsInTransaction } from '../src/services/usageEvents.js'import { signRpcAccessToken } from '../src/utils/auth/rpcAuthToken.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'+ import './realPostgres.requirement.ts'type PrivilegeExpectation = Record<string, Set<string>>@@@ -1734,6 -1734,286 +1735,286 @@@ describeRealPostgres('control-api realexpect(stored.rows).toEqual([{ sequence_no: auditId, subject: 'host:1st:mcp-host/chatllm' }])})+ it('recognizes deployed entity-change aliases without replaying migration DDL', async () => {+ const { CONTROL_API_MIGRATIONS, initDb } = await import('../src/db.js')+ const connector = { connect: () => dbPool.connect() }+ await initDb(connector)++ const feedMigration = CONTROL_API_MIGRATIONS.find(+ migration => migration.version === '0122_durable_entity_change_feed'+ )+ const checkpointMigration = CONTROL_API_MIGRATIONS.find(+ migration => migration.version === '0123_entity_change_checkpoint_cursor_convergence'+ )+ expect(feedMigration?.legacyVersions).toEqual([+ '0116_durable_entity_change_feed',+ '0117_durable_entity_change_feed',+ '0119_durable_entity_change_feed',+ ])+ expect(checkpointMigration?.legacyVersions).toEqual([+ '0120_entity_change_checkpoint_cursor_convergence',+ ])+ if (!feedMigration || !checkpointMigration) return++ const captureDefinition = await dbPool.query<{ definition: string }>(+ `SELECT pg_get_functiondef('entity_change_capture_resource()'::regprocedure) AS definition`+ )+ const checkpointDefinition = await dbPool.query<{ definition: string }>(+ `SELECT pg_get_functiondef(+ 'entity_change_read_checkpoint(uuid,integer)'::regprocedure+ ) AS definition`+ )++ const setLegacyVersion = async (legacyVersion: string, versions: readonly string[]) => {+ await dbPool.query(`DELETE FROM schema_migrations WHERE version = ANY($1::text[])`, [+ versions,+ ])+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1)', [legacyVersion])+ }++ try {+ for (const legacyVersion of feedMigration.legacyVersions ?? []) {+ await setLegacyVersion(legacyVersion, [+ feedMigration.version,+ ...(feedMigration.legacyVersions ?? []),+ ])+ await dbPool.query(`+ CREATE OR REPLACE FUNCTION entity_change_capture_resource() RETURNS trigger+ LANGUAGE plpgsql AS $$+ BEGIN+ RAISE EXCEPTION 'legacy entity-change feed migration was reapplied';+ END;+ $$+ `)++ await initDb(connector)++ const recorded = await dbPool.query<{ version: string }>(+ `SELECT version FROM schema_migrations+ WHERE version = ANY($1::text[]) ORDER BY version`,+ [[feedMigration.version, legacyVersion]]+ )+ expect(recorded.rows.map(row => row.version)).toContain(feedMigration.version)+ await expect(+ dbPool.query(+ `INSERT INTO gfs_resources (drive, name, kind)+ VALUES ($1, 'legacy-alias-probe', 'file')`,+ [`legacy-alias-${randomUUID()}`]+ )+ ).rejects.toThrow('legacy entity-change feed migration was reapplied')++ await dbPool.query(captureDefinition.rows[0]!.definition)+ }++ await setLegacyVersion(checkpointMigration.legacyVersions?.[0] ?? '', [+ checkpointMigration.version,+ ...(checkpointMigration.legacyVersions ?? []),+ ])+ await dbPool.query(`+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(+ requested_cursor UUID,+ maximum_recovery_events INTEGER+ ) RETURNS TABLE (+ needs_resync BOOLEAN,+ current_cursor UUID,+ current_sequence BIGINT,+ invalidated_scopes TEXT[]+ ) LANGUAGE plpgsql AS $$+ BEGIN+ RAISE EXCEPTION 'legacy entity-change checkpoint migration was reapplied';+ END;+ $$+ `)++ await initDb(connector)++ const checkpointVersions = await dbPool.query<{ version: string }>(+ `SELECT version FROM schema_migrations+ WHERE version = ANY($1::text[]) ORDER BY version`,+ [[checkpointMigration.version, ...(checkpointMigration.legacyVersions ?? [])]]+ )+ expect(checkpointVersions.rows.map(row => row.version)).toContain(checkpointMigration.version)+ await expect(+ dbPool.query(+ `SELECT * FROM entity_change_read_checkpoint(+ '00000000-0000-0000-0000-000000000000'::uuid, 100+ )`+ )+ ).rejects.toThrow('legacy entity-change checkpoint migration was reapplied')+ } finally {+ await dbPool.query(captureDefinition.rows[0]!.definition)+ await dbPool.query(checkpointDefinition.rows[0]!.definition)+ await dbPool.query(`DELETE FROM schema_migrations WHERE version = ANY($1::text[])`, [+ [+ feedMigration.version,+ ...(feedMigration.legacyVersions ?? []),+ checkpointMigration.version,+ ...(checkpointMigration.legacyVersions ?? []),+ ],+ ])+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1), ($2)', [+ feedMigration.version,+ checkpointMigration.version,+ ])+ }+ })++ it('applies checkpoint convergence after a legacy feed-only migration record', async () => {+ const { CONTROL_API_MIGRATIONS, initDb } = await import('../src/db.js')+ const connector = { connect: () => dbPool.connect() }+ await initDb(connector)++ const feedMigration = CONTROL_API_MIGRATIONS.find(+ migration => migration.version === '0122_durable_entity_change_feed'+ )+ const checkpointMigration = CONTROL_API_MIGRATIONS.find(+ migration => migration.version === '0123_entity_change_checkpoint_cursor_convergence'+ )+ expect(feedMigration?.legacyVersions).toContain('0116_durable_entity_change_feed')+ expect(checkpointMigration?.legacyVersions).toEqual([+ '0120_entity_change_checkpoint_cursor_convergence',+ ])+ if (!feedMigration || !checkpointMigration) return++ const oldVersions = [+ feedMigration.version,+ ...(feedMigration.legacyVersions ?? []),+ checkpointMigration.version,+ ...(checkpointMigration.legacyVersions ?? []),+ ]+ try {+ await dbPool.query('DELETE FROM schema_migrations WHERE version = ANY($1::text[])', [+ oldVersions,+ ])+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1)', [+ '0116_durable_entity_change_feed',+ ])+ await dbPool.query(`+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(+ requested_cursor UUID,+ maximum_recovery_events INTEGER+ ) RETURNS TABLE (+ needs_resync BOOLEAN,+ current_cursor UUID,+ current_sequence BIGINT,+ invalidated_scopes TEXT[]+ ) LANGUAGE plpgsql AS $$+ BEGIN+ RAISE EXCEPTION 'checkpoint convergence migration was not applied';+ END;+ $$+ `)++ await initDb(connector)++ const recordedVersions = await dbPool.query<{ version: string }>(+ `SELECT version FROM schema_migrations WHERE version = ANY($1::text[]) ORDER BY version`,+ [oldVersions]+ )+ expect(recordedVersions.rows.map(row => row.version)).toEqual([+ '0116_durable_entity_change_feed',+ feedMigration.version,+ checkpointMigration.version,+ ])+ const checkpoint = await dbPool.query(+ `SELECT * FROM entity_change_read_checkpoint(+ '00000000-0000-0000-0000-000000000000'::uuid, 100+ )`+ )+ expect(checkpoint.rows).toHaveLength(1)+ expect(typeof checkpoint.rows[0]?.needs_resync).toBe('boolean')+ expect(checkpoint.rows[0]?.current_cursor).toMatch(+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i+ )+ } finally {+ await dbPool.query('DELETE FROM schema_migrations WHERE version = ANY($1::text[])', [+ oldVersions,+ ])+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1), ($2)', [+ feedMigration.version,+ checkpointMigration.version,+ ])+ }+ })++ it('prevents temp-type shadowing in entity-change SECURITY DEFINER triggers', async () => {+ const { initDb } = await import('../src/db.js')+ await initDb({ connect: () => dbPool.connect() })++ const attackerRole = `entity_change_shadow_${randomBytes(5).toString('hex')}`+ const attackerPassword = randomBytes(24).toString('hex')+ let attackerPool: Pool | undefined+ try {+ await adminPool.query(+ `CREATE ROLE ${quoteIdent(attackerRole)} LOGIN PASSWORD '${attackerPassword}'`+ )+ await adminPool.query(`GRANT gfs_controller TO ${quoteIdent(attackerRole)}`)+ const attackerUrl = new URL(connectionString)+ attackerUrl.username = attackerRole+ attackerUrl.password = attackerPassword+ attackerPool = new Pool({ connectionString: attackerUrl.toString(), max: 1 })++ const attacker = await attackerPool.connect()+ try {+ await attacker.query('CREATE TEMP TABLE temp_schema_bootstrap (id INTEGER)')+ await attacker.query(`+ CREATE FUNCTION pg_temp.entity_change_shadow_check(value pg_catalog.text)+ RETURNS BOOLEAN LANGUAGE plpgsql AS $$+ BEGIN+ RAISE EXCEPTION 'temporary type shadow executed as %', current_user;+ END;+ $$+ `)+ await attacker.query(`+ CREATE DOMAIN pg_temp.text AS pg_catalog.text+ CHECK (pg_temp.entity_change_shadow_check(VALUE))+ `)++ const resourceId = randomUUID()+ await expect(+ attacker.query(+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)+ VALUES ($1, $2, 'shadow-probe', 'file', '/shadow-probe')`,+ [resourceId, `shadow-${randomUUID()}`]+ )+ ).resolves.toBeDefined()+ const stored = await dbPool.query('SELECT 1 FROM gfs_resources WHERE resource_id = $1', [+ resourceId,+ ])+ expect(stored.rowCount).toBe(1)+ } finally {+ attacker.release()+ }++ const migrationVersions = await dbPool.query<{ version: string }>(+ `SELECT version FROM schema_migrations WHERE version = $1`,+ ['0124_entity_change_definer_search_path']+ )+ expect(migrationVersions.rows).toHaveLength(1)++ const functionPaths = await dbPool.query<{ function_name: string; config: string[] | null }>(`+ SELECT procedure.proname AS function_name, procedure.proconfig AS config+ FROM pg_proc procedure+ JOIN pg_namespace namespace ON namespace.oid = procedure.pronamespace+ WHERE namespace.nspname = 'public'+ AND procedure.proname IN (+ 'entity_change_capture_resource',+ 'entity_change_capture_scope',+ 'entity_change_dispatch_batch',+ 'entity_change_read_checkpoint'+ )+ `)+ expect(functionPaths.rows).toHaveLength(4)+ for (const { config } of functionPaths.rows) {+ expect(config).toContain('search_path=pg_catalog, public, pg_temp')+ }+ } finally {- await attackerPool?.end()++ await endPoolAndWaitForClients(attackerPool)+ await dbPool.query(`DROP OWNED BY ${quoteIdent(attackerRole)}`)+ await adminPool.query(`DROP ROLE IF EXISTS ${quoteIdent(attackerRole)}`)+ }+ })+it('commits GFS grant/share batches atomically and invalidates only after commit', async () => {const operator = await dbPool.query<{ id: string }>(`SELECT id::text FROM control_admin_usersdiff --cc control-api/test/entityChangeFeed.realPostgres.integration.test.tsindex 000000000,96f3e1172..4513377c4mode 000000,100644..100644--- a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts+++ b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts@@@ -1,0 -1,549 +1,550 @@@+ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'+ import { randomBytes, randomUUID } from 'node:crypto'+ import { Pool } from 'pg'+ import { initDb } from '../src/db.js'+ import { DbSeedResourceStore, seedRootDirectories } from '../src/gfs/seedResources.js'++import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'+ import './realPostgres.requirement.ts'++ const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+ const describeRealPostgres = adminUrl ? describe : describe.skip++ function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+ }++ function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+ }++ describeRealPostgres('entity change feed real PostgreSQL contract', () => {+ const database = `control_api_entity_change_${randomBytes(6).toString('hex')}`+ let connectionString = ''+ let adminPool: Pool+ let instancePool: Pool+ let replicaPool: Pool++ beforeAll(async () => {+ connectionString = databaseUrl(adminUrl!, database)+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ instancePool = new Pool({ connectionString })+ replicaPool = new Pool({ connectionString })+ await initDb({ connect: () => instancePool.connect() })+ })++ beforeEach(async () => {+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ })++ afterAll(async () => {- await instancePool?.end()- await replicaPool?.end()++ await endPoolAndWaitForClients(instancePool)++ await endPoolAndWaitForClients(replicaPool)+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ }+ })++ it('captures nested GFS root provisioning and stays idempotent on reseed', async () => {+ const client = await instancePool.connect()+ const drive = `entity-change-seed-${randomUUID()}`+ try {+ const before = await client.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ await client.query('BEGIN')+ const seeded = await seedRootDirectories(new DbSeedResourceStore(client), drive, [+ '/org/team/project',+ ])+ await client.query('COMMIT')+ expect(seeded.rootResourceId).toBeTruthy()+ expect(seeded.byPath['/org/team/project']).toBeTruthy()+ const afterCreate = await client.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ expect(Number(afterCreate.rows[0]?.count) - Number(before.rows[0]?.count)).toBe(4)++ await client.query('BEGIN')+ await seedRootDirectories(new DbSeedResourceStore(client), drive, ['/org/team/project'])+ await client.query('COMMIT')+ const afterReseed = await client.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ expect(afterReseed.rows[0]?.count).toBe(afterCreate.rows[0]?.count)+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('captures committed create, replace, rename/move, delete, grant, and membership changes', async () => {+ const client = await instancePool.connect()+ const drive = `entity-change-${randomUUID()}`+ const rootId = randomUUID()+ const parentId = randomUUID()+ const movedParentId = randomUUID()+ const fileId = randomUUID()+ const userId = randomUUID()+ const teamId = randomUUID()+ try {+ await client.query('BEGIN')+ await client.query('SET ROLE gfs_controller')+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)+ VALUES ($1, $2, '/', 'directory', '/')`,+ [rootId, drive]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'parent', 'directory', '/parent')`,+ [parentId, drive, rootId]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'destination', 'directory', '/destination')`,+ [movedParentId, drive, rootId]+ )+ await client.query(+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)+ VALUES ($1, $2, $3, 'draft.md', 'file', '/parent/draft.md')`,+ [fileId, drive, parentId]+ )+ await client.query('RESET ROLE')+ await client.query('SET ROLE control_api_runtime')+ await client.query(+ `INSERT INTO gfs_grants (drive, resource_id, subject_type, subject_id, permissions)+ VALUES ($1, $2, 'team', $3, ARRAY['read'])`,+ [drive, fileId, teamId]+ )+ await client.query(+ `INSERT INTO gfs_shares (drive, resource_id, subject_type, subject_id, permissions)+ VALUES ($1, $2, 'user', $3, ARRAY['read'])`,+ [drive, fileId, userId]+ )+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ const created = await instancePool.query<{ count: string }>(+ `SELECT count(*)::text AS count FROM entity_change_outbox WHERE scope = 'gfs'`+ )+ expect(Number(created.rows[0]?.count)).toBe(4)++ await client.query('BEGIN')+ await client.query('SET ROLE control_api_runtime')+ await client.query(+ `UPDATE gfs_resources SET version = version + 1, bytes = 45 WHERE resource_id = $1`,+ [fileId]+ )+ await client.query(+ `UPDATE gfs_resources SET parent_resource_id = $2, name = 'renamed.md',+ path_cache = '/destination/renamed.md' WHERE resource_id = $1`,+ [fileId, movedParentId]+ )+ await client.query('RESET ROLE')+ await client.query('SET ROLE gfs_controller')+ await client.query(+ `UPDATE gfs_resources SET deleted_at = clock_timestamp() WHERE resource_id = $1`,+ [fileId]+ )+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ await client.query('BEGIN')+ await client.query('SET ROLE control_api_runtime')+ await client.query(`INSERT INTO users (id, email) VALUES ($1, $2)`, [+ userId,+ `${userId}@entity-change.invalid`,+ ])+ await client.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])+ await client.query(+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,+ [teamId, userId]+ )+ await client.query('DELETE FROM team_members WHERE team_id = $1 AND user_id = $2', [+ teamId,+ userId,+ ])+ await client.query('DELETE FROM gfs_grants WHERE drive = $1 AND resource_id = $2', [+ drive,+ fileId,+ ])+ await client.query('DELETE FROM gfs_shares WHERE drive = $1 AND resource_id = $2', [+ drive,+ fileId,+ ])+ await client.query('RESET ROLE')+ await client.query('COMMIT')++ const outbox = await instancePool.query<{ scope: string; entity_id: string | null }>(+ 'SELECT scope, entity_id FROM entity_change_outbox ORDER BY id'+ )+ expect(outbox.rows.filter(row => row.scope === 'gfs')).toHaveLength(7)+ expect(outbox.rows.filter(row => row.entity_id === fileId)).toHaveLength(4)+ expect(outbox.rows.filter(row => row.scope === 'authorization')).toHaveLength(7)+ // Event evidence contains no file content, path, or authorization subject.+ expect(JSON.stringify(outbox.rows)).not.toContain('draft.md')+ expect(JSON.stringify(outbox.rows)).not.toContain(userId)+ expect(JSON.stringify(outbox.rows)).not.toContain(teamId)+ } finally {+ client.release()+ }+ })++ it('rolls capture back with the producer transaction and publishes only committed facts', async () => {+ const client = await instancePool.connect()+ const drive = `entity-change-rollback-${randomUUID()}`+ try {+ const before = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ await client.query('BEGIN')+ await client.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'rolled-back', 'file')`,+ [drive]+ )+ await client.query('ROLLBACK')+ const after = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_outbox'+ )+ expect(after.rows[0]?.count).toBe(before.rows[0]?.count)++ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'committed', 'file')`,+ [drive]+ )+ const firstDispatch = await instancePool.query<{ feed_sequence: string }>(+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'+ )+ const feedAfterFirst = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_feed'+ )+ const duplicateDispatch = await instancePool.query<{ feed_sequence: string }>(+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'+ )+ const feedAfterDuplicate = await instancePool.query<{ count: string }>(+ 'SELECT count(*)::text AS count FROM entity_change_feed'+ )+ expect(firstDispatch.rows).toHaveLength(1)+ expect(duplicateDispatch.rows).toHaveLength(0)+ expect(feedAfterDuplicate.rows[0]?.count).toBe(feedAfterFirst.rows[0]?.count)+ } finally {+ client.release()+ }+ })++ it('uses dispatcher order rather than identity allocation as public sequence order', async () => {+ const pending = await instancePool.connect()+ const earlyDrive = `entity-change-early-${randomUUID()}`+ try {+ await pending.query('BEGIN')+ await pending.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'early', 'file')`,+ [earlyDrive]+ )+ const pendingId = await pending.query<{ id: string }>(+ `SELECT max(id)::text AS id FROM entity_change_outbox WHERE entity_id IS NOT NULL`+ )+ const watermarkBefore = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ const userId = randomUUID()+ const teamId = randomUUID()+ await instancePool.query('INSERT INTO users (id, email) VALUES ($1, $2)', [+ userId,+ `${userId}@ordering.invalid`,+ ])+ await instancePool.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])+ await instancePool.query(+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,+ [teamId, userId]+ )+ const committedMax = await instancePool.query<{ id: string }>(+ 'SELECT max(id)::text AS id FROM entity_change_outbox'+ )+ expect(Number(pendingId.rows[0]?.id)).toBeLessThan(Number(committedMax.rows[0]?.id))+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const firstFeed = await instancePool.query<{ sequence: string; scope: string }>(+ `SELECT sequence::text, scope FROM entity_change_feed+ WHERE sequence > $1 ORDER BY sequence`,+ [watermarkBefore.rows[0]?.sequence]+ )+ expect(firstFeed.rows.map(row => row.scope)).toEqual(['authorization'])+ await pending.query('COMMIT')+ await replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const feed = await instancePool.query<{ sequence: string; scope: string }>(+ `SELECT sequence::text, scope FROM entity_change_feed+ WHERE sequence > $1 ORDER BY sequence`,+ [watermarkBefore.rows[0]?.sequence]+ )+ expect(feed.rows.map(row => row.scope)).toEqual(['authorization', 'gfs'])+ expect(feed.rows.map(row => Number(row.sequence))).toEqual([+ Number(watermarkBefore.rows[0]?.sequence) + 1,+ Number(watermarkBefore.rows[0]?.sequence) + 2,+ ])+ } finally {+ await pending.query('ROLLBACK').catch(() => undefined)+ pending.release()+ }+ })++ it('keeps checkpoint scopes and cursor on one committed watermark snapshot', async () => {+ const drive = `entity-change-checkpoint-lock-${randomUUID()}`+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint-lock', 'file')`,+ [drive]+ )+ const baseline = await instancePool.query<{ current_cursor: string }>(+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'+ )+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'pending-change', 'file')`,+ [`${drive}-pending`]+ )++ const dispatcher = await instancePool.connect()+ const reader = await replicaPool.connect()+ let checkpointPending: Promise<{+ rows: Array<{+ needs_resync: boolean+ current_cursor: string+ invalidated_scopes: string[]+ }>+ }> | null = null+ try {+ await dispatcher.query('BEGIN')+ await dispatcher.query(+ 'SELECT sequence FROM entity_change_watermark WHERE singleton = true FOR UPDATE'+ )+ const readerPid = await reader.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')+ let settled = false+ checkpointPending = reader+ .query<{+ needs_resync: boolean+ current_cursor: string+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ baseline.rows[0]?.current_cursor,+ 10000,+ ])+ .then(result => {+ settled = true+ return result+ })++ let waitingOnWatermark = false+ for (let attempt = 0; attempt < 80; attempt += 1) {+ const activity = await adminPool.query<{ wait_event_type: string | null }>(+ 'SELECT wait_event_type FROM pg_stat_activity WHERE pid = $1',+ [readerPid.rows[0]?.pid]+ )+ if (activity.rows[0]?.wait_event_type === 'Lock') {+ waitingOnWatermark = true+ break+ }+ if (settled) break+ await new Promise(resolve => setTimeout(resolve, 25))+ }+ expect(waitingOnWatermark).toBe(true)+ expect(settled).toBe(false)++ await dispatcher.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ await dispatcher.query('COMMIT')+ const checkpoint = await checkpointPending+ expect(checkpoint.rows[0]?.needs_resync).toBe(false)+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])+ const watermark = await instancePool.query<{ current_cursor: string }>(+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'+ )+ expect(checkpoint.rows[0]?.current_cursor).toBe(watermark.rows[0]?.current_cursor)+ } finally {+ await dispatcher.query('ROLLBACK').catch(() => undefined)+ await checkpointPending?.catch(() => undefined)+ dispatcher.release()+ reader.release()+ }+ })++ it('recovers missed wakeups and expired cursors with a coarse authorized checkpoint', async () => {+ const resource = await instancePool.query<{ resource_id: string }>(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint', 'file')+ RETURNING resource_id`,+ [`entity-change-checkpoint-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const staleEvent = await instancePool.query<{ cursor: string; sequence: string }>(+ 'SELECT cursor::text, sequence::text FROM entity_change_feed ORDER BY sequence DESC LIMIT 1'+ )+ // Advance the watermark beyond the cursor that will expire. Otherwise the+ // cursor under test is itself the current watermark, which correctly means+ // the client is caught up and must not receive a resync.+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint-newer', 'file')`,+ [`entity-change-checkpoint-newer-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const current = await instancePool.query<{ current_cursor: string; sequence: string }>(+ `SELECT current_cursor::text, sequence::text+ FROM entity_change_watermark WHERE singleton = true`+ )+ expect(current.rows[0]?.current_cursor).not.toBe(staleEvent.rows[0]?.cursor)++ // No listener is involved here: the durable query is the fallback after a missed NOTIFY.+ const recovery = await instancePool.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ staleEvent.rows[0]?.cursor,+ 10000,+ ])+ expect(recovery.rows[0]?.needs_resync).toBe(false)++ await instancePool.query(+ `UPDATE entity_change_feed+ SET created_at = clock_timestamp() - interval '2 days'+ WHERE sequence <= $1`,+ [current.rows[0]?.sequence]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const expired = await instancePool.query<{+ needs_resync: boolean+ current_cursor: string+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ staleEvent.rows[0]?.cursor,+ 10000,+ ])+ expect(expired.rows[0]?.needs_resync).toBe(true)+ expect(expired.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])+ expect(JSON.stringify(expired.rows)).not.toContain(resource.rows[0]?.resource_id)++ // The resync response advances to the current watermark. Reusing that+ // server-issued cursor is caught up even when retention has pruned every+ // corresponding feed row; an idle client must not receive resync forever.+ const returnedCursor = expired.rows[0]?.current_cursor+ expect(returnedCursor).toBeTruthy()+ for (let attempt = 0; attempt < 3; attempt += 1) {+ const resumed = await instancePool.query<{+ needs_resync: boolean+ current_cursor: string+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [returnedCursor, 10000])+ expect(resumed.rows[0]?.needs_resync).toBe(false)+ expect(resumed.rows[0]?.current_cursor).toBe(returnedCursor)+ expect(resumed.rows[0]?.invalidated_scopes).toEqual([])+ }++ const freshClient = await instancePool.query<{+ needs_resync: boolean+ current_cursor: string+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ '00000000-0000-0000-0000-000000000000',+ 10000,+ ])+ expect(freshClient.rows[0]?.needs_resync).toBe(true)+ expect(freshClient.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])+ })++ it('coalesces latest-state invalidations and keeps feed storage behind the runtime boundary', async () => {+ // Establish a retained cursor in this test instead of depending on the+ // preceding expiry test's intentionally pruned watermark cursor.+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'coalesce-baseline', 'file')`,+ [`entity-change-coalesce-baseline-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const baseline = await instancePool.query<{ current_cursor: string }>(+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'+ )+ const inserted = await instancePool.query<{ resource_id: string }>(+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'many-updates', 'file')+ RETURNING resource_id`,+ [`entity-change-coalesce-${randomUUID()}`]+ )+ const resourceId = inserted.rows[0]?.resource_id+ await instancePool.query(+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',+ [resourceId]+ )+ await instancePool.query(+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',+ [resourceId]+ )+ const before = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ await Promise.all([+ instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),+ replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),+ ])+ const after = await instancePool.query<{ sequence: string }>(+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'+ )+ const published = await instancePool.query<{ scope: string; count: string }>(+ `SELECT scope, count(*)::text AS count FROM entity_change_feed+ WHERE sequence > $1 GROUP BY scope`,+ [before.rows[0]?.sequence]+ )+ expect(Number(after.rows[0]?.sequence) - Number(before.rows[0]?.sequence)).toBe(1)+ expect(published.rows).toEqual([{ scope: 'gfs', count: '1' }])++ const restricted = await instancePool.connect()+ try {+ await restricted.query('SET ROLE control_api_runtime')+ await expect(restricted.query('SELECT * FROM entity_change_feed')).rejects.toThrow()+ const checkpoint = await restricted.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ baseline.rows[0]?.current_cursor,+ 10000,+ ])+ expect(checkpoint.rows[0]?.needs_resync).toBe(false)+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])+ expect(Object.keys(checkpoint.rows[0] ?? {}).sort()).toEqual([+ 'current_cursor',+ 'current_sequence',+ 'invalidated_scopes',+ 'needs_resync',+ ])+ } finally {+ await restricted.query('RESET ROLE').catch(() => undefined)+ restricted.release()+ }+ })++ it('resyncs instead of scanning an oversized retained replay window', async () => {+ const baseline = await instancePool.query<{ current_cursor: string }>(+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'+ )+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind)+ VALUES ($1, 'bounded-window', 'file')`,+ [`entity-change-bounded-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ await instancePool.query(+ `INSERT INTO gfs_resources (drive, name, kind)+ VALUES ($1, 'bounded-window-2', 'file')`,+ [`entity-change-bounded-${randomUUID()}`]+ )+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')+ const bounded = await instancePool.query<{+ needs_resync: boolean+ invalidated_scopes: string[]+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [+ baseline.rows[0]?.current_cursor,+ 1,+ ])+ expect(bounded.rows[0]?.needs_resync).toBe(true)+ expect(bounded.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])+ })+ })