Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

chore: integrate current dev with PR 855 regression fixes

Alfredo Lopez82143e59d6dc
diff --cc control-api/package-lock.json
index c57d40107,fe6e679b9..4e19aa7aa
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@@ -1,12 -1,12 +1,12 @@@
{
"name": "clerum-control-api",
- "version": "0.1.692",
- "version": "0.1.699",
++ "version": "0.1.700",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.692",
- "version": "0.1.699",
++ "version": "0.1.700",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --cc control-api/package.json
index 8acf89846,be18ec391..2d2de50fa
--- a/control-api/package.json
+++ b/control-api/package.json
@@@ -1,6 -1,6 +1,6 @@@
{
"name": "clerum-control-api",
- "version": "0.1.692",
- "version": "0.1.699",
++ "version": "0.1.700",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --cc control-api/test/db.realPostgresMigration.integration.test.ts
index 7c02d431d,25ebd9eb9..a54a7ebd6
--- a/control-api/test/db.realPostgresMigration.integration.test.ts
+++ b/control-api/test/db.realPostgresMigration.integration.test.ts
@@@ -30,7 -30,7 +30,8 @@@ import { PostgresGovernedSessionReplayR
import { projectAcceptedUsageEvents } from '../src/services/tracing/usageProjection.js'
import { ingestUsageEventsInTransaction } from '../src/services/usageEvents.js'
import { signRpcAccessToken } from '../src/utils/auth/rpcAuthToken.js'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+ import './realPostgres.requirement.ts'
type PrivilegeExpectation = Record<string, Set<string>>
@@@ -1734,6 -1734,286 +1735,286 @@@ describeRealPostgres('control-api real
expect(stored.rows).toEqual([{ sequence_no: auditId, subject: 'host:1st:mcp-host/chatllm' }])
})
+ it('recognizes deployed entity-change aliases without replaying migration DDL', async () => {
+ const { CONTROL_API_MIGRATIONS, initDb } = await import('../src/db.js')
+ const connector = { connect: () => dbPool.connect() }
+ await initDb(connector)
+
+ const feedMigration = CONTROL_API_MIGRATIONS.find(
+ migration => migration.version === '0122_durable_entity_change_feed'
+ )
+ const checkpointMigration = CONTROL_API_MIGRATIONS.find(
+ migration => migration.version === '0123_entity_change_checkpoint_cursor_convergence'
+ )
+ expect(feedMigration?.legacyVersions).toEqual([
+ '0116_durable_entity_change_feed',
+ '0117_durable_entity_change_feed',
+ '0119_durable_entity_change_feed',
+ ])
+ expect(checkpointMigration?.legacyVersions).toEqual([
+ '0120_entity_change_checkpoint_cursor_convergence',
+ ])
+ if (!feedMigration || !checkpointMigration) return
+
+ const captureDefinition = await dbPool.query<{ definition: string }>(
+ `SELECT pg_get_functiondef('entity_change_capture_resource()'::regprocedure) AS definition`
+ )
+ const checkpointDefinition = await dbPool.query<{ definition: string }>(
+ `SELECT pg_get_functiondef(
+ 'entity_change_read_checkpoint(uuid,integer)'::regprocedure
+ ) AS definition`
+ )
+
+ const setLegacyVersion = async (legacyVersion: string, versions: readonly string[]) => {
+ await dbPool.query(`DELETE FROM schema_migrations WHERE version = ANY($1::text[])`, [
+ versions,
+ ])
+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1)', [legacyVersion])
+ }
+
+ try {
+ for (const legacyVersion of feedMigration.legacyVersions ?? []) {
+ await setLegacyVersion(legacyVersion, [
+ feedMigration.version,
+ ...(feedMigration.legacyVersions ?? []),
+ ])
+ await dbPool.query(`
+ CREATE OR REPLACE FUNCTION entity_change_capture_resource() RETURNS trigger
+ LANGUAGE plpgsql AS $$
+ BEGIN
+ RAISE EXCEPTION 'legacy entity-change feed migration was reapplied';
+ END;
+ $$
+ `)
+
+ await initDb(connector)
+
+ const recorded = await dbPool.query<{ version: string }>(
+ `SELECT version FROM schema_migrations
+ WHERE version = ANY($1::text[]) ORDER BY version`,
+ [[feedMigration.version, legacyVersion]]
+ )
+ expect(recorded.rows.map(row => row.version)).toContain(feedMigration.version)
+ await expect(
+ dbPool.query(
+ `INSERT INTO gfs_resources (drive, name, kind)
+ VALUES ($1, 'legacy-alias-probe', 'file')`,
+ [`legacy-alias-${randomUUID()}`]
+ )
+ ).rejects.toThrow('legacy entity-change feed migration was reapplied')
+
+ await dbPool.query(captureDefinition.rows[0]!.definition)
+ }
+
+ await setLegacyVersion(checkpointMigration.legacyVersions?.[0] ?? '', [
+ checkpointMigration.version,
+ ...(checkpointMigration.legacyVersions ?? []),
+ ])
+ await dbPool.query(`
+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(
+ requested_cursor UUID,
+ maximum_recovery_events INTEGER
+ ) RETURNS TABLE (
+ needs_resync BOOLEAN,
+ current_cursor UUID,
+ current_sequence BIGINT,
+ invalidated_scopes TEXT[]
+ ) LANGUAGE plpgsql AS $$
+ BEGIN
+ RAISE EXCEPTION 'legacy entity-change checkpoint migration was reapplied';
+ END;
+ $$
+ `)
+
+ await initDb(connector)
+
+ const checkpointVersions = await dbPool.query<{ version: string }>(
+ `SELECT version FROM schema_migrations
+ WHERE version = ANY($1::text[]) ORDER BY version`,
+ [[checkpointMigration.version, ...(checkpointMigration.legacyVersions ?? [])]]
+ )
+ expect(checkpointVersions.rows.map(row => row.version)).toContain(checkpointMigration.version)
+ await expect(
+ dbPool.query(
+ `SELECT * FROM entity_change_read_checkpoint(
+ '00000000-0000-0000-0000-000000000000'::uuid, 100
+ )`
+ )
+ ).rejects.toThrow('legacy entity-change checkpoint migration was reapplied')
+ } finally {
+ await dbPool.query(captureDefinition.rows[0]!.definition)
+ await dbPool.query(checkpointDefinition.rows[0]!.definition)
+ await dbPool.query(`DELETE FROM schema_migrations WHERE version = ANY($1::text[])`, [
+ [
+ feedMigration.version,
+ ...(feedMigration.legacyVersions ?? []),
+ checkpointMigration.version,
+ ...(checkpointMigration.legacyVersions ?? []),
+ ],
+ ])
+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1), ($2)', [
+ feedMigration.version,
+ checkpointMigration.version,
+ ])
+ }
+ })
+
+ it('applies checkpoint convergence after a legacy feed-only migration record', async () => {
+ const { CONTROL_API_MIGRATIONS, initDb } = await import('../src/db.js')
+ const connector = { connect: () => dbPool.connect() }
+ await initDb(connector)
+
+ const feedMigration = CONTROL_API_MIGRATIONS.find(
+ migration => migration.version === '0122_durable_entity_change_feed'
+ )
+ const checkpointMigration = CONTROL_API_MIGRATIONS.find(
+ migration => migration.version === '0123_entity_change_checkpoint_cursor_convergence'
+ )
+ expect(feedMigration?.legacyVersions).toContain('0116_durable_entity_change_feed')
+ expect(checkpointMigration?.legacyVersions).toEqual([
+ '0120_entity_change_checkpoint_cursor_convergence',
+ ])
+ if (!feedMigration || !checkpointMigration) return
+
+ const oldVersions = [
+ feedMigration.version,
+ ...(feedMigration.legacyVersions ?? []),
+ checkpointMigration.version,
+ ...(checkpointMigration.legacyVersions ?? []),
+ ]
+ try {
+ await dbPool.query('DELETE FROM schema_migrations WHERE version = ANY($1::text[])', [
+ oldVersions,
+ ])
+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1)', [
+ '0116_durable_entity_change_feed',
+ ])
+ await dbPool.query(`
+ CREATE OR REPLACE FUNCTION entity_change_read_checkpoint(
+ requested_cursor UUID,
+ maximum_recovery_events INTEGER
+ ) RETURNS TABLE (
+ needs_resync BOOLEAN,
+ current_cursor UUID,
+ current_sequence BIGINT,
+ invalidated_scopes TEXT[]
+ ) LANGUAGE plpgsql AS $$
+ BEGIN
+ RAISE EXCEPTION 'checkpoint convergence migration was not applied';
+ END;
+ $$
+ `)
+
+ await initDb(connector)
+
+ const recordedVersions = await dbPool.query<{ version: string }>(
+ `SELECT version FROM schema_migrations WHERE version = ANY($1::text[]) ORDER BY version`,
+ [oldVersions]
+ )
+ expect(recordedVersions.rows.map(row => row.version)).toEqual([
+ '0116_durable_entity_change_feed',
+ feedMigration.version,
+ checkpointMigration.version,
+ ])
+ const checkpoint = await dbPool.query(
+ `SELECT * FROM entity_change_read_checkpoint(
+ '00000000-0000-0000-0000-000000000000'::uuid, 100
+ )`
+ )
+ expect(checkpoint.rows).toHaveLength(1)
+ expect(typeof checkpoint.rows[0]?.needs_resync).toBe('boolean')
+ expect(checkpoint.rows[0]?.current_cursor).toMatch(
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+ )
+ } finally {
+ await dbPool.query('DELETE FROM schema_migrations WHERE version = ANY($1::text[])', [
+ oldVersions,
+ ])
+ await dbPool.query('INSERT INTO schema_migrations(version) VALUES ($1), ($2)', [
+ feedMigration.version,
+ checkpointMigration.version,
+ ])
+ }
+ })
+
+ it('prevents temp-type shadowing in entity-change SECURITY DEFINER triggers', async () => {
+ const { initDb } = await import('../src/db.js')
+ await initDb({ connect: () => dbPool.connect() })
+
+ const attackerRole = `entity_change_shadow_${randomBytes(5).toString('hex')}`
+ const attackerPassword = randomBytes(24).toString('hex')
+ let attackerPool: Pool | undefined
+ try {
+ await adminPool.query(
+ `CREATE ROLE ${quoteIdent(attackerRole)} LOGIN PASSWORD '${attackerPassword}'`
+ )
+ await adminPool.query(`GRANT gfs_controller TO ${quoteIdent(attackerRole)}`)
+ const attackerUrl = new URL(connectionString)
+ attackerUrl.username = attackerRole
+ attackerUrl.password = attackerPassword
+ attackerPool = new Pool({ connectionString: attackerUrl.toString(), max: 1 })
+
+ const attacker = await attackerPool.connect()
+ try {
+ await attacker.query('CREATE TEMP TABLE temp_schema_bootstrap (id INTEGER)')
+ await attacker.query(`
+ CREATE FUNCTION pg_temp.entity_change_shadow_check(value pg_catalog.text)
+ RETURNS BOOLEAN LANGUAGE plpgsql AS $$
+ BEGIN
+ RAISE EXCEPTION 'temporary type shadow executed as %', current_user;
+ END;
+ $$
+ `)
+ await attacker.query(`
+ CREATE DOMAIN pg_temp.text AS pg_catalog.text
+ CHECK (pg_temp.entity_change_shadow_check(VALUE))
+ `)
+
+ const resourceId = randomUUID()
+ await expect(
+ attacker.query(
+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)
+ VALUES ($1, $2, 'shadow-probe', 'file', '/shadow-probe')`,
+ [resourceId, `shadow-${randomUUID()}`]
+ )
+ ).resolves.toBeDefined()
+ const stored = await dbPool.query('SELECT 1 FROM gfs_resources WHERE resource_id = $1', [
+ resourceId,
+ ])
+ expect(stored.rowCount).toBe(1)
+ } finally {
+ attacker.release()
+ }
+
+ const migrationVersions = await dbPool.query<{ version: string }>(
+ `SELECT version FROM schema_migrations WHERE version = $1`,
+ ['0124_entity_change_definer_search_path']
+ )
+ expect(migrationVersions.rows).toHaveLength(1)
+
+ const functionPaths = await dbPool.query<{ function_name: string; config: string[] | null }>(`
+ SELECT procedure.proname AS function_name, procedure.proconfig AS config
+ FROM pg_proc procedure
+ JOIN pg_namespace namespace ON namespace.oid = procedure.pronamespace
+ WHERE namespace.nspname = 'public'
+ AND procedure.proname IN (
+ 'entity_change_capture_resource',
+ 'entity_change_capture_scope',
+ 'entity_change_dispatch_batch',
+ 'entity_change_read_checkpoint'
+ )
+ `)
+ expect(functionPaths.rows).toHaveLength(4)
+ for (const { config } of functionPaths.rows) {
+ expect(config).toContain('search_path=pg_catalog, public, pg_temp')
+ }
+ } finally {
- await attackerPool?.end()
++ await endPoolAndWaitForClients(attackerPool)
+ await dbPool.query(`DROP OWNED BY ${quoteIdent(attackerRole)}`)
+ await adminPool.query(`DROP ROLE IF EXISTS ${quoteIdent(attackerRole)}`)
+ }
+ })
+
it('commits GFS grant/share batches atomically and invalidates only after commit', async () => {
const operator = await dbPool.query<{ id: string }>(
`SELECT id::text FROM control_admin_users
diff --cc control-api/test/entityChangeFeed.realPostgres.integration.test.ts
index 000000000,96f3e1172..4513377c4
mode 000000,100644..100644
--- a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
+++ b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
@@@ -1,0 -1,549 +1,550 @@@
+ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+ import { randomBytes, randomUUID } from 'node:crypto'
+ import { Pool } from 'pg'
+ import { initDb } from '../src/db.js'
+ import { DbSeedResourceStore, seedRootDirectories } from '../src/gfs/seedResources.js'
++import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+ import './realPostgres.requirement.ts'
+
+ const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+ const describeRealPostgres = adminUrl ? describe : describe.skip
+
+ function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+ }
+
+ function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+ }
+
+ describeRealPostgres('entity change feed real PostgreSQL contract', () => {
+ const database = `control_api_entity_change_${randomBytes(6).toString('hex')}`
+ let connectionString = ''
+ let adminPool: Pool
+ let instancePool: Pool
+ let replicaPool: Pool
+
+ beforeAll(async () => {
+ connectionString = databaseUrl(adminUrl!, database)
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ instancePool = new Pool({ connectionString })
+ replicaPool = new Pool({ connectionString })
+ await initDb({ connect: () => instancePool.connect() })
+ })
+
+ beforeEach(async () => {
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ })
+
+ afterAll(async () => {
- await instancePool?.end()
- await replicaPool?.end()
++ await endPoolAndWaitForClients(instancePool)
++ await endPoolAndWaitForClients(replicaPool)
+ if (adminPool) {
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ }
+ })
+
+ it('captures nested GFS root provisioning and stays idempotent on reseed', async () => {
+ const client = await instancePool.connect()
+ const drive = `entity-change-seed-${randomUUID()}`
+ try {
+ const before = await client.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ await client.query('BEGIN')
+ const seeded = await seedRootDirectories(new DbSeedResourceStore(client), drive, [
+ '/org/team/project',
+ ])
+ await client.query('COMMIT')
+ expect(seeded.rootResourceId).toBeTruthy()
+ expect(seeded.byPath['/org/team/project']).toBeTruthy()
+ const afterCreate = await client.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ expect(Number(afterCreate.rows[0]?.count) - Number(before.rows[0]?.count)).toBe(4)
+
+ await client.query('BEGIN')
+ await seedRootDirectories(new DbSeedResourceStore(client), drive, ['/org/team/project'])
+ await client.query('COMMIT')
+ const afterReseed = await client.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ expect(afterReseed.rows[0]?.count).toBe(afterCreate.rows[0]?.count)
+ } finally {
+ await client.query('ROLLBACK').catch(() => undefined)
+ client.release()
+ }
+ })
+
+ it('captures committed create, replace, rename/move, delete, grant, and membership changes', async () => {
+ const client = await instancePool.connect()
+ const drive = `entity-change-${randomUUID()}`
+ const rootId = randomUUID()
+ const parentId = randomUUID()
+ const movedParentId = randomUUID()
+ const fileId = randomUUID()
+ const userId = randomUUID()
+ const teamId = randomUUID()
+ try {
+ await client.query('BEGIN')
+ await client.query('SET ROLE gfs_controller')
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, name, kind, path_cache)
+ VALUES ($1, $2, '/', 'directory', '/')`,
+ [rootId, drive]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'parent', 'directory', '/parent')`,
+ [parentId, drive, rootId]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'destination', 'directory', '/destination')`,
+ [movedParentId, drive, rootId]
+ )
+ await client.query(
+ `INSERT INTO gfs_resources (resource_id, drive, parent_resource_id, name, kind, path_cache)
+ VALUES ($1, $2, $3, 'draft.md', 'file', '/parent/draft.md')`,
+ [fileId, drive, parentId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(
+ `INSERT INTO gfs_grants (drive, resource_id, subject_type, subject_id, permissions)
+ VALUES ($1, $2, 'team', $3, ARRAY['read'])`,
+ [drive, fileId, teamId]
+ )
+ await client.query(
+ `INSERT INTO gfs_shares (drive, resource_id, subject_type, subject_id, permissions)
+ VALUES ($1, $2, 'user', $3, ARRAY['read'])`,
+ [drive, fileId, userId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ const created = await instancePool.query<{ count: string }>(
+ `SELECT count(*)::text AS count FROM entity_change_outbox WHERE scope = 'gfs'`
+ )
+ expect(Number(created.rows[0]?.count)).toBe(4)
+
+ await client.query('BEGIN')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(
+ `UPDATE gfs_resources SET version = version + 1, bytes = 45 WHERE resource_id = $1`,
+ [fileId]
+ )
+ await client.query(
+ `UPDATE gfs_resources SET parent_resource_id = $2, name = 'renamed.md',
+ path_cache = '/destination/renamed.md' WHERE resource_id = $1`,
+ [fileId, movedParentId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('SET ROLE gfs_controller')
+ await client.query(
+ `UPDATE gfs_resources SET deleted_at = clock_timestamp() WHERE resource_id = $1`,
+ [fileId]
+ )
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ await client.query('BEGIN')
+ await client.query('SET ROLE control_api_runtime')
+ await client.query(`INSERT INTO users (id, email) VALUES ($1, $2)`, [
+ userId,
+ `${userId}@entity-change.invalid`,
+ ])
+ await client.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])
+ await client.query(
+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,
+ [teamId, userId]
+ )
+ await client.query('DELETE FROM team_members WHERE team_id = $1 AND user_id = $2', [
+ teamId,
+ userId,
+ ])
+ await client.query('DELETE FROM gfs_grants WHERE drive = $1 AND resource_id = $2', [
+ drive,
+ fileId,
+ ])
+ await client.query('DELETE FROM gfs_shares WHERE drive = $1 AND resource_id = $2', [
+ drive,
+ fileId,
+ ])
+ await client.query('RESET ROLE')
+ await client.query('COMMIT')
+
+ const outbox = await instancePool.query<{ scope: string; entity_id: string | null }>(
+ 'SELECT scope, entity_id FROM entity_change_outbox ORDER BY id'
+ )
+ expect(outbox.rows.filter(row => row.scope === 'gfs')).toHaveLength(7)
+ expect(outbox.rows.filter(row => row.entity_id === fileId)).toHaveLength(4)
+ expect(outbox.rows.filter(row => row.scope === 'authorization')).toHaveLength(7)
+ // Event evidence contains no file content, path, or authorization subject.
+ expect(JSON.stringify(outbox.rows)).not.toContain('draft.md')
+ expect(JSON.stringify(outbox.rows)).not.toContain(userId)
+ expect(JSON.stringify(outbox.rows)).not.toContain(teamId)
+ } finally {
+ client.release()
+ }
+ })
+
+ it('rolls capture back with the producer transaction and publishes only committed facts', async () => {
+ const client = await instancePool.connect()
+ const drive = `entity-change-rollback-${randomUUID()}`
+ try {
+ const before = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ await client.query('BEGIN')
+ await client.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'rolled-back', 'file')`,
+ [drive]
+ )
+ await client.query('ROLLBACK')
+ const after = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_outbox'
+ )
+ expect(after.rows[0]?.count).toBe(before.rows[0]?.count)
+
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'committed', 'file')`,
+ [drive]
+ )
+ const firstDispatch = await instancePool.query<{ feed_sequence: string }>(
+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'
+ )
+ const feedAfterFirst = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_feed'
+ )
+ const duplicateDispatch = await instancePool.query<{ feed_sequence: string }>(
+ 'SELECT feed_sequence FROM entity_change_dispatch_batch(1000, 86400)'
+ )
+ const feedAfterDuplicate = await instancePool.query<{ count: string }>(
+ 'SELECT count(*)::text AS count FROM entity_change_feed'
+ )
+ expect(firstDispatch.rows).toHaveLength(1)
+ expect(duplicateDispatch.rows).toHaveLength(0)
+ expect(feedAfterDuplicate.rows[0]?.count).toBe(feedAfterFirst.rows[0]?.count)
+ } finally {
+ client.release()
+ }
+ })
+
+ it('uses dispatcher order rather than identity allocation as public sequence order', async () => {
+ const pending = await instancePool.connect()
+ const earlyDrive = `entity-change-early-${randomUUID()}`
+ try {
+ await pending.query('BEGIN')
+ await pending.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'early', 'file')`,
+ [earlyDrive]
+ )
+ const pendingId = await pending.query<{ id: string }>(
+ `SELECT max(id)::text AS id FROM entity_change_outbox WHERE entity_id IS NOT NULL`
+ )
+ const watermarkBefore = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ const userId = randomUUID()
+ const teamId = randomUUID()
+ await instancePool.query('INSERT INTO users (id, email) VALUES ($1, $2)', [
+ userId,
+ `${userId}@ordering.invalid`,
+ ])
+ await instancePool.query('INSERT INTO teams (id, name) VALUES ($1, $2)', [teamId, teamId])
+ await instancePool.query(
+ `INSERT INTO team_members (team_id, user_id, role) VALUES ($1, $2, 'member')`,
+ [teamId, userId]
+ )
+ const committedMax = await instancePool.query<{ id: string }>(
+ 'SELECT max(id)::text AS id FROM entity_change_outbox'
+ )
+ expect(Number(pendingId.rows[0]?.id)).toBeLessThan(Number(committedMax.rows[0]?.id))
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const firstFeed = await instancePool.query<{ sequence: string; scope: string }>(
+ `SELECT sequence::text, scope FROM entity_change_feed
+ WHERE sequence > $1 ORDER BY sequence`,
+ [watermarkBefore.rows[0]?.sequence]
+ )
+ expect(firstFeed.rows.map(row => row.scope)).toEqual(['authorization'])
+ await pending.query('COMMIT')
+ await replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const feed = await instancePool.query<{ sequence: string; scope: string }>(
+ `SELECT sequence::text, scope FROM entity_change_feed
+ WHERE sequence > $1 ORDER BY sequence`,
+ [watermarkBefore.rows[0]?.sequence]
+ )
+ expect(feed.rows.map(row => row.scope)).toEqual(['authorization', 'gfs'])
+ expect(feed.rows.map(row => Number(row.sequence))).toEqual([
+ Number(watermarkBefore.rows[0]?.sequence) + 1,
+ Number(watermarkBefore.rows[0]?.sequence) + 2,
+ ])
+ } finally {
+ await pending.query('ROLLBACK').catch(() => undefined)
+ pending.release()
+ }
+ })
+
+ it('keeps checkpoint scopes and cursor on one committed watermark snapshot', async () => {
+ const drive = `entity-change-checkpoint-lock-${randomUUID()}`
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint-lock', 'file')`,
+ [drive]
+ )
+ const baseline = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'pending-change', 'file')`,
+ [`${drive}-pending`]
+ )
+
+ const dispatcher = await instancePool.connect()
+ const reader = await replicaPool.connect()
+ let checkpointPending: Promise<{
+ rows: Array<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>
+ }> | null = null
+ try {
+ await dispatcher.query('BEGIN')
+ await dispatcher.query(
+ 'SELECT sequence FROM entity_change_watermark WHERE singleton = true FOR UPDATE'
+ )
+ const readerPid = await reader.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')
+ let settled = false
+ checkpointPending = reader
+ .query<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ baseline.rows[0]?.current_cursor,
+ 10000,
+ ])
+ .then(result => {
+ settled = true
+ return result
+ })
+
+ let waitingOnWatermark = false
+ for (let attempt = 0; attempt < 80; attempt += 1) {
+ const activity = await adminPool.query<{ wait_event_type: string | null }>(
+ 'SELECT wait_event_type FROM pg_stat_activity WHERE pid = $1',
+ [readerPid.rows[0]?.pid]
+ )
+ if (activity.rows[0]?.wait_event_type === 'Lock') {
+ waitingOnWatermark = true
+ break
+ }
+ if (settled) break
+ await new Promise(resolve => setTimeout(resolve, 25))
+ }
+ expect(waitingOnWatermark).toBe(true)
+ expect(settled).toBe(false)
+
+ await dispatcher.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ await dispatcher.query('COMMIT')
+ const checkpoint = await checkpointPending
+ expect(checkpoint.rows[0]?.needs_resync).toBe(false)
+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])
+ const watermark = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ expect(checkpoint.rows[0]?.current_cursor).toBe(watermark.rows[0]?.current_cursor)
+ } finally {
+ await dispatcher.query('ROLLBACK').catch(() => undefined)
+ await checkpointPending?.catch(() => undefined)
+ dispatcher.release()
+ reader.release()
+ }
+ })
+
+ it('recovers missed wakeups and expired cursors with a coarse authorized checkpoint', async () => {
+ const resource = await instancePool.query<{ resource_id: string }>(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint', 'file')
+ RETURNING resource_id`,
+ [`entity-change-checkpoint-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const staleEvent = await instancePool.query<{ cursor: string; sequence: string }>(
+ 'SELECT cursor::text, sequence::text FROM entity_change_feed ORDER BY sequence DESC LIMIT 1'
+ )
+ // Advance the watermark beyond the cursor that will expire. Otherwise the
+ // cursor under test is itself the current watermark, which correctly means
+ // the client is caught up and must not receive a resync.
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint-newer', 'file')`,
+ [`entity-change-checkpoint-newer-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const current = await instancePool.query<{ current_cursor: string; sequence: string }>(
+ `SELECT current_cursor::text, sequence::text
+ FROM entity_change_watermark WHERE singleton = true`
+ )
+ expect(current.rows[0]?.current_cursor).not.toBe(staleEvent.rows[0]?.cursor)
+
+ // No listener is involved here: the durable query is the fallback after a missed NOTIFY.
+ const recovery = await instancePool.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ staleEvent.rows[0]?.cursor,
+ 10000,
+ ])
+ expect(recovery.rows[0]?.needs_resync).toBe(false)
+
+ await instancePool.query(
+ `UPDATE entity_change_feed
+ SET created_at = clock_timestamp() - interval '2 days'
+ WHERE sequence <= $1`,
+ [current.rows[0]?.sequence]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const expired = await instancePool.query<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ staleEvent.rows[0]?.cursor,
+ 10000,
+ ])
+ expect(expired.rows[0]?.needs_resync).toBe(true)
+ expect(expired.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])
+ expect(JSON.stringify(expired.rows)).not.toContain(resource.rows[0]?.resource_id)
+
+ // The resync response advances to the current watermark. Reusing that
+ // server-issued cursor is caught up even when retention has pruned every
+ // corresponding feed row; an idle client must not receive resync forever.
+ const returnedCursor = expired.rows[0]?.current_cursor
+ expect(returnedCursor).toBeTruthy()
+ for (let attempt = 0; attempt < 3; attempt += 1) {
+ const resumed = await instancePool.query<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [returnedCursor, 10000])
+ expect(resumed.rows[0]?.needs_resync).toBe(false)
+ expect(resumed.rows[0]?.current_cursor).toBe(returnedCursor)
+ expect(resumed.rows[0]?.invalidated_scopes).toEqual([])
+ }
+
+ const freshClient = await instancePool.query<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ '00000000-0000-0000-0000-000000000000',
+ 10000,
+ ])
+ expect(freshClient.rows[0]?.needs_resync).toBe(true)
+ expect(freshClient.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])
+ })
+
+ it('coalesces latest-state invalidations and keeps feed storage behind the runtime boundary', async () => {
+ // Establish a retained cursor in this test instead of depending on the
+ // preceding expiry test's intentionally pruned watermark cursor.
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'coalesce-baseline', 'file')`,
+ [`entity-change-coalesce-baseline-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const baseline = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ const inserted = await instancePool.query<{ resource_id: string }>(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'many-updates', 'file')
+ RETURNING resource_id`,
+ [`entity-change-coalesce-${randomUUID()}`]
+ )
+ const resourceId = inserted.rows[0]?.resource_id
+ await instancePool.query(
+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',
+ [resourceId]
+ )
+ await instancePool.query(
+ 'UPDATE gfs_resources SET version = version + 1 WHERE resource_id = $1',
+ [resourceId]
+ )
+ const before = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await Promise.all([
+ instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),
+ replicaPool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)'),
+ ])
+ const after = await instancePool.query<{ sequence: string }>(
+ 'SELECT sequence::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ const published = await instancePool.query<{ scope: string; count: string }>(
+ `SELECT scope, count(*)::text AS count FROM entity_change_feed
+ WHERE sequence > $1 GROUP BY scope`,
+ [before.rows[0]?.sequence]
+ )
+ expect(Number(after.rows[0]?.sequence) - Number(before.rows[0]?.sequence)).toBe(1)
+ expect(published.rows).toEqual([{ scope: 'gfs', count: '1' }])
+
+ const restricted = await instancePool.connect()
+ try {
+ await restricted.query('SET ROLE control_api_runtime')
+ await expect(restricted.query('SELECT * FROM entity_change_feed')).rejects.toThrow()
+ const checkpoint = await restricted.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ baseline.rows[0]?.current_cursor,
+ 10000,
+ ])
+ expect(checkpoint.rows[0]?.needs_resync).toBe(false)
+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])
+ expect(Object.keys(checkpoint.rows[0] ?? {}).sort()).toEqual([
+ 'current_cursor',
+ 'current_sequence',
+ 'invalidated_scopes',
+ 'needs_resync',
+ ])
+ } finally {
+ await restricted.query('RESET ROLE').catch(() => undefined)
+ restricted.release()
+ }
+ })
+
+ it('resyncs instead of scanning an oversized retained replay window', async () => {
+ const baseline = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind)
+ VALUES ($1, 'bounded-window', 'file')`,
+ [`entity-change-bounded-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind)
+ VALUES ($1, 'bounded-window-2', 'file')`,
+ [`entity-change-bounded-${randomUUID()}`]
+ )
+ await instancePool.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ const bounded = await instancePool.query<{
+ needs_resync: boolean
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ baseline.rows[0]?.current_cursor,
+ 1,
+ ])
+ expect(bounded.rows[0]?.needs_resync).toBe(true)
+ expect(bounded.rows[0]?.invalidated_scopes).toEqual(['gfs', 'authorization'])
+ })
+ })