Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): keep change checkpoints snapshot-consistent

Josué752ce7f03b4b
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index aa041be5d..07501440f 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.605",
+ "version": "0.1.606",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.605",
+ "version": "0.1.606",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index adeb0d846..0f2f7f103 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.605",
+ "version": "0.1.606",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/entityChangeSchema.ts b/control-api/src/services/entityChangeSchema.ts
index 5a3e9b8df..47ecac513 100644
--- a/control-api/src/services/entityChangeSchema.ts
+++ b/control-api/src/services/entityChangeSchema.ts
@@ -248,7 +248,9 @@ export async function applyEntityChangeSchema(db: DbClient): Promise<void> {
END IF;
SELECT watermark.sequence, watermark.pruned_through, watermark.current_cursor
INTO current_watermark, pruned_watermark, current_cursor
- FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;
+ FROM public.entity_change_watermark watermark
+ WHERE watermark.singleton = true
+ FOR SHARE;
IF requested_cursor IS NULL THEN
needs_resync := true;
current_sequence := current_watermark;
@@ -280,8 +282,6 @@ export async function applyEntityChangeSchema(db: DbClient): Promise<void> {
COALESCE(max(sequence), cursor_sequence)
INTO invalidated_scopes, current_sequence
FROM public.entity_change_feed WHERE sequence > cursor_sequence;
- SELECT watermark.current_cursor INTO current_cursor
- FROM public.entity_change_watermark watermark WHERE watermark.singleton = true;
RETURN NEXT;
END;
$$;
diff --git a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
index fa5ab9884..829e23a24 100644
--- a/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
+++ b/control-api/test/entityChangeFeed.realPostgres.integration.test.ts
@@ -263,6 +263,83 @@ describeRealPostgres('entity change feed real PostgreSQL contract', () => {
}
})
+ it('keeps checkpoint scopes and cursor on one committed watermark snapshot', async () => {
+ const drive = `entity-change-checkpoint-lock-${randomUUID()}`
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint-lock', 'file')`,
+ [drive]
+ )
+ const baseline = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ await instancePool.query(
+ `INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'pending-change', 'file')`,
+ [`${drive}-pending`]
+ )
+
+ const dispatcher = await instancePool.connect()
+ const reader = await replicaPool.connect()
+ let checkpointPending: Promise<{
+ rows: Array<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>
+ }> | null = null
+ try {
+ await dispatcher.query('BEGIN')
+ await dispatcher.query(
+ 'SELECT sequence FROM entity_change_watermark WHERE singleton = true FOR UPDATE'
+ )
+ const readerPid = await reader.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')
+ let settled = false
+ checkpointPending = reader
+ .query<{
+ needs_resync: boolean
+ current_cursor: string
+ invalidated_scopes: string[]
+ }>('SELECT * FROM entity_change_read_checkpoint($1::uuid, $2)', [
+ baseline.rows[0]?.current_cursor,
+ 10000,
+ ])
+ .then(result => {
+ settled = true
+ return result
+ })
+
+ let waitingOnWatermark = false
+ for (let attempt = 0; attempt < 80; attempt += 1) {
+ const activity = await adminPool.query<{ wait_event_type: string | null }>(
+ 'SELECT wait_event_type FROM pg_stat_activity WHERE pid = $1',
+ [readerPid.rows[0]?.pid]
+ )
+ if (activity.rows[0]?.wait_event_type === 'Lock') {
+ waitingOnWatermark = true
+ break
+ }
+ if (settled) break
+ await new Promise(resolve => setTimeout(resolve, 25))
+ }
+ expect(waitingOnWatermark).toBe(true)
+ expect(settled).toBe(false)
+
+ await dispatcher.query('SELECT * FROM entity_change_dispatch_batch(1000, 86400)')
+ await dispatcher.query('COMMIT')
+ const checkpoint = await checkpointPending
+ expect(checkpoint.rows[0]?.needs_resync).toBe(false)
+ expect(checkpoint.rows[0]?.invalidated_scopes).toEqual(['gfs'])
+ const watermark = await instancePool.query<{ current_cursor: string }>(
+ 'SELECT current_cursor::text FROM entity_change_watermark WHERE singleton = true'
+ )
+ expect(checkpoint.rows[0]?.current_cursor).toBe(watermark.rows[0]?.current_cursor)
+ } finally {
+ await dispatcher.query('ROLLBACK').catch(() => undefined)
+ await checkpointPending?.catch(() => undefined)
+ dispatcher.release()
+ reader.release()
+ }
+ })
+
it('recovers missed wakeups and expired cursors with a coarse authorized checkpoint', async () => {
const resource = await instancePool.query<{ resource_id: string }>(
`INSERT INTO gfs_resources (drive, name, kind) VALUES ($1, 'checkpoint', 'file')