Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): r4-m2 treat fatal commit responses as unknown

Josué1a1944b754a1
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 62861e1d1..1c36f4bae 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.754",
+ "version": "0.1.755",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.754",
+ "version": "0.1.755",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 7c81030c1..b8e823509 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.754",
+ "version": "0.1.755",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 264264f68..70e4c046c 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -6747,7 +6747,7 @@ function isDefinitivePostgresCommitErrorResponse(error: unknown): boolean {
response.name === 'error' &&
typeof response.code === 'string' &&
/^[0-9A-Z]{5}$/.test(response.code) &&
- typeof response.severity === 'string' &&
+ response.severity === 'ERROR' &&
!UNKNOWN_COMMIT_OUTCOME_SQLSTATES.has(response.code)
)
}
@@ -6905,10 +6905,9 @@ export async function withTransaction<T>(
releaseError = rollbackError instanceof Error ? rollbackError : true
}
}
- // Most PostgreSQL ErrorResponses prove COMMIT failed, but SQLSTATE 08007
- // (transaction_resolution_unknown) and 40003 (statement_completion_unknown)
- // explicitly leave the durable outcome uncertain. Lost transport and malformed
- // responses are uncertain as well.
+ // Only an ERROR-severity PostgreSQL response proves COMMIT was rejected.
+ // FATAL/PANIC may terminate a backend after COMMIT succeeded while its
+ // acknowledgement is withheld, so those and SQLSTATE 08007/40003 stay unknown.
if (commitSent && !transactionFinished && !isDefinitivePostgresCommitErrorResponse(error)) {
options.onCommitOutcomeUnknown?.()
}
diff --git a/control-api/test/helpers/realPostgresFatalCommit.ts b/control-api/test/helpers/realPostgresFatalCommit.ts
new file mode 100644
index 000000000..f7c3afd91
--- /dev/null
+++ b/control-api/test/helpers/realPostgresFatalCommit.ts
@@ -0,0 +1,163 @@
+import { once } from 'node:events'
+import { type Socket, createConnection, createServer } from 'node:net'
+
+type ErrorFields = {
+ code: string | null
+ severity: string | null
+ message: string | null
+}
+
+function parseErrorFields(frame: Buffer): ErrorFields {
+ const fields: ErrorFields = { code: null, severity: null, message: null }
+ let offset = 5
+ while (offset < frame.length && frame[offset] !== 0) {
+ const field = String.fromCharCode(frame[offset]!)
+ const end = frame.indexOf(0, offset + 1)
+ if (end < 0) break
+ const value = frame.toString('utf8', offset + 1, end)
+ if (field === 'C') fields.code = value
+ if (field === 'S') fields.severity = value
+ if (field === 'M') fields.message = value
+ offset = end + 1
+ }
+ return fields
+}
+
+/** Drop a real successful COMMIT reply, then expose PostgreSQL's backend FATAL. */
+export async function createPostgresFatalCommitReplyInjector(connectionString: string) {
+ const target = new URL(connectionString)
+ const targetHost = target.searchParams.get('host') || target.hostname.replace(/^\[|\]$/g, '')
+ const targetPort = Number(target.searchParams.get('port') || target.port || 5432)
+ const sockets = new Set<Socket>()
+ const commands: string[] = []
+ let commitAckWithheldResolve!: () => void
+ let backendReadyResolve!: (pid: number) => void
+ let fatalResponseResolve!: (fields: ErrorFields) => void
+ const commitAckWithheld = new Promise<void>(resolve => {
+ commitAckWithheldResolve = resolve
+ })
+ const backendReady = new Promise<number>(resolve => {
+ backendReadyResolve = resolve
+ })
+ const fatalResponse = new Promise<ErrorFields>(resolve => {
+ fatalResponseResolve = resolve
+ })
+ let backendPid: number | undefined
+ let withheldCommitReply = false
+
+ const server = createServer({ allowHalfOpen: true }, front => {
+ const back = targetHost.startsWith('/')
+ ? createConnection({
+ path: `${targetHost.replace(/\/$/, '')}/.s.PGSQL.${targetPort}`,
+ allowHalfOpen: true,
+ })
+ : createConnection({ host: targetHost, port: targetPort, allowHalfOpen: true })
+ for (const socket of [front, back]) {
+ sockets.add(socket)
+ socket.once('close', () => sockets.delete(socket))
+ }
+ front.on('error', () => back.destroy())
+ back.on('error', () => front.destroy())
+ front.on('end', () => back.end())
+ back.on('end', () => front.end())
+
+ let clientStartupPending = true
+ let clientBuffer = Buffer.alloc(0)
+ front.on('data', chunk => {
+ clientBuffer = Buffer.concat([clientBuffer, chunk])
+ if (clientStartupPending) {
+ if (clientBuffer.length < 4) return
+ const startupLength = clientBuffer.readInt32BE(0)
+ if (clientBuffer.length < startupLength) return
+ back.write(clientBuffer.subarray(0, startupLength))
+ clientBuffer = clientBuffer.subarray(startupLength)
+ clientStartupPending = false
+ }
+ while (clientBuffer.length >= 5) {
+ const length = clientBuffer.readInt32BE(1)
+ const frameLength = length + 1
+ if (clientBuffer.length < frameLength) break
+ const frame = clientBuffer.subarray(0, frameLength)
+ if (frame[0] === 'Q'.charCodeAt(0)) {
+ const query = frame.toString('utf8', 5, frame.length - 1).trim()
+ if (query === 'BEGIN' || query === 'COMMIT' || query === 'ROLLBACK') {
+ commands.push(query)
+ }
+ if (query === 'COMMIT') withheldCommitReply = true
+ }
+ back.write(frame)
+ clientBuffer = clientBuffer.subarray(frameLength)
+ }
+ })
+
+ let serverBuffer = Buffer.alloc(0)
+ let commitCommandComplete = false
+ back.on('data', chunk => {
+ serverBuffer = Buffer.concat([serverBuffer, chunk])
+ while (serverBuffer.length >= 5) {
+ const length = serverBuffer.readInt32BE(1)
+ const frameLength = length + 1
+ if (serverBuffer.length < frameLength) break
+ const frame = serverBuffer.subarray(0, frameLength)
+ const type = String.fromCharCode(frame[0]!)
+ let forward = true
+ if (type === 'K' && frame.length >= 13) {
+ backendPid = frame.readInt32BE(5)
+ backendReadyResolve(backendPid)
+ }
+ if (
+ withheldCommitReply &&
+ type === 'C' &&
+ frame.toString('utf8', 5, frame.length - 1) === 'COMMIT'
+ ) {
+ commitCommandComplete = true
+ forward = false
+ } else if (withheldCommitReply && type === 'Z' && commitCommandComplete) {
+ withheldCommitReply = false
+ commitCommandComplete = false
+ forward = false
+ commitAckWithheldResolve()
+ }
+ if (type === 'E') {
+ const fields = parseErrorFields(frame)
+ if (
+ fields.code === '57P01' ||
+ fields.severity === 'FATAL' ||
+ fields.severity === 'PANIC'
+ ) {
+ fatalResponseResolve(fields)
+ }
+ }
+ if (forward) front.write(frame)
+ serverBuffer = serverBuffer.subarray(frameLength)
+ }
+ })
+ })
+
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('missing PostgreSQL proxy address')
+ const proxied = new URL(connectionString)
+ proxied.hostname = '127.0.0.1'
+ proxied.port = String(address.port)
+ proxied.searchParams.delete('host')
+ proxied.searchParams.delete('port')
+
+ return {
+ connectionString: proxied.toString(),
+ backendReady,
+ commitAckWithheld,
+ fatalResponse,
+ commands,
+ get backendPid() {
+ return backendPid
+ },
+ close: async () => {
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ },
+ }
+}
diff --git a/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.ts b/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.ts
new file mode 100644
index 000000000..5fb1b5d27
--- /dev/null
+++ b/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.ts
@@ -0,0 +1,204 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import bcrypt from 'bcryptjs'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+import request from 'supertest'
+import { initDb } from '../src/db.js'
+import { sendPasswordAdmissionError } from '../src/middleware/passwordLoginAdmission.js'
+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'
+import {
+ createSilentInvitationForTeams,
+ setInvitationPasswordForEmail,
+} from '../src/services/directory/membership.js'
+import { createPostgresFatalCommitReplyInjector } from './helpers/realPostgresFatalCommit.js'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+
+const holder = vi.hoisted(() => ({ pool: null as unknown as Pool, unknownCommitOutcomes: 0 }))
+
+vi.mock('../src/db.js', async importOriginal => {
+ const real = await importOriginal<typeof import('../src/db.js')>()
+ const proxy = {
+ query: (...args: unknown[]) => (holder.pool.query as Function)(...args),
+ connect: () => holder.pool.connect(),
+ }
+ return {
+ ...real,
+ pool: proxy,
+ corePool: proxy,
+ rateLimitPool: proxy,
+ withTransaction: (
+ work: Parameters<typeof real.withTransaction>[0],
+ _txPool?: Parameters<typeof real.withTransaction>[1],
+ options?: Parameters<typeof real.withTransaction>[2]
+ ) =>
+ real.withTransaction(work, holder.pool, {
+ ...options,
+ onCommitOutcomeUnknown: () => {
+ holder.unknownCommitOutcomes += 1
+ options?.onCommitOutcomeUnknown?.()
+ },
+ }),
+ }
+})
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const realPg = adminUrl ? describe : describe.skip
+
+realPg('R4-M2 PostgreSQL session termination after COMMIT dispatch', () => {
+ const database = `r4m2_fatal_commit_${randomBytes(6).toString('hex')}`
+ const email = `r4m2-${randomBytes(8).toString('hex')}@example.invalid`
+ const originalPassword = 'Synthetic-R4-M2-Original-Password'
+ const newPassword = 'Synthetic-R4-M2-Replacement-Password'
+ let admin: Pool
+ let observer: Pool
+ let normalPool: Pool
+ let proxyPool: Pool | undefined
+ let fault: Awaited<ReturnType<typeof createPostgresFatalCommitReplyInjector>> | undefined
+ let connectionString: string
+ let resetInvitationId: string
+ let siblingInvitationId: string
+ let userId: string
+ let originalHash: string
+
+ beforeAll(async () => {
+ admin = new Pool({ connectionString: adminUrl })
+ await admin.query(`CREATE DATABASE "${database}"`)
+ const databaseUrl = new URL(adminUrl!)
+ databaseUrl.pathname = `/${database}`
+ connectionString = databaseUrl.toString()
+ normalPool = new Pool({ connectionString })
+ observer = new Pool({ connectionString })
+ holder.pool = normalPool
+ await initDb(normalPool)
+
+ originalHash = await bcrypt.hash(originalPassword, 12)
+ const user = await normalPool.query(
+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',
+ [email, originalHash]
+ )
+ userId = user.rows[0].id as string
+ const primary = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R4-M2 member',
+ email,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+ const sibling = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R4-M2 member',
+ email,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+ resetInvitationId = primary.id
+ siblingInvitationId = sibling.id
+ }, 60_000)
+
+ afterAll(async () => {
+ try {
+ await fault?.close()
+ if (proxyPool) await endPoolAndWaitForClients(proxyPool)
+ if (normalPool) await endPoolAndWaitForClients(normalPool)
+ if (observer) await endPoolAndWaitForClients(observer)
+ if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)
+ } finally {
+ await admin?.end()
+ }
+ })
+
+ it('treats a post-COMMIT FATAL as unknown and does not issue a replay session', async () => {
+ holder.unknownCommitOutcomes = 0
+ const lease = await acquirePasswordWork()
+ expect(lease).not.toBeNull()
+ if (!lease) throw new Error('password-work owner was not acquired')
+
+ fault = await createPostgresFatalCommitReplyInjector(connectionString)
+ proxyPool = new Pool({ connectionString: fault.connectionString, max: 2 })
+ holder.pool = proxyPool
+
+ const resetOutcome = setInvitationPasswordForEmail(
+ email,
+ resetInvitationId,
+ newPassword,
+ userId,
+ lease
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+
+ const backendPid = await fault.backendReady
+ await fault.commitAckWithheld
+ const committed = await observer.query(
+ `SELECT primary_invite.status AS reset_status,
+ sibling.status AS sibling_status,
+ member_user.password_hash,
+ (SELECT COUNT(*)::int FROM password_verification_work) AS owners
+ FROM invitations primary_invite
+ JOIN invitations sibling ON sibling.id = $2
+ JOIN users member_user ON member_user.id = $3
+ WHERE primary_invite.id = $1`,
+ [resetInvitationId, siblingInvitationId, userId]
+ )
+ expect(committed.rows[0]).toMatchObject({
+ reset_status: 'accepted',
+ sibling_status: 'revoked',
+ owners: 0,
+ })
+ expect(await bcrypt.compare(newPassword, committed.rows[0].password_hash as string)).toBe(true)
+
+ const terminated = await observer.query<{ terminated: boolean }>(
+ 'SELECT pg_terminate_backend($1) AS terminated',
+ [backendPid]
+ )
+ expect(terminated.rows[0]?.terminated).toBe(true)
+ expect(await fault.fatalResponse).toMatchObject({ code: '57P01', severity: 'FATAL' })
+
+ const outcome = await resetOutcome
+ expect(outcome).toMatchObject({
+ error: {
+ status: 503,
+ reason: 'authority_failure',
+ publicError: 'recovery_outcome_unknown',
+ },
+ })
+ expect(holder.unknownCommitOutcomes).toBe(1)
+ expect(fault.commands.filter(command => command === 'COMMIT')).toHaveLength(1)
+
+ const publicApp = express()
+ publicApp.post('/password-reset', (_req, res) => {
+ if ('error' in outcome && sendPasswordAdmissionError(outcome.error, res)) return
+ res.status(500).json({ error: 'unexpected test outcome' })
+ })
+ const publicResponse = await request(publicApp).post('/password-reset').send({})
+ expect(publicResponse.status).toBe(503)
+ expect(publicResponse.body).toEqual({
+ error: 'recovery_outcome_unknown',
+ retryAfterSeconds: 2,
+ })
+ expect(publicResponse.headers['retry-after']).toBe('2')
+ expect(publicResponse.headers['cache-control']).toBe('no-store')
+ expect(publicResponse.headers['set-cookie']).toBeUndefined()
+
+ holder.pool = normalPool
+ const replayLease = await acquirePasswordWork()
+ expect(replayLease).not.toBeNull()
+ if (!replayLease) throw new Error('password-work owner was not reacquired')
+ const replay = await setInvitationPasswordForEmail(
+ email,
+ resetInvitationId,
+ newPassword,
+ userId,
+ replayLease
+ )
+ expect(replay).toEqual({ error: 'not_pending' })
+ expect(
+ await observer.query('SELECT status FROM invitations WHERE id = $1', [resetInvitationId])
+ ).toMatchObject({ rows: [{ status: 'accepted' }] })
+ expect(
+ await observer.query('SELECT COUNT(*)::int AS count FROM password_verification_work')
+ ).toMatchObject({ rows: [{ count: 0 }] })
+ }, 30_000)
+})