Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): r4-m2 treat fatal commit responses as unknown
Josué1a1944b754a1
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 62861e1d1..1c36f4bae 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.754",+ "version": "0.1.755","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.754",+ "version": "0.1.755","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 7c81030c1..b8e823509 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.754",+ "version": "0.1.755","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 264264f68..70e4c046c 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -6747,7 +6747,7 @@ function isDefinitivePostgresCommitErrorResponse(error: unknown): boolean {response.name === 'error' &&typeof response.code === 'string' &&/^[0-9A-Z]{5}$/.test(response.code) &&- typeof response.severity === 'string' &&+ response.severity === 'ERROR' &&!UNKNOWN_COMMIT_OUTCOME_SQLSTATES.has(response.code))}@@ -6905,10 +6905,9 @@ export async function withTransaction<T>(releaseError = rollbackError instanceof Error ? rollbackError : true}}- // Most PostgreSQL ErrorResponses prove COMMIT failed, but SQLSTATE 08007- // (transaction_resolution_unknown) and 40003 (statement_completion_unknown)- // explicitly leave the durable outcome uncertain. Lost transport and malformed- // responses are uncertain as well.+ // Only an ERROR-severity PostgreSQL response proves COMMIT was rejected.+ // FATAL/PANIC may terminate a backend after COMMIT succeeded while its+ // acknowledgement is withheld, so those and SQLSTATE 08007/40003 stay unknown.if (commitSent && !transactionFinished && !isDefinitivePostgresCommitErrorResponse(error)) {options.onCommitOutcomeUnknown?.()}diff --git a/control-api/test/helpers/realPostgresFatalCommit.ts b/control-api/test/helpers/realPostgresFatalCommit.tsnew file mode 100644index 000000000..f7c3afd91--- /dev/null+++ b/control-api/test/helpers/realPostgresFatalCommit.ts@@ -0,0 +1,163 @@+import { once } from 'node:events'+import { type Socket, createConnection, createServer } from 'node:net'++type ErrorFields = {+ code: string | null+ severity: string | null+ message: string | null+}++function parseErrorFields(frame: Buffer): ErrorFields {+ const fields: ErrorFields = { code: null, severity: null, message: null }+ let offset = 5+ while (offset < frame.length && frame[offset] !== 0) {+ const field = String.fromCharCode(frame[offset]!)+ const end = frame.indexOf(0, offset + 1)+ if (end < 0) break+ const value = frame.toString('utf8', offset + 1, end)+ if (field === 'C') fields.code = value+ if (field === 'S') fields.severity = value+ if (field === 'M') fields.message = value+ offset = end + 1+ }+ return fields+}++/** Drop a real successful COMMIT reply, then expose PostgreSQL's backend FATAL. */+export async function createPostgresFatalCommitReplyInjector(connectionString: string) {+ const target = new URL(connectionString)+ const targetHost = target.searchParams.get('host') || target.hostname.replace(/^\[|\]$/g, '')+ const targetPort = Number(target.searchParams.get('port') || target.port || 5432)+ const sockets = new Set<Socket>()+ const commands: string[] = []+ let commitAckWithheldResolve!: () => void+ let backendReadyResolve!: (pid: number) => void+ let fatalResponseResolve!: (fields: ErrorFields) => void+ const commitAckWithheld = new Promise<void>(resolve => {+ commitAckWithheldResolve = resolve+ })+ const backendReady = new Promise<number>(resolve => {+ backendReadyResolve = resolve+ })+ const fatalResponse = new Promise<ErrorFields>(resolve => {+ fatalResponseResolve = resolve+ })+ let backendPid: number | undefined+ let withheldCommitReply = false++ const server = createServer({ allowHalfOpen: true }, front => {+ const back = targetHost.startsWith('/')+ ? createConnection({+ path: `${targetHost.replace(/\/$/, '')}/.s.PGSQL.${targetPort}`,+ allowHalfOpen: true,+ })+ : createConnection({ host: targetHost, port: targetPort, allowHalfOpen: true })+ for (const socket of [front, back]) {+ sockets.add(socket)+ socket.once('close', () => sockets.delete(socket))+ }+ front.on('error', () => back.destroy())+ back.on('error', () => front.destroy())+ front.on('end', () => back.end())+ back.on('end', () => front.end())++ let clientStartupPending = true+ let clientBuffer = Buffer.alloc(0)+ front.on('data', chunk => {+ clientBuffer = Buffer.concat([clientBuffer, chunk])+ if (clientStartupPending) {+ if (clientBuffer.length < 4) return+ const startupLength = clientBuffer.readInt32BE(0)+ if (clientBuffer.length < startupLength) return+ back.write(clientBuffer.subarray(0, startupLength))+ clientBuffer = clientBuffer.subarray(startupLength)+ clientStartupPending = false+ }+ while (clientBuffer.length >= 5) {+ const length = clientBuffer.readInt32BE(1)+ const frameLength = length + 1+ if (clientBuffer.length < frameLength) break+ const frame = clientBuffer.subarray(0, frameLength)+ if (frame[0] === 'Q'.charCodeAt(0)) {+ const query = frame.toString('utf8', 5, frame.length - 1).trim()+ if (query === 'BEGIN' || query === 'COMMIT' || query === 'ROLLBACK') {+ commands.push(query)+ }+ if (query === 'COMMIT') withheldCommitReply = true+ }+ back.write(frame)+ clientBuffer = clientBuffer.subarray(frameLength)+ }+ })++ let serverBuffer = Buffer.alloc(0)+ let commitCommandComplete = false+ back.on('data', chunk => {+ serverBuffer = Buffer.concat([serverBuffer, chunk])+ while (serverBuffer.length >= 5) {+ const length = serverBuffer.readInt32BE(1)+ const frameLength = length + 1+ if (serverBuffer.length < frameLength) break+ const frame = serverBuffer.subarray(0, frameLength)+ const type = String.fromCharCode(frame[0]!)+ let forward = true+ if (type === 'K' && frame.length >= 13) {+ backendPid = frame.readInt32BE(5)+ backendReadyResolve(backendPid)+ }+ if (+ withheldCommitReply &&+ type === 'C' &&+ frame.toString('utf8', 5, frame.length - 1) === 'COMMIT'+ ) {+ commitCommandComplete = true+ forward = false+ } else if (withheldCommitReply && type === 'Z' && commitCommandComplete) {+ withheldCommitReply = false+ commitCommandComplete = false+ forward = false+ commitAckWithheldResolve()+ }+ if (type === 'E') {+ const fields = parseErrorFields(frame)+ if (+ fields.code === '57P01' ||+ fields.severity === 'FATAL' ||+ fields.severity === 'PANIC'+ ) {+ fatalResponseResolve(fields)+ }+ }+ if (forward) front.write(frame)+ serverBuffer = serverBuffer.subarray(frameLength)+ }+ })+ })++ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('missing PostgreSQL proxy address')+ const proxied = new URL(connectionString)+ proxied.hostname = '127.0.0.1'+ proxied.port = String(address.port)+ proxied.searchParams.delete('host')+ proxied.searchParams.delete('port')++ return {+ connectionString: proxied.toString(),+ backendReady,+ commitAckWithheld,+ fatalResponse,+ commands,+ get backendPid() {+ return backendPid+ },+ close: async () => {+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ },+ }+}diff --git a/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.ts b/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.tsnew file mode 100644index 000000000..5fb1b5d27--- /dev/null+++ b/control-api/test/r4m2.passwordResetFatalCommit.realPostgres.integration.test.ts@@ -0,0 +1,204 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import express from 'express'+import bcrypt from 'bcryptjs'+import { randomBytes } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'+import { initDb } from '../src/db.js'+import { sendPasswordAdmissionError } from '../src/middleware/passwordLoginAdmission.js'+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'+import {+ createSilentInvitationForTeams,+ setInvitationPasswordForEmail,+} from '../src/services/directory/membership.js'+import { createPostgresFatalCommitReplyInjector } from './helpers/realPostgresFatalCommit.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'++const holder = vi.hoisted(() => ({ pool: null as unknown as Pool, unknownCommitOutcomes: 0 }))++vi.mock('../src/db.js', async importOriginal => {+ const real = await importOriginal<typeof import('../src/db.js')>()+ const proxy = {+ query: (...args: unknown[]) => (holder.pool.query as Function)(...args),+ connect: () => holder.pool.connect(),+ }+ return {+ ...real,+ pool: proxy,+ corePool: proxy,+ rateLimitPool: proxy,+ withTransaction: (+ work: Parameters<typeof real.withTransaction>[0],+ _txPool?: Parameters<typeof real.withTransaction>[1],+ options?: Parameters<typeof real.withTransaction>[2]+ ) =>+ real.withTransaction(work, holder.pool, {+ ...options,+ onCommitOutcomeUnknown: () => {+ holder.unknownCommitOutcomes += 1+ options?.onCommitOutcomeUnknown?.()+ },+ }),+ }+})++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const realPg = adminUrl ? describe : describe.skip++realPg('R4-M2 PostgreSQL session termination after COMMIT dispatch', () => {+ const database = `r4m2_fatal_commit_${randomBytes(6).toString('hex')}`+ const email = `r4m2-${randomBytes(8).toString('hex')}@example.invalid`+ const originalPassword = 'Synthetic-R4-M2-Original-Password'+ const newPassword = 'Synthetic-R4-M2-Replacement-Password'+ let admin: Pool+ let observer: Pool+ let normalPool: Pool+ let proxyPool: Pool | undefined+ let fault: Awaited<ReturnType<typeof createPostgresFatalCommitReplyInjector>> | undefined+ let connectionString: string+ let resetInvitationId: string+ let siblingInvitationId: string+ let userId: string+ let originalHash: string++ beforeAll(async () => {+ admin = new Pool({ connectionString: adminUrl })+ await admin.query(`CREATE DATABASE "${database}"`)+ const databaseUrl = new URL(adminUrl!)+ databaseUrl.pathname = `/${database}`+ connectionString = databaseUrl.toString()+ normalPool = new Pool({ connectionString })+ observer = new Pool({ connectionString })+ holder.pool = normalPool+ await initDb(normalPool)++ originalHash = await bcrypt.hash(originalPassword, 12)+ const user = await normalPool.query(+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',+ [email, originalHash]+ )+ userId = user.rows[0].id as string+ const primary = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R4-M2 member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const sibling = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R4-M2 member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ resetInvitationId = primary.id+ siblingInvitationId = sibling.id+ }, 60_000)++ afterAll(async () => {+ try {+ await fault?.close()+ if (proxyPool) await endPoolAndWaitForClients(proxyPool)+ if (normalPool) await endPoolAndWaitForClients(normalPool)+ if (observer) await endPoolAndWaitForClients(observer)+ if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)+ } finally {+ await admin?.end()+ }+ })++ it('treats a post-COMMIT FATAL as unknown and does not issue a replay session', async () => {+ holder.unknownCommitOutcomes = 0+ const lease = await acquirePasswordWork()+ expect(lease).not.toBeNull()+ if (!lease) throw new Error('password-work owner was not acquired')++ fault = await createPostgresFatalCommitReplyInjector(connectionString)+ proxyPool = new Pool({ connectionString: fault.connectionString, max: 2 })+ holder.pool = proxyPool++ const resetOutcome = setInvitationPasswordForEmail(+ email,+ resetInvitationId,+ newPassword,+ userId,+ lease+ ).then(+ value => ({ value }),+ error => ({ error })+ )++ const backendPid = await fault.backendReady+ await fault.commitAckWithheld+ const committed = await observer.query(+ `SELECT primary_invite.status AS reset_status,+ sibling.status AS sibling_status,+ member_user.password_hash,+ (SELECT COUNT(*)::int FROM password_verification_work) AS owners+ FROM invitations primary_invite+ JOIN invitations sibling ON sibling.id = $2+ JOIN users member_user ON member_user.id = $3+ WHERE primary_invite.id = $1`,+ [resetInvitationId, siblingInvitationId, userId]+ )+ expect(committed.rows[0]).toMatchObject({+ reset_status: 'accepted',+ sibling_status: 'revoked',+ owners: 0,+ })+ expect(await bcrypt.compare(newPassword, committed.rows[0].password_hash as string)).toBe(true)++ const terminated = await observer.query<{ terminated: boolean }>(+ 'SELECT pg_terminate_backend($1) AS terminated',+ [backendPid]+ )+ expect(terminated.rows[0]?.terminated).toBe(true)+ expect(await fault.fatalResponse).toMatchObject({ code: '57P01', severity: 'FATAL' })++ const outcome = await resetOutcome+ expect(outcome).toMatchObject({+ error: {+ status: 503,+ reason: 'authority_failure',+ publicError: 'recovery_outcome_unknown',+ },+ })+ expect(holder.unknownCommitOutcomes).toBe(1)+ expect(fault.commands.filter(command => command === 'COMMIT')).toHaveLength(1)++ const publicApp = express()+ publicApp.post('/password-reset', (_req, res) => {+ if ('error' in outcome && sendPasswordAdmissionError(outcome.error, res)) return+ res.status(500).json({ error: 'unexpected test outcome' })+ })+ const publicResponse = await request(publicApp).post('/password-reset').send({})+ expect(publicResponse.status).toBe(503)+ expect(publicResponse.body).toEqual({+ error: 'recovery_outcome_unknown',+ retryAfterSeconds: 2,+ })+ expect(publicResponse.headers['retry-after']).toBe('2')+ expect(publicResponse.headers['cache-control']).toBe('no-store')+ expect(publicResponse.headers['set-cookie']).toBeUndefined()++ holder.pool = normalPool+ const replayLease = await acquirePasswordWork()+ expect(replayLease).not.toBeNull()+ if (!replayLease) throw new Error('password-work owner was not reacquired')+ const replay = await setInvitationPasswordForEmail(+ email,+ resetInvitationId,+ newPassword,+ userId,+ replayLease+ )+ expect(replay).toEqual({ error: 'not_pending' })+ expect(+ await observer.query('SELECT status FROM invitations WHERE id = $1', [resetInvitationId])+ ).toMatchObject({ rows: [{ status: 'accepted' }] })+ expect(+ await observer.query('SELECT COUNT(*)::int AS count FROM password_verification_work')+ ).toMatchObject({ rows: [{ count: 0 }] })+ }, 30_000)+})