Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): report uncertain reset commit outcomes

Josué75fa5841ec67
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 5a1d149f9..39ae925e3 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.746",
+ "version": "0.1.747",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.746",
+ "version": "0.1.747",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 00003d11b..a945ea8ab 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.746",
+ "version": "0.1.747",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 83f3b8e5e..125313e44 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -6734,6 +6734,8 @@ export type DbTransactionOptions = {
signal?: AbortSignal
/** Called only after PostgreSQL acknowledges COMMIT. Must be synchronous and non-throwing. */
onCommitAcknowledged?: () => void
+ /** Called when COMMIT was sent but PostgreSQL did not acknowledge its outcome. */
+ onCommitOutcomeUnknown?: () => void
}
export async function withTransaction<T>(
@@ -6891,6 +6893,9 @@ export async function withTransaction<T>(
}
// Once COMMIT was sent, a missing reply is an unknown durable outcome.
// Cancellation does not establish rollback, no spend or no dispatch.
+ if (commitSent && !transactionFinished) {
+ options.onCommitOutcomeUnknown?.()
+ }
signal?.throwIfAborted()
throw error
} finally {
diff --git a/control-api/src/middleware/passwordLoginAdmission.ts b/control-api/src/middleware/passwordLoginAdmission.ts
index 6624dc302..171497ef7 100644
--- a/control-api/src/middleware/passwordLoginAdmission.ts
+++ b/control-api/src/middleware/passwordLoginAdmission.ts
@@ -16,7 +16,7 @@ export function sendPasswordAdmissionError(error: unknown, res: Response): boole
res.setHeader('Cache-Control', 'no-store')
res.setHeader('Retry-After', String(error.retryAfterSeconds))
res.status(error.status).json({
- error: error.status === 429 ? 'rate_limited' : 'authority_unavailable',
+ error: error.publicError,
retryAfterSeconds: error.retryAfterSeconds,
})
return true
diff --git a/control-api/src/services/auth/passwordCredentialVerification.ts b/control-api/src/services/auth/passwordCredentialVerification.ts
index abd293899..e3dde5ff7 100644
--- a/control-api/src/services/auth/passwordCredentialVerification.ts
+++ b/control-api/src/services/auth/passwordCredentialVerification.ts
@@ -22,7 +22,11 @@ export class PasswordAdmissionError extends Error {
constructor(
readonly status: 429 | 503,
readonly retryAfterSeconds: number,
- readonly reason: PasswordAdmissionDenialReason
+ readonly reason: PasswordAdmissionDenialReason,
+ readonly publicError:
+ | 'rate_limited'
+ | 'authority_unavailable'
+ | 'recovery_outcome_unknown' = status === 429 ? 'rate_limited' : 'authority_unavailable'
) {
super(status === 429 ? 'rate_limited' : 'authority_unavailable')
passwordAdmissionDenialsTotal.inc({ reason })
diff --git a/control-api/src/services/directory/membership.ts b/control-api/src/services/directory/membership.ts
index 8a69a5b2b..912a55a88 100644
--- a/control-api/src/services/directory/membership.ts
+++ b/control-api/src/services/directory/membership.ts
@@ -220,9 +220,11 @@ export async function withMemberPasswordWorkLease<T>(
/** Commit member credential mutations and durable-owner release as one outcome. */
async function withPasswordWorkTransaction<T>(
lease: PasswordWorkLease,
- work: (db: DbTransactionClient) => Promise<T>
+ work: (db: DbTransactionClient) => Promise<T>,
+ options: { exposeUnknownCommitOutcome?: boolean } = {}
): Promise<T> {
let commitAcknowledged = false
+ let commitOutcomeUnknown = false
let hasOutcome = false
let outcome!: T
try {
@@ -239,9 +241,19 @@ async function withPasswordWorkTransaction<T>(
commitAcknowledged = true
lease.confirmReleaseWithinCommitted()
},
+ onCommitOutcomeUnknown: () => {
+ commitOutcomeUnknown = true
+ },
}
)
} catch (error) {
+ if (commitOutcomeUnknown && options.exposeUnknownCommitOutcome) {
+ passwordSetupLogger.warn(
+ { event: 'password_recovery_commit_outcome_unknown' },
+ 'password recovery commit outcome is unknown'
+ )
+ throw new PasswordAdmissionError(503, 2, 'authority_failure', 'recovery_outcome_unknown')
+ }
if (commitAcknowledged && hasOutcome) {
passwordSetupLogger.warn(
{ event: 'password_post_commit_cleanup_failed' },
@@ -251,7 +263,17 @@ async function withPasswordWorkTransaction<T>(
}
throw error
} finally {
- if (!commitAcknowledged) await lease.release()
+ if (!commitAcknowledged) {
+ try {
+ await lease.release()
+ } catch (error) {
+ if (!commitOutcomeUnknown) throw error
+ passwordSetupLogger.warn(
+ { event: 'password_owner_release_after_unknown_commit_failed' },
+ 'password owner release after unknown commit failed'
+ )
+ }
+ }
}
}
@@ -1317,71 +1339,75 @@ export async function setInvitationPasswordForEmail(
const lease = passwordWorkLease ?? (await acquirePasswordSetupLease())
try {
- return await withPasswordWorkTransaction(lease, async db => {
- const candidate = await getInvitationRecordById(db, trimmedInvitationId)
- if (!candidate || candidate.purpose !== 'password_reset') {
- return { error: 'not_found' as const }
- }
- if (candidate.email.toLowerCase() !== normalizedEmail) {
- return { error: 'forbidden' as const }
- }
-
- const user = await lockPasswordResetUser(db, normalizedEmail)
- if (!user) return { error: 'not_found' as const }
- if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }
- if (expectedUserId && user.id !== expectedUserId.trim()) {
- return { error: 'forbidden' as const }
- }
- const invitation = await getInvitationRecordById(db, trimmedInvitationId, true)
- if (!invitation || invitation.purpose !== 'password_reset') {
- return { error: 'not_found' as const }
- }
- if (invitation.email.toLowerCase() !== normalizedEmail) {
- return { error: 'forbidden' as const }
- }
- if (invitation.status !== 'pending') return { error: 'not_pending' as const }
- if (invitation.expires_at.getTime() <= Date.now()) return { error: 'expired' as const }
- if (invitation.accepted_user_id && invitation.accepted_user_id !== user.id) {
- return { error: 'forbidden' as const }
- }
-
- const passwordHash = await bcrypt.hash(trimmedPassword, 12)
- const update = await db.query(
- `UPDATE users
+ return await withPasswordWorkTransaction(
+ lease,
+ async db => {
+ const candidate = await getInvitationRecordById(db, trimmedInvitationId)
+ if (!candidate || candidate.purpose !== 'password_reset') {
+ return { error: 'not_found' as const }
+ }
+ if (candidate.email.toLowerCase() !== normalizedEmail) {
+ return { error: 'forbidden' as const }
+ }
+
+ const user = await lockPasswordResetUser(db, normalizedEmail)
+ if (!user) return { error: 'not_found' as const }
+ if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }
+ if (expectedUserId && user.id !== expectedUserId.trim()) {
+ return { error: 'forbidden' as const }
+ }
+ const invitation = await getInvitationRecordById(db, trimmedInvitationId, true)
+ if (!invitation || invitation.purpose !== 'password_reset') {
+ return { error: 'not_found' as const }
+ }
+ if (invitation.email.toLowerCase() !== normalizedEmail) {
+ return { error: 'forbidden' as const }
+ }
+ if (invitation.status !== 'pending') return { error: 'not_pending' as const }
+ if (invitation.expires_at.getTime() <= Date.now()) return { error: 'expired' as const }
+ if (invitation.accepted_user_id && invitation.accepted_user_id !== user.id) {
+ return { error: 'forbidden' as const }
+ }
+
+ const passwordHash = await bcrypt.hash(trimmedPassword, 12)
+ const update = await db.query(
+ `UPDATE users
SET password_hash = $2,
password_set_at = NOW(),
updated_at = NOW()
WHERE id = $1
AND lifecycle_state = 'active'
RETURNING id`,
- [user.id, passwordHash]
- )
- if (update.rows.length !== 1) return { error: 'user_retired' as const }
+ [user.id, passwordHash]
+ )
+ if (update.rows.length !== 1) return { error: 'user_retired' as const }
- const redeemed = await db.query(
- `UPDATE invitations
+ const redeemed = await db.query(
+ `UPDATE invitations
SET status = 'accepted',
accepted_at = NOW(),
accepted_user_id = $2
WHERE id = $1 AND status = 'pending'
RETURNING id`,
- [invitation.id, user.id]
- )
- if (redeemed.rows.length !== 1) return { error: 'not_pending' as const }
- await revokeSiblingPasswordResets(db, normalizedEmail, invitation.id, user.id)
-
- const refreshed = await getInvitationRecordById(db, invitation.id)
- if (!refreshed) return { error: 'not_found' as const }
- const teams = await loadInvitationTeams(db, refreshed.id, refreshed)
- const sessionContext = await getMemberSessionContext(db, user)
- return {
- data: {
- passwordUpdated: true,
- sessionContext,
- ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),
- },
- }
- })
+ [invitation.id, user.id]
+ )
+ if (redeemed.rows.length !== 1) return { error: 'not_pending' as const }
+ await revokeSiblingPasswordResets(db, normalizedEmail, invitation.id, user.id)
+
+ const refreshed = await getInvitationRecordById(db, invitation.id)
+ if (!refreshed) return { error: 'not_found' as const }
+ const teams = await loadInvitationTeams(db, refreshed.id, refreshed)
+ const sessionContext = await getMemberSessionContext(db, user)
+ return {
+ data: {
+ passwordUpdated: true,
+ sessionContext,
+ ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),
+ },
+ }
+ },
+ { exposeUnknownCommitOutcome: true }
+ )
} catch (error) {
if (error instanceof PasswordAdmissionError) throw error
passwordSetupLogger.warn(
diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.ts
index 98bc625be..b60b5f018 100644
--- a/control-api/test/helpers/realPostgresCancellation.ts
+++ b/control-api/test/helpers/realPostgresCancellation.ts
@@ -2,17 +2,29 @@ import { once } from 'node:events'
import { type Socket, createConnection, createServer } from 'node:net'
/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */
-export async function createPostgresCommitReplyBlackhole(connectionString: string) {
- return createPostgresProtocolBlackhole(connectionString, true)
+export async function createPostgresCommitReplyBlackhole(
+ connectionString: string,
+ dropCommitNumber = 1
+) {
+ return createPostgresProtocolBlackhole(connectionString, true, dropCommitNumber)
}
/** Hide replies and client disconnects without ending the real backend socket. */
export async function createPostgresDisconnectBlackhole(connectionString: string) {
- return createPostgresProtocolBlackhole(connectionString, false)
+ return createPostgresProtocolBlackhole(connectionString, false, 1)
}
-async function createPostgresProtocolBlackhole(connectionString: string, dropCommitReply: boolean) {
+async function createPostgresProtocolBlackhole(
+ connectionString: string,
+ dropCommitReply: boolean,
+ dropCommitNumber: number
+) {
+ if (!Number.isSafeInteger(dropCommitNumber) || dropCommitNumber < 1) {
+ throw new Error('dropCommitNumber must be a positive integer')
+ }
const target = new URL(connectionString)
+ const targetHost = target.searchParams.get('host') || target.hostname.replace(/^\[|\]$/g, '')
+ const targetPort = Number(target.searchParams.get('port') || target.port || 5432)
const sslMode = target.searchParams.get('sslmode')
if (sslMode && sslMode !== 'disable') {
throw new Error('COMMIT reply observation requires the harness plaintext PostgreSQL connection')
@@ -24,14 +36,21 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropCom
commit = resolve
})
let armed = dropCommitReply
- let dropping = false
+ let commitCount = 0
+ let droppingBackend: Socket | undefined
+ let droppedFrontend: Socket | undefined
+ let dropAllReplies = false
let discardedReplyBytes = 0
let discardedFrontendBytes = 0
let suppressDisconnects = false
let suppressedDisconnects = 0
const server = createServer({ allowHalfOpen: true }, front => {
- const host = target.hostname.replace(/^\[|\]$/g, '')
- const back = createConnection({ host, port: Number(target.port || 5432), allowHalfOpen: true })
+ const back = targetHost.startsWith('/')
+ ? createConnection({
+ path: `${targetHost.replace(/\/$/, '')}/.s.PGSQL.${targetPort}`,
+ allowHalfOpen: true,
+ })
+ : createConnection({ host: targetHost, port: targetPort, allowHalfOpen: true })
for (const socket of [front, back]) {
sockets.add(socket)
socket.once('close', () => sockets.delete(socket))
@@ -66,9 +85,11 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropCom
if (buffered[0] === 'Q'.charCodeAt(0)) {
const sql = buffered.toString('utf8', 5, length)
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') commands.push(sql)
- if (sql === 'COMMIT' && armed) {
+ if (sql === 'COMMIT') commitCount++
+ if (sql === 'COMMIT' && armed && commitCount === dropCommitNumber) {
armed = false
- dropping = true
+ droppingBackend = back
+ droppedFrontend = front
commit()
}
}
@@ -77,7 +98,7 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropCom
back.write(chunk)
})
back.on('data', chunk => {
- if (dropping) discardedReplyBytes += chunk.length
+ if (dropAllReplies || back === droppingBackend) discardedReplyBytes += chunk.length
else front.write(chunk)
})
})
@@ -88,6 +109,8 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropCom
const proxied = new URL(connectionString)
proxied.hostname = '127.0.0.1'
proxied.port = String(address.port)
+ proxied.searchParams.delete('host')
+ proxied.searchParams.delete('port')
return {
connectionString: proxied.toString(),
commitForwarded,
@@ -101,14 +124,23 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropCom
get suppressedDisconnects() {
return suppressedDisconnects
},
+ failClientAfterCommitForwarded: () => {
+ const front = droppedFrontend
+ if (!front) throw new Error('no targeted COMMIT reply is being withheld')
+ droppingBackend = undefined
+ droppedFrontend = undefined
+ front.destroy(new Error('simulated lost COMMIT acknowledgement'))
+ },
dropRepliesAndDisconnects: () => {
armed = false
- dropping = true
+ dropAllReplies = true
suppressDisconnects = true
},
allowReplies: () => {
- dropping = false
+ dropAllReplies = false
suppressDisconnects = false
+ droppingBackend = undefined
+ droppedFrontend = undefined
},
close: async () => {
for (const socket of sockets) socket.destroy()
diff --git a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts
new file mode 100644
index 000000000..009903200
--- /dev/null
+++ b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts
@@ -0,0 +1,182 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import bcrypt from 'bcryptjs'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+import request from 'supertest'
+import { initDb } from '../src/db.js'
+import { sendPasswordAdmissionError } from '../src/middleware/passwordLoginAdmission.js'
+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'
+import {
+ createSilentInvitationForTeams,
+ setInvitationPasswordForEmail,
+} from '../src/services/directory/membership.js'
+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+
+const holder = vi.hoisted(() => ({ pool: null as unknown as Pool }))
+
+vi.mock('../src/db.js', async importOriginal => {
+ const real = await importOriginal<typeof import('../src/db.js')>()
+ const proxy = {
+ query: (...args: unknown[]) => (holder.pool.query as Function)(...args),
+ connect: () => holder.pool.connect(),
+ }
+ return {
+ ...real,
+ pool: proxy,
+ corePool: proxy,
+ rateLimitPool: proxy,
+ withTransaction: (
+ fn: Parameters<typeof real.withTransaction>[0],
+ _txPool?: Parameters<typeof real.withTransaction>[1],
+ options?: Parameters<typeof real.withTransaction>[2]
+ ) => real.withTransaction(fn, holder.pool, options),
+ }
+})
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const realPg = adminUrl ? describe : describe.skip
+
+realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () => {
+ const database = `r2h2_reset_commit_${randomBytes(6).toString('hex')}`
+ const oldPassword = 'Synthetic-R2-H2-Old-Password'
+ const newPassword = 'Synthetic-R2-H2-New-Password'
+ const email = `r2h2-${randomBytes(8).toString('hex')}@example.invalid`
+ let admin: Pool
+ let observer: Pool
+ let normalPool: Pool
+ let proxyPool: Pool | undefined
+ let blackhole: Awaited<ReturnType<typeof createPostgresCommitReplyBlackhole>> | undefined
+ let connectionString: string
+ let resetInvitationId: string
+ let siblingInvitationId: string
+ let userId: string
+ let originalHash: string
+
+ beforeAll(async () => {
+ admin = new Pool({ connectionString: adminUrl })
+ await admin.query(`CREATE DATABASE "${database}"`)
+ const databaseUrl = new URL(adminUrl!)
+ databaseUrl.pathname = `/${database}`
+ databaseUrl.searchParams.set('options', '-c lock_timeout=1000ms')
+ connectionString = databaseUrl.toString()
+ normalPool = new Pool({ connectionString })
+ observer = new Pool({ connectionString })
+ holder.pool = normalPool
+ await initDb(normalPool)
+
+ originalHash = await bcrypt.hash(oldPassword, 12)
+ const user = await normalPool.query(
+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',
+ [email, originalHash]
+ )
+ userId = user.rows[0].id as string
+
+ const primary = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R2-H2 member',
+ email,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+ const sibling = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R2-H2 member',
+ email,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+ resetInvitationId = primary.id
+ siblingInvitationId = sibling.id
+ }, 60_000)
+
+ afterAll(async () => {
+ try {
+ blackhole?.allowReplies()
+ await blackhole?.close()
+ if (proxyPool) await endPoolAndWaitForClients(proxyPool)
+ if (normalPool) await endPoolAndWaitForClients(normalPool)
+ if (observer) await endPoolAndWaitForClients(observer)
+ if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)
+ } finally {
+ await admin?.end()
+ }
+ })
+
+ it('reports an unknown outcome without replay when reset committed but PostgreSQL reply was lost', async () => {
+ const lease = await acquirePasswordWork()
+ expect(lease).not.toBeNull()
+ if (!lease) throw new Error('password-work owner was not acquired')
+
+ blackhole = await createPostgresCommitReplyBlackhole(connectionString)
+ proxyPool = new Pool({ connectionString: blackhole.connectionString, max: 4 })
+ holder.pool = proxyPool
+
+ const outcomePromise = setInvitationPasswordForEmail(
+ email,
+ resetInvitationId,
+ newPassword,
+ userId,
+ lease
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+
+ await blackhole.commitForwarded
+ const committed = await observer.query(
+ `SELECT i.status AS reset_status,
+ sibling.status AS sibling_status,
+ u.password_hash,
+ (SELECT COUNT(*)::int FROM password_verification_work) AS owners
+ FROM invitations i
+ JOIN invitations sibling ON sibling.id = $2
+ JOIN users u ON u.id = $3
+ WHERE i.id = $1`,
+ [resetInvitationId, siblingInvitationId, userId]
+ )
+ expect(committed.rows[0]).toMatchObject({
+ reset_status: 'accepted',
+ sibling_status: 'revoked',
+ owners: 0,
+ })
+ expect(await bcrypt.compare(newPassword, committed.rows[0].password_hash as string)).toBe(true)
+
+ const replyDeadline = Date.now() + 2_000
+ while (blackhole.discardedReplyBytes === 0 && Date.now() < replyDeadline) {
+ await new Promise(resolve => setTimeout(resolve, 10))
+ }
+ expect(blackhole.discardedReplyBytes).toBeGreaterThan(0)
+ blackhole.failClientAfterCommitForwarded()
+
+ const outcome = await outcomePromise
+ expect(outcome).toMatchObject({
+ error: {
+ status: 503,
+ reason: 'authority_failure',
+ publicError: 'recovery_outcome_unknown',
+ },
+ })
+ if ('error' in outcome) {
+ expect(outcome.error.message).toBe('authority_unavailable')
+ expect(outcome.error.message).not.toMatch(/password|sql|invitation|database/i)
+ }
+ expect(blackhole.commands.filter(command => command === 'COMMIT')).toHaveLength(1)
+
+ const responseApp = express()
+ responseApp.post('/password-reset', (_req, res) => {
+ if ('error' in outcome) sendPasswordAdmissionError(outcome.error, res)
+ else res.status(500).json({ error: 'unexpected successful response' })
+ })
+ const publicResponse = await request(responseApp).post('/password-reset').send({})
+ expect(publicResponse.status).toBe(503)
+ expect(publicResponse.body).toEqual({
+ error: 'recovery_outcome_unknown',
+ retryAfterSeconds: 2,
+ })
+ expect(publicResponse.headers['retry-after']).toBe('2')
+ expect(publicResponse.headers['cache-control']).toBe('no-store')
+ expect(publicResponse.headers['set-cookie']).toBeUndefined()
+ }, 20_000)
+})