Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): report uncertain reset commit outcomes
Josué75fa5841ec67
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 5a1d149f9..39ae925e3 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.746",+ "version": "0.1.747","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.746",+ "version": "0.1.747","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 00003d11b..a945ea8ab 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.746",+ "version": "0.1.747","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 83f3b8e5e..125313e44 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -6734,6 +6734,8 @@ export type DbTransactionOptions = {signal?: AbortSignal/** Called only after PostgreSQL acknowledges COMMIT. Must be synchronous and non-throwing. */onCommitAcknowledged?: () => void+ /** Called when COMMIT was sent but PostgreSQL did not acknowledge its outcome. */+ onCommitOutcomeUnknown?: () => void}export async function withTransaction<T>(@@ -6891,6 +6893,9 @@ export async function withTransaction<T>(}// Once COMMIT was sent, a missing reply is an unknown durable outcome.// Cancellation does not establish rollback, no spend or no dispatch.+ if (commitSent && !transactionFinished) {+ options.onCommitOutcomeUnknown?.()+ }signal?.throwIfAborted()throw error} finally {diff --git a/control-api/src/middleware/passwordLoginAdmission.ts b/control-api/src/middleware/passwordLoginAdmission.tsindex 6624dc302..171497ef7 100644--- a/control-api/src/middleware/passwordLoginAdmission.ts+++ b/control-api/src/middleware/passwordLoginAdmission.ts@@ -16,7 +16,7 @@ export function sendPasswordAdmissionError(error: unknown, res: Response): booleres.setHeader('Cache-Control', 'no-store')res.setHeader('Retry-After', String(error.retryAfterSeconds))res.status(error.status).json({- error: error.status === 429 ? 'rate_limited' : 'authority_unavailable',+ error: error.publicError,retryAfterSeconds: error.retryAfterSeconds,})return truediff --git a/control-api/src/services/auth/passwordCredentialVerification.ts b/control-api/src/services/auth/passwordCredentialVerification.tsindex abd293899..e3dde5ff7 100644--- a/control-api/src/services/auth/passwordCredentialVerification.ts+++ b/control-api/src/services/auth/passwordCredentialVerification.ts@@ -22,7 +22,11 @@ export class PasswordAdmissionError extends Error {constructor(readonly status: 429 | 503,readonly retryAfterSeconds: number,- readonly reason: PasswordAdmissionDenialReason+ readonly reason: PasswordAdmissionDenialReason,+ readonly publicError:+ | 'rate_limited'+ | 'authority_unavailable'+ | 'recovery_outcome_unknown' = status === 429 ? 'rate_limited' : 'authority_unavailable') {super(status === 429 ? 'rate_limited' : 'authority_unavailable')passwordAdmissionDenialsTotal.inc({ reason })diff --git a/control-api/src/services/directory/membership.ts b/control-api/src/services/directory/membership.tsindex 8a69a5b2b..912a55a88 100644--- a/control-api/src/services/directory/membership.ts+++ b/control-api/src/services/directory/membership.ts@@ -220,9 +220,11 @@ export async function withMemberPasswordWorkLease<T>(/** Commit member credential mutations and durable-owner release as one outcome. */async function withPasswordWorkTransaction<T>(lease: PasswordWorkLease,- work: (db: DbTransactionClient) => Promise<T>+ work: (db: DbTransactionClient) => Promise<T>,+ options: { exposeUnknownCommitOutcome?: boolean } = {}): Promise<T> {let commitAcknowledged = false+ let commitOutcomeUnknown = falselet hasOutcome = falselet outcome!: Ttry {@@ -239,9 +241,19 @@ async function withPasswordWorkTransaction<T>(commitAcknowledged = truelease.confirmReleaseWithinCommitted()},+ onCommitOutcomeUnknown: () => {+ commitOutcomeUnknown = true+ },})} catch (error) {+ if (commitOutcomeUnknown && options.exposeUnknownCommitOutcome) {+ passwordSetupLogger.warn(+ { event: 'password_recovery_commit_outcome_unknown' },+ 'password recovery commit outcome is unknown'+ )+ throw new PasswordAdmissionError(503, 2, 'authority_failure', 'recovery_outcome_unknown')+ }if (commitAcknowledged && hasOutcome) {passwordSetupLogger.warn({ event: 'password_post_commit_cleanup_failed' },@@ -251,7 +263,17 @@ async function withPasswordWorkTransaction<T>(}throw error} finally {- if (!commitAcknowledged) await lease.release()+ if (!commitAcknowledged) {+ try {+ await lease.release()+ } catch (error) {+ if (!commitOutcomeUnknown) throw error+ passwordSetupLogger.warn(+ { event: 'password_owner_release_after_unknown_commit_failed' },+ 'password owner release after unknown commit failed'+ )+ }+ }}}@@ -1317,71 +1339,75 @@ export async function setInvitationPasswordForEmail(const lease = passwordWorkLease ?? (await acquirePasswordSetupLease())try {- return await withPasswordWorkTransaction(lease, async db => {- const candidate = await getInvitationRecordById(db, trimmedInvitationId)- if (!candidate || candidate.purpose !== 'password_reset') {- return { error: 'not_found' as const }- }- if (candidate.email.toLowerCase() !== normalizedEmail) {- return { error: 'forbidden' as const }- }-- const user = await lockPasswordResetUser(db, normalizedEmail)- if (!user) return { error: 'not_found' as const }- if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }- if (expectedUserId && user.id !== expectedUserId.trim()) {- return { error: 'forbidden' as const }- }- const invitation = await getInvitationRecordById(db, trimmedInvitationId, true)- if (!invitation || invitation.purpose !== 'password_reset') {- return { error: 'not_found' as const }- }- if (invitation.email.toLowerCase() !== normalizedEmail) {- return { error: 'forbidden' as const }- }- if (invitation.status !== 'pending') return { error: 'not_pending' as const }- if (invitation.expires_at.getTime() <= Date.now()) return { error: 'expired' as const }- if (invitation.accepted_user_id && invitation.accepted_user_id !== user.id) {- return { error: 'forbidden' as const }- }-- const passwordHash = await bcrypt.hash(trimmedPassword, 12)- const update = await db.query(- `UPDATE users+ return await withPasswordWorkTransaction(+ lease,+ async db => {+ const candidate = await getInvitationRecordById(db, trimmedInvitationId)+ if (!candidate || candidate.purpose !== 'password_reset') {+ return { error: 'not_found' as const }+ }+ if (candidate.email.toLowerCase() !== normalizedEmail) {+ return { error: 'forbidden' as const }+ }++ const user = await lockPasswordResetUser(db, normalizedEmail)+ if (!user) return { error: 'not_found' as const }+ if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }+ if (expectedUserId && user.id !== expectedUserId.trim()) {+ return { error: 'forbidden' as const }+ }+ const invitation = await getInvitationRecordById(db, trimmedInvitationId, true)+ if (!invitation || invitation.purpose !== 'password_reset') {+ return { error: 'not_found' as const }+ }+ if (invitation.email.toLowerCase() !== normalizedEmail) {+ return { error: 'forbidden' as const }+ }+ if (invitation.status !== 'pending') return { error: 'not_pending' as const }+ if (invitation.expires_at.getTime() <= Date.now()) return { error: 'expired' as const }+ if (invitation.accepted_user_id && invitation.accepted_user_id !== user.id) {+ return { error: 'forbidden' as const }+ }++ const passwordHash = await bcrypt.hash(trimmedPassword, 12)+ const update = await db.query(+ `UPDATE usersSET password_hash = $2,password_set_at = NOW(),updated_at = NOW()WHERE id = $1AND lifecycle_state = 'active'RETURNING id`,- [user.id, passwordHash]- )- if (update.rows.length !== 1) return { error: 'user_retired' as const }+ [user.id, passwordHash]+ )+ if (update.rows.length !== 1) return { error: 'user_retired' as const }- const redeemed = await db.query(- `UPDATE invitations+ const redeemed = await db.query(+ `UPDATE invitationsSET status = 'accepted',accepted_at = NOW(),accepted_user_id = $2WHERE id = $1 AND status = 'pending'RETURNING id`,- [invitation.id, user.id]- )- if (redeemed.rows.length !== 1) return { error: 'not_pending' as const }- await revokeSiblingPasswordResets(db, normalizedEmail, invitation.id, user.id)-- const refreshed = await getInvitationRecordById(db, invitation.id)- if (!refreshed) return { error: 'not_found' as const }- const teams = await loadInvitationTeams(db, refreshed.id, refreshed)- const sessionContext = await getMemberSessionContext(db, user)- return {- data: {- passwordUpdated: true,- sessionContext,- ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),- },- }- })+ [invitation.id, user.id]+ )+ if (redeemed.rows.length !== 1) return { error: 'not_pending' as const }+ await revokeSiblingPasswordResets(db, normalizedEmail, invitation.id, user.id)++ const refreshed = await getInvitationRecordById(db, invitation.id)+ if (!refreshed) return { error: 'not_found' as const }+ const teams = await loadInvitationTeams(db, refreshed.id, refreshed)+ const sessionContext = await getMemberSessionContext(db, user)+ return {+ data: {+ passwordUpdated: true,+ sessionContext,+ ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),+ },+ }+ },+ { exposeUnknownCommitOutcome: true }+ )} catch (error) {if (error instanceof PasswordAdmissionError) throw errorpasswordSetupLogger.warn(diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.tsindex 98bc625be..b60b5f018 100644--- a/control-api/test/helpers/realPostgresCancellation.ts+++ b/control-api/test/helpers/realPostgresCancellation.ts@@ -2,17 +2,29 @@ import { once } from 'node:events'import { type Socket, createConnection, createServer } from 'node:net'/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */-export async function createPostgresCommitReplyBlackhole(connectionString: string) {- return createPostgresProtocolBlackhole(connectionString, true)+export async function createPostgresCommitReplyBlackhole(+ connectionString: string,+ dropCommitNumber = 1+) {+ return createPostgresProtocolBlackhole(connectionString, true, dropCommitNumber)}/** Hide replies and client disconnects without ending the real backend socket. */export async function createPostgresDisconnectBlackhole(connectionString: string) {- return createPostgresProtocolBlackhole(connectionString, false)+ return createPostgresProtocolBlackhole(connectionString, false, 1)}-async function createPostgresProtocolBlackhole(connectionString: string, dropCommitReply: boolean) {+async function createPostgresProtocolBlackhole(+ connectionString: string,+ dropCommitReply: boolean,+ dropCommitNumber: number+) {+ if (!Number.isSafeInteger(dropCommitNumber) || dropCommitNumber < 1) {+ throw new Error('dropCommitNumber must be a positive integer')+ }const target = new URL(connectionString)+ const targetHost = target.searchParams.get('host') || target.hostname.replace(/^\[|\]$/g, '')+ const targetPort = Number(target.searchParams.get('port') || target.port || 5432)const sslMode = target.searchParams.get('sslmode')if (sslMode && sslMode !== 'disable') {throw new Error('COMMIT reply observation requires the harness plaintext PostgreSQL connection')@@ -24,14 +36,21 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropComcommit = resolve})let armed = dropCommitReply- let dropping = false+ let commitCount = 0+ let droppingBackend: Socket | undefined+ let droppedFrontend: Socket | undefined+ let dropAllReplies = falselet discardedReplyBytes = 0let discardedFrontendBytes = 0let suppressDisconnects = falselet suppressedDisconnects = 0const server = createServer({ allowHalfOpen: true }, front => {- const host = target.hostname.replace(/^\[|\]$/g, '')- const back = createConnection({ host, port: Number(target.port || 5432), allowHalfOpen: true })+ const back = targetHost.startsWith('/')+ ? createConnection({+ path: `${targetHost.replace(/\/$/, '')}/.s.PGSQL.${targetPort}`,+ allowHalfOpen: true,+ })+ : createConnection({ host: targetHost, port: targetPort, allowHalfOpen: true })for (const socket of [front, back]) {sockets.add(socket)socket.once('close', () => sockets.delete(socket))@@ -66,9 +85,11 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropComif (buffered[0] === 'Q'.charCodeAt(0)) {const sql = buffered.toString('utf8', 5, length)if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') commands.push(sql)- if (sql === 'COMMIT' && armed) {+ if (sql === 'COMMIT') commitCount+++ if (sql === 'COMMIT' && armed && commitCount === dropCommitNumber) {armed = false- dropping = true+ droppingBackend = back+ droppedFrontend = frontcommit()}}@@ -77,7 +98,7 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropComback.write(chunk)})back.on('data', chunk => {- if (dropping) discardedReplyBytes += chunk.length+ if (dropAllReplies || back === droppingBackend) discardedReplyBytes += chunk.lengthelse front.write(chunk)})})@@ -88,6 +109,8 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropComconst proxied = new URL(connectionString)proxied.hostname = '127.0.0.1'proxied.port = String(address.port)+ proxied.searchParams.delete('host')+ proxied.searchParams.delete('port')return {connectionString: proxied.toString(),commitForwarded,@@ -101,14 +124,23 @@ async function createPostgresProtocolBlackhole(connectionString: string, dropComget suppressedDisconnects() {return suppressedDisconnects},+ failClientAfterCommitForwarded: () => {+ const front = droppedFrontend+ if (!front) throw new Error('no targeted COMMIT reply is being withheld')+ droppingBackend = undefined+ droppedFrontend = undefined+ front.destroy(new Error('simulated lost COMMIT acknowledgement'))+ },dropRepliesAndDisconnects: () => {armed = false- dropping = true+ dropAllReplies = truesuppressDisconnects = true},allowReplies: () => {- dropping = false+ dropAllReplies = falsesuppressDisconnects = false+ droppingBackend = undefined+ droppedFrontend = undefined},close: async () => {for (const socket of sockets) socket.destroy()diff --git a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.tsnew file mode 100644index 000000000..009903200--- /dev/null+++ b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts@@ -0,0 +1,182 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import express from 'express'+import bcrypt from 'bcryptjs'+import { randomBytes } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'+import { initDb } from '../src/db.js'+import { sendPasswordAdmissionError } from '../src/middleware/passwordLoginAdmission.js'+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'+import {+ createSilentInvitationForTeams,+ setInvitationPasswordForEmail,+} from '../src/services/directory/membership.js'+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'++const holder = vi.hoisted(() => ({ pool: null as unknown as Pool }))++vi.mock('../src/db.js', async importOriginal => {+ const real = await importOriginal<typeof import('../src/db.js')>()+ const proxy = {+ query: (...args: unknown[]) => (holder.pool.query as Function)(...args),+ connect: () => holder.pool.connect(),+ }+ return {+ ...real,+ pool: proxy,+ corePool: proxy,+ rateLimitPool: proxy,+ withTransaction: (+ fn: Parameters<typeof real.withTransaction>[0],+ _txPool?: Parameters<typeof real.withTransaction>[1],+ options?: Parameters<typeof real.withTransaction>[2]+ ) => real.withTransaction(fn, holder.pool, options),+ }+})++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const realPg = adminUrl ? describe : describe.skip++realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () => {+ const database = `r2h2_reset_commit_${randomBytes(6).toString('hex')}`+ const oldPassword = 'Synthetic-R2-H2-Old-Password'+ const newPassword = 'Synthetic-R2-H2-New-Password'+ const email = `r2h2-${randomBytes(8).toString('hex')}@example.invalid`+ let admin: Pool+ let observer: Pool+ let normalPool: Pool+ let proxyPool: Pool | undefined+ let blackhole: Awaited<ReturnType<typeof createPostgresCommitReplyBlackhole>> | undefined+ let connectionString: string+ let resetInvitationId: string+ let siblingInvitationId: string+ let userId: string+ let originalHash: string++ beforeAll(async () => {+ admin = new Pool({ connectionString: adminUrl })+ await admin.query(`CREATE DATABASE "${database}"`)+ const databaseUrl = new URL(adminUrl!)+ databaseUrl.pathname = `/${database}`+ databaseUrl.searchParams.set('options', '-c lock_timeout=1000ms')+ connectionString = databaseUrl.toString()+ normalPool = new Pool({ connectionString })+ observer = new Pool({ connectionString })+ holder.pool = normalPool+ await initDb(normalPool)++ originalHash = await bcrypt.hash(oldPassword, 12)+ const user = await normalPool.query(+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',+ [email, originalHash]+ )+ userId = user.rows[0].id as string++ const primary = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R2-H2 member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const sibling = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R2-H2 member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ resetInvitationId = primary.id+ siblingInvitationId = sibling.id+ }, 60_000)++ afterAll(async () => {+ try {+ blackhole?.allowReplies()+ await blackhole?.close()+ if (proxyPool) await endPoolAndWaitForClients(proxyPool)+ if (normalPool) await endPoolAndWaitForClients(normalPool)+ if (observer) await endPoolAndWaitForClients(observer)+ if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)+ } finally {+ await admin?.end()+ }+ })++ it('reports an unknown outcome without replay when reset committed but PostgreSQL reply was lost', async () => {+ const lease = await acquirePasswordWork()+ expect(lease).not.toBeNull()+ if (!lease) throw new Error('password-work owner was not acquired')++ blackhole = await createPostgresCommitReplyBlackhole(connectionString)+ proxyPool = new Pool({ connectionString: blackhole.connectionString, max: 4 })+ holder.pool = proxyPool++ const outcomePromise = setInvitationPasswordForEmail(+ email,+ resetInvitationId,+ newPassword,+ userId,+ lease+ ).then(+ value => ({ value }),+ error => ({ error })+ )++ await blackhole.commitForwarded+ const committed = await observer.query(+ `SELECT i.status AS reset_status,+ sibling.status AS sibling_status,+ u.password_hash,+ (SELECT COUNT(*)::int FROM password_verification_work) AS owners+ FROM invitations i+ JOIN invitations sibling ON sibling.id = $2+ JOIN users u ON u.id = $3+ WHERE i.id = $1`,+ [resetInvitationId, siblingInvitationId, userId]+ )+ expect(committed.rows[0]).toMatchObject({+ reset_status: 'accepted',+ sibling_status: 'revoked',+ owners: 0,+ })+ expect(await bcrypt.compare(newPassword, committed.rows[0].password_hash as string)).toBe(true)++ const replyDeadline = Date.now() + 2_000+ while (blackhole.discardedReplyBytes === 0 && Date.now() < replyDeadline) {+ await new Promise(resolve => setTimeout(resolve, 10))+ }+ expect(blackhole.discardedReplyBytes).toBeGreaterThan(0)+ blackhole.failClientAfterCommitForwarded()++ const outcome = await outcomePromise+ expect(outcome).toMatchObject({+ error: {+ status: 503,+ reason: 'authority_failure',+ publicError: 'recovery_outcome_unknown',+ },+ })+ if ('error' in outcome) {+ expect(outcome.error.message).toBe('authority_unavailable')+ expect(outcome.error.message).not.toMatch(/password|sql|invitation|database/i)+ }+ expect(blackhole.commands.filter(command => command === 'COMMIT')).toHaveLength(1)++ const responseApp = express()+ responseApp.post('/password-reset', (_req, res) => {+ if ('error' in outcome) sendPasswordAdmissionError(outcome.error, res)+ else res.status(500).json({ error: 'unexpected successful response' })+ })+ const publicResponse = await request(responseApp).post('/password-reset').send({})+ expect(publicResponse.status).toBe(503)+ expect(publicResponse.body).toEqual({+ error: 'recovery_outcome_unknown',+ retryAfterSeconds: 2,+ })+ expect(publicResponse.headers['retry-after']).toBe('2')+ expect(publicResponse.headers['cache-control']).toBe('no-store')+ expect(publicResponse.headers['set-cookie']).toBeUndefined()+ }, 20_000)+})