Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): classify rejected commits as definite failures
Josuéb66a5d6e6ced
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex ee317e212..da78ca66e 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.750",+ "version": "0.1.751","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.750",+ "version": "0.1.751","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex b375972b9..62fe86662 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.750",+ "version": "0.1.751","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 125313e44..a050a7ca6 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -6738,6 +6738,17 @@ export type DbTransactionOptions = {onCommitOutcomeUnknown?: () => void}+function isPostgresErrorResponse(error: unknown): boolean {+ if (!error || typeof error !== 'object') return false+ const response = error as { name?: unknown; code?: unknown; severity?: unknown }+ return (+ response.name === 'error' &&+ typeof response.code === 'string' &&+ /^[0-9A-Z]{5}$/.test(response.code) &&+ typeof response.severity === 'string'+ )+}+export async function withTransaction<T>(work: (db: DbTransactionClient) => Promise<T>,// Injectable only so the carrier's client lifecycle can be unit-tested against@@ -6891,9 +6902,9 @@ export async function withTransaction<T>(releaseError = rollbackError instanceof Error ? rollbackError : true}}- // Once COMMIT was sent, a missing reply is an unknown durable outcome.- // Cancellation does not establish rollback, no spend or no dispatch.- if (commitSent && !transactionFinished) {+ // A PostgreSQL ErrorResponse proves COMMIT failed. Only a lost transport or+ // malformed response leaves the durable outcome unknown after COMMIT was sent.+ if (commitSent && !transactionFinished && !isPostgresErrorResponse(error)) {options.onCommitOutcomeUnknown?.()}signal?.throwIfAborted()diff --git a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.tsindex 009903200..64f49c630 100644--- a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts+++ b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts@@ -14,7 +14,7 @@ import {import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'-const holder = vi.hoisted(() => ({ pool: null as unknown as Pool }))+const holder = vi.hoisted(() => ({ pool: null as unknown as Pool, unknownCommitOutcomes: 0 }))vi.mock('../src/db.js', async importOriginal => {const real = await importOriginal<typeof import('../src/db.js')>()@@ -31,7 +31,18 @@ vi.mock('../src/db.js', async importOriginal => {fn: Parameters<typeof real.withTransaction>[0],_txPool?: Parameters<typeof real.withTransaction>[1],options?: Parameters<typeof real.withTransaction>[2]- ) => real.withTransaction(fn, holder.pool, options),+ ) =>+ real.withTransaction(+ fn,+ holder.pool,+ options && {+ ...options,+ onCommitOutcomeUnknown: () => {+ holder.unknownCommitOutcomes += 1+ options.onCommitOutcomeUnknown?.()+ },+ }+ ),}})@@ -105,6 +116,7 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =})it('reports an unknown outcome without replay when reset committed but PostgreSQL reply was lost', async () => {+ holder.unknownCommitOutcomes = 0const lease = await acquirePasswordWork()expect(lease).not.toBeNull()if (!lease) throw new Error('password-work owner was not acquired')@@ -163,6 +175,7 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =expect(outcome.error.message).not.toMatch(/password|sql|invitation|database/i)}expect(blackhole.commands.filter(command => command === 'COMMIT')).toHaveLength(1)+ expect(holder.unknownCommitOutcomes).toBe(1)const responseApp = express()responseApp.post('/password-reset', (_req, res) => {@@ -179,4 +192,105 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =expect(publicResponse.headers['cache-control']).toBe('no-store')expect(publicResponse.headers['set-cookie']).toBeUndefined()}, 20_000)++ it('keeps reset proof retryable when PostgreSQL rejects COMMIT definitively', async () => {+ const retryEmail = `r2h3-${randomBytes(8).toString('hex')}@example.invalid`+ const retryUserHash = await bcrypt.hash(oldPassword, 12)+ const retryUser = await normalPool.query(+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',+ [retryEmail, retryUserHash]+ )+ const retryUserId = retryUser.rows[0].id as string+ const retryReset = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R2-H3 member',+ email: retryEmail,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const retrySibling = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic R2-H3 member',+ email: retryEmail,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })++ await normalPool.query(`+ CREATE FUNCTION reject_r2h3_user_update() RETURNS trigger AS $$+ BEGIN+ RAISE EXCEPTION 'synthetic deferred commit rejection' USING ERRCODE = '23514';+ END;+ $$ LANGUAGE plpgsql+ `)+ await normalPool.query(`+ CREATE CONSTRAINT TRIGGER reject_r2h3_user_update+ AFTER UPDATE ON users DEFERRABLE INITIALLY DEFERRED+ FOR EACH ROW EXECUTE FUNCTION reject_r2h3_user_update()+ `)++ holder.unknownCommitOutcomes = 0+ let resetError: unknown+ try {+ await setInvitationPasswordForEmail(retryEmail, retryReset.id, newPassword, retryUserId)+ } catch (error) {+ resetError = error+ }+ expect(resetError).toMatchObject({+ status: 503,+ publicError: 'authority_unavailable',+ })+ expect(holder.unknownCommitOutcomes).toBe(0)+ const errorApp = express()+ errorApp.post('/password-reset', (_req, res) => {+ if (!sendPasswordAdmissionError(resetError, res)) {+ res.status(500).json({ error: 'unexpected response contract' })+ }+ })+ const publicError = await request(errorApp).post('/password-reset').send({})+ expect(publicError.status).toBe(503)+ expect(publicError.body).toEqual({ error: 'authority_unavailable', retryAfterSeconds: 2 })+ expect(publicError.headers['retry-after']).toBe('2')+ expect(publicError.headers['cache-control']).toBe('no-store')++ const unchanged = await observer.query(+ `SELECT i.status AS reset_status, sibling.status AS sibling_status,+ u.password_hash, (SELECT COUNT(*)::int FROM password_verification_work) AS owners+ FROM invitations i+ JOIN invitations sibling ON sibling.id = $2+ JOIN users u ON u.id = $3+ WHERE i.id = $1`,+ [retryReset.id, retrySibling.id, retryUserId]+ )+ expect(unchanged.rows[0]).toMatchObject({+ reset_status: 'pending',+ sibling_status: 'pending',+ owners: 0,+ password_hash: retryUserHash,+ })++ await normalPool.query('DROP TRIGGER reject_r2h3_user_update ON users')+ await normalPool.query('DROP FUNCTION reject_r2h3_user_update()')+ const retried = await setInvitationPasswordForEmail(+ retryEmail,+ retryReset.id,+ newPassword,+ retryUserId+ )+ expect(retried).toMatchObject({ data: { passwordUpdated: true } })+ expect(holder.unknownCommitOutcomes).toBe(0)+ const recovered = await observer.query(+ `SELECT i.status AS reset_status, sibling.status AS sibling_status, u.password_hash+ FROM invitations i+ JOIN invitations sibling ON sibling.id = $2+ JOIN users u ON u.id = $3+ WHERE i.id = $1`,+ [retryReset.id, retrySibling.id, retryUserId]+ )+ expect(recovered.rows[0]).toMatchObject({+ reset_status: 'accepted',+ sibling_status: 'revoked',+ })+ expect(await bcrypt.compare(newPassword, recovered.rows[0].password_hash as string)).toBe(true)+ }, 20_000)})