Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): classify rejected commits as definite failures

Josuéb66a5d6e6ced
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index ee317e212..da78ca66e 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.750",
+ "version": "0.1.751",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.750",
+ "version": "0.1.751",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index b375972b9..62fe86662 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.750",
+ "version": "0.1.751",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 125313e44..a050a7ca6 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -6738,6 +6738,17 @@ export type DbTransactionOptions = {
onCommitOutcomeUnknown?: () => void
}
+function isPostgresErrorResponse(error: unknown): boolean {
+ if (!error || typeof error !== 'object') return false
+ const response = error as { name?: unknown; code?: unknown; severity?: unknown }
+ return (
+ response.name === 'error' &&
+ typeof response.code === 'string' &&
+ /^[0-9A-Z]{5}$/.test(response.code) &&
+ typeof response.severity === 'string'
+ )
+}
+
export async function withTransaction<T>(
work: (db: DbTransactionClient) => Promise<T>,
// Injectable only so the carrier's client lifecycle can be unit-tested against
@@ -6891,9 +6902,9 @@ export async function withTransaction<T>(
releaseError = rollbackError instanceof Error ? rollbackError : true
}
}
- // Once COMMIT was sent, a missing reply is an unknown durable outcome.
- // Cancellation does not establish rollback, no spend or no dispatch.
- if (commitSent && !transactionFinished) {
+ // A PostgreSQL ErrorResponse proves COMMIT failed. Only a lost transport or
+ // malformed response leaves the durable outcome unknown after COMMIT was sent.
+ if (commitSent && !transactionFinished && !isPostgresErrorResponse(error)) {
options.onCommitOutcomeUnknown?.()
}
signal?.throwIfAborted()
diff --git a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts
index 009903200..64f49c630 100644
--- a/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts
+++ b/control-api/test/r2h2.passwordResetLostCommit.realPostgres.integration.test.ts
@@ -14,7 +14,7 @@ import {
import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
-const holder = vi.hoisted(() => ({ pool: null as unknown as Pool }))
+const holder = vi.hoisted(() => ({ pool: null as unknown as Pool, unknownCommitOutcomes: 0 }))
vi.mock('../src/db.js', async importOriginal => {
const real = await importOriginal<typeof import('../src/db.js')>()
@@ -31,7 +31,18 @@ vi.mock('../src/db.js', async importOriginal => {
fn: Parameters<typeof real.withTransaction>[0],
_txPool?: Parameters<typeof real.withTransaction>[1],
options?: Parameters<typeof real.withTransaction>[2]
- ) => real.withTransaction(fn, holder.pool, options),
+ ) =>
+ real.withTransaction(
+ fn,
+ holder.pool,
+ options && {
+ ...options,
+ onCommitOutcomeUnknown: () => {
+ holder.unknownCommitOutcomes += 1
+ options.onCommitOutcomeUnknown?.()
+ },
+ }
+ ),
}
})
@@ -105,6 +116,7 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =
})
it('reports an unknown outcome without replay when reset committed but PostgreSQL reply was lost', async () => {
+ holder.unknownCommitOutcomes = 0
const lease = await acquirePasswordWork()
expect(lease).not.toBeNull()
if (!lease) throw new Error('password-work owner was not acquired')
@@ -163,6 +175,7 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =
expect(outcome.error.message).not.toMatch(/password|sql|invitation|database/i)
}
expect(blackhole.commands.filter(command => command === 'COMMIT')).toHaveLength(1)
+ expect(holder.unknownCommitOutcomes).toBe(1)
const responseApp = express()
responseApp.post('/password-reset', (_req, res) => {
@@ -179,4 +192,105 @@ realPg('R2-H2 lost password-reset COMMIT acknowledgement on PostgreSQL 16', () =
expect(publicResponse.headers['cache-control']).toBe('no-store')
expect(publicResponse.headers['set-cookie']).toBeUndefined()
}, 20_000)
+
+ it('keeps reset proof retryable when PostgreSQL rejects COMMIT definitively', async () => {
+ const retryEmail = `r2h3-${randomBytes(8).toString('hex')}@example.invalid`
+ const retryUserHash = await bcrypt.hash(oldPassword, 12)
+ const retryUser = await normalPool.query(
+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',
+ [retryEmail, retryUserHash]
+ )
+ const retryUserId = retryUser.rows[0].id as string
+ const retryReset = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R2-H3 member',
+ email: retryEmail,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+ const retrySibling = await createSilentInvitationForTeams({
+ inviteeName: 'Synthetic R2-H3 member',
+ email: retryEmail,
+ purpose: 'password_reset',
+ teamAssignments: [],
+ fallbackRole: 'member',
+ })
+
+ await normalPool.query(`
+ CREATE FUNCTION reject_r2h3_user_update() RETURNS trigger AS $$
+ BEGIN
+ RAISE EXCEPTION 'synthetic deferred commit rejection' USING ERRCODE = '23514';
+ END;
+ $$ LANGUAGE plpgsql
+ `)
+ await normalPool.query(`
+ CREATE CONSTRAINT TRIGGER reject_r2h3_user_update
+ AFTER UPDATE ON users DEFERRABLE INITIALLY DEFERRED
+ FOR EACH ROW EXECUTE FUNCTION reject_r2h3_user_update()
+ `)
+
+ holder.unknownCommitOutcomes = 0
+ let resetError: unknown
+ try {
+ await setInvitationPasswordForEmail(retryEmail, retryReset.id, newPassword, retryUserId)
+ } catch (error) {
+ resetError = error
+ }
+ expect(resetError).toMatchObject({
+ status: 503,
+ publicError: 'authority_unavailable',
+ })
+ expect(holder.unknownCommitOutcomes).toBe(0)
+ const errorApp = express()
+ errorApp.post('/password-reset', (_req, res) => {
+ if (!sendPasswordAdmissionError(resetError, res)) {
+ res.status(500).json({ error: 'unexpected response contract' })
+ }
+ })
+ const publicError = await request(errorApp).post('/password-reset').send({})
+ expect(publicError.status).toBe(503)
+ expect(publicError.body).toEqual({ error: 'authority_unavailable', retryAfterSeconds: 2 })
+ expect(publicError.headers['retry-after']).toBe('2')
+ expect(publicError.headers['cache-control']).toBe('no-store')
+
+ const unchanged = await observer.query(
+ `SELECT i.status AS reset_status, sibling.status AS sibling_status,
+ u.password_hash, (SELECT COUNT(*)::int FROM password_verification_work) AS owners
+ FROM invitations i
+ JOIN invitations sibling ON sibling.id = $2
+ JOIN users u ON u.id = $3
+ WHERE i.id = $1`,
+ [retryReset.id, retrySibling.id, retryUserId]
+ )
+ expect(unchanged.rows[0]).toMatchObject({
+ reset_status: 'pending',
+ sibling_status: 'pending',
+ owners: 0,
+ password_hash: retryUserHash,
+ })
+
+ await normalPool.query('DROP TRIGGER reject_r2h3_user_update ON users')
+ await normalPool.query('DROP FUNCTION reject_r2h3_user_update()')
+ const retried = await setInvitationPasswordForEmail(
+ retryEmail,
+ retryReset.id,
+ newPassword,
+ retryUserId
+ )
+ expect(retried).toMatchObject({ data: { passwordUpdated: true } })
+ expect(holder.unknownCommitOutcomes).toBe(0)
+ const recovered = await observer.query(
+ `SELECT i.status AS reset_status, sibling.status AS sibling_status, u.password_hash
+ FROM invitations i
+ JOIN invitations sibling ON sibling.id = $2
+ JOIN users u ON u.id = $3
+ WHERE i.id = $1`,
+ [retryReset.id, retrySibling.id, retryUserId]
+ )
+ expect(recovered.rows[0]).toMatchObject({
+ reset_status: 'accepted',
+ sibling_status: 'revoked',
+ })
+ expect(await bcrypt.compare(newPassword, recovered.rows[0].password_hash as string)).toBe(true)
+ }, 20_000)
})