Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: classify password callback failures safely (R17-M1)

Josuéa0d406b9961b
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index f0e1c8757..0627c44e9 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.781",
+ "version": "0.1.782",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.781",
+ "version": "0.1.782",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index b92979931..e2e1f12a7 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.781",
+ "version": "0.1.782",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/http/errorHandler.ts b/control-api/src/http/errorHandler.ts
index 876f2f263..1af0d79b0 100644
--- a/control-api/src/http/errorHandler.ts
+++ b/control-api/src/http/errorHandler.ts
@@ -2,6 +2,7 @@ import type { NextFunction, Request, Response } from 'express'
import { ApiException } from '@kubernetes/client-node'
import { STATUS_CODES } from 'node:http'
import { rootLogger } from '../observability/logger.js'
+import { PasswordMutationInternalError } from '../services/auth/passwordAdmissionError.js'
import { memberRegistrationErrorResponse } from '../services/memberRegistrationErrors.js'
// Only errors we construct with these codes are ever forwarded verbatim by the
@@ -158,6 +159,11 @@ export function clerumErrorHandler(
const correlationId = req.correlationId ?? Math.random().toString(36).slice(2, 10)
const log = req.log ?? rootLogger
+ if (err instanceof PasswordMutationInternalError) {
+ res.status(500).json({ error: err.code, correlationId })
+ return
+ }
+
// Typed member-registration failures map to 503 (spec §8.6) — scoped
// instanceof check; the generic 5xx-collapse below stays intact.
const memberRegistration = memberRegistrationErrorResponse(err)
diff --git a/control-api/src/services/auth/passwordAdmissionError.ts b/control-api/src/services/auth/passwordAdmissionError.ts
index b7c3283ab..2bafff79a 100644
--- a/control-api/src/services/auth/passwordAdmissionError.ts
+++ b/control-api/src/services/auth/passwordAdmissionError.ts
@@ -6,10 +6,20 @@ export class PasswordAdmissionError extends Error {
readonly retryAfterSeconds: number,
readonly reason: PasswordAdmissionDenialReason,
readonly publicError:
- | 'rate_limited'
- | 'authority_unavailable'
- | 'recovery_outcome_unknown' = status === 429 ? 'rate_limited' : 'authority_unavailable'
+ 'rate_limited' | 'authority_unavailable' | 'recovery_outcome_unknown' = status === 429
+ ? 'rate_limited'
+ : 'authority_unavailable'
) {
super(status === 429 ? 'rate_limited' : 'authority_unavailable')
}
}
+
+/** Safe, non-retryable result for an unexpected password mutation callback failure. */
+export class PasswordMutationInternalError extends Error {
+ readonly code = 'internal_error'
+
+ constructor() {
+ super('internal_error')
+ this.name = 'PasswordMutationInternalError'
+ }
+}
diff --git a/control-api/src/services/auth/passwordWorkOwnership.ts b/control-api/src/services/auth/passwordWorkOwnership.ts
index a1b93a947..11d07bcb1 100644
--- a/control-api/src/services/auth/passwordWorkOwnership.ts
+++ b/control-api/src/services/auth/passwordWorkOwnership.ts
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto'
import { hostname } from 'node:os'
import { type DbTransactionClient, withTransaction } from '../../db.js'
import { rootLogger } from '../../observability/logger.js'
-import { PasswordAdmissionError } from './passwordAdmissionError.js'
+import { PasswordAdmissionError, PasswordMutationInternalError } from './passwordAdmissionError.js'
import { PASSWORD_ADMISSION_POLICY } from './passwordAdmissionState.js'
export type PasswordCommitMode = 'standard' | 'one_use_recovery'
@@ -29,7 +29,9 @@ function warn(event: string, message: string) {
logger.warn({ event }, message)
}
function sanitize(error: unknown): never {
- if (error instanceof PasswordAdmissionError) throw error
+ if (error instanceof PasswordAdmissionError || error instanceof PasswordMutationInternalError) {
+ throw error
+ }
warn('password_authority_unavailable', 'password authority unavailable')
throw authorityFailure()
}
@@ -86,7 +88,30 @@ async function reservePasswordWork() {
try {
return await withTransaction(
async db => {
- outcome = await work(db)
+ const queryFailures = new Set<unknown>()
+ const mutationDb: DbTransactionClient = {
+ ...db,
+ query: async (text, values) => {
+ try {
+ return await db.query(text, values)
+ } catch (error) {
+ queryFailures.add(error)
+ throw error
+ }
+ },
+ } as DbTransactionClient
+ try {
+ outcome = await work(mutationDb)
+ } catch (error) {
+ if (error instanceof PasswordAdmissionError || queryFailures.has(error)) {
+ throw error
+ }
+ logger.error(
+ { event: 'password_mutation_callback_failed' },
+ 'password mutation callback failed'
+ )
+ throw new PasswordMutationInternalError()
+ }
await releaseOwned(db)
return outcome
},
diff --git a/control-api/test/passwordRecovery.http.realPostgres.integration.test.ts b/control-api/test/passwordRecovery.http.realPostgres.integration.test.ts
index ca987daa6..f1acc5a6c 100644
--- a/control-api/test/passwordRecovery.http.realPostgres.integration.test.ts
+++ b/control-api/test/passwordRecovery.http.realPostgres.integration.test.ts
@@ -27,6 +27,7 @@ realPg('R2-M1/R2-M2 public password authority contract on PostgreSQL 16', () =>
let publicProcess: ExternalRestHttpProcess
let bcrypt: typeof import('bcryptjs').default
let sourceCounter = 1
+ let oneUseResetMutationPosts = 0
const invitationFlowFixtures = new Map<string, { email: string; invitationUuid: string }>()
beforeAll(async () => {
@@ -64,18 +65,26 @@ realPg('R2-M1/R2-M2 public password authority contract on PostgreSQL 16', () =>
await db.initDb()
const { requireInternalService, requireInternalToken } =
await import('../src/middleware/internalServiceAuth.js')
+ const { clerumErrorHandler } = await import('../src/http/errorHandler.js')
const { createExternalInvitationsRouter } =
await import('../src/routes/external/invitations.js')
const { createExternalUsersRouter } = await import('../src/routes/external/users.js')
controlApp = express()
controlApp.use(express.json())
+ controlApp.use((req, _res, next) => {
+ if (req.method === 'POST' && req.path === '/api/v1/external/invitations/password-token') {
+ oneUseResetMutationPosts++
+ }
+ next()
+ })
controlApp.use(requireInternalToken)
const controlApi = express.Router()
controlApi.use('/external', requireInternalService('external-rest-api'))
controlApi.use(createExternalInvitationsRouter())
controlApi.use(createExternalUsersRouter({} as Parameters<typeof createExternalUsersRouter>[0]))
controlApp.use('/api/v1', controlApi)
+ controlApp.use(clerumErrorHandler)
controlServer = controlApp.listen(0, '127.0.0.1')
await new Promise<void>(resolve => controlServer.once('listening', resolve))
@@ -254,6 +263,88 @@ realPg('R2-M1/R2-M2 public password authority contract on PostgreSQL 16', () =>
).toBe(true)
})
+ it('rolls back a callback TypeError and returns a sanitized non-retryable 500', async () => {
+ const session = await acceptedInvitationSession()
+ const failure = new TypeError('private SQL and credential diagnostic')
+ const hash = vi.spyOn(bcrypt, 'hash').mockRejectedValueOnce(failure)
+
+ let response: Awaited<ReturnType<typeof invitationPasswordRequest>>
+ try {
+ response = await invitationPasswordRequest(session, nextSourceIp())
+ } finally {
+ hash.mockRestore()
+ }
+
+ expect(response.status).toBe(500)
+ expect(response.body).toMatchObject({ error: 'internal_error' })
+ expect(response.headers['retry-after']).toBeUndefined()
+ expect(response.headers['set-cookie']).toBeUndefined()
+ expect(response.headers['cache-control']).toContain('no-store')
+ expect(response.text).not.toContain('private SQL')
+ expect(response.text).not.toContain('credential diagnostic')
+ expect(
+ (await db.pool.query('SELECT status FROM invitations WHERE id = $1', [session.invitationId]))
+ .rows[0].status
+ ).toBe('accepted')
+ expect(
+ (await db.pool.query('SELECT password_hash FROM users WHERE id = $1', [session.userId]))
+ .rows[0].password_hash
+ ).toBeNull()
+ expect(
+ (await db.pool.query('SELECT COUNT(*)::int AS count FROM password_verification_work')).rows[0]
+ .count
+ ).toBe(0)
+ })
+
+ it('rolls back a one-use reset callback TypeError without retrying the proof', async () => {
+ const email = `r2m1-reset-callback-${randomBytes(8).toString('hex')}@example.invalid`
+ const user = await db.pool.query('INSERT INTO users(email) VALUES ($1) RETURNING id', [email])
+ const { createSilentPasswordResetInvitation } =
+ await import('../src/services/directory/passwordResetInvitations.js')
+ const invitation = await createSilentPasswordResetInvitation({
+ inviteeName: 'Synthetic callback-failure reset',
+ email,
+ })
+ const flowToken = `synthetic-reset-callback-${randomBytes(20).toString('hex')}`
+ invitationFlowFixtures.set(flowToken, { email, invitationUuid: invitation.token })
+ const mutationPostsBefore = oneUseResetMutationPosts
+ const failure = new TypeError('private reset SQL and proof diagnostic')
+ const hash = vi.spyOn(bcrypt, 'hash').mockRejectedValueOnce(failure)
+
+ let response: Awaited<ReturnType<typeof invitationPasswordRequestWithoutSession>>
+ try {
+ response = await invitationPasswordRequestWithoutSession(
+ { id: invitation.id, token: flowToken },
+ email,
+ nextSourceIp()
+ )
+ } finally {
+ hash.mockRestore()
+ }
+
+ expect(response.status).toBe(500)
+ expect(response.body).toMatchObject({ error: 'internal_error' })
+ expect(response.headers['retry-after']).toBeUndefined()
+ expect(response.headers['set-cookie']).toBeUndefined()
+ expect(response.text).not.toContain('private reset SQL')
+ expect(response.text).not.toContain('proof diagnostic')
+ expect(oneUseResetMutationPosts - mutationPostsBefore).toBe(1)
+ expect(
+ (
+ await db.pool.query(
+ `SELECT i.status, u.password_hash
+ FROM invitations i JOIN users u ON u.id = $2
+ WHERE i.id = $1`,
+ [invitation.id, user.rows[0].id]
+ )
+ ).rows[0]
+ ).toMatchObject({ status: 'pending', password_hash: null })
+ expect(
+ (await db.pool.query('SELECT COUNT(*)::int AS count FROM password_verification_work')).rows[0]
+ .count
+ ).toBe(0)
+ })
+
it.each(['member_invitation', 'admin_desktop_access'] as const)(
'composes token-based %s onboarding through the ordinary endpoint and persists its identity',
async purpose => {
diff --git a/control-api/test/passwordWorkOwnership.operation.test.ts b/control-api/test/passwordWorkOwnership.operation.test.ts
index cbb3d1e52..ad47d3fca 100644
--- a/control-api/test/passwordWorkOwnership.operation.test.ts
+++ b/control-api/test/passwordWorkOwnership.operation.test.ts
@@ -73,6 +73,87 @@ describe('password-work reservation preservation', () => {
).rejects.toBe(failure)
})
+ it.each([
+ ['TypeError', () => new TypeError('private callback details')],
+ ['Error', () => new Error('private callback details')],
+ ])(
+ 'classifies an unexpected %s from commitWith as a non-retryable internal error',
+ async (_name, makeError) => {
+ await expect(
+ withPasswordWorkLease(lease =>
+ lease.commitWith(
+ async () => {
+ throw makeError()
+ },
+ { mode: 'one_use_recovery' }
+ )
+ )
+ ).rejects.toMatchObject({
+ name: 'PasswordMutationInternalError',
+ code: 'internal_error',
+ message: 'internal_error',
+ })
+ expect(query.mock.calls.filter(([sql]) => sql.includes('DELETE FROM'))).toHaveLength(1)
+ }
+ )
+
+ it('keeps a TypeError from the transaction query boundary as authority unavailable', async () => {
+ query.mockImplementation(async (sql: string) => {
+ if (sql === 'SELECT synthetic driver failure') {
+ throw new TypeError('private driver diagnostic')
+ }
+ if (sql.includes('INSERT INTO password_verification_work')) {
+ return { rows: [{ singleton: true }] }
+ }
+ if (sql.includes('DELETE FROM password_verification_work')) {
+ return { rows: [{ singleton: true }] }
+ }
+ return { rows: [] }
+ })
+
+ await expect(
+ runPasswordMutation(
+ async db => {
+ await db.query('SELECT synthetic driver failure')
+ return 'unreachable'
+ },
+ { mode: 'standard' }
+ )
+ ).rejects.toMatchObject({
+ status: 503,
+ publicError: 'authority_unavailable',
+ message: 'authority_unavailable',
+ })
+ })
+
+ it('classifies a distinct callback failure after a caught query error as internal', async () => {
+ query.mockImplementation(async (sql: string) => {
+ if (sql === 'SELECT synthetic driver failure') {
+ throw new TypeError('private driver diagnostic')
+ }
+ if (sql.includes('INSERT INTO password_verification_work')) {
+ return { rows: [{ singleton: true }] }
+ }
+ if (sql.includes('DELETE FROM password_verification_work')) {
+ return { rows: [{ singleton: true }] }
+ }
+ return { rows: [] }
+ })
+
+ await expect(
+ runPasswordMutation(
+ async db => {
+ await db.query('SELECT synthetic driver failure').catch(() => undefined)
+ throw new TypeError('private callback defect')
+ },
+ { mode: 'standard' }
+ )
+ ).rejects.toMatchObject({
+ name: 'PasswordMutationInternalError',
+ code: 'internal_error',
+ })
+ })
+
it.each([false, true])('release failure wins over outer work failure=%s safely', async fails => {
state.failRelease = true
const outcome = withMemberPasswordWorkLease(async () => {
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index eab0a3e2d..0a01f956f 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.129",
+ "version": "0.1.130",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.129",
+ "version": "0.1.130",
"dependencies": {
"@clerum/jwt-key-policy": "file:../packages/jwt-key-policy",
"cors": "^2.8.5",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index 195f25710..cd2c73e1f 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.129",
+ "version": "0.1.130",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/http/passwordAdmissionError.ts b/external-rest-api/src/http/passwordAdmissionError.ts
index 7a3bc1d9f..87cdb3429 100644
--- a/external-rest-api/src/http/passwordAdmissionError.ts
+++ b/external-rest-api/src/http/passwordAdmissionError.ts
@@ -59,6 +59,24 @@ export function sendPasswordAuthorityError(error: unknown, res: Response): boole
)
return true
}
+ if (error.status === 500 && error.code === 'internal_error') {
+ for (const name of [
+ 'Retry-After',
+ 'X-RateLimit-Limit',
+ 'X-RateLimit-Remaining',
+ 'X-RateLimit-Reset',
+ 'RateLimit',
+ 'RateLimit-Policy',
+ 'RateLimit-Limit',
+ 'RateLimit-Remaining',
+ 'RateLimit-Reset',
+ ]) {
+ res.removeHeader(name)
+ }
+ res.setHeader('Cache-Control', 'no-store')
+ res.status(500).json({ error: 'internal_error' })
+ return true
+ }
if (error.status >= 500) {
sendPublicPasswordAuthorityError(
res,
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 25201aad1..e15863de5 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -8,7 +8,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.11.0',
- externalRestApiVersion: '0.1.129',
+ externalRestApiVersion: '0.1.130',
rpcProxyVersion: '0.1.126',
desktopVersion: '0.11.0',
minimumDesktopVersion: '0.1.252',
diff --git a/external-rest-api/src/services/invitationPasswordService.ts b/external-rest-api/src/services/invitationPasswordService.ts
index 1240ac269..73987faf9 100644
--- a/external-rest-api/src/services/invitationPasswordService.ts
+++ b/external-rest-api/src/services/invitationPasswordService.ts
@@ -13,12 +13,7 @@ type AuthContext = {
}
type ResetPasswordDomainError =
- | 'not_found'
- | 'forbidden'
- | 'not_pending'
- | 'expired'
- | 'invalid_invitation'
- | 'invalid_password'
+ 'not_found' | 'forbidden' | 'not_pending' | 'expired' | 'invalid_invitation' | 'invalid_password'
function invitationPasswordDomainError(
error: ControlApiError
@@ -82,12 +77,7 @@ export async function setupInvitationPassword(
password: string
): Promise<{
error?:
- | 'not_found'
- | 'forbidden'
- | 'not_accepted'
- | 'not_pending'
- | 'expired'
- | 'invalid_password'
+ 'not_found' | 'forbidden' | 'not_accepted' | 'not_pending' | 'expired' | 'invalid_password'
data?: InvitationPreview & { passwordUpdated: boolean }
}> {
try {
@@ -107,6 +97,13 @@ export async function setupInvitationPassword(
return { data }
} catch (error) {
if (error instanceof ControlApiTransportError) throw passwordAuthorityUnavailable()
+ if (
+ error instanceof ControlApiError &&
+ error.status === 500 &&
+ error.code === 'internal_error'
+ ) {
+ throw error
+ }
if (error instanceof ControlApiError && [429, 503].includes(error.status)) throw error
if (error instanceof ControlApiError) {
const domainError = invitationPasswordDomainError(error)
@@ -146,6 +143,7 @@ export async function setupInvitationPasswordWithToken(
if (error instanceof ControlApiTransportError) throw unknownRecoveryOutcome()
if (error instanceof ControlApiError && error.status === 429) throw error
if (error instanceof ControlApiError) {
+ if (error.status === 500 && error.code === 'internal_error') throw error
if (
error.status === 503 &&
(error.code === 'authority_unavailable' || error.code === 'recovery_outcome_unknown')
diff --git a/external-rest-api/test/invitationAdmission.public.test.ts b/external-rest-api/test/invitationAdmission.public.test.ts
index 6c4a0291d..bac6eef4f 100644
--- a/external-rest-api/test/invitationAdmission.public.test.ts
+++ b/external-rest-api/test/invitationAdmission.public.test.ts
@@ -116,4 +116,26 @@ describe('accepted invitation password authority outcomes', () => {
expect(JSON.stringify(response.body)).not.toContain('database')
expect(fetchMock).toHaveBeenCalledOnce()
})
+
+ it('returns a sanitized non-retryable internal error for callback programming failures', async () => {
+ fetchMock.mockResolvedValueOnce(
+ upstreamResponse(
+ 500,
+ { error: 'internal_error', detail: 'private SQL and credential diagnostic' },
+ { 'retry-after': '1200', 'x-ratelimit-remaining': '0' }
+ )
+ )
+
+ const response = await submitPassword()
+
+ expect(response.status).toBe(500)
+ expect(response.body).toEqual({ error: 'internal_error' })
+ expect(response.headers['retry-after']).toBeUndefined()
+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()
+ expect(response.headers['cache-control']).toContain('no-store')
+ expect(response.headers['set-cookie']).toBeUndefined()
+ expect(response.text).not.toContain('private SQL')
+ expect(response.text).not.toContain('credential diagnostic')
+ expect(fetchMock).toHaveBeenCalledOnce()
+ })
})
diff --git a/external-rest-api/test/invitations.passwordRecovery.public.test.ts b/external-rest-api/test/invitations.passwordRecovery.public.test.ts
index 3c262ae10..bdb29d78b 100644
--- a/external-rest-api/test/invitations.passwordRecovery.public.test.ts
+++ b/external-rest-api/test/invitations.passwordRecovery.public.test.ts
@@ -165,6 +165,31 @@ describe('password reset preview authority failures', () => {
expect(fetchMock).toHaveBeenCalledTimes(1)
})
+ it('preserves a definite internal callback failure without retry or proof replay', async () => {
+ let resetPosts = 0
+ await startUpstream((req, res) => {
+ if (req.method === 'POST') resetPosts += 1
+ res.writeHead(500, {
+ 'content-type': 'application/json',
+ 'retry-after': '1200',
+ 'x-ratelimit-remaining': '0',
+ })
+ res.end(JSON.stringify({ error: 'internal_error', detail: 'private reset diagnostic' }))
+ })
+
+ const response = await submitDedicatedReset()
+
+ expect(response.status).toBe(500)
+ expect(response.body).toEqual({ error: 'internal_error' })
+ expect(response.headers['retry-after']).toBeUndefined()
+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()
+ expect(response.headers['cache-control']).toContain('no-store')
+ expect(response.headers['set-cookie']).toBeUndefined()
+ expect(response.text).not.toContain('private reset diagnostic')
+ expect(response.text).not.toContain(recoveryProof)
+ expect(resetPosts).toBe(1)
+ })
+
it('sanitizes an unrecognized producer client response without a retry hint', async () => {
await startUpstream((_req, res) => {
res.writeHead(400, { 'content-type': 'application/json' })