Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): restore verified password recovery
Josué06e5dd357f08
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 86e53095b..092d6afee 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.737",+ "version": "0.1.738","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.737",+ "version": "0.1.738","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex d584b970e..2b01bf382 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.737",+ "version": "0.1.738","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/passwordLoginAdmission.ts b/control-api/src/middleware/passwordLoginAdmission.tsindex 27a80a628..6624dc302 100644--- a/control-api/src/middleware/passwordLoginAdmission.ts+++ b/control-api/src/middleware/passwordLoginAdmission.ts@@ -51,7 +51,7 @@ export const passwordLoginSourceAdmission = rateLimitMiddleware({},onLimited: (_req, res, retryAfterSeconds) => {sendPasswordAdmissionError(- new PasswordAdmissionError(429, Math.min(60, retryAfterSeconds)),+ new PasswordAdmissionError(429, Math.min(60, retryAfterSeconds), 'source_rate'),res)},diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 21c338f14..37ae61d77 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -663,3 +663,17 @@ export const llmAllowlistConfigMapWriteFailuresTotal = getOrCreateCounter({help: 'Count of failed clerum-llm-allowed-models ConfigMap writes, labelled by phase.',labelNames: ['phase'] as const as Array<'phase'>, // mutation | boot | sync})++export type PasswordAdmissionDenialReason =+ | 'source_rate'+ | 'identifier_attempts'+ | 'identifier_cooldown'+ | 'global_pace'+ | 'verification_busy'+ | 'authority_failure'++export const passwordAdmissionDenialsTotal = getOrCreateCounter({+ name: 'password_admission_denials_total',+ help: 'Password admission denials by bounded internal reason; never labelled by identity.',+ labelNames: ['reason'] as const as Array<'reason'>,+})diff --git a/control-api/src/routes/external/invitations.ts b/control-api/src/routes/external/invitations.tsindex 502363b5f..665393328 100644--- a/control-api/src/routes/external/invitations.ts+++ b/control-api/src/routes/external/invitations.ts@@ -102,13 +102,25 @@ export function createExternalInvitationsRouter(): Router {if (memberRegistrationErrorResponse(error)) throw errorreturn res.status(400).json({ error: 'invalid_invitation' })}- if (validation.invitationUuid !== invitationId) {+ // The trusted Member Registration flow binds invitationUuid to the+ // invitation token. The UI submits the distinct database row ID.+ // Resolve the trusted token first, then bind the supplied row ID to+ // that exact invitation before mutating credentials.+ const invitation = await getInvitationByToken(validation.invitationUuid)+ if (!invitation || invitation.id !== invitationId) {return res.status(403).json({ error: 'forbidden' })}+ if (+ invitation.email.toLowerCase() !== validation.email.toLowerCase() ||+ invitation.purpose !== 'password_reset' ||+ invitation.status !== 'pending'+ ) {+ return res.status(409).json({ error: 'invitation_not_pending' })+ }const result = await setInvitationPasswordForEmail(validation.email,- validation.invitationUuid,+ invitation.id,password)if ('error' in result) {@@ -118,9 +130,6 @@ export function createExternalInvitationsRouter(): Router {if (result.error === 'forbidden') {return res.status(403).json({ error: 'forbidden' })}- if (result.error === 'not_accepted') {- return res.status(409).json({ error: 'invitation_not_accepted' })- }if (result.error === 'not_pending') {return res.status(409).json({ error: 'invitation_not_pending' })}@@ -137,28 +146,25 @@ export function createExternalInvitationsRouter(): Router {if (!userId) {return res.status(409).json({ error: 'invitation_not_ready' })}- const authGeneration = Number(result.data.authGeneration)+ const session = result.data.sessionContextif (- result.data.lifecycleState !== 'active' ||- !Number.isSafeInteger(authGeneration) ||- authGeneration < 1+ !session ||+ session.userId !== userId ||+ !Number.isSafeInteger(session.authGeneration) ||+ session.authGeneration < 1) {return res.status(409).json({ error: 'invitation_not_ready' })}- const sessionToken = signExternalSessionToken({- userId,- email: result.data.email,- teamId: result.data.teamId || null,- role: result.data.role,- authGeneration,- })+ const sessionToken = signExternalSessionToken(session)+ const { sessionContext: _sessionContext, ...response } = result.datareturn res.status(200).json({- ...result.data,+ ...response,token: sessionToken,})} catch (error) {+ if (sendPasswordAdmissionError(error, res)) returnreturn next(error)}}@@ -205,6 +211,7 @@ export function createExternalInvitationsRouter(): Router {}return res.status(200).json(result.data)} catch (error) {+ if (sendPasswordAdmissionError(error, res)) returnreturn next(error)}}diff --git a/control-api/src/services/auth/passwordCredentialVerification.ts b/control-api/src/services/auth/passwordCredentialVerification.tsindex d53e7c9d4..abd293899 100644--- a/control-api/src/services/auth/passwordCredentialVerification.ts+++ b/control-api/src/services/auth/passwordCredentialVerification.ts@@ -1,6 +1,10 @@import bcrypt from 'bcryptjs'import { type DbTransactionClient, pool, withTransaction } from '../../db.js'import { rootLogger } from '../../observability/logger.js'+import {+ type PasswordAdmissionDenialReason,+ passwordAdmissionDenialsTotal,+} from '../../observability/metrics.js'import {type PasswordIdentifierState,admitPasswordIdentifier,@@ -17,13 +21,19 @@ const logger = rootLogger.child({ module: 'password-admission' })export class PasswordAdmissionError extends Error {constructor(readonly status: 429 | 503,- readonly retryAfterSeconds: number+ readonly retryAfterSeconds: number,+ readonly reason: PasswordAdmissionDenialReason) {super(status === 429 ? 'rate_limited' : 'authority_unavailable')+ passwordAdmissionDenialsTotal.inc({ reason })}}-function denied(retryMs: number): never {- throw new PasswordAdmissionError(429, Math.min(900, Math.max(1, Math.ceil(retryMs / 1000))))+function denied(retryMs: number, reason: PasswordAdmissionDenialReason): never {+ throw new PasswordAdmissionError(+ 429,+ Math.min(900, Math.max(1, Math.ceil(retryMs / 1000))),+ reason+ )}export type PasswordUser = {id: string@@ -106,16 +116,20 @@ export async function capturePasswordEvaluation(const now = await databaseNow(db)const decision = admitPasswordIdentifier(stateOf(row), now, chargeAttempt)// A denied request never writes: it cannot move the rolling denial horizon.- if (decision.retryMs) return { retryMs: decision.retryMs, row, principal }+ if (decision.retryMs) {+ const reason: PasswordAdmissionDenialReason =+ Number(row.locked_until_ms) > now ? 'identifier_cooldown' : 'identifier_attempts'+ return { retryMs: decision.retryMs, reason, row, principal }+ }const expiresAt = now + policy.evaluationMsawait db.query('UPDATE password_identifier_state SET attempts = $2, failures = $3, locked_until_ms = $4, retained_until_ms = greatest(retained_until_ms, $5) WHERE identifier_key = $1',[key, decision.state.attempts, decision.state.failures, decision.state.lockedUntil, expiresAt])- return { retryMs: 0, row, expiresAt, principal }+ return { retryMs: 0, reason: null, row, expiresAt, principal }})if (!snapshot) return null- if (snapshot.retryMs) denied(snapshot.retryMs)+ if (snapshot.retryMs) denied(snapshot.retryMs, snapshot.reason ?? 'identifier_attempts')// No transaction or row lock survives into bcrypt.let user = snapshot.principalif (userId === undefined) {@@ -149,7 +163,7 @@ export async function reservePasswordPace(): Promise<void> {const remaining = await pool.query(`SELECT greatest(1, ceil(extract(epoch FROM next_permit - clock_timestamp()) * 1000)) AS retry_ms FROM password_verification_pace WHERE singleton`)- denied(Number(remaining.rows[0]?.retry_ms ?? policy.paceMs))+ denied(Number(remaining.rows[0]?.retry_ms ?? policy.paceMs), 'global_pace')}}@@ -207,11 +221,11 @@ export async function verifyMemberPassword(if (!capture) return nullawait reservePasswordPace()// The pace permit is deliberately not refunded on a busy lease.- if (busy) denied(policy.paceMs)+ if (busy) denied(policy.paceMs, 'verification_busy')busy = truetry {const lease = await acquirePasswordWork()- if (!lease) denied(policy.paceMs)+ if (!lease) denied(policy.paceMs, 'verification_busy')const user = capture.userlet authenticated = falsetry {@@ -249,7 +263,7 @@ export async function verifyMemberPassword(if (error instanceof PasswordAdmissionError) throw error// Do not log SQL, identifiers, hashes, submitted credentials or backend messages.logger.warn({ event: 'password_authority_unavailable' }, 'password authority unavailable')- throw new PasswordAdmissionError(503, 2)+ throw new PasswordAdmissionError(503, 2, 'authority_failure')}}diff --git a/control-api/src/services/auth/passwordWorkOwnership.ts b/control-api/src/services/auth/passwordWorkOwnership.tsindex 3d5c8655d..538fa5f1d 100644--- a/control-api/src/services/auth/passwordWorkOwnership.ts+++ b/control-api/src/services/auth/passwordWorkOwnership.ts@@ -1,12 +1,15 @@import { randomUUID } from 'node:crypto'import { hostname } from 'node:os'-import { withTransaction } from '../../db.js'+import { type DbTransactionClient, withTransaction } from '../../db.js'// Process-incarnation identity is distinct from reusable host/PID diagnostics.let ownerInstance: string | undefined/** Null means immediately busy. A persistence error propagates for sanitized fail-closed handling. */-export async function acquirePasswordWork(): Promise<{ release: () => Promise<void> } | null> {+export async function acquirePasswordWork(): Promise<{+ release: () => Promise<void>+ releaseWithin: (db: Pick<DbTransactionClient, 'query'>) => Promise<void>+} | null> {const owner = (ownerInstance ??= randomUUID())const operation = randomUUID()const admitted = await withTransaction(async db => {@@ -22,18 +25,24 @@ export async function acquirePasswordWork(): Promise<{ release: () => Promise<voreturn result.rows.length === 1})if (!admitted) return null+ const releaseOwned = async (db: Pick<DbTransactionClient, 'query'>) => {+ await db.query('SET LOCAL synchronous_commit = on')+ const result = await db.query(+ `DELETE FROM password_verification_work+ WHERE singleton AND operation_id = $1 AND owner_instance = $2 RETURNING singleton`,+ [operation, owner]+ )+ if (result.rows.length !== 1) throw new Error('password work ownership lost')+ }let released: Promise<void> | undefinedreturn {+ // Password credential updates can release the durable owner in the same+ // transaction as their commit. If that transaction rolls back, callers+ // must fall back to release() so the persisted owner does not linger.+ releaseWithin: releaseOwned,release: () => {released ??= withTransaction(async db => {- await db.query('SET LOCAL synchronous_commit = on')- const result = await db.query(- `DELETE FROM password_verification_work- WHERE singleton AND operation_id = $1 AND owner_instance = $2 RETURNING singleton`,- [operation, owner]- )- // An externally removed/replaced owner is not proof that our authority was preserved.- if (result.rows.length !== 1) throw new Error('password work ownership lost')+ await releaseOwned(db)})return released},diff --git a/control-api/src/services/directory/login.ts b/control-api/src/services/directory/login.tsindex 1249f4099..d38540a65 100644--- a/control-api/src/services/directory/login.ts+++ b/control-api/src/services/directory/login.ts@@ -1,4 +1,4 @@-import { withTransaction } from '../../db.js'+import { type DbTransactionClient, withTransaction } from '../../db.js'import { verifyMemberPassword } from '../auth/passwordCredentialVerification.js'import type { TeamRole } from './types.js'@@ -131,6 +131,19 @@ function teamlessMemberMembership(): MembershipRow {}}+async function currentMemberMembership(+ db: Pick<DbTransactionClient, 'query'>,+ user: { id: string; email: string }+): Promise<MembershipRow> {+ let membership = await findFirstActiveMembership(db, user.id)+ if ((membership.rowCount ?? 0) === 0) {+ await healAcceptedInvitationMemberships(db, user.id, user.email)+ await linkAcceptedInvitationsToUser(db, user.id, user.email)+ membership = await findFirstActiveMembership(db, user.id)+ }+ return (membership.rows[0] as MembershipRow | undefined) || teamlessMemberMembership()+}+export async function googleLoginData(input: { email: string; name?: string; picture?: string }) {return withTransaction(async db => {const existing = await db.query(@@ -202,26 +215,6 @@ export async function passwordLoginData(input: { email: string; password: stringconst user = await verifyMemberPassword(input.email, input.password, { publicLogin: true })if (!user) return nullreturn withTransaction(async db => {- let membership = await findFirstActiveMembership(db, user.id)- if ((membership.rowCount ?? 0) === 0) {- await healAcceptedInvitationMemberships(db, user.id, user.email)- await linkAcceptedInvitationsToUser(db, user.id, user.email)- membership = await findFirstActiveMembership(db, user.id)- }-- if ((membership.rowCount ?? 0) === 0) {- return {- authGeneration: Number(user.lifecycle_version || 0),- user: {- id: user.id,- email: user.email,- name: user.name,- picture: user.picture,- },- membership: teamlessMemberMembership(),- }- }-return {authGeneration: Number(user.lifecycle_version || 0),user: {@@ -230,11 +223,26 @@ export async function passwordLoginData(input: { email: string; password: stringname: user.name,picture: user.picture,},- membership: membership.rows[0] as MembershipRow,+ membership: await currentMemberMembership(db, user),}})}+/** Resolve ordinary session claims in the same transaction as password recovery. */+export async function getMemberSessionContext(+ db: Pick<DbTransactionClient, 'query'>,+ user: { id: string; email: string; lifecycle_version?: number | string | null }+) {+ const membership = await currentMemberMembership(db, user)+ return {+ userId: user.id,+ email: user.email,+ teamId: membership.team_id,+ role: membership.role,+ authGeneration: Number(user.lifecycle_version || 0),+ }+}+export async function verifyUserPassword(input: {userId: stringemail: stringdiff --git a/control-api/src/services/directory/membership.ts b/control-api/src/services/directory/membership.tsindex 181fc5443..5af83587c 100644--- a/control-api/src/services/directory/membership.ts+++ b/control-api/src/services/directory/membership.ts@@ -1,7 +1,12 @@import bcrypt from 'bcryptjs'import { type DbClient, pool, withTransaction } from '../../db.js'+import { rootLogger } from '../../observability/logger.js'+import { PASSWORD_ADMISSION_POLICY } from '../auth/passwordAdmissionState.js'+import { PasswordAdmissionError } from '../auth/passwordCredentialVerification.js'+import { acquirePasswordWork } from '../auth/passwordWorkOwnership.js'import { registerAndSendInvitation } from '../invitationFlowRegistrationService.js'import { appendControlApiPermissionEventsInTransaction } from '../tracing/controlApiPermissionEvents.js'+import { getMemberSessionContext } from './login.js'import type { InviteRole, TeamRole } from './types.js'import {normalizeChannels,@@ -16,6 +21,9 @@ const DRAFT_INVITATION_CLEANUP_HOURS = 24const DRAFT_INVITATION_CLEANUP_INTERVAL_MS = 60 * 60 * 1000let draftInvitationCleanupTimer: ReturnType<typeof setInterval> | null = null+class PasswordResetPrincipalUnavailableError extends Error {}+const passwordSetupLogger = rootLogger.child({ module: 'password-admission' })+type InvitationRow = {id: stringteam_id: string | null@@ -144,7 +152,8 @@ async function getInvitationRecordByTokenOrId(async function getInvitationRecordById(db: Pick<DbClient, 'query'>,- invitationId: string+ invitationId: string,+ lockForUpdate = false): Promise<InvitationRow | null> {const normalizedInvitationId = invitationId.trim()if (!normalizedInvitationId) return null@@ -156,12 +165,60 @@ async function getInvitationRecordById(FROM invitations iLEFT JOIN teams t ON t.id = i.team_idWHERE i.id::text = $1- LIMIT 1`,+ LIMIT 1+ ${lockForUpdate ? 'FOR UPDATE OF i' : ''}`,[normalizedInvitationId])return (result.rows[0] as InvitationRow | undefined) || null}+async function acquirePasswordSetupLease() {+ const retryAfterSeconds = Math.ceil(PASSWORD_ADMISSION_POLICY.paceMs / 1000)+ try {+ const lease = await acquirePasswordWork()+ if (!lease) {+ throw new PasswordAdmissionError(429, retryAfterSeconds, 'verification_busy')+ }+ return lease+ } catch (error) {+ if (error instanceof PasswordAdmissionError) throw error+ throw new PasswordAdmissionError(503, 2, 'authority_failure')+ }+}++async function lockPasswordResetUser(+ db: Pick<DbClient, 'query'>,+ email: string+): Promise<InvitationUserRow | null> {+ const result = await db.query(+ `SELECT id, email, name, picture, password_hash, lifecycle_state, lifecycle_version+ FROM users+ WHERE email = $1+ LIMIT 1+ FOR UPDATE`,+ [email]+ )+ return (result.rows[0] as InvitationUserRow | undefined) ?? null+}++async function revokeSiblingPasswordResets(+ db: Pick<DbClient, 'query'>,+ email: string,+ redeemedInvitationId: string,+ userId: string+): Promise<void> {+ await db.query(+ `UPDATE invitations+ SET status = 'revoked'+ WHERE lower(email) = $1+ AND purpose = 'password_reset'+ AND status IN ('draft', 'pending')+ AND id <> $2::uuid+ AND (accepted_user_id IS NULL OR accepted_user_id = $3)`,+ [email, redeemedInvitationId, userId]+ )+}+async function ensureInvitationUser(db: Pick<DbClient, 'query'>,email: string,@@ -674,13 +731,19 @@ async function createInvitationForTeamsRecord() {cleanupStaleDraftInvitations().catch(() => undefined)const purpose = normalizeInvitationPurpose(input.purpose)- const inserted = await withTransaction(async db =>- insertInvitationForTeams(db, {+ const inserted = await withTransaction(async db => {+ if (purpose === 'password_reset') {+ const principal = await lockPasswordResetUser(db, input.email.trim().toLowerCase())+ if (!principal || invitationUserIsRetired(principal)) {+ throw new PasswordResetPrincipalUnavailableError()+ }+ }+ return insertInvitationForTeams(db, {...input,purpose,status: options.sendEmail ? 'draft' : 'pending',})- )+ })if (!options.sendEmail) {return invitationForTeamsResponse(inserted.invitation, inserted.teams)@@ -710,6 +773,12 @@ async function createInvitationForTeamsRecord()const pending = activated.rows[0] as InvitationRow | undefinedif (!pending) {+ if (purpose === 'password_reset') {+ const current = await getInvitationRecordById(pool, inserted.invitation.id)+ if (current?.status === 'revoked') {+ return invitationForTeamsResponse(current, inserted.teams)+ }+ }throw new Error('Failed to activate invitation after registration')}return invitationForTeamsResponse(@@ -859,13 +928,17 @@ export async function requestProfilePasswordReset(email: string): Promise<{ requreturn { requested: true }}- await createInvitationForTeams({- inviteeName: user.name || user.email.split('@')[0] || user.email,- email: user.email,- purpose: 'password_reset',- teamAssignments: [],- fallbackRole: 'member',- })+ try {+ await createInvitationForTeams({+ inviteeName: user.name || user.email.split('@')[0] || user.email,+ email: user.email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ } catch (error) {+ if (!(error instanceof PasswordResetPrincipalUnavailableError)) throw error+ }return { requested: true }}@@ -1025,8 +1098,7 @@ export async function revokePendingInvitation(return (result.rows[0] as { id: string; email: string } | undefined) || null}-export async function getInvitationByToken(token: string) {- const invitation = await getInvitationRecordByToken(pool, token.trim())+async function invitationResponseForActiveRecord(invitation: InvitationRow | null) {if (!invitation) return nullif (invitation.status === 'draft' || invitation.status === 'revoked') return nullif (invitation.status === 'pending' && invitation.expires_at.getTime() <= Date.now()) {@@ -1055,6 +1127,10 @@ export async function getInvitationByToken(token: string) {return invitationResponse(invitation, user, teams)}+export async function getInvitationByToken(token: string) {+ return invitationResponseForActiveRecord(await getInvitationRecordByToken(pool, token.trim()))+}+export async function setInvitationPasswordForUser(userId: string,email: string,@@ -1072,57 +1148,52 @@ export async function setInvitationPasswordForUser(return { error: 'invalid_password' as const }}+ const candidate = await getInvitationRecordById(pool, trimmedInvitationId)+ if (candidate?.purpose === 'password_reset') {+ if (candidate.email.toLowerCase() !== normalizedEmail) {+ return { error: 'forbidden' as const }+ }+ const result = await setInvitationPasswordForEmail(+ normalizedEmail,+ trimmedInvitationId,+ trimmedPassword,+ trimmedUserId+ )+ if ('error' in result) return result+ const { sessionContext: _sessionContext, ...data } = result.data+ return { data }+ }+return withTransaction(async db => {const invitation = await getInvitationRecordById(db, trimmedInvitationId)- if (!invitation) {- return { error: 'not_found' as const }- }+ if (!invitation) return { error: 'not_found' as const }if (invitation.email.toLowerCase() !== normalizedEmail) {return { error: 'forbidden' as const }}- if (invitation.purpose === 'password_reset') {- if (invitation.status !== 'pending') {- return { error: 'not_pending' as const }- }- if (invitation.expires_at.getTime() <= Date.now()) {- return { error: 'expired' as const }- }- } else if (- invitation.purpose === 'member_invitation' ||- invitation.purpose === 'admin_desktop_access'+ if (+ invitation.purpose !== 'member_invitation' &&+ invitation.purpose !== 'admin_desktop_access') {- if (invitation.status !== 'accepted') {- return { error: 'not_accepted' as const }- }- } else {return { error: 'not_found' as const }}-+ if (invitation.status !== 'accepted') return { error: 'not_accepted' as const }const user = await ensureInvitationUser(db, invitation.email, invitation.invitee_name)if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }- if (user.id !== trimmedUserId) {- return { error: 'forbidden' as const }- }+ if (user.id !== trimmedUserId) return { error: 'forbidden' as const }const passwordHash = await bcrypt.hash(trimmedPassword, 12)- await db.query(+ const update = await db.query(`UPDATE users- SET password_hash = $2,- password_set_at = NOW(),- updated_at = NOW()- WHERE id = $1- AND lifecycle_state = 'active'`,+ SET password_hash = $2,+ password_set_at = NOW(),+ updated_at = NOW()+ WHERE id = $1+ AND lifecycle_state = 'active'+ RETURNING id`,[user.id, passwordHash])- if (invitation.purpose === 'password_reset' && invitation.status === 'pending') {- await db.query(- `UPDATE invitations- SET status = 'accepted',- accepted_at = NOW(),- accepted_user_id = $2- WHERE id = $1`,- [invitation.id, user.id]- )- } else if (!invitation.accepted_user_id) {+ if (update.rows.length !== 1) return { error: 'user_retired' as const }++ if (!invitation.accepted_user_id) {await db.query(`UPDATE invitations SET accepted_user_id = $2 WHERE id = $1`, [invitation.id,user.id,@@ -1130,10 +1201,7 @@ export async function setInvitationPasswordForUser(}const refreshed = await getInvitationRecordById(db, invitation.id)- if (!refreshed) {- return { error: 'not_found' as const }- }-+ if (!refreshed) return { error: 'not_found' as const }const teams = await loadInvitationTeams(db, refreshed.id, refreshed)return {data: {@@ -1147,7 +1215,8 @@ export async function setInvitationPasswordForUser(export async function setInvitationPasswordForEmail(email: string,invitationId: string,- password: string+ password: string,+ expectedUserId?: string) {const normalizedEmail = email.trim().toLowerCase()const trimmedInvitationId = invitationId.trim()@@ -1159,73 +1228,99 @@ export async function setInvitationPasswordForEmail(return { error: 'invalid_password' as const }}- return withTransaction(async db => {- const invitation = await getInvitationRecordById(db, trimmedInvitationId)- if (!invitation) {- return { error: 'not_found' as const }- }- if (invitation.email.toLowerCase() !== normalizedEmail) {- return { error: 'forbidden' as const }- }- if (invitation.purpose === 'password_reset') {- if (invitation.status !== 'pending') {- return { error: 'not_pending' as const }+ const lease = await acquirePasswordSetupLease()+ let releaseStaged = false+ let releasedWithCommit = false+ try {+ const result = await withTransaction(async db => {+ const candidate = await getInvitationRecordById(db, trimmedInvitationId)+ if (!candidate || candidate.purpose !== 'password_reset') {+ return { error: 'not_found' as const }}- if (invitation.expires_at.getTime() <= Date.now()) {- return { error: 'expired' as const }+ if (candidate.email.toLowerCase() !== normalizedEmail) {+ return { error: 'forbidden' as const }}- } else if (- invitation.purpose === 'member_invitation' ||- invitation.purpose === 'admin_desktop_access'- ) {- if (invitation.status !== 'accepted') {- return { error: 'not_accepted' as const }++ const user = await lockPasswordResetUser(db, normalizedEmail)+ if (!user) return { error: 'not_found' as const }+ if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }+ if (expectedUserId && user.id !== expectedUserId.trim()) {+ return { error: 'forbidden' as const }+ }+ const invitation = await getInvitationRecordById(db, trimmedInvitationId, true)+ if (!invitation || invitation.purpose !== 'password_reset') {+ return { error: 'not_found' as const }+ }+ if (invitation.email.toLowerCase() !== normalizedEmail) {+ return { error: 'forbidden' as const }+ }+ if (invitation.status !== 'pending') return { error: 'not_pending' as const }+ if (invitation.expires_at.getTime() <= Date.now()) return { error: 'expired' as const }+ if (invitation.accepted_user_id && invitation.accepted_user_id !== user.id) {+ return { error: 'forbidden' as const }}- } else {- return { error: 'not_found' as const }- }- const user = await ensureInvitationUser(db, invitation.email, invitation.invitee_name)- if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }- const passwordHash = await bcrypt.hash(trimmedPassword, 12)- await db.query(- `UPDATE users- SET password_hash = $2,- password_set_at = NOW(),- updated_at = NOW()- WHERE id = $1- AND lifecycle_state = 'active'`,- [user.id, passwordHash]- )- if (invitation.purpose === 'password_reset' && invitation.status === 'pending') {- await db.query(+ const passwordHash = await bcrypt.hash(trimmedPassword, 12)+ const update = await db.query(+ `UPDATE users+ SET password_hash = $2,+ password_set_at = NOW(),+ updated_at = NOW()+ WHERE id = $1+ AND lifecycle_state = 'active'+ RETURNING id`,+ [user.id, passwordHash]+ )+ if (update.rows.length !== 1) return { error: 'user_retired' as const }++ const redeemed = await db.query(`UPDATE invitationsSET status = 'accepted',accepted_at = NOW(),accepted_user_id = $2- WHERE id = $1`,+ WHERE id = $1 AND status = 'pending'+ RETURNING id`,[invitation.id, user.id])- } else if (!invitation.accepted_user_id) {- await db.query(`UPDATE invitations SET accepted_user_id = $2 WHERE id = $1`, [- invitation.id,- user.id,- ])- }-- const refreshed = await getInvitationRecordById(db, invitation.id)- if (!refreshed) {- return { error: 'not_found' as const }- }-- const teams = await loadInvitationTeams(db, refreshed.id, refreshed)- return {- data: {- passwordUpdated: true,- ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),- },+ if (redeemed.rows.length !== 1) return { error: 'not_pending' as const }+ await revokeSiblingPasswordResets(db, normalizedEmail, invitation.id, user.id)++ const refreshed = await getInvitationRecordById(db, invitation.id)+ if (!refreshed) return { error: 'not_found' as const }+ const teams = await loadInvitationTeams(db, refreshed.id, refreshed)+ const sessionContext = await getMemberSessionContext(db, user)+ await lease.releaseWithin(db)+ releaseStaged = true+ return {+ data: {+ passwordUpdated: true,+ sessionContext,+ ...invitationResponse(refreshed, { ...user, password_hash: passwordHash }, teams),+ },+ }+ })+ if (releaseStaged) releasedWithCommit = true+ return result+ } catch (error) {+ if (error instanceof PasswordAdmissionError) throw error+ passwordSetupLogger.warn(+ { event: 'password_authority_unavailable' },+ 'password authority unavailable'+ )+ throw new PasswordAdmissionError(503, 2, 'authority_failure')+ } finally {+ if (!releasedWithCommit) {+ try {+ await lease.release()+ } catch {+ passwordSetupLogger.warn(+ { event: 'password_authority_unavailable' },+ 'password authority unavailable'+ )+ throw new PasswordAdmissionError(503, 2, 'authority_failure')+ }}- })+ }}export async function acceptInvitation(@@ -1242,6 +1337,9 @@ export async function acceptInvitation(if (invitation.email.toLowerCase() !== email.toLowerCase()) {return { error: 'forbidden' as const }}+ if (invitation.purpose === 'password_reset') {+ return { error: 'not_pending' as const }+ }if (invitation.status === 'revoked') {return { error: 'not_pending' as const }}@@ -1258,7 +1356,7 @@ export async function acceptInvitation(return { error: 'forbidden' as const }}- if (invitation.status === 'pending' && invitation.purpose !== 'password_reset') {+ if (invitation.status === 'pending') {await applyInvitationMemberships(db, invitation, user.id)await db.query(@@ -1301,6 +1399,9 @@ export async function acceptInvitationForEmail(email: string, token: string, invif (invitation.email.toLowerCase() !== email.toLowerCase()) {return { error: 'forbidden' as const }}+ if (invitation.purpose === 'password_reset') {+ return { error: 'not_pending' as const }+ }if (invitation.status === 'revoked' || invitation.status === 'draft') {return { error: 'not_pending' as const }}@@ -1311,7 +1412,7 @@ export async function acceptInvitationForEmail(email: string, token: string, invconst user = await ensureInvitationUser(db, invitation.email, invitation.invitee_name)if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }- if (invitation.status === 'pending' && invitation.purpose !== 'password_reset') {+ if (invitation.status === 'pending') {await applyInvitationMemberships(db, invitation, user.id)await db.query(@@ -1355,6 +1456,9 @@ export async function acceptInvitationById(email: string, invitationId: string)if (invitation.email.toLowerCase() !== email.toLowerCase()) {return { error: 'forbidden' as const }}+ if (invitation.purpose === 'password_reset') {+ return { error: 'not_pending' as const }+ }if (invitation.status === 'revoked' || invitation.status === 'draft') {return { error: 'not_pending' as const }}@@ -1365,7 +1469,7 @@ export async function acceptInvitationById(email: string, invitationId: string)const user = await ensureInvitationUser(db, invitation.email, invitation.invitee_name)if (invitationUserIsRetired(user)) return { error: 'user_retired' as const }- if (invitation.status === 'pending' && invitation.purpose !== 'password_reset') {+ if (invitation.status === 'pending') {await applyInvitationMemberships(db, invitation, user.id)await db.query(@@ -1928,14 +2032,17 @@ export async function updateUserPassword(}const result = await pool.query(- `SELECT password_hash+ `SELECT password_hash, password_auth_generationFROM usersWHERE id = $1AND email = $2LIMIT 1`,[normalizedUserId, normalizedEmail])- const user = (result.rows[0] as { password_hash: string | null } | undefined) || null+ const user =+ (result.rows[0] as+ | { password_hash: string | null; password_auth_generation: string | number }+ | undefined) || nullif (!user) {return { error: 'not_found' as const }}diff --git a/control-api/test/bug192.passwordLogin.realPostgres.integration.test.ts b/control-api/test/bug192.passwordLogin.realPostgres.integration.test.tsindex f5243eb3f..75c66e4a6 100644--- a/control-api/test/bug192.passwordLogin.realPostgres.integration.test.ts+++ b/control-api/test/bug192.passwordLogin.realPostgres.integration.test.ts@@ -5,8 +5,13 @@ import { createPublicKey, randomBytes } from 'node:crypto'import { Pool } from 'pg'import request from 'supertest'import { initDb } from '../src/db.js'+import { passwordAdmissionDenialsTotal } from '../src/observability/metrics.js'import { createExternalAuthRouter } from '../src/routes/external/auth.js'+import { createExternalInvitationsRouter } from '../src/routes/external/invitations.js'+import { passwordIdentifierKey } from '../src/services/auth/passwordAdmissionState.js'+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'import { verifyUserPassword } from '../src/services/directory/login.js'+import { createInvitationForTeams } from '../src/services/directory/membership.js'import { verifyExternalSessionToken } from '../src/utils/auth/externalSessionAuthToken.js'import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'@@ -14,6 +19,19 @@ const holder = vi.hoisted(() => ({pool: null as unknown as Pool,leaseClient: null as unknown as import('pg').PoolClient,}))+const invitationFlow = vi.hoisted(() => ({+ registerAndSendInvitation: vi.fn().mockResolvedValue(undefined),+ validateInvitationFlowToken: vi.fn(),+}))+vi.mock('../src/services/invitationFlowRegistrationService.js', async importOriginal => {+ const real =+ await importOriginal<typeof import('../src/services/invitationFlowRegistrationService.js')>()+ return {+ ...real,+ registerAndSendInvitation: invitationFlow.registerAndSendInvitation,+ validateInvitationFlowToken: invitationFlow.validateInvitationFlowToken,+ }+})vi.mock('../src/db.js', async importOriginal => {const real = await importOriginal<typeof import('../src/db.js')>()const proxy = {@@ -67,6 +85,78 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>.set('x-external-client-ip', ip).send({ email, password: submitted })}+ async function denialCount(reason: string) {+ const metric = await passwordAdmissionDenialsTotal.get()+ return metric.values.find(value => value.labels.reason === reason)?.value ?? 0+ }+ async function recoverPassword(nextPassword: string) {+ const invitation = await createInvitationForTeams({+ inviteeName: 'Synthetic member',+ email: '[email protected]',+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const proof = 'synthetic-trusted-reset-proof'+ invitationFlow.validateInvitationFlowToken.mockResolvedValue({+ email: '[email protected]',+ invitationUuid: invitation.token,+ })+ const lookup = await request(app).get(`/external/invitations/token/${proof}`).expect(200)+ expect(lookup.body).toMatchObject({+ id: invitation.id,+ purpose: 'password_reset',+ status: 'pending',+ })+ expect(lookup.body).not.toHaveProperty('token')++ const originalPasswordHash = (+ await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])+ ).rows[0].password_hash+ await request(app)+ .post('/external/invitations/password-token')+ .send({+ email: '[email protected]',+ token: proof,+ invitationId: '00000000-0000-4000-8000-000000000000',+ password: nextPassword,+ })+ .expect(403)+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(originalPasswordHash)++ const reset = await request(app)+ .post('/external/invitations/password-token')+ .send({+ email: '[email protected]',+ token: proof,+ invitationId: invitation.id,+ password: nextPassword,+ })+ .expect(200)+ expect(reset.body).not.toHaveProperty('sessionContext')+ expect(reset.body.token).toEqual(expect.any(String))++ const replay = await request(app)+ .post('/external/invitations/password-token')+ .send({+ email: '[email protected]',+ token: proof,+ invitationId: invitation.id,+ password: 'Synthetic-replay-password',+ })+ .expect(409)+ expect(replay.body).not.toHaveProperty('token')++ const access = await request(app)+ .post('/external/auth/verify')+ .send({ token: reset.body.token })+ expect(access.status).toBe(200)+ expect(access.body.claims).toMatchObject({ userId, email: '[email protected]' })+ return access.body.claims+ }beforeAll(async () => {admin = new Pool({ connectionString: adminUrl })await admin.query(`CREATE DATABASE "${database}"`)@@ -82,6 +172,7 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>req.internalService = { name: 'external-rest-api' }next()})+ app.use(createExternalInvitationsRouter())app.use(createExternalAuthRouter({} as Parameters<typeof createExternalAuthRouter>[0]))app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) =>@@ -90,6 +181,8 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>}, 60_000)beforeEach(async () => {vi.restoreAllMocks()+ invitationFlow.registerAndSendInvitation.mockReset().mockResolvedValue(undefined)+ invitationFlow.validateInvitationFlowToken.mockReset()await holder.pool.query('DELETE FROM team_members')await holder.pool.query('DELETE FROM invitations')await holder.pool.query('DELETE FROM teams')@@ -158,13 +251,16 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>it('denies a sixth source attempt before bcrypt', async () => {for (let i = 0; i < 5; i++) {await openPace()- await login(`source-${i}@example.invalid`)+ expect((await login(`source-${i}@example.invalid`)).status).toBe(401)}+ expect(compare).toHaveBeenCalledTimes(5)+ const sourceDenials = await denialCount('source_rate')const before = compare.mock.calls.lengthconst response = await login('[email protected]')expect(response.status).toBe(429)expect(response.body.error).toBe('rate_limited')expect(compare.mock.calls.length).toBe(before)+ expect(await denialCount('source_rate')).toBe(sourceDenials + 1)})it('denies rotating sources and canonical identifier variants after five attempts', async () => {for (let i = 0; i < 5; i++) {@@ -175,19 +271,129 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>`192.0.2.${i + 1}`)}+ const cooldownDenials = await denialCount('identifier_cooldown')const before = compare.mock.calls.lengthconst response = await login('[email protected]', 'wrong', '192.0.2.100')expect(response.status).toBe(429)expect(response.body.error).toBe('rate_limited')expect(compare.mock.calls.length).toBe(before)+ expect(await denialCount('identifier_cooldown')).toBe(cooldownDenials + 1)})it('applies the same pace to unknown and existing evaluations with burst one', async () => {expect((await login('[email protected]')).status).toBe(401)+ const globalPaceDenials = await denialCount('global_pace')const response = await login('[email protected]', 'wrong', '192.0.2.2')expect(response.status).toBe(429)expect(compare).toHaveBeenCalledTimes(1)expect(Number(response.headers['retry-after'])).toBeGreaterThan(0)expect(Number(response.headers['retry-after'])).toBeLessThanOrEqual(8)+ expect(await denialCount('global_pace')).toBe(globalPaceDenials + 1)+ })+ it('restores authenticated access after global pace exhaustion without refunding attempts', async () => {+ expect((await login('[email protected]', 'wrong', '192.0.2.31')).status).toBe(401)+ const denied = await login('[email protected]', password, '192.0.2.32')+ expect(denied.status).toBe(429)+ expect(compare).toHaveBeenCalledTimes(1)++ const attemptsBefore = (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey('[email protected]')]+ )+ ).rows[0].attempts+ await recoverPassword('Synthetic-BUG192-recovered-under-global-saturation')+ const attemptsAfter = (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey('[email protected]')]+ )+ ).rows[0].attempts+ expect(attemptsAfter).toEqual(attemptsBefore)+ expect(+ (+ await login(+ '[email protected]',+ 'Synthetic-BUG192-recovered-under-global-saturation',+ '192.0.2.33'+ )+ ).status+ ).toBe(429)+ })+ it('restores authenticated access during identifier cooldown and preserves public attempts', async () => {+ for (let i = 0; i < 5; i++) {+ await openPace()+ await login('[email protected]', 'wrong', `192.0.2.${40 + i}`)+ }+ const before = compare.mock.calls.length+ await openPace()+ const denied = await login('[email protected]', password, '192.0.2.49')+ expect(denied.status).toBe(429)+ expect(compare.mock.calls.length).toBe(before)++ const stateBeforeRecovery = (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey('[email protected]')]+ )+ ).rows[0].attempts+ expect(stateBeforeRecovery).toHaveLength(5)+ const teamId = (+ await holder.pool.query(+ "INSERT INTO teams(name) VALUES ('Synthetic recovery team') RETURNING id"+ )+ ).rows[0].id+ await holder.pool.query(+ "INSERT INTO team_members(team_id, user_id, role, status) VALUES ($1, $2, 'admin', 'active')",+ [teamId, userId]+ )+ const claims = await recoverPassword('Synthetic-BUG192-recovered-during-identifier-cooldown')+ expect(claims).toMatchObject({ teamId, role: 'admin' })+ const stateAfterRecovery = (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey('[email protected]')]+ )+ ).rows[0].attempts+ expect(stateAfterRecovery).toEqual(stateBeforeRecovery)++ const attemptDenials = await denialCount('identifier_attempts')+ const comparisons = compare.mock.calls.length+ await openPace()+ const stillDenied = await login(+ '[email protected]',+ 'Synthetic-BUG192-recovered-during-identifier-cooldown',+ '192.0.2.50'+ )+ expect(stillDenied.status).toBe(429)+ expect(compare.mock.calls.length).toBe(comparisons)+ expect(await denialCount('identifier_attempts')).toBe(attemptDenials + 1)+ })+ it('keeps the account attempt budget after correct-password successes', async () => {+ for (let i = 0; i < 5; i++) {+ await openPace()+ expect((await login('[email protected]', password, `192.0.2.${60 + i}`)).status).toBe(+ 200+ )+ }+ const attemptDenials = await denialCount('identifier_attempts')+ await openPace()+ const denied = await login('[email protected]', password, '192.0.2.69')+ expect(denied.status).toBe(429)+ expect(await denialCount('identifier_attempts')).toBe(attemptDenials + 1)+ })+ it('records durable verification-owner contention without running bcrypt', async () => {+ const lease = await acquirePasswordWork()+ expect(lease).not.toBeNull()+ try {+ const busyDenials = await denialCount('verification_busy')+ const before = compare.mock.calls.length+ const response = await login('[email protected]', password)+ expect(response.status).toBe(429)+ expect(compare.mock.calls.length).toBe(before)+ expect(await denialCount('verification_busy')).toBe(busyDenials + 1)+ } finally {+ await lease?.release()+ }})it('keeps real and shadow cooldown status/body/retry metadata identical', async () => {// Backdate admitted attempts and completed failures using the DB clock,@@ -270,6 +476,7 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>// Real database producer: temporarily revoke table access by making its// relation unavailable, then restore it even when the assertion fails.const exists = await newStateExists()+ const authorityFailures = await denialCount('authority_failure')if (exists)await holder.pool.query('ALTER TABLE password_identifier_state RENAME TO password_identifier_state_unavailable'@@ -281,6 +488,7 @@ realPg('BUG-192 observable password-login regressions on real PostgreSQL', () =>expect(b.status).toBe(503)expect(a.body).toEqual(b.body)expect(compare).not.toHaveBeenCalled()+ expect(await denialCount('authority_failure')).toBe(authorityFailures + 2)} finally {if (exists)await holder.pool.query(diff --git a/control-api/test/bug192.passwordPersistence.realPostgres.integration.test.ts b/control-api/test/bug192.passwordPersistence.realPostgres.integration.test.tsindex 688c22ef0..8505a981d 100644--- a/control-api/test/bug192.passwordPersistence.realPostgres.integration.test.ts+++ b/control-api/test/bug192.passwordPersistence.realPostgres.integration.test.ts@@ -16,8 +16,14 @@ import {reservePasswordPace,verifyMemberPassword,} from '../src/services/auth/passwordCredentialVerification.js'+import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'import {+ acceptInvitationForEmail,+ createInvitationForTeams,+ createSilentInvitationForTeams,+ getInvitationByToken,setInvitationPasswordForEmail,+ setInvitationPasswordForUser,updateUserPassword,} from '../src/services/directory/membership.js'import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'@@ -26,6 +32,10 @@ const holder = vi.hoisted(() => ({pool: null as unknown as Pool,leaseClient: null as unknown as import('pg').PoolClient,}))+const invitationFlow = vi.hoisted(() => ({+ registerAndSendInvitation: vi.fn().mockResolvedValue(undefined),+}))+vi.mock('../src/services/invitationFlowRegistrationService.js', () => invitationFlow)vi.mock('../src/db.js', async importOriginal => {const real = await importOriginal<typeof import('../src/db.js')>()const proxy = {@@ -69,6 +79,7 @@ realPg('Spec 043 persistence, migration, atomicity and fencing design acceptance}, 60_000)beforeEach(async () => {vi.restoreAllMocks()+ invitationFlow.registerAndSendInvitation.mockReset().mockResolvedValue(undefined)await holder.pool.query('DELETE FROM users')await holder.pool.query('UPDATE password_identifier_state SET retained_until_ms=0')await holder.pool.query('DELETE FROM password_identifier_state')@@ -252,6 +263,337 @@ realPg('Spec 043 persistence, migration, atomicity and fencing design acceptance).rows[0]expect(bcrypt.getRounds(credential.password_hash)).toBe(12)})+ it('completes authenticated reset with a one-connection PostgreSQL pool', async () => {+ const invitation = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const previousPool = holder.pool+ const singleConnectionPool = new Pool({+ connectionString,+ max: 1,+ connectionTimeoutMillis: 2500,+ })+ holder.pool = singleConnectionPool+ try {+ const result = await setInvitationPasswordForUser(+ userId,+ email,+ invitation.id,+ 'Synthetic-BUG192-single-pool-reset'+ )+ expect(result).not.toHaveProperty('error')+ const after = (+ await singleConnectionPool.query(+ 'SELECT password_hash, password_auth_generation FROM users WHERE id = $1',+ [userId]+ )+ ).rows[0]+ expect(after.password_hash).not.toBe(hash)+ expect(Number(after.password_auth_generation)).toBeGreaterThan(1)+ expect(+ (+ await singleConnectionPool.query('SELECT status FROM invitations WHERE id = $1', [+ invitation.id,+ ])+ ).rows[0].status+ ).toBe('accepted')+ } finally {+ holder.pool = previousPool+ await endPoolAndWaitForClients(singleConnectionPool)+ }+ })+ it('does not accept a pending password-reset link through generic invitation acceptance', async () => {+ const invitation = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })++ await expect(acceptInvitationForEmail(email, '', invitation.id)).resolves.toEqual({+ error: 'not_pending',+ })+ expect(+ (await holder.pool.query('SELECT status FROM invitations WHERE id = $1', [invitation.id]))+ .rows[0].status+ ).toBe('pending')+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(hash)+ })+ it('revokes every other pending password-reset link for the same account only', async () => {+ const makeReset = (address: string) =>+ createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email: address,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const redeemed = await makeReset(email)+ const sibling = await makeReset(email)+ await expect(getInvitationByToken(redeemed.token)).resolves.toMatchObject({+ id: redeemed.id,+ purpose: 'password_reset',+ status: 'pending',+ })+ const otherEmail = '[email protected]'+ await holder.pool.query('INSERT INTO users(email) VALUES ($1)', [otherEmail])+ const otherMember = await makeReset(otherEmail)+ const before = (+ await holder.pool.query('SELECT password_auth_generation FROM users WHERE id = $1', [userId])+ ).rows[0].password_auth_generation++ const result = await setInvitationPasswordForEmail(+ email,+ redeemed.id,+ 'Synthetic-BUG192-recovered-password'+ )++ expect(result).not.toHaveProperty('error')+ const rows = (+ await holder.pool.query(+ 'SELECT id, email, purpose, status FROM invitations WHERE id = ANY($1::uuid[]) ORDER BY id',+ [[redeemed.id, sibling.id, otherMember.id]]+ )+ ).rows+ expect(rows).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ id: redeemed.id,+ email,+ purpose: 'password_reset',+ status: 'accepted',+ }),+ expect.objectContaining({+ id: sibling.id,+ email,+ purpose: 'password_reset',+ status: 'revoked',+ }),+ expect.objectContaining({+ id: otherMember.id,+ email: otherEmail,+ purpose: 'password_reset',+ status: 'pending',+ }),+ ])+ )+ const after = (+ await holder.pool.query('SELECT password_auth_generation FROM users WHERE id = $1', [userId])+ ).rows[0].password_auth_generation+ expect(Number(after)).toBe(Number(before) + 1)+ const replayHash = vi.spyOn(bcrypt, 'hash')+ await expect(+ setInvitationPasswordForEmail(email, redeemed.id, 'Synthetic-BUG192-used-link-replay')+ ).resolves.toMatchObject({ error: 'not_pending' })+ await expect(+ setInvitationPasswordForEmail(email, sibling.id, 'Synthetic-BUG192-replay-password')+ ).resolves.toMatchObject({ error: 'not_pending' })+ expect(replayHash).not.toHaveBeenCalled()+ expect(+ Number(+ (+ await holder.pool.query('SELECT password_auth_generation FROM users WHERE id = $1', [+ userId,+ ])+ ).rows[0].password_auth_generation+ )+ ).toBe(Number(after))+ })++ it('maps the email producer token to its distinct recovery row ID', async () => {+ const invitation = await createInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ expect(invitationFlow.registerAndSendInvitation).toHaveBeenCalledWith(+ email,+ invitation.token,+ null,+ expect.any(String),+ expect.any(String),+ expect.objectContaining({ purpose: 'password_reset' })+ )+ expect(invitation.token).toBeTruthy()+ expect(invitation.token).not.toBe(invitation.id)++ const preview = await getInvitationByToken(invitation.token)+ expect(preview).toMatchObject({+ id: invitation.id,+ email,+ purpose: 'password_reset',+ status: 'pending',+ })++ const result = await setInvitationPasswordForEmail(+ email,+ preview!.id,+ 'Synthetic-BUG192-token-mapped-recovery'+ )+ expect(result).not.toHaveProperty('error')+ expect(+ (await holder.pool.query('SELECT status FROM invitations WHERE id = $1', [invitation.id]))+ .rows[0].status+ ).toBe('accepted')+ })++ it('allows at most one credential mutation when sibling reset links race', async () => {+ const makeReset = () =>+ createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const first = await makeReset()+ const second = await makeReset()+ const hash = vi.spyOn(bcrypt, 'hash')+ const before = (+ await holder.pool.query('SELECT password_auth_generation FROM users WHERE id = $1', [userId])+ ).rows[0].password_auth_generation++ const outcomes = await Promise.allSettled([+ setInvitationPasswordForEmail(email, first.id, 'Synthetic-BUG192-race-password-a'),+ setInvitationPasswordForEmail(email, second.id, 'Synthetic-BUG192-race-password-b'),+ ])++ expect(+ outcomes.filter(result => result.status === 'fulfilled' && !('error' in result.value))+ ).toHaveLength(1)+ expect(hash).toHaveBeenCalledTimes(1)+ const after = (+ await holder.pool.query('SELECT password_auth_generation FROM users WHERE id = $1', [userId])+ ).rows[0].password_auth_generation+ expect(Number(after)).toBe(Number(before) + 1)+ expect(+ (+ await holder.pool.query('SELECT status FROM invitations WHERE id = ANY($1::uuid[])', [+ [first.id, second.id],+ ])+ ).rows+ .map(row => row.status)+ .sort()+ ).toEqual(['accepted', 'revoked'])+ const redeemedId = (+ await holder.pool.query(+ "SELECT id FROM invitations WHERE id = ANY($1::uuid[]) AND status = 'accepted'",+ [[first.id, second.id]]+ )+ ).rows[0].id+ const siblingId = redeemedId === first.id ? second.id : first.id+ await expect(+ setInvitationPasswordForEmail(email, siblingId, 'Synthetic-BUG192-race-replay')+ ).resolves.toMatchObject({ error: 'not_pending' })+ })+ it('rejects expired, revoked, wrong-purpose and wrong-account recovery credentials', async () => {+ const makeInvitation = (purpose: 'password_reset' | 'member_invitation') =>+ createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose,+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const expired = await makeInvitation('password_reset')+ const revoked = await makeInvitation('password_reset')+ const wrongAccount = await makeInvitation('password_reset')+ const memberInvite = await makeInvitation('member_invitation')+ await holder.pool.query(+ "UPDATE invitations SET expires_at = NOW() - interval '1 second' WHERE id = $1",+ [expired.id]+ )+ await holder.pool.query("UPDATE invitations SET status = 'revoked' WHERE id = $1", [revoked.id])++ await expect(+ setInvitationPasswordForEmail(email, expired.id, 'Synthetic-BUG192-expired-reset')+ ).resolves.toMatchObject({ error: 'expired' })+ await expect(+ setInvitationPasswordForEmail(email, revoked.id, 'Synthetic-BUG192-revoked-reset')+ ).resolves.toMatchObject({ error: 'not_pending' })+ await expect(+ setInvitationPasswordForEmail(+ '[email protected]',+ wrongAccount.id,+ 'Synthetic-BUG192-wrong-account'+ )+ ).resolves.toMatchObject({ error: 'forbidden' })+ await expect(+ setInvitationPasswordForEmail(email, memberInvite.id, 'Synthetic-BUG192-wrong-purpose')+ ).resolves.toMatchObject({ error: 'not_found' })+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(hash)+ })+ it('fails recovery closed when durable password work authority is unavailable', async () => {+ const invitation = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ await holder.pool.query(+ 'ALTER TABLE password_verification_work RENAME TO password_verification_work_unavailable'+ )+ try {+ await expect(+ setInvitationPasswordForEmail(email, invitation.id, 'Synthetic-BUG192-no-authority')+ ).rejects.toMatchObject({ status: 503 })+ expect(+ (await holder.pool.query('SELECT status FROM invitations WHERE id = $1', [invitation.id]))+ .rows[0].status+ ).toBe('pending')+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(hash)+ } finally {+ await holder.pool.query(+ 'ALTER TABLE password_verification_work_unavailable RENAME TO password_verification_work'+ )+ }+ })+ it('bounds recovery hashing with the shared owner and does not hash while it is busy', async () => {+ const invitation = await createSilentInvitationForTeams({+ inviteeName: 'Synthetic member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const lease = await acquirePasswordWork()+ expect(lease).not.toBeNull()+ const hashSpy = vi.spyOn(bcrypt, 'hash')+ try {+ await expect(+ setInvitationPasswordForEmail(email, invitation.id, 'Synthetic-BUG192-owner-busy')+ ).rejects.toMatchObject({ status: 429, reason: 'verification_busy' })+ expect(hashSpy).not.toHaveBeenCalled()+ expect(+ (await holder.pool.query('SELECT status FROM invitations WHERE id = $1', [invitation.id]))+ .rows[0].status+ ).toBe('pending')+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(hash)+ } finally {+ hashSpy.mockRestore()+ await lease?.release()+ }+ })it('provisioning and lifecycle changes fence stale completions at the database boundary', async () => {for (const update of ['password_hash = NULL', 'lifecycle_version = lifecycle_version + 1']) {const stale = await capturePasswordEvaluation(email, false)diff --git a/control-api/test/bug192.passwordRecovery.memberRegistration.realPostgres.integration.test.ts b/control-api/test/bug192.passwordRecovery.memberRegistration.realPostgres.integration.test.tsnew file mode 100644index 000000000..a85303373--- /dev/null+++ b/control-api/test/bug192.passwordRecovery.memberRegistration.realPostgres.integration.test.ts@@ -0,0 +1,236 @@+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import bcrypt from 'bcryptjs'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'+import { initDb } from '../src/db.js'+import { createExternalAuthRouter } from '../src/routes/external/auth.js'+import { createExternalInvitationsRouter } from '../src/routes/external/invitations.js'+import { passwordIdentifierKey } from '../src/services/auth/passwordAdmissionState.js'+import { createInvitationForTeams } from '../src/services/directory/membership.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'++const holder = vi.hoisted(() => ({+ pool: null as unknown as Pool,+ leaseClient: null as unknown as import('pg').PoolClient,+}))++vi.mock('../src/db.js', async importOriginal => {+ const real = await importOriginal<typeof import('../src/db.js')>()+ const proxy = {+ query: (...args: unknown[]) => (holder.pool.query as Function)(...args),+ connect: () => holder.pool.connect(),+ }+ return {+ ...real,+ pool: proxy,+ rateLimitPool: proxy,+ withTransaction: (fn: Parameters<typeof real.withTransaction>[0]) =>+ real.withTransaction(fn, holder.pool),+ }+})++vi.mock('../src/services/rateLimiterService.js', async importOriginal => {+ const real = await importOriginal<typeof import('../src/services/rateLimiterService.js')>()+ return {+ ...real,+ acquireRateLimitConcurrencyLease: (+ requirements: Parameters<typeof real.acquireRateLimitConcurrencyLease>[0]+ ) => real.acquireRateLimitConcurrencyLease(requirements, { client: holder.leaseClient }),+ }+})++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const memberRegistrationDbUrl = process.env.CONTROL_API_REAL_MEMBER_REGISTRATION_PG_URL+const realCrossService =+ adminUrl &&+ memberRegistrationDbUrl &&+ process.env.CONTROL_API_MEMBER_REGISTRATION_SERVICE_BASE_URL &&+ process.env.CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET+ ? describe+ : describe.skip+const password = 'Synthetic-Issue1055-previous-password'++realCrossService(+ 'BUG-192 recovery through the real Member Registration producer on PostgreSQL',+ () => {+ const database = `bug192_flow_${randomBytes(6).toString('hex')}`+ const email = `issue1055-${randomUUID()}@example.invalid`+ let admin: Pool+ let memberRegistrationDb: Pool+ let app: express.Express+ let userId: string+ let realHash: string++ beforeAll(async () => {+ admin = new Pool({ connectionString: adminUrl })+ await admin.query(`CREATE DATABASE "${database}"`)+ const url = new URL(adminUrl!)+ url.pathname = `/${database}`+ holder.pool = new Pool({ connectionString: url.toString() })+ await initDb(holder.pool)+ holder.leaseClient = await holder.pool.connect()+ memberRegistrationDb = new Pool({ connectionString: memberRegistrationDbUrl })+ realHash = await bcrypt.hash(password, 12)+ app = express()+ app.use(express.json())+ app.use((req, _res, next) => {+ req.internalService = { name: 'external-rest-api' }+ next()+ })+ app.use(createExternalInvitationsRouter())+ app.use(createExternalAuthRouter({} as Parameters<typeof createExternalAuthRouter>[0]))+ app.use(+ (err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) =>+ res.status(500).json({ error: 'unexpected_test_failure' })+ )+ }, 60_000)++ beforeEach(async () => {+ await holder.pool.query('DELETE FROM team_members')+ await holder.pool.query('DELETE FROM invitations')+ await holder.pool.query('DELETE FROM teams')+ await holder.pool.query('DELETE FROM users')+ await holder.pool.query('DELETE FROM rate_limit_buckets')+ await holder.pool.query('DELETE FROM password_identifier_state')+ await holder.pool.query('DELETE FROM password_verification_pace')+ userId = (+ await holder.pool.query(+ 'INSERT INTO users(email, password_hash) VALUES ($1, $2) RETURNING id',+ [email, realHash]+ )+ ).rows[0].id+ await holder.pool.query(+ `INSERT INTO password_verification_pace(singleton, next_permit)+ VALUES (TRUE, clock_timestamp() + interval '1 minute')`+ )+ })++ afterAll(async () => {+ try {+ try {+ await memberRegistrationDb?.query(+ 'DELETE FROM invitation_flow_registrations WHERE email = $1',+ [email]+ )+ holder.leaseClient?.release()+ await endPoolAndWaitForClients(holder.pool)+ if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)+ } finally {+ await memberRegistrationDb?.end()+ }+ } finally {+ await admin?.end()+ }+ })++ it('recovers with the trusted signed proof while anonymous login pace is saturated', async () => {+ const deniedLogin = await request(app)+ .post('/external/auth/password-login')+ .set('x-external-client-ip', '192.0.2.155')+ .send({ email, password })+ .expect(429)+ expect(deniedLogin.body.error).toBe('rate_limited')++ const invitation = await createInvitationForTeams({+ inviteeName: 'Synthetic issue 1055 member',+ email,+ purpose: 'password_reset',+ teamAssignments: [],+ fallbackRole: 'member',+ })+ const flowRows = await memberRegistrationDb.query(+ `SELECT invitation_jwt+ FROM invitation_flow_registrations+ WHERE email = $1 AND invitation_uuid = $2::uuid+ ORDER BY created_at DESC, id DESC+ LIMIT 1`,+ [email, invitation.token]+ )+ const proof = flowRows.rows[0]?.invitation_jwt+ expect(proof).toEqual(expect.any(String))++ const preview = await request(app)+ .get(`/external/invitations/token/${encodeURIComponent(proof)}`)+ .expect(200)+ expect(preview.body).toMatchObject({+ id: invitation.id,+ purpose: 'password_reset',+ status: 'pending',+ })+ expect(preview.body).not.toHaveProperty('token')++ const wrongEmail = await request(app)+ .post('/external/invitations/password-token')+ .send({+ email: '[email protected]',+ token: proof,+ invitationId: invitation.id,+ password: 'Synthetic-Issue1055-new-password',+ })+ .expect(400)+ expect(wrongEmail.body.error).toBe('invalid_invitation')++ const originalHash = (+ await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])+ ).rows[0].password_hash+ const wrongRow = await request(app)+ .post('/external/invitations/password-token')+ .send({+ email,+ token: proof,+ invitationId: randomUUID(),+ password: 'Synthetic-Issue1055-new-password',+ })+ .expect(403)+ expect(wrongRow.body.error).toBe('forbidden')+ expect(+ (await holder.pool.query('SELECT password_hash FROM users WHERE id = $1', [userId])).rows[0]+ .password_hash+ ).toBe(originalHash)++ const attemptsBeforeRecovery = (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey(email)]+ )+ ).rows[0].attempts+ const reset = await request(app)+ .post('/external/invitations/password-token')+ .send({+ email,+ token: proof,+ invitationId: invitation.id,+ password: 'Synthetic-Issue1055-new-password',+ })+ .expect(200)+ expect(reset.body.token).toEqual(expect.any(String))+ expect(reset.body).not.toHaveProperty('sessionContext')+ expect(+ (+ await holder.pool.query(+ 'SELECT attempts FROM password_identifier_state WHERE identifier_key = $1',+ [passwordIdentifierKey(email)]+ )+ ).rows[0].attempts+ ).toEqual(attemptsBeforeRecovery)++ const authenticated = await request(app)+ .post('/external/auth/verify')+ .send({ token: reset.body.token })+ .expect(200)+ expect(authenticated.body.claims).toMatchObject({ userId, email })++ await holder.pool.query(+ "UPDATE password_verification_pace SET next_permit = clock_timestamp() - interval '1 second'"+ )+ const newPasswordLogin = await request(app)+ .post('/external/auth/password-login')+ .set('x-external-client-ip', '192.0.2.156')+ .send({ email, password: 'Synthetic-Issue1055-new-password' })+ .expect(200)+ expect(newPasswordLogin.body).toHaveProperty('token')+ }, 30_000)+ }+)diff --git a/control-api/test/routes.externalInvitations.unavailable.test.ts b/control-api/test/routes.externalInvitations.unavailable.test.tsindex 2ea8607e8..daff89712 100644--- a/control-api/test/routes.externalInvitations.unavailable.test.ts+++ b/control-api/test/routes.externalInvitations.unavailable.test.ts@@ -40,9 +40,8 @@ vi.mock('../src/middleware/externalSessionAuth.js', () => ({next: express.NextFunction) => next(),}))-vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({- signExternalSessionToken: vi.fn(() => 'session-token'),-}))+const sessionToken = vi.hoisted(() => ({ signExternalSessionToken: vi.fn(() => 'session-token') }))+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => sessionToken)// Router-level harness: proves the ROUTE GUARDS rethrow instead of swallowing.// The real app.ts middleware is covered separately in@@ -90,6 +89,157 @@ describe('external invitation routes when the hub is unavailable', () => {expect(res.body.error).toBe('invalid_invitation')})+ it('issues a normal session only after trusted password-reset validation and completion', async () => {+ flow.validateInvitationFlowToken.mockResolvedValue({+ email: '[email protected]',+ invitationUuid: 'producer-token-1',+ })+ directory.getInvitationByToken.mockResolvedValue({+ id: 'database-row-1',+ token: 'producer-token-1',+ email: '[email protected]',+ purpose: 'password_reset',+ status: 'pending',+ })+ directory.setInvitationPasswordForEmail.mockResolvedValue({+ data: {+ id: 'database-row-1',+ userId: 'user-1',+ email: '[email protected]',+ teamId: null,+ role: 'member',+ purpose: 'password_reset',+ status: 'accepted',+ lifecycleState: 'active',+ authGeneration: 7,+ passwordUpdated: true,+ sessionContext: {+ userId: 'user-1',+ email: '[email protected]',+ teamId: 'team-1',+ role: 'admin',+ authGeneration: 7,+ },+ },+ })++ const res = await request(app())+ .post('/external/invitations/password-token')+ .send({+ token: 'trusted-reset-proof',+ email: '[email protected]',+ invitationId: 'database-row-1',+ password: 'Synthetic-new-password',+ })+ .expect(200)++ expect(flow.validateInvitationFlowToken).toHaveBeenCalledWith(+ 'trusted-reset-proof',+ )+ expect(directory.setInvitationPasswordForEmail).toHaveBeenCalledWith(+ '[email protected]',+ 'database-row-1',+ 'Synthetic-new-password'+ )+ expect(sessionToken.signExternalSessionToken).toHaveBeenCalledWith({+ userId: 'user-1',+ email: '[email protected]',+ teamId: 'team-1',+ role: 'admin',+ authGeneration: 7,+ })+ expect(res.body.token).toBe('session-token')+ expect(res.body).not.toHaveProperty('sessionContext')+ })++ it('does not issue a session when reset proof or pending state is invalid', async () => {+ directory.setInvitationPasswordForEmail.mockReset()+ sessionToken.signExternalSessionToken.mockReset().mockReturnValue('session-token')+ flow.validateInvitationFlowToken.mockRejectedValue(new Error('invalid'))+ const invalidProof = await request(app())+ .post('/external/invitations/password-token')+ .send({+ token: 'invalid-reset-proof',+ email: '[email protected]',+ invitationId: 'reset-1',+ password: 'Synthetic-new-password',+ })+ .expect(400)+ expect(invalidProof.body.error).toBe('invalid_invitation')++ flow.validateInvitationFlowToken.mockResolvedValue({+ email: '[email protected]',+ invitationUuid: 'producer-token-1',+ })+ directory.getInvitationByToken.mockResolvedValue({+ id: 'database-row-1',+ token: 'producer-token-1',+ email: '[email protected]',+ purpose: 'password_reset',+ status: 'accepted',+ })+ await request(app())+ .post('/external/invitations/password-token')+ .send({+ token: 'redeemed-reset-proof',+ email: '[email protected]',+ invitationId: 'database-row-1',+ password: 'Synthetic-new-password',+ })+ .expect(409)++ expect(directory.setInvitationPasswordForEmail).not.toHaveBeenCalled()+ expect(sessionToken.signExternalSessionToken).not.toHaveBeenCalled()++ directory.getInvitationByToken.mockReset().mockResolvedValue({+ id: 'database-row-1',+ token: 'producer-token-1',+ email: '[email protected]',+ purpose: 'password_reset',+ status: 'pending',+ })+ directory.setInvitationPasswordForEmail.mockResolvedValue({ error: 'not_pending' })+ await request(app())+ .post('/external/invitations/password-token')+ .send({+ token: 'raced-reset-proof',+ email: '[email protected]',+ invitationId: 'database-row-1',+ password: 'Synthetic-new-password',+ })+ .expect(409)+ expect(sessionToken.signExternalSessionToken).not.toHaveBeenCalled()+ })++ it('rejects a database row ID that does not match the trusted flow token', async () => {+ flow.validateInvitationFlowToken.mockResolvedValue({+ email: '[email protected]',+ invitationUuid: 'producer-token-1',+ })+ directory.getInvitationByToken.mockResolvedValue({+ id: 'database-row-1',+ token: 'producer-token-1',+ email: '[email protected]',+ purpose: 'password_reset',+ status: 'pending',+ })++ const res = await request(app())+ .post('/external/invitations/password-token')+ .send({+ token: 'trusted-reset-proof',+ email: '[email protected]',+ invitationId: 'another-row',+ password: 'Synthetic-new-password',+ })+ .expect(403)++ expect(res.body.error).toBe('forbidden')+ expect(directory.setInvitationPasswordForEmail).not.toHaveBeenCalled()+ expect(sessionToken.signExternalSessionToken).not.toHaveBeenCalled()+ })+it('POST desktop-authorization returns 503, NOT 404, when the hub-rejection message shape collides with the "(404)" not_found string match', async () => {// enroll() builds messages like `...rejected enrollment for '<domain>'// (${response.status})`. A hub 404 during boot/on-demand enrollment