Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): harden replace-inviter acceptance from deep review

Jose Gabriel Ramirez Encinase4829ac7ac65
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 865f6235f..0a4fd7424 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.646",
+ "version": "0.1.647",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.646",
+ "version": "0.1.647",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 5817feb3f..731cc8961 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.646",
+ "version": "0.1.647",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/adminAuthService.ts b/control-api/src/services/adminAuthService.ts
index 896fa7f61..caa2f0b18 100644
--- a/control-api/src/services/adminAuthService.ts
+++ b/control-api/src/services/adminAuthService.ts
@@ -583,7 +583,7 @@ export async function deleteControlAdmin(
*/
export async function retireControlAdminInTransaction(
db: DbClient,
- input: { actorAdminId: string; adminId: string; reason: string }
+ input: { actorAdminId: string; adminId: string; reason: string; sourceStatusRef?: string }
): Promise<{ deleted: true } | { error: 'not_found' }> {
const { actorAdminId, adminId } = input
const actorResult = await db.query(
@@ -693,7 +693,12 @@ export async function retireControlAdminInTransaction(
kind: 'service_action',
sourceEventId: `control_admin_deletion_audit:${audit.id}`,
occurredAt: new Date().toISOString(),
- reasonCode: 'control_admin_access_revoked',
+ // A hand-over records why the new admin appears as the deleting actor.
+ reasonCode:
+ input.reason === CONTROL_ADMIN_REPLACED_REASON
+ ? CONTROL_ADMIN_REPLACED_REASON
+ : 'control_admin_access_revoked',
+ ...(input.sourceStatusRef ? { sourceStatusRef: input.sourceStatusRef } : {}),
payload: {
resource_class: 'control_admin_access',
status: 'revoked',
@@ -981,13 +986,27 @@ export async function completeControlAdminInvitation(input: {
invitationId: string
username: string
passwordHash: string
-}): Promise<AdminUserRecord | { error: 'duplicate_email' | 'duplicate_username' | 'not_found' }> {
+}): Promise<
+ | AdminUserRecord
+ | {
+ error: 'duplicate_email' | 'duplicate_username' | 'not_found' | ReplaceInviterRefusal
+ }
+> {
const email = normalizeEmail(input.email)
const username = input.username.trim()
return withTransaction(async db => {
const invitation = await getPendingControlAdminInvitation(db, input.invitationId, email)
if (!invitation) return { error: 'not_found' as const }
+ // A replace-inviter acceptance locks the inviter before its invitation, so
+ // two acceptances from one inviter queue here instead of deadlocking when
+ // step (4) revokes the other's invitation.
+ if (invitation.replaceInviter && invitation.invitedByAdminId) {
+ await db.query(`SELECT 1 FROM control_admin_users WHERE id = $1 FOR UPDATE`, [
+ invitation.invitedByAdminId,
+ ])
+ }
+
// Lock and re-check before creating anything: getPendingControlAdminInvitation
// does not lock, and the accept UPDATE below has no status guard, so a revoke
// that commits meanwhile (DELETE route, MCC resend/cancel, or a replace-inviter
@@ -1031,17 +1050,30 @@ export async function completeControlAdminInvitation(input: {
)
if (invitation.replaceInviter && invitation.invitedByAdminId) {
- await replaceInviterInTransaction(db, {
+ const refusal = await replaceInviterInTransaction(db, {
invitationId: invitation.id,
inviterAdminId: invitation.invitedByAdminId,
newAdmin: admin,
})
+ if (refusal) throw new ReplaceInviterRefused(refusal)
}
return admin
+ }).catch((error: unknown) => {
+ // Rolls back the new admin and the accepted invitation; nothing is retired.
+ if (error instanceof ReplaceInviterRefused) return { error: error.reason }
+ throw error
})
}
+export type ReplaceInviterRefusal = 'replace_inviter_receiver_missing'
+
+class ReplaceInviterRefused extends Error {
+ constructor(readonly reason: ReplaceInviterRefusal) {
+ super(reason)
+ }
+}
+
export const CONTROL_ADMIN_REPLACED_REASON = 'control_admin_replaced'
// admin > inviter > member; used so a copied role never downgrades a target.
@@ -1056,7 +1088,7 @@ const TEAM_ROLE_RANK = (column: string) =>
async function replaceInviterInTransaction(
db: DbClient,
input: { invitationId: string; inviterAdminId: string; newAdmin: AdminUserRecord }
-): Promise<void> {
+): Promise<ReplaceInviterRefusal | null> {
const inviterResult = await db.query(
`SELECT id::text AS id, email, status
FROM control_admin_users
@@ -1067,24 +1099,58 @@ async function replaceInviterInTransaction(
const inviter = inviterResult.rows[0] as
| { id: string; email: string | null; status: 'active' | 'disabled' }
| undefined
- if (!inviter || inviter.status !== 'active') return
-
- const inviterDesktop = inviter.email
- ? await db.query(
- `SELECT id::text AS id
- FROM users
- WHERE lower(email) = lower($1)
- AND lifecycle_state = 'active'
- LIMIT 1
- FOR UPDATE`,
- [inviter.email]
- )
- : { rows: [], rowCount: 0 }
+ if (!inviter || inviter.status !== 'active') return null
+
+ // The active operator link names the inviter's desktop user exactly; an admin
+ // email change (completeControlAdminEmailChangeRequest) never updates
+ // users.email. Invited admins have no link (links come only from
+ // initial_setup), so the shared email is their only mapping.
+ const linkedDesktop = await db.query(
+ `SELECT u.id::text AS id
+ FROM gfs_desktop_operator_links link
+ JOIN users u ON u.id = link.user_id
+ WHERE link.control_admin_id = $1::uuid
+ AND link.state = 'active'
+ AND u.lifecycle_state = 'active'
+ FOR UPDATE OF u`,
+ [inviter.id]
+ )
+ const inviterDesktop =
+ (linkedDesktop.rowCount ?? 0) > 0 || !inviter.email
+ ? linkedDesktop
+ : await db.query(
+ `SELECT id::text AS id
+ FROM users
+ WHERE lower(email) = lower($1)
+ AND lifecycle_state = 'active'
+ LIMIT 1
+ FOR UPDATE`,
+ [inviter.email]
+ )
const inviterDesktopUserId = (inviterDesktop.rows[0] as { id: string } | undefined)?.id ?? null
// completeControlAdminInvitation always inserts the normalized invitation email.
const newAdminEmail = input.newAdmin.email ?? ''
if (inviterDesktopUserId) {
+ // Retiring the inviter must not drop team access nobody receives: without a
+ // desktop user or pending desktop invitation for the invitee, refuse.
+ const receiver = await db.query(
+ `SELECT
+ EXISTS(SELECT 1 FROM team_members WHERE user_id = $1::uuid AND status = 'active') AS has_roles,
+ EXISTS(SELECT 1 FROM users WHERE lower(email) = lower($2) AND lifecycle_state = 'active')
+ OR EXISTS(SELECT 1 FROM invitations
+ WHERE lower(email) = lower($2)
+ AND purpose = 'admin_desktop_access'
+ AND status = 'pending'
+ AND expires_at > NOW()) AS has_receiver`,
+ [inviterDesktopUserId, newAdminEmail]
+ )
+ const { has_roles: hasRoles, has_receiver: hasReceiver } = receiver.rows[0] as {
+ has_roles: boolean
+ has_receiver: boolean
+ }
+ if (hasRoles && !hasReceiver) return 'replace_inviter_receiver_missing'
+
// (1a) The invitee already has a desktop user: raise its memberships.
await db.query(
`INSERT INTO team_members(team_id, user_id, role, status)
@@ -1109,6 +1175,8 @@ async function replaceInviterInTransaction(
// (1b) The desktop invitation is accepted after this transaction commits
// (auth.ts:504-507) and upserts role = EXCLUDED.role (membership.ts:319-326),
// so the roles must also live on the invitation or 1a would be downgraded.
+ // Each copied role is the higher of the inviter's and any role the invitee's
+ // desktop user already holds, which acceptance would otherwise overwrite.
// Materialize a legacy single-team invitation first so adding rows cannot
// hide its fallback team (loadInvitationTeams, membership.ts:242-267).
await db.query(
@@ -1124,11 +1192,24 @@ async function replaceInviterInTransaction(
)
await db.query(
`INSERT INTO invitation_teams(invitation_id, team_id, role)
- SELECT i.id, source.team_id, source.role
+ SELECT i.id, source.team_id,
+ CASE
+ WHEN held.role IS NOT NULL
+ AND ${TEAM_ROLE_RANK('held.role')} > ${TEAM_ROLE_RANK('source.role')}
+ THEN held.role
+ ELSE source.role
+ END
FROM invitations i
JOIN team_members source
ON source.user_id = $1::uuid
AND source.status = 'active'
+ LEFT JOIN users target
+ ON lower(target.email) = lower(i.email)
+ AND target.lifecycle_state = 'active'
+ LEFT JOIN team_members held
+ ON held.user_id = target.id
+ AND held.team_id = source.team_id
+ AND held.status = 'active'
WHERE lower(i.email) = lower($2)
AND i.purpose = 'admin_desktop_access'
AND i.status = 'pending'
@@ -1177,6 +1258,7 @@ async function replaceInviterInTransaction(
actorAdminId: input.newAdmin.id,
adminId: inviter.id,
reason: CONTROL_ADMIN_REPLACED_REASON,
+ sourceStatusRef: `control_admin_invitation:${input.invitationId}`,
})
if ('error' in retired) {
throw new Error('inviting control admin changed during replace-inviter acceptance')
@@ -1201,6 +1283,7 @@ async function replaceInviterInTransaction(
AND lower(email) IN (SELECT lower(email) FROM revoked)`,
[inviter.id, input.invitationId]
)
+ return null
}
export async function completeControlAdminEmailChangeRequest(input: {
diff --git a/control-api/test/services.adminAuthDelete.realPostgres.integration.test.ts b/control-api/test/services.adminAuthDelete.realPostgres.integration.test.ts
index 3480fdb51..be5db320f 100644
--- a/control-api/test/services.adminAuthDelete.realPostgres.integration.test.ts
+++ b/control-api/test/services.adminAuthDelete.realPostgres.integration.test.ts
@@ -3,6 +3,7 @@ import { randomBytes, randomUUID } from 'node:crypto'
import { Pool } from 'pg'
import { initDb, pool } from '../src/db.js'
import { deleteControlAdmin } from '../src/services/adminAuthService.js'
+import './realPostgres.requirement.ts'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
diff --git a/control-api/test/services.controlAdminReplaceInviter.realPostgres.integration.test.ts b/control-api/test/services.controlAdminReplaceInviter.realPostgres.integration.test.ts
index 2ac7a0214..a2bfb0a7e 100644
--- a/control-api/test/services.controlAdminReplaceInviter.realPostgres.integration.test.ts
+++ b/control-api/test/services.controlAdminReplaceInviter.realPostgres.integration.test.ts
@@ -20,6 +20,7 @@ import {
acceptInvitationById,
createSilentInvitationForTeams,
} from '../src/services/directory/membership.js'
+import './realPostgres.requirement.ts'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
@@ -109,6 +110,14 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
return invitation.id
}
+ async function teamRole(teamId: string, userId: string): Promise<string | undefined> {
+ const result = await testPool.query(
+ `SELECT role FROM team_members WHERE team_id = $1::uuid AND user_id = $2::uuid AND status = 'active'`,
+ [teamId, userId]
+ )
+ return (result.rows[0] as { role?: string } | undefined)?.role
+ }
+
async function adminInvitationStatus(id: string): Promise<string | undefined> {
const result = await testPool.query(
`SELECT status FROM control_admin_invitations WHERE id = $1::uuid`,
@@ -200,12 +209,19 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
if ('error' in invitation) throw new Error(invitation.error)
const desktopInvitationId = await inviteDesktopAccess(clientEmail)
const unrelatedDesktopInvitationId = await inviteDesktopAccess(uniqueEmail('unrelated'))
+ const memberInvitation = await testPool.query(
+ `INSERT INTO invitations (email, role, status, purpose)
+ VALUES ($1, 'member', 'pending', 'member_invitation') RETURNING id::text AS id`,
+ [clientEmail]
+ )
+ const memberInvitationId = (memberInvitation.rows[0] as { id: string }).id
await revokeControlAdminInvitation(invitation.id)
expect(await adminInvitationStatus(invitation.id)).toBe('revoked')
expect(await desktopInvitationStatus(desktopInvitationId)).toBe('revoked')
expect(await desktopInvitationStatus(unrelatedDesktopInvitationId)).toBe('pending')
+ expect(await desktopInvitationStatus(memberInvitationId)).toBe('pending')
expect(await adminStatus(inviter.id)).toBe('active')
expect(await desktopLifecycle(inviterDesktopId)).toBe('active')
await expect(
@@ -232,6 +248,17 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
VALUES ($1::uuid, 'telegram', $2, 'ops-approval-channel', NOW())`,
[inviterDesktopId, `tg-${inviterDesktopId}`]
)
+ await testPool.query(
+ `INSERT INTO workflow_approval_medium_challenges
+ (user_id, medium, provider_user_id, code_hash, expires_at)
+ VALUES ($1::uuid, 'telegram', $2, 'example-code-hash', NOW() + INTERVAL '10 minutes')`,
+ [inviterDesktopId, `tg-${inviterDesktopId}`]
+ )
+ const otherAdmin = await seedAdmin('other-ops')
+ const otherEmail = uniqueEmail('other-client')
+ const otherInvitation = await createControlAdminInvitation(otherEmail, otherAdmin.id)
+ if ('error' in otherInvitation) throw new Error(otherInvitation.error)
+ const otherDesktopInvitationId = await inviteDesktopAccess(otherEmail)
const clientEmail = uniqueEmail('client')
const handover = await createControlAdminInvitation(clientEmail, inviter.id, {
@@ -301,17 +328,175 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
[inviterDesktopId]
)
expect(approvalMedium.rows).toEqual([{ disabled: true }])
+ const challenge = await testPool.query(
+ `SELECT consumed_at IS NOT NULL AS consumed
+ FROM workflow_approval_medium_challenges WHERE user_id = $1::uuid`,
+ [inviterDesktopId]
+ )
+ expect(challenge.rows).toEqual([{ consumed: true }])
const audit = await testPool.query(
`SELECT actor_admin_id::text AS actor, target_admin_id::text AS target
FROM control_admin_deletion_audit WHERE target_admin_id = $1::uuid`,
[inviter.id]
)
expect(audit.rows).toEqual([{ actor: completed.id, target: inviter.id }])
+ const deletedEvent = await testPool.query(
+ `SELECT payload_metadata->>'reason_code' AS reason_code,
+ payload_metadata->>'detail_ref' AS detail_ref
+ FROM administrative_events
+ WHERE action = 'control_admin_deleted' AND target_human_sub = $1`,
+ [inviter.id]
+ )
+ expect(deletedEvent.rows).toEqual([
+ {
+ reason_code: 'control_admin_replaced',
+ detail_ref: `control_admin_invitation:${handover.id}`,
+ },
+ ])
expect(await adminInvitationStatus(handover.id)).toBe('accepted')
expect(await desktopInvitationStatus(clientDesktopInvitationId)).toBe('accepted')
expect(await adminInvitationStatus(stray.id)).toBe('revoked')
expect(await desktopInvitationStatus(strayDesktopInvitationId)).toBe('revoked')
+ // Only the inviter's own invitations are revoked.
+ expect(await adminInvitationStatus(otherInvitation.id)).toBe('pending')
+ expect(await desktopInvitationStatus(otherDesktopInvitationId)).toBe('pending')
+ })
+
+ it('finds the inviter desktop user by operator link after an admin email change', async () => {
+ const inviter = await seedAdmin('moved-ops')
+ const linkedDesktopId = await seedDesktopUser(inviter.email)
+ await seedOperatorLink(linkedDesktopId, inviter.id)
+ const team = await seedTeam('Moved')
+ await addTeamMember(team, linkedDesktopId, 'admin')
+ // completeControlAdminEmailChangeRequest updates control_admin_users only.
+ const movedEmail = uniqueEmail('moved-ops')
+ await testPool.query(`UPDATE control_admin_users SET email = $2 WHERE id = $1::uuid`, [
+ inviter.id,
+ movedEmail,
+ ])
+ const bystanderId = await seedDesktopUser(movedEmail)
+ const bystanderTeam = await seedTeam('Bystander')
+ await addTeamMember(bystanderTeam, bystanderId, 'member')
+
+ const clientEmail = uniqueEmail('moved-client')
+ const handover = await createControlAdminInvitation(clientEmail, inviter.id, {
+ replaceInviter: true,
+ })
+ if ('error' in handover) throw new Error(handover.error)
+ const clientDesktopInvitationId = await inviteDesktopAccess(clientEmail)
+ const completed = await completeControlAdminInvitation({
+ email: clientEmail,
+ invitationId: handover.id,
+ username: `moved-${randomUUID().slice(0, 8)}`,
+ passwordHash: 'real-pg-replace-client',
+ })
+ if ('error' in completed) throw new Error(completed.error)
+ const accepted = await acceptInvitationById(clientEmail, clientDesktopInvitationId)
+ if ('error' in accepted) throw new Error(String(accepted.error))
+
+ expect(await desktopLifecycle(linkedDesktopId)).toBe('retired')
+ expect(await teamRole(team, accepted.data.userId)).toBe('admin')
+ expect(await desktopLifecycle(bystanderId)).toBe('active')
+ expect(await teamRole(bystanderTeam, accepted.data.userId)).toBeUndefined()
+ })
+
+ it('never downgrades a role the new admin already holds or was invited with', async () => {
+ const inviter = await seedAdmin('keep-ops')
+ const inviterDesktopId = await seedDesktopUser(inviter.email)
+ const heldTeam = await seedTeam('Held')
+ const invitedTeam = await seedTeam('Invited')
+ await addTeamMember(heldTeam, inviterDesktopId, 'member')
+ await addTeamMember(invitedTeam, inviterDesktopId, 'member')
+ const clientEmail = uniqueEmail('keep-client')
+ const clientDesktopId = await seedDesktopUser(clientEmail)
+ await addTeamMember(heldTeam, clientDesktopId, 'admin')
+ const handover = await createControlAdminInvitation(clientEmail, inviter.id, {
+ replaceInviter: true,
+ })
+ if ('error' in handover) throw new Error(handover.error)
+ const desktopInvitation = await createSilentInvitationForTeams({
+ inviteeName: clientEmail,
+ email: clientEmail,
+ teamAssignments: [{ teamId: invitedTeam, role: 'admin' }],
+ fallbackRole: 'member',
+ purpose: 'admin_desktop_access',
+ })
+
+ const completed = await completeControlAdminInvitation({
+ email: clientEmail,
+ invitationId: handover.id,
+ username: `keep-${randomUUID().slice(0, 8)}`,
+ passwordHash: 'real-pg-replace-client',
+ })
+ if ('error' in completed) throw new Error(completed.error)
+ expect(await teamRole(heldTeam, clientDesktopId)).toBe('admin')
+ const accepted = await acceptInvitationById(clientEmail, desktopInvitation.id)
+ if ('error' in accepted) throw new Error(String(accepted.error))
+
+ expect(accepted.data.userId).toBe(clientDesktopId)
+ expect(await teamRole(heldTeam, clientDesktopId)).toBe('admin')
+ expect(await teamRole(invitedTeam, clientDesktopId)).toBe('admin')
+ })
+
+ it('refuses a hand-over that has nobody to receive the inviter team roles', async () => {
+ const inviter = await seedAdmin('nobody-ops')
+ const inviterDesktopId = await seedDesktopUser(inviter.email)
+ const team = await seedTeam('Nobody')
+ await addTeamMember(team, inviterDesktopId, 'admin')
+ const clientEmail = uniqueEmail('nobody-client')
+ const handover = await createControlAdminInvitation(clientEmail, inviter.id, {
+ replaceInviter: true,
+ })
+ if ('error' in handover) throw new Error(handover.error)
+
+ await expect(
+ completeControlAdminInvitation({
+ email: clientEmail,
+ invitationId: handover.id,
+ username: `nobody-${randomUUID().slice(0, 8)}`,
+ passwordHash: 'real-pg-replace-client',
+ })
+ ).resolves.toEqual({ error: 'replace_inviter_receiver_missing' })
+
+ expect(await adminInvitationStatus(handover.id)).toBe('pending')
+ expect(await adminStatus(inviter.id)).toBe('active')
+ expect(await desktopLifecycle(inviterDesktopId)).toBe('active')
+ expect(await teamRole(team, inviterDesktopId)).toBe('admin')
+ const admins = await testPool.query(
+ `SELECT 1 FROM control_admin_users WHERE lower(email) = lower($1)`,
+ [clientEmail]
+ )
+ expect(admins.rowCount).toBe(0)
+ })
+
+ it('accepts one of two concurrent hand-overs from the same inviter without a deadlock', async () => {
+ const inviter = await seedAdmin('twin-ops')
+ const emails = [uniqueEmail('twin-a'), uniqueEmail('twin-b')]
+ const invitations = []
+ for (const email of emails) {
+ const invitation = await createControlAdminInvitation(email, inviter.id, {
+ replaceInviter: true,
+ })
+ if ('error' in invitation) throw new Error(invitation.error)
+ invitations.push(invitation)
+ }
+
+ const results = await Promise.all(
+ invitations.map((invitation, index) =>
+ completeControlAdminInvitation({
+ email: emails[index],
+ invitationId: invitation.id,
+ username: `twin-${index}-${randomUUID().slice(0, 8)}`,
+ passwordHash: 'real-pg-replace-client',
+ })
+ )
+ )
+
+ const winners = results.filter(result => !('error' in result))
+ expect(winners).toHaveLength(1)
+ expect(results.filter(result => 'error' in result)).toEqual([{ error: 'not_found' }])
+ expect(await adminStatus(inviter.id)).toBe('disabled')
})
it('denies the retired inviter team-granted host access although its team rows stay active', async () => {
@@ -353,6 +538,7 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
replaceInviter: true,
})
if ('error' in handover) throw new Error(handover.error)
+ await inviteDesktopAccess(clientEmail)
const completed = await completeControlAdminInvitation({
email: clientEmail,
invitationId: handover.id,
@@ -481,6 +667,8 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
username: `race-${randomUUID().slice(0, 8)}`,
passwordHash: 'real-pg-replace-client',
})
+ // Observe an early rejection here instead of as an unhandled rejection.
+ completion.catch(() => undefined)
for (let attempt = 0; ; attempt += 1) {
const waiting = await adminPool.query(
`SELECT 1 FROM pg_stat_activity WHERE datname = $1 AND wait_event_type = 'Lock'`,
@@ -504,5 +692,5 @@ describeRealPostgres('control admin replace-inviter invitations on real PostgreS
[email]
)
expect(admins.rowCount).toBe(0)
- })
+ }, 15_000)
})