Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(auth): preserve RP-1010-04 bcrypt work ownership

Josué5bfc9144e5ea
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 5607f2eb2..40685ff8f 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.731",
+ "version": "0.1.732",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.731",
+ "version": "0.1.732",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index c518dfd37..edf241fcb 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.731",
+ "version": "0.1.732",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 60dfa542e..8b9a08d4a 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -11,6 +11,7 @@ import {
applyPasswordAdmissionSchema,
applyPasswordEvaluationRetentionSchema,
} from './services/auth/passwordAdmissionSchema.js'
+import { applyPasswordWorkOwnershipSchema } from './services/auth/passwordWorkOwnershipSchema.js'
import { applyCodexCatalogModelsSchema } from './services/codexSubscriptionCatalog.js'
import {
applyCodexChatgptAccountIdSchema,
@@ -6344,6 +6345,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
version: '0127_password_evaluation_retention',
apply: applyPasswordEvaluationRetentionSchema,
},
+ {
+ version: '0128_password_work_ownership',
+ apply: applyPasswordWorkOwnershipSchema,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
diff --git a/control-api/src/services/auth/passwordCredentialVerification.ts b/control-api/src/services/auth/passwordCredentialVerification.ts
index c859025bb..88230144e 100644
--- a/control-api/src/services/auth/passwordCredentialVerification.ts
+++ b/control-api/src/services/auth/passwordCredentialVerification.ts
@@ -1,7 +1,6 @@
import bcrypt from 'bcryptjs'
import { type DbTransactionClient, pool, withTransaction } from '../../db.js'
import { rootLogger } from '../../observability/logger.js'
-import { acquireRateLimitConcurrencyLease } from '../rateLimiterService.js'
import {
type PasswordIdentifierState,
admitPasswordIdentifier,
@@ -9,6 +8,7 @@ import {
passwordIdentifierKey,
PASSWORD_ADMISSION_POLICY as policy,
} from './passwordAdmissionState.js'
+import { acquirePasswordWork } from './passwordWorkOwnership.js'
// Synthetic, repository-controlled cost-12 hash, precomputed once, never per request.
export const PASSWORD_DUMMY_HASH = '$2b$12$DdpJSFUb/dYx43plh72Ia.k9RJ8krESZ/DtTRUbdQq/ngzfJuyCUu'
@@ -178,12 +178,8 @@ export async function verifyMemberPassword(
if (busy) denied(policy.paceMs)
busy = true
try {
- const lease = await acquireRateLimitConcurrencyLease(
- [{ bucketKey: 'password-verification-global', maxConcurrent: policy.concurrency }],
- { releaseIdleClient: true }
- )
- if (!lease.backendAvailable) throw new PasswordAdmissionError(503, 2)
- if (!lease.allowed) denied(policy.paceMs)
+ const lease = await acquirePasswordWork()
+ if (!lease) denied(policy.paceMs)
const user = capture.user
let authenticated = false
try {
@@ -214,7 +210,7 @@ export async function verifyMemberPassword(
} finally {
await lease.release()
}
- // Bcrypt is finished. Release its pool session before completion borrows a client.
+ // Bcrypt is finished and its durable owner is released before credential completion.
const current = await completePasswordEvaluation(capture, authenticated)
return current && authenticated ? user : null
} finally {
diff --git a/control-api/src/services/auth/passwordWorkOwnership.ts b/control-api/src/services/auth/passwordWorkOwnership.ts
new file mode 100644
index 000000000..3d5c8655d
--- /dev/null
+++ b/control-api/src/services/auth/passwordWorkOwnership.ts
@@ -0,0 +1,41 @@
+import { randomUUID } from 'node:crypto'
+import { hostname } from 'node:os'
+import { withTransaction } from '../../db.js'
+
+// Process-incarnation identity is distinct from reusable host/PID diagnostics.
+let ownerInstance: string | undefined
+
+/** Null means immediately busy. A persistence error propagates for sanitized fail-closed handling. */
+export async function acquirePasswordWork(): Promise<{ release: () => Promise<void> } | null> {
+ const owner = (ownerInstance ??= randomUUID())
+ const operation = randomUUID()
+ const admitted = await withTransaction(async db => {
+ // This must survive a database/process crash; never use the disposable limiter pool.
+ await db.query('SET LOCAL synchronous_commit = on')
+ const result = await db.query(
+ `INSERT INTO password_verification_work
+ (singleton, operation_id, owner_instance, owner_host, owner_pid)
+ VALUES (TRUE, $1, $2, $3, $4) ON CONFLICT (singleton) DO NOTHING
+ RETURNING singleton`,
+ [operation, owner, hostname(), process.pid]
+ )
+ return result.rows.length === 1
+ })
+ if (!admitted) return null
+ let released: Promise<void> | undefined
+ return {
+ release: () => {
+ released ??= withTransaction(async db => {
+ await db.query('SET LOCAL synchronous_commit = on')
+ const result = await db.query(
+ `DELETE FROM password_verification_work
+ WHERE singleton AND operation_id = $1 AND owner_instance = $2 RETURNING singleton`,
+ [operation, owner]
+ )
+ // An externally removed/replaced owner is not proof that our authority was preserved.
+ if (result.rows.length !== 1) throw new Error('password work ownership lost')
+ })
+ return released
+ },
+ }
+}
diff --git a/control-api/src/services/auth/passwordWorkOwnershipSchema.ts b/control-api/src/services/auth/passwordWorkOwnershipSchema.ts
new file mode 100644
index 000000000..c047352c0
--- /dev/null
+++ b/control-api/src/services/auth/passwordWorkOwnershipSchema.ts
@@ -0,0 +1,16 @@
+import type { DbClient } from '../../db.js'
+
+/** Dedicated password authority. Never reclaimed by age, heartbeat or session loss. */
+export async function applyPasswordWorkOwnershipSchema(db: Pick<DbClient, 'query'>): Promise<void> {
+ await db.query(`
+ CREATE TABLE IF NOT EXISTS password_verification_work (
+ singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton),
+ operation_id UUID NOT NULL,
+ owner_instance UUID NOT NULL,
+ owner_host TEXT NOT NULL,
+ owner_pid INTEGER NOT NULL CHECK (owner_pid > 0),
+ acquired_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ );
+ GRANT SELECT, INSERT, DELETE ON password_verification_work TO control_api_runtime;
+ `)
+}
diff --git a/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts b/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts
index 5abff6756..14d579b05 100644
--- a/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts
+++ b/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts
@@ -118,13 +118,15 @@ realPg('RP-001 production credential pool ownership', () => {
const connect = db!.pool.connect.bind(db!.pool)
const spy = vi.spyOn(db!.pool, 'connect')
let completing = false
+ let afterComparisonBorrows = 0
vi.spyOn(bcrypt, 'compare').mockImplementation(async (submitted: string, stored: string) => {
const result = await realCompare(submitted, stored)
completing = true
return result
})
spy.mockImplementation((...args: unknown[]) => {
- if (completing) {
+ // Durable owner release now borrows first; fault the credential completion, not release.
+ if (completing && ++afterComparisonBorrows === 2) {
completing = false
return Promise.reject(new Error('synthetic completion failure'))
}
diff --git a/control-api/test/rp004.passwordWork.realPostgres.integration.test.ts b/control-api/test/rp004.passwordWork.realPostgres.integration.test.ts
new file mode 100644
index 000000000..1fb957de1
--- /dev/null
+++ b/control-api/test/rp004.passwordWork.realPostgres.integration.test.ts
@@ -0,0 +1,253 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import bcrypt from 'bcryptjs'
+import { type ChildProcess, spawn } from 'node:child_process'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const realPg = adminUrl ? describe : describe.skip
+const password = 'Synthetic-RP004-correct-password'
+const email = '[email protected]'
+
+// Both processes execute the source producer and real cost-12 bcrypt. No built artifacts required.
+const childScript = `
+require('ts-node').register({project:'./tsconfig.json',experimentalResolver:true,transpileOnly:true});
+const bcrypt=require('bcryptjs');
+const compare=bcrypt.compare.bind(bcrypt);
+// Isolated fault harness keeps the owner alive after its idle pool reports session termination.
+require('./src/db.ts').pool.on('error',()=>{});
+bcrypt.compare=async (...args)=>{
+ const work=compare(...args);
+ process.send({kind:'entered'});
+ if(process.env.PAUSE_COMPARISON==='1') process.kill(process.pid,'SIGSTOP');
+ return work;
+};
+const {verifyMemberPassword}=require('./src/services/auth/passwordCredentialVerification.ts');
+verifyMemberPassword(process.env.TEST_EMAIL,process.env.TEST_PASSWORD,{publicLogin:false})
+ .then(user=>process.send({kind:'result',success:!!user},()=>process.exit(0)),
+ error=>process.send({kind:'result',status:error.status},()=>process.exit(0)));
+`
+
+realPg('RP-1010-04 durable expensive-work ownership', () => {
+ const database = `rp004_${randomBytes(6).toString('hex')}`
+ let admin: Pool
+ let db: typeof import('../src/db.js')
+ let url: URL
+ const children = new Set<ChildProcess>()
+ beforeAll(async () => {
+ admin = new Pool({ connectionString: adminUrl })
+ await admin.query(`CREATE DATABASE "${database}"`)
+ url = new URL(adminUrl!)
+ url.pathname = `/${database}`
+ vi.stubEnv('CONTROL_API_PG_CONNECTION_STRING', url.toString())
+ vi.resetModules()
+ db = await import('../src/db.js')
+ await db.initDb()
+ await db.pool.query('INSERT INTO users(email,password_hash) VALUES ($1,$2)', [
+ email,
+ await bcrypt.hash(password, 12),
+ ])
+ }, 60_000)
+ beforeEach(async () => {
+ await db.pool.query('TRUNCATE password_identifier_state, password_verification_pace')
+ if (
+ (await db.pool.query("SELECT to_regclass('password_verification_work') AS present")).rows[0]
+ .present
+ )
+ await db.pool.query('DELETE FROM password_verification_work')
+ })
+ afterAll(async () => {
+ try {
+ for (const child of children) {
+ const exited = new Promise<void>(resolve => child.once('exit', () => resolve()))
+ child.kill('SIGCONT')
+ child.kill('SIGKILL')
+ await exited
+ }
+ if (db) {
+ await endPoolAndWaitForClients(db.pool)
+ await endPoolAndWaitForClients(db.rateLimitPool)
+ }
+ if (admin) await admin.query(`DROP DATABASE "${database}"`)
+ } finally {
+ await admin?.end()
+ vi.unstubAllEnvs()
+ }
+ })
+
+ function start(paused: boolean) {
+ const name = `rp004_${randomBytes(5).toString('hex')}`
+ const childUrl = new URL(url)
+ childUrl.searchParams.set('application_name', name)
+ const child = spawn(process.execPath, ['-e', childScript], {
+ cwd: process.cwd(),
+ env: {
+ ...process.env,
+ CONTROL_API_PG_CONNECTION_STRING: childUrl.toString(),
+ TEST_EMAIL: email,
+ TEST_PASSWORD: password,
+ PAUSE_COMPARISON: paused ? '1' : '0',
+ },
+ stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
+ })
+ children.add(child)
+ const messages: Array<{ kind: string; success?: boolean; status?: number }> = []
+ let notify = () => {}
+ child.on('message', value => {
+ messages.push(value as (typeof messages)[number])
+ notify()
+ })
+ let stderr = ''
+ child.stderr!.on('data', value => {
+ stderr += String(value)
+ })
+ const exited = new Promise<number | null>(resolve =>
+ child.once('exit', code => {
+ children.delete(child)
+ resolve(code)
+ notify()
+ })
+ )
+ async function wait(kind: string) {
+ const deadline = Date.now() + 20_000
+ while (!messages.some(value => value.kind === kind)) {
+ if (!children.has(child)) throw new Error(`producer exited before ${kind}: ${stderr}`)
+ if (Date.now() >= deadline) throw new Error(`producer timeout: ${kind}`)
+ await new Promise<void>(resolve => {
+ const timer = setTimeout(resolve, 50)
+ notify = () => {
+ clearTimeout(timer)
+ resolve()
+ }
+ })
+ }
+ return messages.find(value => value.kind === kind)!
+ }
+ return { child, name, wait, exited }
+ }
+ it('does not admit replacement bcrypt after the paused owner loses all database sessions', async () => {
+ const first = start(true)
+ try {
+ await first.wait('entered')
+ await admin.query(
+ 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1',
+ [first.name]
+ )
+ // Natural pace expiry; do not pre-open/reset next_permit and mask the reported scenario.
+ await new Promise(resolve => setTimeout(resolve, 7_600))
+ const second = start(false)
+ const result = await second.wait('result')
+ expect(result).toMatchObject({ status: 429 })
+ expect(await second.exited).toBe(0)
+ } finally {
+ first.child.kill('SIGCONT')
+ await first.wait('result')
+ expect(await first.exited).toBe(0)
+ }
+ }, 60_000)
+ it('admits exactly one competing owner and releases idempotently', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const leases = await Promise.all(Array.from({ length: 8 }, () => acquirePasswordWork()))
+ expect(leases.filter(Boolean)).toHaveLength(1)
+ await Promise.all([leases.find(Boolean)!.release(), leases.find(Boolean)!.release()])
+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(0)
+ })
+ it('never reclaims aged work through password or legacy bucket cleanup', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const lease = await acquirePasswordWork()
+ await db.pool.query(
+ "UPDATE password_verification_work SET acquired_at=clock_timestamp()-interval '1 day'"
+ )
+ const { cleanupExpiredBuckets } = await import('../src/services/rateLimiterService.js')
+ await cleanupExpiredBuckets()
+ expect(await acquirePasswordWork()).toBeNull()
+ await lease!.release()
+ })
+ it('does not release a newer owner with an old operation token', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const old = await acquirePasswordWork()
+ // Fault seam: no bcrypt was started; emulate an authorized recovery/replacement.
+ await db.pool.query('DELETE FROM password_verification_work')
+ const current = await acquirePasswordWork()
+ await expect(old!.release()).rejects.toThrow('ownership lost')
+ expect(await acquirePasswordWork()).toBeNull()
+ await current!.release()
+ })
+ it('retains ownership when release persistence is unavailable', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const lease = await acquirePasswordWork()
+ const fault = vi
+ .spyOn(db, 'withTransaction')
+ .mockRejectedValueOnce(new Error('synthetic unavailable'))
+ try {
+ await expect(lease!.release()).rejects.toThrow('synthetic unavailable')
+ } finally {
+ fault.mockRestore()
+ }
+ expect(await acquirePasswordWork()).toBeNull()
+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(1)
+ })
+ it('denies busy work without bcrypt or a pace refund', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const engine = await import('../src/services/auth/passwordCredentialVerification.js')
+ const lease = await acquirePasswordWork()
+ const compare = vi.spyOn(bcrypt, 'compare')
+ try {
+ await expect(
+ engine.verifyMemberPassword(email, password, { publicLogin: false })
+ ).rejects.toMatchObject({ status: 429 })
+ expect(compare).not.toHaveBeenCalled()
+ expect(
+ (
+ await db.pool.query(
+ 'SELECT next_permit > clock_timestamp() AS consumed FROM password_verification_pace'
+ )
+ ).rows[0].consumed
+ ).toBe(true)
+ } finally {
+ compare.mockRestore()
+ await lease!.release()
+ }
+ })
+ it('keeps a crashed process reservation until verified recovery', async () => {
+ const first = start(true)
+ await first.wait('entered')
+ first.child.kill('SIGKILL')
+ await first.exited
+ await new Promise(resolve => setTimeout(resolve, 7_600))
+ const next = start(false)
+ expect(await next.wait('result')).toMatchObject({ status: 429 })
+ expect(await next.exited).toBe(0)
+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(1)
+ // Test owns and has observed termination of every producer; recover only that observed token.
+ const row = (
+ await db.pool.query('SELECT operation_id,owner_instance FROM password_verification_work')
+ ).rows[0]
+ await db.pool.query(
+ 'DELETE FROM password_verification_work WHERE operation_id=$1 AND owner_instance=$2',
+ [row.operation_id, row.owner_instance]
+ )
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const recovered = await acquirePasswordWork()
+ expect(recovered).not.toBeNull()
+ await recovered!.release()
+ }, 60_000)
+ it('applies the forward ownership schema idempotently without clearing work', async () => {
+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')
+ const lease = await acquirePasswordWork()
+ const { applyPasswordWorkOwnershipSchema } =
+ await import('../src/services/auth/passwordWorkOwnershipSchema.js')
+ await applyPasswordWorkOwnershipSchema(db.pool)
+ await applyPasswordWorkOwnershipSchema(db.pool)
+ expect(await acquirePasswordWork()).toBeNull()
+ await lease!.release()
+ expect(
+ (
+ await db.pool.query(
+ "SELECT 1 FROM schema_migrations WHERE version='0128_password_work_ownership'"
+ )
+ ).rows
+ ).toHaveLength(1)
+ })
+})
diff --git a/docs/how-to/password-work-ownership-recovery.md b/docs/how-to/password-work-ownership-recovery.md
new file mode 100644
index 000000000..b08d12376
--- /dev/null
+++ b/docs/how-to/password-work-ownership-recovery.md
@@ -0,0 +1,50 @@
+# Recover password verification ownership
+
+Member password login and Desktop password reverification use the persisted
+`password_verification_work` singleton to bound actual bcrypt work across processes.
+The slot has no expiry. Database connection loss, a missed heartbeat, capture expiry,
+or a process restart does not prove that the old computation stopped.
+
+A crashed owner or an ambiguous acquisition/release can therefore leave password
+verification unavailable. Other authentication methods and credential recovery keep
+their existing policies. Never clear this record from a request, cleanup job, startup
+hook, or automated timeout. Do not reset the pace or identifier history to recover it.
+
+## Roll out the first ownership-aware version
+
+Use the existing Control API `Recreate` deployment strategy. Drain and stop every
+previous password-verifier instance before routing to this version. An older binary
+does not use this slot, so an overlapping rolling deployment cannot provide the new
+guarantee. Apply the forward migration before starting the new instances. Do not
+roll back to an older verifier while any ownership-aware verification can still run.
+
+`Recreate` is not proof of termination on a disconnected node. If a node is unreachable,
+do not rely on forced Pod deletion or an empty Kubernetes listing. Require verified
+container/process termination from the node runtime, or infrastructure fencing that
+prevents that node/process from resuming. If that evidence is unavailable, leave
+password verification fail-closed and escalate to the infrastructure owner.
+
+## Recover an abandoned slot
+
+This is a privileged, separately authorized operational change, not an API endpoint.
+
+1. Stop password-verifier traffic and prevent new verifier instances from starting.
+2. With an authorized database connection, inspect `singleton`, `operation_id`,
+ `owner_instance`, `owner_host`, `owner_pid` and `acquired_at` in the work record.
+ Keep this diagnostic identity in restricted operational evidence, not public logs.
+3. Prove that **all** previous verifier instances can no longer execute, including
+ suspended processes, old replicas and disconnected nodes. Host/PID alone is not
+ sufficient: PIDs and hostnames are reusable. Record the instance/container identity
+ and termination or fencing evidence. Neither session termination nor elapsed time
+ meets this requirement.
+4. Only after that proof, delete the exact observed record with predicates for both
+ its `operation_id` and `owner_instance`, in a transaction with
+ `SET LOCAL synchronous_commit = on`. Require exactly one returned row. A changed
+ record or missing row means the observation is stale; stop and investigate.
+5. Keep the aggregate pace and identifier state intact. Resume only ownership-aware
+ instances and verify ordinary password authentication and current service health.
+
+No generic `TRUNCATE`, unconditional deletion, database-session kill, TTL reclaim or
+automatic retry of bcrypt is a recovery mechanism. Database restoration/failover
+must preserve acknowledged work reservations; recovering an older database snapshot
+requires the same verifier termination/fencing procedure before reopening traffic.