Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(auth): preserve RP-1010-04 bcrypt work ownership
Josué5bfc9144e5ea
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 5607f2eb2..40685ff8f 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.731",+ "version": "0.1.732","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.731",+ "version": "0.1.732","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex c518dfd37..edf241fcb 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.731",+ "version": "0.1.732","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 60dfa542e..8b9a08d4a 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -11,6 +11,7 @@ import {applyPasswordAdmissionSchema,applyPasswordEvaluationRetentionSchema,} from './services/auth/passwordAdmissionSchema.js'+import { applyPasswordWorkOwnershipSchema } from './services/auth/passwordWorkOwnershipSchema.js'import { applyCodexCatalogModelsSchema } from './services/codexSubscriptionCatalog.js'import {applyCodexChatgptAccountIdSchema,@@ -6344,6 +6345,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [version: '0127_password_evaluation_retention',apply: applyPasswordEvaluationRetentionSchema,},+ {+ version: '0128_password_work_ownership',+ apply: applyPasswordWorkOwnershipSchema,+ },]async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {diff --git a/control-api/src/services/auth/passwordCredentialVerification.ts b/control-api/src/services/auth/passwordCredentialVerification.tsindex c859025bb..88230144e 100644--- a/control-api/src/services/auth/passwordCredentialVerification.ts+++ b/control-api/src/services/auth/passwordCredentialVerification.ts@@ -1,7 +1,6 @@import bcrypt from 'bcryptjs'import { type DbTransactionClient, pool, withTransaction } from '../../db.js'import { rootLogger } from '../../observability/logger.js'-import { acquireRateLimitConcurrencyLease } from '../rateLimiterService.js'import {type PasswordIdentifierState,admitPasswordIdentifier,@@ -9,6 +8,7 @@ import {passwordIdentifierKey,PASSWORD_ADMISSION_POLICY as policy,} from './passwordAdmissionState.js'+import { acquirePasswordWork } from './passwordWorkOwnership.js'// Synthetic, repository-controlled cost-12 hash, precomputed once, never per request.export const PASSWORD_DUMMY_HASH = '$2b$12$DdpJSFUb/dYx43plh72Ia.k9RJ8krESZ/DtTRUbdQq/ngzfJuyCUu'@@ -178,12 +178,8 @@ export async function verifyMemberPassword(if (busy) denied(policy.paceMs)busy = truetry {- const lease = await acquireRateLimitConcurrencyLease(- [{ bucketKey: 'password-verification-global', maxConcurrent: policy.concurrency }],- { releaseIdleClient: true }- )- if (!lease.backendAvailable) throw new PasswordAdmissionError(503, 2)- if (!lease.allowed) denied(policy.paceMs)+ const lease = await acquirePasswordWork()+ if (!lease) denied(policy.paceMs)const user = capture.userlet authenticated = falsetry {@@ -214,7 +210,7 @@ export async function verifyMemberPassword(} finally {await lease.release()}- // Bcrypt is finished. Release its pool session before completion borrows a client.+ // Bcrypt is finished and its durable owner is released before credential completion.const current = await completePasswordEvaluation(capture, authenticated)return current && authenticated ? user : null} finally {diff --git a/control-api/src/services/auth/passwordWorkOwnership.ts b/control-api/src/services/auth/passwordWorkOwnership.tsnew file mode 100644index 000000000..3d5c8655d--- /dev/null+++ b/control-api/src/services/auth/passwordWorkOwnership.ts@@ -0,0 +1,41 @@+import { randomUUID } from 'node:crypto'+import { hostname } from 'node:os'+import { withTransaction } from '../../db.js'++// Process-incarnation identity is distinct from reusable host/PID diagnostics.+let ownerInstance: string | undefined++/** Null means immediately busy. A persistence error propagates for sanitized fail-closed handling. */+export async function acquirePasswordWork(): Promise<{ release: () => Promise<void> } | null> {+ const owner = (ownerInstance ??= randomUUID())+ const operation = randomUUID()+ const admitted = await withTransaction(async db => {+ // This must survive a database/process crash; never use the disposable limiter pool.+ await db.query('SET LOCAL synchronous_commit = on')+ const result = await db.query(+ `INSERT INTO password_verification_work+ (singleton, operation_id, owner_instance, owner_host, owner_pid)+ VALUES (TRUE, $1, $2, $3, $4) ON CONFLICT (singleton) DO NOTHING+ RETURNING singleton`,+ [operation, owner, hostname(), process.pid]+ )+ return result.rows.length === 1+ })+ if (!admitted) return null+ let released: Promise<void> | undefined+ return {+ release: () => {+ released ??= withTransaction(async db => {+ await db.query('SET LOCAL synchronous_commit = on')+ const result = await db.query(+ `DELETE FROM password_verification_work+ WHERE singleton AND operation_id = $1 AND owner_instance = $2 RETURNING singleton`,+ [operation, owner]+ )+ // An externally removed/replaced owner is not proof that our authority was preserved.+ if (result.rows.length !== 1) throw new Error('password work ownership lost')+ })+ return released+ },+ }+}diff --git a/control-api/src/services/auth/passwordWorkOwnershipSchema.ts b/control-api/src/services/auth/passwordWorkOwnershipSchema.tsnew file mode 100644index 000000000..c047352c0--- /dev/null+++ b/control-api/src/services/auth/passwordWorkOwnershipSchema.ts@@ -0,0 +1,16 @@+import type { DbClient } from '../../db.js'++/** Dedicated password authority. Never reclaimed by age, heartbeat or session loss. */+export async function applyPasswordWorkOwnershipSchema(db: Pick<DbClient, 'query'>): Promise<void> {+ await db.query(`+ CREATE TABLE IF NOT EXISTS password_verification_work (+ singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton),+ operation_id UUID NOT NULL,+ owner_instance UUID NOT NULL,+ owner_host TEXT NOT NULL,+ owner_pid INTEGER NOT NULL CHECK (owner_pid > 0),+ acquired_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()+ );+ GRANT SELECT, INSERT, DELETE ON password_verification_work TO control_api_runtime;+ `)+}diff --git a/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts b/control-api/test/rp001.passwordPool.realPostgres.integration.test.tsindex 5abff6756..14d579b05 100644--- a/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts+++ b/control-api/test/rp001.passwordPool.realPostgres.integration.test.ts@@ -118,13 +118,15 @@ realPg('RP-001 production credential pool ownership', () => {const connect = db!.pool.connect.bind(db!.pool)const spy = vi.spyOn(db!.pool, 'connect')let completing = false+ let afterComparisonBorrows = 0vi.spyOn(bcrypt, 'compare').mockImplementation(async (submitted: string, stored: string) => {const result = await realCompare(submitted, stored)completing = truereturn result})spy.mockImplementation((...args: unknown[]) => {- if (completing) {+ // Durable owner release now borrows first; fault the credential completion, not release.+ if (completing && ++afterComparisonBorrows === 2) {completing = falsereturn Promise.reject(new Error('synthetic completion failure'))}diff --git a/control-api/test/rp004.passwordWork.realPostgres.integration.test.ts b/control-api/test/rp004.passwordWork.realPostgres.integration.test.tsnew file mode 100644index 000000000..1fb957de1--- /dev/null+++ b/control-api/test/rp004.passwordWork.realPostgres.integration.test.ts@@ -0,0 +1,253 @@+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'+import bcrypt from 'bcryptjs'+import { type ChildProcess, spawn } from 'node:child_process'+import { randomBytes } from 'node:crypto'+import { Pool } from 'pg'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const realPg = adminUrl ? describe : describe.skip+const password = 'Synthetic-RP004-correct-password'+const email = '[email protected]'++// Both processes execute the source producer and real cost-12 bcrypt. No built artifacts required.+const childScript = `+require('ts-node').register({project:'./tsconfig.json',experimentalResolver:true,transpileOnly:true});+const bcrypt=require('bcryptjs');+const compare=bcrypt.compare.bind(bcrypt);+// Isolated fault harness keeps the owner alive after its idle pool reports session termination.+require('./src/db.ts').pool.on('error',()=>{});+bcrypt.compare=async (...args)=>{+ const work=compare(...args);+ process.send({kind:'entered'});+ if(process.env.PAUSE_COMPARISON==='1') process.kill(process.pid,'SIGSTOP');+ return work;+};+const {verifyMemberPassword}=require('./src/services/auth/passwordCredentialVerification.ts');+verifyMemberPassword(process.env.TEST_EMAIL,process.env.TEST_PASSWORD,{publicLogin:false})+ .then(user=>process.send({kind:'result',success:!!user},()=>process.exit(0)),+ error=>process.send({kind:'result',status:error.status},()=>process.exit(0)));+`++realPg('RP-1010-04 durable expensive-work ownership', () => {+ const database = `rp004_${randomBytes(6).toString('hex')}`+ let admin: Pool+ let db: typeof import('../src/db.js')+ let url: URL+ const children = new Set<ChildProcess>()+ beforeAll(async () => {+ admin = new Pool({ connectionString: adminUrl })+ await admin.query(`CREATE DATABASE "${database}"`)+ url = new URL(adminUrl!)+ url.pathname = `/${database}`+ vi.stubEnv('CONTROL_API_PG_CONNECTION_STRING', url.toString())+ vi.resetModules()+ db = await import('../src/db.js')+ await db.initDb()+ await db.pool.query('INSERT INTO users(email,password_hash) VALUES ($1,$2)', [+ email,+ await bcrypt.hash(password, 12),+ ])+ }, 60_000)+ beforeEach(async () => {+ await db.pool.query('TRUNCATE password_identifier_state, password_verification_pace')+ if (+ (await db.pool.query("SELECT to_regclass('password_verification_work') AS present")).rows[0]+ .present+ )+ await db.pool.query('DELETE FROM password_verification_work')+ })+ afterAll(async () => {+ try {+ for (const child of children) {+ const exited = new Promise<void>(resolve => child.once('exit', () => resolve()))+ child.kill('SIGCONT')+ child.kill('SIGKILL')+ await exited+ }+ if (db) {+ await endPoolAndWaitForClients(db.pool)+ await endPoolAndWaitForClients(db.rateLimitPool)+ }+ if (admin) await admin.query(`DROP DATABASE "${database}"`)+ } finally {+ await admin?.end()+ vi.unstubAllEnvs()+ }+ })++ function start(paused: boolean) {+ const name = `rp004_${randomBytes(5).toString('hex')}`+ const childUrl = new URL(url)+ childUrl.searchParams.set('application_name', name)+ const child = spawn(process.execPath, ['-e', childScript], {+ cwd: process.cwd(),+ env: {+ ...process.env,+ CONTROL_API_PG_CONNECTION_STRING: childUrl.toString(),+ TEST_EMAIL: email,+ TEST_PASSWORD: password,+ PAUSE_COMPARISON: paused ? '1' : '0',+ },+ stdio: ['ignore', 'ignore', 'pipe', 'ipc'],+ })+ children.add(child)+ const messages: Array<{ kind: string; success?: boolean; status?: number }> = []+ let notify = () => {}+ child.on('message', value => {+ messages.push(value as (typeof messages)[number])+ notify()+ })+ let stderr = ''+ child.stderr!.on('data', value => {+ stderr += String(value)+ })+ const exited = new Promise<number | null>(resolve =>+ child.once('exit', code => {+ children.delete(child)+ resolve(code)+ notify()+ })+ )+ async function wait(kind: string) {+ const deadline = Date.now() + 20_000+ while (!messages.some(value => value.kind === kind)) {+ if (!children.has(child)) throw new Error(`producer exited before ${kind}: ${stderr}`)+ if (Date.now() >= deadline) throw new Error(`producer timeout: ${kind}`)+ await new Promise<void>(resolve => {+ const timer = setTimeout(resolve, 50)+ notify = () => {+ clearTimeout(timer)+ resolve()+ }+ })+ }+ return messages.find(value => value.kind === kind)!+ }+ return { child, name, wait, exited }+ }+ it('does not admit replacement bcrypt after the paused owner loses all database sessions', async () => {+ const first = start(true)+ try {+ await first.wait('entered')+ await admin.query(+ 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1',+ [first.name]+ )+ // Natural pace expiry; do not pre-open/reset next_permit and mask the reported scenario.+ await new Promise(resolve => setTimeout(resolve, 7_600))+ const second = start(false)+ const result = await second.wait('result')+ expect(result).toMatchObject({ status: 429 })+ expect(await second.exited).toBe(0)+ } finally {+ first.child.kill('SIGCONT')+ await first.wait('result')+ expect(await first.exited).toBe(0)+ }+ }, 60_000)+ it('admits exactly one competing owner and releases idempotently', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const leases = await Promise.all(Array.from({ length: 8 }, () => acquirePasswordWork()))+ expect(leases.filter(Boolean)).toHaveLength(1)+ await Promise.all([leases.find(Boolean)!.release(), leases.find(Boolean)!.release()])+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(0)+ })+ it('never reclaims aged work through password or legacy bucket cleanup', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const lease = await acquirePasswordWork()+ await db.pool.query(+ "UPDATE password_verification_work SET acquired_at=clock_timestamp()-interval '1 day'"+ )+ const { cleanupExpiredBuckets } = await import('../src/services/rateLimiterService.js')+ await cleanupExpiredBuckets()+ expect(await acquirePasswordWork()).toBeNull()+ await lease!.release()+ })+ it('does not release a newer owner with an old operation token', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const old = await acquirePasswordWork()+ // Fault seam: no bcrypt was started; emulate an authorized recovery/replacement.+ await db.pool.query('DELETE FROM password_verification_work')+ const current = await acquirePasswordWork()+ await expect(old!.release()).rejects.toThrow('ownership lost')+ expect(await acquirePasswordWork()).toBeNull()+ await current!.release()+ })+ it('retains ownership when release persistence is unavailable', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const lease = await acquirePasswordWork()+ const fault = vi+ .spyOn(db, 'withTransaction')+ .mockRejectedValueOnce(new Error('synthetic unavailable'))+ try {+ await expect(lease!.release()).rejects.toThrow('synthetic unavailable')+ } finally {+ fault.mockRestore()+ }+ expect(await acquirePasswordWork()).toBeNull()+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(1)+ })+ it('denies busy work without bcrypt or a pace refund', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const engine = await import('../src/services/auth/passwordCredentialVerification.js')+ const lease = await acquirePasswordWork()+ const compare = vi.spyOn(bcrypt, 'compare')+ try {+ await expect(+ engine.verifyMemberPassword(email, password, { publicLogin: false })+ ).rejects.toMatchObject({ status: 429 })+ expect(compare).not.toHaveBeenCalled()+ expect(+ (+ await db.pool.query(+ 'SELECT next_permit > clock_timestamp() AS consumed FROM password_verification_pace'+ )+ ).rows[0].consumed+ ).toBe(true)+ } finally {+ compare.mockRestore()+ await lease!.release()+ }+ })+ it('keeps a crashed process reservation until verified recovery', async () => {+ const first = start(true)+ await first.wait('entered')+ first.child.kill('SIGKILL')+ await first.exited+ await new Promise(resolve => setTimeout(resolve, 7_600))+ const next = start(false)+ expect(await next.wait('result')).toMatchObject({ status: 429 })+ expect(await next.exited).toBe(0)+ expect((await db.pool.query('SELECT * FROM password_verification_work')).rows).toHaveLength(1)+ // Test owns and has observed termination of every producer; recover only that observed token.+ const row = (+ await db.pool.query('SELECT operation_id,owner_instance FROM password_verification_work')+ ).rows[0]+ await db.pool.query(+ 'DELETE FROM password_verification_work WHERE operation_id=$1 AND owner_instance=$2',+ [row.operation_id, row.owner_instance]+ )+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const recovered = await acquirePasswordWork()+ expect(recovered).not.toBeNull()+ await recovered!.release()+ }, 60_000)+ it('applies the forward ownership schema idempotently without clearing work', async () => {+ const { acquirePasswordWork } = await import('../src/services/auth/passwordWorkOwnership.js')+ const lease = await acquirePasswordWork()+ const { applyPasswordWorkOwnershipSchema } =+ await import('../src/services/auth/passwordWorkOwnershipSchema.js')+ await applyPasswordWorkOwnershipSchema(db.pool)+ await applyPasswordWorkOwnershipSchema(db.pool)+ expect(await acquirePasswordWork()).toBeNull()+ await lease!.release()+ expect(+ (+ await db.pool.query(+ "SELECT 1 FROM schema_migrations WHERE version='0128_password_work_ownership'"+ )+ ).rows+ ).toHaveLength(1)+ })+})diff --git a/docs/how-to/password-work-ownership-recovery.md b/docs/how-to/password-work-ownership-recovery.mdnew file mode 100644index 000000000..b08d12376--- /dev/null+++ b/docs/how-to/password-work-ownership-recovery.md@@ -0,0 +1,50 @@+# Recover password verification ownership++Member password login and Desktop password reverification use the persisted+`password_verification_work` singleton to bound actual bcrypt work across processes.+The slot has no expiry. Database connection loss, a missed heartbeat, capture expiry,+or a process restart does not prove that the old computation stopped.++A crashed owner or an ambiguous acquisition/release can therefore leave password+verification unavailable. Other authentication methods and credential recovery keep+their existing policies. Never clear this record from a request, cleanup job, startup+hook, or automated timeout. Do not reset the pace or identifier history to recover it.++## Roll out the first ownership-aware version++Use the existing Control API `Recreate` deployment strategy. Drain and stop every+previous password-verifier instance before routing to this version. An older binary+does not use this slot, so an overlapping rolling deployment cannot provide the new+guarantee. Apply the forward migration before starting the new instances. Do not+roll back to an older verifier while any ownership-aware verification can still run.++`Recreate` is not proof of termination on a disconnected node. If a node is unreachable,+do not rely on forced Pod deletion or an empty Kubernetes listing. Require verified+container/process termination from the node runtime, or infrastructure fencing that+prevents that node/process from resuming. If that evidence is unavailable, leave+password verification fail-closed and escalate to the infrastructure owner.++## Recover an abandoned slot++This is a privileged, separately authorized operational change, not an API endpoint.++1. Stop password-verifier traffic and prevent new verifier instances from starting.+2. With an authorized database connection, inspect `singleton`, `operation_id`,+ `owner_instance`, `owner_host`, `owner_pid` and `acquired_at` in the work record.+ Keep this diagnostic identity in restricted operational evidence, not public logs.+3. Prove that **all** previous verifier instances can no longer execute, including+ suspended processes, old replicas and disconnected nodes. Host/PID alone is not+ sufficient: PIDs and hostnames are reusable. Record the instance/container identity+ and termination or fencing evidence. Neither session termination nor elapsed time+ meets this requirement.+4. Only after that proof, delete the exact observed record with predicates for both+ its `operation_id` and `owner_instance`, in a transaction with+ `SET LOCAL synchronous_commit = on`. Require exactly one returned row. A changed+ record or missing row means the observation is stale; stop and investigate.+5. Keep the aggregate pace and identifier state intact. Resume only ownership-aware+ instances and verify ordinary password authentication and current service health.++No generic `TRUNCATE`, unconditional deletion, database-session kill, TTL reclaim or+automatic retry of bcrypt is a recovery mechanism. Database restoration/failover+must preserve acknowledged work reservations; recovering an older database snapshot+requires the same verifier termination/fencing procedure before reopening traffic.