Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
refactor(control-api): read the live CR uid after the dcr row read and CAS on cr_uid (RP-R4-LC-01)
Diego Zacarias6619778d8042
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex ace292e60..cb2c2fbf4 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.666",+ "version": "0.1.667","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.666",+ "version": "0.1.667","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex ebcadcb60..a96510aea 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.666",+ "version": "0.1.667","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/oauth/dynamicClientStore.ts b/control-api/src/oauth/dynamicClientStore.tsindex a783f4fa3..6f7212777 100644--- a/control-api/src/oauth/dynamicClientStore.ts+++ b/control-api/src/oauth/dynamicClientStore.ts@@ -37,6 +37,8 @@ function resolveOwnerKind(input: { ownerKind?: 'mcpserver' }): 'mcpserver' {/** One decrypted row of `dynamic_clients`. */export interface DynamicClientRow {+ /** Surrogate row id; part of the identity a reclaim must still find unchanged. */+ id: stringownerKind: 'mcpserver'serverNamespace: stringserverName: string@@ -59,6 +61,12 @@ export interface DynamicClientRow {crUid?: stringcreatedAt: DateupdatedAt: Date+ /**+ * Age of the row (`NOW() - updated_at`) as measured by the database at read+ * time, so the pending-TTL decision never mixes the pod clock with the DB clock+ * that stamped `updated_at`.+ */+ ageMs: number}export type UpsertDynamicClientInput = DynamicClientKey & {@@ -173,6 +181,7 @@ export async function upsertDynamicClient(}interface DynamicClientDbRow {+ id: string | numberowner_kind: 'mcpserver'server_namespace: stringserver_name: string@@ -188,12 +197,14 @@ interface DynamicClientDbRow {cr_uid: string | nullcreated_at: Dateupdated_at: Date+ age_ms: number}-const SELECT_COLUMNS = `owner_kind, server_namespace, server_name, issuer, client_id, client_mode,+const SELECT_COLUMNS = `id, owner_kind, server_namespace, server_name, issuer, client_id, client_mode,client_secret_encrypted, registration_access_token_encrypted,registration_client_uri, client_id_issued_at, client_secret_expires_at,- install_id, cr_uid, created_at, updated_at`+ install_id, cr_uid, created_at, updated_at,+ (EXTRACT(EPOCH FROM (NOW() - updated_at)) * 1000)::float8 AS age_ms`/** Read one dynamic client, decrypting its secret and registration token. */export async function getDynamicClient(@@ -211,6 +222,7 @@ export async function getDynamicClient(if (result.rows.length === 0) return nullconst row = result.rows[0] as DynamicClientDbRowreturn {+ id: String(row.id),ownerKind: row.owner_kind,serverNamespace: row.server_namespace,serverName: row.server_name,@@ -230,6 +242,7 @@ export async function getDynamicClient(crUid: row.cr_uid ?? undefined,createdAt: row.created_at,updatedAt: row.updated_at,+ ageMs: Number(row.age_ms),}}@@ -311,9 +324,13 @@ export type ExistingDynamicClientClass = 'in-use' | 'in-progress' | 'reclaimable/*** Decide who owns a conflicting `dynamic_clients` row — the one rule for "whose- * row is this?" (D4). Pure and total; the caller supplies wall-clock `now`, the- * pending TTL, and `liveCrUid` (the uid of a live McpServer with this name, when- * one exists at conflict time).+ * row is this?" (D4). Pure and total; the caller supplies the row's DB-measured+ * age, the pending TTL, and `liveCrUid` (the uid of a live McpServer with this+ * name, when one exists).+ *+ * `liveCrUid` is only meaningful if it was read AFTER the row it is classified+ * with: a CR created and bound between an earlier K8s read and the row read would+ * otherwise make a live install's row look orphaned.** A live CR owns the name unconditionally: it covers a bound-live row AND the* pre-check→INSERT TOCTOU race where the CR appeared after the pending INSERT.@@ -323,16 +340,15 @@ export type ExistingDynamicClientClass = 'in-use' | 'in-progress' | 'reclaimable* today — all reclaimable except the fresh pending one.*/export function classifyExistingDynamicClient(- row: { installId: string | null; crUid: string | null; updatedAt: Date },- ctx: { now: Date; pendingTtlMs: number; liveCrUid?: string }+ row: { installId: string | null; crUid: string | null; ageMs: number },+ ctx: { pendingTtlMs: number; liveCrUid?: string }): ExistingDynamicClientClass {if (ctx.liveCrUid !== undefined) return 'in-use'if (row.crUid != null) return 'reclaimable'if (row.installId != null) {- // In-progress iff `updated_at > now − TTL` (i.e. age strictly under the TTL);- // at exactly the TTL the row is expired and reclaimable.- const ageMs = ctx.now.getTime() - row.updatedAt.getTime()- return ageMs < ctx.pendingTtlMs ? 'in-progress' : 'reclaimable'+ // In-progress iff age is strictly under the TTL; at exactly the TTL the row is+ // expired and reclaimable.+ return row.ageMs < ctx.pendingTtlMs ? 'in-progress' : 'reclaimable'}return 'reclaimable'}@@ -380,25 +396,42 @@ export async function insertDynamicClientPending(}/**- * Take over a `reclaimable` row (orphan or legacy) for a new install, atomically.- * Opens a transaction, `SELECT … FOR UPDATE`s the row, re-runs- * {@link classifyExistingDynamicClient} on the LOCKED row (its state may have- * changed since the losing INSERT), and only if still `reclaimable` overwrites the- * credentials with the new install's `install_id` and `cr_uid = NULL`.+ * The identity of a `dynamic_clients` row as the caller read and classified it.+ * A reclaim only applies to the row in exactly this state.+ */+export type ObservedDynamicClient = Pick<DynamicClientRow, 'id' | 'installId' | 'crUid'>++/**+ * Take over a row the caller classified `reclaimable` (orphan or legacy) for a+ * new install, atomically. Does NOT classify: the caller did, from `observed` and+ * a live-CR read taken after it. This applies the takeover only if the row still+ * has the observed `id`, `install_id` AND `cr_uid` — one guard, evaluated on the+ * row-locked latest version, so:+ * - a concurrent reclaim that won first changed `install_id` → we lose;+ * - the owner binding its pending row (`cr_uid` NULL → uid) after our read →+ * we lose, and a live install's client is never overwritten or revoked;+ * - the row deleted (teardown) → 0 rows → we lose (conservative; the operator's+ * retry goes through the INSERT).+ * Re-classifying under the lock is deliberately absent: it would re-decide with a+ * K8s read that predates the locked version, which is exactly the stale-uid bug.*- * The UPDATE is a CAS on `id` + `install_id` read under the lock, so two- * concurrent reclaims cannot both win: the second serializes behind the first,- * re-reads a now-fresh pending row, and classifies it `in-progress` →- * `{ reclaimed: false }`. Returns the OLD row's RFC 7592 handle (decrypted, when- * present) so the caller can best-effort revoke the superseded client at the AS.+ * The guard does not compare `updated_at`: every production writer that moves it+ * also changes `id` (INSERT), `install_id` (reclaim) or `cr_uid` (bind), so the+ * observed age can only have grown and an expired pending row stays expired.+ * `upsertDynamicClient` refreshes `updated_at` with identity intact and would break+ * that premise; it has no production caller and must not gain one on this table.+ *+ * Returns the OLD row's RFC 7592 handle (decrypted, when present) so the caller+ * can best-effort revoke the superseded client at the AS.+ *+ * The reclaim is a single statement, so the transaction adds no atomicity; the+ * runner is kept only as the injection point.*- * No leading `db` param: all work runs inside the transaction this function owns.* `runInTransaction` is a REQUIRED injected runner (pass `withTransaction` from* db.ts) rather than a default import — this module sits on the config init path* (config → tokenHelper → callback → here), so importing db.ts's `withTransaction`* value at module top-level would close an init cycle back to `config` and leave- * it undefined. The injection also lets tests bind the runner to a disposable pool,- * matching the `runInTransaction` idiom in reactiveTokenHelper.+ * it undefined. The injection also lets tests bind the runner to a disposable pool.*/export async function reclaimOrphanDynamicClient(encryptionKey: Buffer,@@ -406,7 +439,7 @@ export async function reclaimOrphanDynamicClient(key: DynamicClientKeynewCredentials: UpsertDynamicClientInputnewInstallId: string- ctx: { now: Date; pendingTtlMs: number; liveCrUid?: string }+ observed: ObservedDynamicClient},runInTransaction: <T>(work: (tx: DbTransactionClient) => Promise<T>) => Promise<T>): Promise<{ reclaimed: boolean; oldHandle?: DynamicClientRegistrationHandle }> {@@ -414,32 +447,17 @@ export async function reclaimOrphanDynamicClient(const creds = encodeCredentialColumns(encryptionKey, input.newCredentials)return runInTransaction(async tx => {- const locked = await tx.query(- `SELECT id, install_id, cr_uid, updated_at,- registration_client_uri, registration_access_token_encrypted- FROM dynamic_clients- WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3- FOR UPDATE`,- [ownerKind, input.key.serverNamespace, input.key.serverName]- )- if (locked.rows.length === 0) return { reclaimed: false }- const row = locked.rows[0] as {- id: string- install_id: string | null- cr_uid: string | null- updated_at: Date- registration_client_uri: string | null- registration_access_token_encrypted: string | null- }-- const klass = classifyExistingDynamicClient(- { installId: row.install_id, crUid: row.cr_uid, updatedAt: row.updated_at },- input.ctx- )- if (klass !== 'reclaimable') return { reclaimed: false }-const updated = await tx.query(- `UPDATE dynamic_clients+ `WITH prev AS (+ SELECT id, registration_client_uri, registration_access_token_encrypted+ FROM dynamic_clients+ WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3+ AND id = $13+ AND install_id IS NOT DISTINCT FROM $14+ AND cr_uid IS NOT DISTINCT FROM $15+ FOR UPDATE+ )+ UPDATE dynamic_clients dSET issuer = $4,client_id = $5,client_mode = $6,@@ -451,8 +469,9 @@ export async function reclaimOrphanDynamicClient(install_id = $12,cr_uid = NULL,updated_at = NOW()- WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3- AND id = $13 AND install_id IS NOT DISTINCT FROM $14`,+ FROM prev+ WHERE d.id = prev.id+ RETURNING prev.registration_client_uri, prev.registration_access_token_encrypted`,[ownerKind,input.key.serverNamespace,@@ -466,19 +485,24 @@ export async function reclaimOrphanDynamicClient(creds.clientIdIssuedAt,creds.clientSecretExpiresAt,input.newInstallId,- row.id,- row.install_id,+ input.observed.id,+ input.observed.installId ?? null,+ input.observed.crUid ?? null,])if ((updated.rowCount ?? 0) !== 1) return { reclaimed: false }+ const prev = updated.rows[0] as {+ registration_client_uri: string | null+ registration_access_token_encrypted: string | null+ }const oldHandle =- row.registration_client_uri && row.registration_access_token_encrypted+ prev.registration_client_uri && prev.registration_access_token_encrypted? {- registrationClientUri: row.registration_client_uri,+ registrationClientUri: prev.registration_client_uri,registrationAccessToken: decryptOAuthSecret(encryptionKey,- row.registration_access_token_encrypted+ prev.registration_access_token_encrypted),}: undefineddiff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.tsindex d613baeb7..81d99900a 100644--- a/control-api/src/routes/admin/remoteMcp.ts+++ b/control-api/src/routes/admin/remoteMcp.ts@@ -808,11 +808,11 @@ export function createAdminRemoteMcpRouter(// confidential client is orphaned at the AS. Mirrors the revoke-on-abort// discipline of persistFailed503 and every explicit return below.try {- // Someone already holds the name. A live CR owns it unconditionally; else the- // persisted row's lifetime decides (in-progress = another saga; reclaimable =- // orphan/legacy). Re-read the live uid HERE (a CR may have appeared since the- // pre-check — the TOCTOU the fence closes).- const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)+ // Someone already holds the name. `existing` is always the LAST row read,+ // and the live-uid read happens only after it: an owner that creates its CR+ // and binds between the two is then seen as `in-use`, and one that binds+ // after the K8s read breaks the reclaim's cr_uid guard. Reading K8s first+ // would classify a just-bound live row as an orphan and revoke its client.let existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)if (!existing) {// The row was deleted between our INSERT-conflict and this read (a teardown@@ -838,13 +838,16 @@ export function createAdminRemoteMcpRouter(res.status(409).json({ error: 'server_name_in_use' })return}+ // A live CR owns the name unconditionally (also covers a CR that appeared+ // since the pre-check).+ const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)const klass = classifyExistingDynamicClient({installId: existing.installId ?? null,crUid: existing.crUid ?? null,- updatedAt: existing.updatedAt,+ ageMs: existing.ageMs,},- { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid }+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid })if (klass === 'in-use' || klass === 'in-progress') {await revokeMintedClient()@@ -857,19 +860,21 @@ export function createAdminRemoteMcpRouter(})return}- // reclaimable: take over the orphan/legacy row atomically (CAS in a tx).+ // reclaimable: take over the orphan/legacy row only if it is still exactly+ // the one we observed (id + install_id + cr_uid).const reclaim = await reclaimOrphanDynamicClient(encryptionKey,{key: dynamicClientKey,newCredentials: dcrCredentials,newInstallId: installId,- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid },+ observed: existing,},runInTransaction)if (!reclaim.reclaimed) {- // Raced: another saga won the reclaim (or re-claimed the freed name).+ // The row changed since we read it: another saga reclaimed it, its owner+ // bound it, or it was deleted.await revokeMintedClient()res.status(409).json({ error: 'install_in_progress' })returndiff --git a/control-api/test/fixtures/remoteOAuthDiscovery.ts b/control-api/test/fixtures/remoteOAuthDiscovery.tsindex 85cd74f09..b397c800e 100644--- a/control-api/test/fixtures/remoteOAuthDiscovery.ts+++ b/control-api/test/fixtures/remoteOAuthDiscovery.ts@@ -383,8 +383,8 @@ export function makeDcrTransport(opts: {** It models the install-identity statements the route now drives: the pending INSERT* (`ON CONFLICT DO NOTHING RETURNING id`), the legacy upsert (`ON CONFLICT DO UPDATE`),- * the bind UPDATE, the reclaim CAS UPDATE, and both DELETE shapes (by-install and- * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the FOR UPDATE + CAS+ * the bind UPDATE, the reclaim guarded UPDATE, and both DELETE shapes (by-install and+ * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the reclaim guard* race — are certified by the real-Postgres suite* (`oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts`); this* harness only lets the route-level wiring (classify → HTTP code, which 7592 DELETE@@ -502,7 +502,9 @@ export function makeInMemoryDynamicClientsDb(): {const key = keyOf(values[0], values[1], values[2])const row = rows.get(key)if (!row) return { rows: [], rowCount: 0 }- // Reclaim CAS: WHERE id = $13 AND install_id IS NOT DISTINCT FROM $14.+ // Reclaim: one guard on the observed identity — id = $13 AND install_id IS+ // NOT DISTINCT FROM $14 AND cr_uid IS NOT DISTINCT FROM $15 — RETURNING the+ // OLD row's RFC 7592 handle.if (text.includes('IS NOT DISTINCT FROM')) {const [,@@ -517,12 +519,21 @@ export function makeInMemoryDynamicClientsDb(): {client_id_issued_at,client_secret_expires_at,newInstallId,- lockedId,- lockedInstallId,+ observedId,+ observedInstallId,+ observedCrUid,] = values- if (row.id !== lockedId || row.install_id !== lockedInstallId) {+ if (+ String(row.id) !== String(observedId) ||+ (row.install_id ?? null) !== (observedInstallId ?? null) ||+ (row.cr_uid ?? null) !== (observedCrUid ?? null)+ ) {return { rows: [], rowCount: 0 }}+ const oldHandle = {+ registration_client_uri: row.registration_client_uri,+ registration_access_token_encrypted: row.registration_access_token_encrypted,+ }Object.assign(row, {issuer,client_id,@@ -536,7 +547,7 @@ export function makeInMemoryDynamicClientsDb(): {cr_uid: null,updated_at: new Date(),})- return { rows: [], rowCount: 1 }+ return { rows: [oldHandle], rowCount: 1 }}// Bind: WHERE install_id = $4 AND cr_uid IS NULL, SET cr_uid = $5.const [, , , installId, crUid] = values@@ -551,9 +562,11 @@ export function makeInMemoryDynamicClientsDb(): {const [owner_kind, server_namespace, server_name] = valuesconst row = rows.get(keyOf(owner_kind, server_namespace, server_name))// Return a SNAPSHOT (shallow copy), as a real SELECT does: a later UPDATE must- // not retroactively mutate a row already read (e.g. the reclaim reads the OLD- // handle from its FOR UPDATE snapshot AFTER issuing the overwriting UPDATE).- return { rows: row ? [{ ...row }] : [], rowCount: row ? 1 : 0 }+ // not retroactively mutate a row already read. `age_ms` mirrors the store's+ // `NOW() - updated_at` column, measured at read time.+ if (!row) return { rows: [], rowCount: 0 }+ const ageMs = Date.now() - (row.updated_at as Date).getTime()+ return { rows: [{ ...row, age_ms: ageMs }], rowCount: 1 }}return { rows: [], rowCount: 0 }},diff --git a/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts b/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.tsindex 99782e42a..9da823f32 100644--- a/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts+++ b/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts@@ -44,38 +44,36 @@ const NS = config.mcpServersNamespace// classifyExistingDynamicClient is pure — cover its exhaustive table with no DB.describe('classifyExistingDynamicClient — ownership decision table', () => {- const now = new Date('2026-01-01T00:00:00.000Z')- const fresh = new Date(now.getTime() - 1000)- const expired = new Date(now.getTime() - PENDING_TTL_MS - 1000)+ const fresh = 1000+ const expired = PENDING_TTL_MS + 1000it('a live CR owns the name regardless of the row state → in-use', () => {expect(classifyExistingDynamicClient(- { installId: null, crUid: null, updatedAt: fresh },- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }+ { installId: null, crUid: null, ageMs: fresh },+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' })).toBe('in-use')expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: 'uid-old', updatedAt: expired },- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }+ { installId: 'i1', crUid: 'uid-old', ageMs: expired },+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' })).toBe('in-use')// A live CR beats even a fresh pending row (the pre-check→INSERT TOCTOU race).expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: null, updatedAt: fresh },- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }+ { installId: 'i1', crUid: null, ageMs: fresh },+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' })).toBe('in-use')})it('at exactly the TTL boundary a pending row is reclaimable (in-progress is strictly under)', () => {- const atBoundary = new Date(now.getTime() - PENDING_TTL_MS)expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: null, updatedAt: atBoundary },- { now, pendingTtlMs: PENDING_TTL_MS }+ { installId: 'i1', crUid: null, ageMs: PENDING_TTL_MS },+ { pendingTtlMs: PENDING_TTL_MS })).toBe('reclaimable')})@@ -83,8 +81,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {it('a fresh pending row with no live CR → in-progress', () => {expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: null, updatedAt: fresh },- { now, pendingTtlMs: PENDING_TTL_MS }+ { installId: 'i1', crUid: null, ageMs: fresh },+ { pendingTtlMs: PENDING_TTL_MS })).toBe('in-progress')})@@ -92,8 +90,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {it('a bound orphan (cr_uid set, no live CR) → reclaimable', () => {expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: 'uid-gone', updatedAt: fresh },- { now, pendingTtlMs: PENDING_TTL_MS }+ { installId: 'i1', crUid: 'uid-gone', ageMs: fresh },+ { pendingTtlMs: PENDING_TTL_MS })).toBe('reclaimable')})@@ -101,8 +99,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {it('an expired pending row → reclaimable', () => {expect(classifyExistingDynamicClient(- { installId: 'i1', crUid: null, updatedAt: expired },- { now, pendingTtlMs: PENDING_TTL_MS }+ { installId: 'i1', crUid: null, ageMs: expired },+ { pendingTtlMs: PENDING_TTL_MS })).toBe('reclaimable')})@@ -110,8 +108,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {it('a legacy row (both null, no live CR) → reclaimable', () => {expect(classifyExistingDynamicClient(- { installId: null, crUid: null, updatedAt: fresh },- { now, pendingTtlMs: PENDING_TTL_MS }+ { installId: null, crUid: null, ageMs: fresh },+ { pendingTtlMs: PENDING_TTL_MS })).toBe('reclaimable')})@@ -198,38 +196,80 @@ describeRealPostgres('dynamicClientStore install identity (real Postgres)', () =expect(after?.registrationClientUri).toBe(`https://as.example.com/reg/${name}`)})- it('reclaimOrphanDynamicClient CAS: takes over a reclaimable row, returns the old handle, second reclaim loses', async () => {+ const keyOf = (serverName: string) => ({ serverNamespace: NS, serverName })++ /**+ * Age a real row past the pending TTL. Only `updated_at` moves (the DB clock the+ * store measures age against); every other column stays as the producer wrote it.+ */+ async function expirePending(serverName: string): Promise<void> {+ await db.query(+ `UPDATE dynamic_clients+ SET updated_at = NOW() - ($1::bigint * INTERVAL '1 millisecond')+ WHERE server_namespace = $2 AND server_name = $3`,+ [PENDING_TTL_MS + 60_000, NS, serverName]+ )+ }++ /** Every persisted column; `ageMs` is a read-time measurement, not row state. */+ const rowState = (row: Awaited<ReturnType<typeof getDynamicClient>>) => {+ if (!row) return row+ const { ageMs: _ageMs, ...state } = row+ return state+ }++ it('the row age is measured by the database: fresh pending → in-progress, aged → reclaimable', async () => {+ const name = `srv-age-${randomUUID().slice(0, 8)}`+ await insertDynamicClientPending(db, KEY, { ...baseCreds(name), installId: randomUUID() })++ const fresh = await getDynamicClient(db, KEY, keyOf(name))+ expect(fresh?.id).toMatch(/^\d+$/)+ expect(fresh?.ageMs).toBeGreaterThanOrEqual(0)+ expect(fresh?.ageMs).toBeLessThan(PENDING_TTL_MS)+ expect(+ classifyExistingDynamicClient(+ { installId: fresh?.installId ?? null, crUid: null, ageMs: fresh?.ageMs ?? 0 },+ { pendingTtlMs: PENDING_TTL_MS }+ )+ ).toBe('in-progress')++ await expirePending(name)+ const aged = await getDynamicClient(db, KEY, keyOf(name))+ expect(aged?.ageMs).toBeGreaterThanOrEqual(PENDING_TTL_MS)+ expect(+ classifyExistingDynamicClient(+ { installId: aged?.installId ?? null, crUid: null, ageMs: aged?.ageMs ?? 0 },+ { pendingTtlMs: PENDING_TTL_MS }+ )+ ).toBe('reclaimable')+ })++ it('reclaimOrphanDynamicClient takes over the observed orphan and returns the old handle', async () => {const name = `srv-reclaim-${randomUUID().slice(0, 8)}`const install1 = randomUUID()- const uidGone = randomUUID()// Seed a BOUND ORPHAN via the real producers: pending insert, then bind to a- // uid whose CR no longer exists (liveCrUid undefined at reclaim time).+ // uid whose CR no longer exists.await insertDynamicClientPending(db, KEY, {...baseCreds(name, { clientId: 'cid-old', clientSecret: 'secret-old' }),registrationAccessToken: 'rat-old',registrationClientUri: 'https://as.example.com/reg/old',installId: install1,})- expect(- (- await bindDynamicClientToResource(- db,- { serverNamespace: NS, serverName: name },- install1,- uidGone- )- ).bound- ).toBe(true)+ expect((await bindDynamicClientToResource(db, keyOf(name), install1, randomUUID())).bound).toBe(+ true+ )+ const observed = await getDynamicClient(db, KEY, keyOf(name))+ if (!observed) throw new Error('seeded row missing')const install2 = randomUUID()const reclaim = await reclaimOrphanDynamicClient(KEY,{- key: { serverNamespace: NS, serverName: name },+ key: keyOf(name),newCredentials: baseCreds(name, { clientId: 'cid-new', clientSecret: 'secret-new' }),newInstallId: install2,- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },+ observed,},runInTransaction)@@ -240,54 +280,131 @@ describeRealPostgres('dynamicClientStore install identity (real Postgres)', () =registrationAccessToken: 'rat-old',})- const owned = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })+ const owned = await getDynamicClient(db, KEY, keyOf(name))+ expect(owned?.id).toBe(observed.id)expect(owned?.installId).toBe(install2)expect(owned?.crUid).toBeUndefined()expect(owned?.clientId).toBe('cid-new')expect(owned?.clientSecret).toBe('secret-new')+ expect(owned?.registrationClientUri).toBe(`https://as.example.com/reg/${name}`)+ })++ // A pending row observed past its TTL (a crashed-looking saga) whose owner then+ // BINDS it to its freshly created CR before the reclaim lands. Only `cr_uid`+ // changes on bind, so the reclaim must refuse on the cr_uid guard — otherwise a+ // live install's credentials are overwritten and its client revoked at the AS.+ it('a row bound by its owner after it was observed is never reclaimed', async () => {+ const name = `srv-bindrace-${randomUUID().slice(0, 8)}`+ const ownerInstall = randomUUID()+ await insertDynamicClientPending(db, KEY, {+ ...baseCreds(name, { clientId: 'cid-owner', clientSecret: 'secret-owner' }),+ installId: ownerInstall,+ })+ await expirePending(name)++ const observed = await getDynamicClient(db, KEY, keyOf(name))+ if (!observed) throw new Error('seeded row missing')+ // No live CR yet at this point → the caller classifies it reclaimable.+ expect(+ classifyExistingDynamicClient(+ {+ installId: observed.installId ?? null,+ crUid: observed.crUid ?? null,+ ageMs: observed.ageMs,+ },+ { pendingTtlMs: PENDING_TTL_MS }+ )+ ).toBe('reclaimable')++ // The owner's CR now exists and it binds its row.+ const ownerUid = randomUUID()+ expect((await bindDynamicClientToResource(db, keyOf(name), ownerInstall, ownerUid)).bound).toBe(+ true+ )+ const bound = await getDynamicClient(db, KEY, keyOf(name))++ const reclaim = await reclaimOrphanDynamicClient(+ KEY,+ {+ key: keyOf(name),+ newCredentials: baseCreds(name, { clientId: 'cid-thief', clientSecret: 'secret-thief' }),+ newInstallId: randomUUID(),+ observed,+ },+ runInTransaction+ )+ expect(reclaim.reclaimed).toBe(false)+ expect(reclaim.oldHandle).toBeUndefined()+ const after = await getDynamicClient(db, KEY, keyOf(name))+ expect(after?.crUid).toBe(ownerUid)+ expect(after?.clientSecret).toBe('secret-owner')+ expect(rowState(after)).toEqual(rowState(bound))+ })++ // Two sagas observe the same reclaimable row; B′ reclaims first. B′'s reclaim+ // leaves cr_uid NULL (as observed) and keeps the id, so only install_id tells B+ // that the row is no longer the one it classified.+ it('the second of two reclaims of the same observed row loses on install_id', async () => {+ const name = `srv-tworeclaims-${randomUUID().slice(0, 8)}`+ await insertDynamicClientPending(db, KEY, {+ ...baseCreds(name, { clientId: 'cid-crashed' }),+ installId: randomUUID(),+ })+ await expirePending(name)+ const observed = await getDynamicClient(db, KEY, keyOf(name))+ if (!observed) throw new Error('seeded row missing')++ const winnerInstall = randomUUID()+ const first = await reclaimOrphanDynamicClient(+ KEY,+ {+ key: keyOf(name),+ newCredentials: baseCreds(name, { clientId: 'cid-winner', clientSecret: 'secret-winner' }),+ newInstallId: winnerInstall,+ observed,+ },+ runInTransaction+ )+ expect(first.reclaimed).toBe(true)+ const won = await getDynamicClient(db, KEY, keyOf(name))- // Row is now fresh-pending (install2). A second reclaim classifies it- // in-progress → loses. Models the concurrent second reclaim with a stale- // old-install expectation.- const install3 = randomUUID()const second = await reclaimOrphanDynamicClient(KEY,{- key: { serverNamespace: NS, serverName: name },- newCredentials: baseCreds(name, { clientId: 'cid-loser' }),- newInstallId: install3,- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },+ key: keyOf(name),+ newCredentials: baseCreds(name, { clientId: 'cid-loser', clientSecret: 'secret-loser' }),+ newInstallId: randomUUID(),+ observed,},runInTransaction)expect(second.reclaimed).toBe(false)- const afterLoser = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })- expect(afterLoser?.installId).toBe(install2)- expect(afterLoser?.clientId).toBe('cid-new')+ const after = await getDynamicClient(db, KEY, keyOf(name))+ expect(after?.installId).toBe(winnerInstall)+ expect(after?.clientId).toBe('cid-winner')+ expect(rowState(after)).toEqual(rowState(won))})- it('reclaimOrphanDynamicClient leaves an in-progress (fresh pending) row alone', async () => {- const name = `srv-noreclaim-${randomUUID().slice(0, 8)}`+ it('reclaimOrphanDynamicClient loses when the observed row was deleted', async () => {+ const name = `srv-reclaimgone-${randomUUID().slice(0, 8)}`const install1 = randomUUID()- await insertDynamicClientPending(db, KEY, {- ...baseCreds(name, { clientId: 'cid-inflight' }),- installId: install1,- })+ await insertDynamicClientPending(db, KEY, { ...baseCreds(name), installId: install1 })+ const observed = await getDynamicClient(db, KEY, keyOf(name))+ if (!observed) throw new Error('seeded row missing')+ await deleteDynamicClientOwnedByInstall(db, keyOf(name), install1)- const result = await reclaimOrphanDynamicClient(+ const reclaim = await reclaimOrphanDynamicClient(KEY,{- key: { serverNamespace: NS, serverName: name },- newCredentials: baseCreds(name, { clientId: 'cid-would-steal' }),+ key: keyOf(name),+ newCredentials: baseCreds(name, { clientId: 'cid-late' }),newInstallId: randomUUID(),- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },+ observed,},runInTransaction)- expect(result.reclaimed).toBe(false)- const row = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })- expect(row?.installId).toBe(install1)- expect(row?.clientId).toBe('cid-inflight')+ expect(reclaim.reclaimed).toBe(false)+ expect(await getDynamicClient(db, KEY, keyOf(name))).toBeNull()})it('bindDynamicClientToResource binds our pending row and refuses a non-matching install', async () => {diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.tsindex adc3d6be9..6bd833579 100644--- a/control-api/test/routes.adminRemoteMcp.test.ts+++ b/control-api/test/routes.adminRemoteMcp.test.ts@@ -3,7 +3,7 @@ import express from 'express'import { randomUUID } from 'node:crypto'import request from 'supertest'import { config } from '../src/config.js'-import type { DbTransactionClient } from '../src/db.js'+import type { DbClient, DbTransactionClient } from '../src/db.js'import type { PinnedTransport } from '../src/http/pinnedFetch.js'import type { K8sGateway } from '../src/k8s.js'import {@@ -12,7 +12,9 @@ import {discoverRemoteOAuth,} from '../src/oauth/discovery.js'import {+ PENDING_TTL_MS,bindDynamicClientToResource,+ claimDeleteDynamicClientForResource,getDynamicClient,insertDynamicClientPending,} from '../src/oauth/dynamicClientStore.js'@@ -1157,6 +1159,13 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {// row. Rows are seeded via the REAL producers (insertDynamicClientPending + bind),// read through the REAL getDynamicClient (T1).+ /** Every persisted column; `ageMs` is a read-time measurement, not row state. */+ const rowState = (row: Awaited<ReturnType<typeof getDynamicClient>>) => {+ if (!row) return row+ const { ageMs: _ageMs, ...state } = row+ return state+ }+const LIVE_ROW = {ownerKind: 'mcpserver' as const,serverNamespace: NS,@@ -1219,7 +1228,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {serverNamespace: NS,serverName: 'live-dcr',})- expect(after).toEqual(before)+ expect(rowState(after)).toEqual(rowState(before))expect(rows.size).toBe(1)})@@ -1281,7 +1290,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {serverNamespace: NS,serverName: 'toctou-dcr',})- expect(after).toEqual(before)+ expect(rowState(after)).toEqual(rowState(before))// The throwaway client minted before the conflict was revoked at the AS, and the// seeded live client's handle was NOT touched.const deleteCalls = calls.filter(c => c.method === 'DELETE')@@ -1382,7 +1391,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {serverNamespace: NS,serverName: 'inprogress-dcr',})- expect(after).toEqual(before)+ expect(rowState(after)).toEqual(rowState(before))expect(rows.size).toBe(1)// Our throwaway minted client was revoked at the AS.expect(calls.some(c => c.method === 'DELETE')).toBe(true)@@ -1440,4 +1449,241 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {const deleteCalls = calls.filter(c => c.method === 'DELETE')expect(deleteCalls.map(c => c.url)).toContain(LIVE_ROW.registrationClientUri)})++ // ── The live-uid read must follow the LAST row read ─────────────────────────+ // Install A of the same name finishes (CR created, row bound) while B is resolving+ // its INSERT conflict. The hook below plays A's steps with the REAL producers+ // (gateway createResource + bindDynamicClientToResource) at the exact point in B's+ // flow under test. Asserted on what is observable: A's client is never revoked at+ // the AS and A's row is byte-identical. The status is 409 either way, so it does+ // not discriminate.++ const RACE_A_ROW = {+ ...LIVE_ROW,+ clientId: 'owner-a-client-id',+ clientSecret: 'owner-a-secret',+ registrationAccessToken: 'owner-a-reg-token',+ registrationClientUri: 'https://as.example.com/register/owner-a',+ }++ /**+ * Wrap the in-memory db so hooks run around B's Nth plain row read+ * (`getDynamicClient`), counted only once armed: `before` effects are visible to+ * that read, `after` effects land once its snapshot is taken.+ */+ function withRowReadHook(+ inner: DbClient,+ plan: Record<number, { before?: () => Promise<void>; after?: () => Promise<void> }>+ ): { db: DbClient; arm: () => void } {+ let armed = false+ let reads = 0+ const db = {+ query: async (text: string, values?: unknown[]) => {+ const isRowRead =+ armed &&+ text.includes('FROM dynamic_clients') &&+ !text.includes('UPDATE') &&+ !text.includes('DELETE') &&+ !text.includes('FOR UPDATE')+ if (!isRowRead) return inner.query(text, values)+ reads += 1+ const step = plan[reads]+ await step?.before?.()+ const result = await inner.query(text, values)+ await step?.after?.()+ return result+ },+ } as unknown as DbClient+ return { db, arm: () => (armed = true) }+ }++ const rowKey = (serverName: string) => `mcpserver/${NS}/${serverName}`++ /**+ * Age a real row past the pending TTL. Only `updated_at` moves (what the store's+ * age column measures); every other column stays as the producer wrote it.+ */+ function expirePending(rows: Map<string, Record<string, unknown>>, serverName: string): void {+ const row = rows.get(rowKey(serverName))+ if (!row) throw new Error(`no row for ${serverName}`)+ row.updated_at = new Date(Date.now() - PENDING_TTL_MS - 60_000)+ }++ /**+ * A's CR appears (real gateway create) and A binds its row to that CR's uid.+ * Returns A's row as persisted right after the bind — the state that must survive.+ */+ async function ownerAFinishes(+ gw: MockGateway,+ inner: DbClient,+ serverName: string,+ installA: string+ ) {+ const cr = (await gw.createResource(+ 'mcpservers',+ { metadata: { name: serverName }, spec: {} },+ NS+ )) as { metadata: { uid: string } }+ const bound = await bindDynamicClientToResource(+ inner,+ { serverNamespace: NS, serverName },+ installA,+ cr.metadata.uid+ )+ if (!bound.bound) throw new Error('owner A failed to bind its row')+ return rowState(await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName }))+ }++ async function installB(gw: MockGateway, db: DbClient, serverName: string) {+ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, {+ db,+ dcr: { transport, resolveDns: PUBLIC_IP },+ runInTransaction: work => work(db as unknown as DbTransactionClient),+ })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName,+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })+ return { res, deleteUrls: calls.filter(c => c.method === 'DELETE').map(c => c.url) }+ }++ it("A binds its row between B's row read and B's live-CR read → A's client is never revoked, A's row untouched", async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db: inner, rows } = makeInMemoryDynamicClientsDb()+ const serverName = 'race-bind-dcr'+ const installA = randomUUID()+ await insertDynamicClientPending(inner, ENC_KEY, {+ ...RACE_A_ROW,+ serverName,+ installId: installA,+ })+ // A's saga has been slow: B reads a pending row already past the TTL.+ expirePending(rows, serverName)++ const gw = gatewayWithContext('ctx-a')+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined+ const { db, arm } = withRowReadHook(inner, {+ 1: {+ after: async () => {+ aState = await ownerAFinishes(gw, inner, serverName, installA)+ },+ },+ })+ arm()+ const { res, deleteUrls } = await installB(gw, db, serverName)++ expect(res.status).toBe(409)+ expect(aState).toBeDefined()+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)+ // B still revokes the throwaway client it minted.+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })+ expect(rowState(afterA)).toEqual(aState)+ })++ // A finishes (CR created, row bound) before B even reads the row. The row B reads+ // is already bound, so only a live-CR read taken AFTER that row read can tell it+ // is in use; with the K8s read first, B sees no CR, classifies the bound row as an+ // orphan, and its reclaim guard matches the row it just read.+ it("A binds its row before B's row read → B's live-CR read follows it, A's client is never revoked", async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db: inner, rows } = makeInMemoryDynamicClientsDb()+ const serverName = 'race-prebind-dcr'+ const installA = randomUUID()+ await insertDynamicClientPending(inner, ENC_KEY, {+ ...RACE_A_ROW,+ serverName,+ installId: installA,+ })+ expirePending(rows, serverName)++ const gw = gatewayWithContext('ctx-a')+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined+ const { db, arm } = withRowReadHook(inner, {+ 1: {+ before: async () => {+ aState = await ownerAFinishes(gw, inner, serverName, installA)+ },+ },+ })+ arm()+ const { res, deleteUrls } = await installB(gw, db, serverName)++ expect(res.status).toBe(409)+ expect(aState).toBeDefined()+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })+ expect(rowState(afterA)).toEqual(aState)+ })++ // Same race through the retry branch: B's first row read finds the row gone (a+ // teardown), A reinstalls the name before B's retried INSERT, and A creates its CR+ // and binds before B's second row read.+ it("retry branch: A reinserts and binds before B's re-read → A's client is never revoked, A's row untouched", async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db: inner } = makeInMemoryDynamicClientsDb()+ const serverName = 'race-retry-dcr'+ // The row B's INSERT first conflicts with: an orphan of an earlier install.+ const orphanInstall = randomUUID()+ await insertDynamicClientPending(inner, ENC_KEY, {+ ...LIVE_ROW,+ serverName,+ installId: orphanInstall,+ })+ await bindDynamicClientToResource(+ inner,+ { serverNamespace: NS, serverName },+ orphanInstall,+ 'uid-orphan-gone'+ )++ const gw = gatewayWithContext('ctx-a')+ const installA = randomUUID()+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined+ const { db, arm } = withRowReadHook(inner, {+ // B's first read finds the orphan torn down; A then claims the name again,+ // ahead of B's retried INSERT.+ 1: {+ before: async () => {+ await claimDeleteDynamicClientForResource(+ inner,+ ENC_KEY,+ { serverNamespace: NS, serverName },+ 'uid-orphan-gone'+ )+ },+ after: async () => {+ await insertDynamicClientPending(inner, ENC_KEY, {+ ...RACE_A_ROW,+ serverName,+ installId: installA,+ })+ },+ },+ // A creates its CR and binds before B re-reads the row.+ 2: {+ before: async () => {+ aState = await ownerAFinishes(gw, inner, serverName, installA)+ },+ },+ })+ arm()+ const { res, deleteUrls } = await installB(gw, db, serverName)++ expect(res.status).toBe(409)+ expect(aState).toBeDefined()+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })+ expect(rowState(afterA)).toEqual(aState)+ })})