Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): retain and cancel authorization work
Alfredo Lopez19093dcd1f1a
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex fab665aa2..a74a93fe0 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.704",+ "version": "0.1.705","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.704",+ "version": "0.1.705","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 04b010bb3..13c95663f 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.704",+ "version": "0.1.705","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 389f549f3..3a04e3fb8 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -1,4 +1,4 @@-import { Pool, type PoolClient } from 'pg'+import { type Client, Pool, type PoolClient } from 'pg'import {type BoundedPoolBudget,type PoolConstructor,@@ -6705,13 +6705,27 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {}}+export type DbTransactionOptions = { signal?: AbortSignal }+export async function withTransaction<T>(work: (db: DbTransactionClient) => Promise<T>,// Injectable only so the carrier's client lifecycle can be unit-tested against// a fake Pool; production always uses the module-level core pool.- txPool: Pool = pool+ txPool: Pool = pool,+ options: DbTransactionOptions = {}): Promise<T> {- const client = (await txPool.connect()) as PoolClient+ const signal = options.signal+ signal?.throwIfAborted()+ // The pool's existing finite connection timeout owns a pending borrow. An+ // abort cannot release its caller until acquisition has settled, including+ // disposal of a client delivered after the signal fired.+ let client: PoolClient & Pick<Client, 'end' | 'connection'>+ try {+ client = (await txPool.connect()) as typeof client+ } catch (error) {+ signal?.throwIfAborted()+ throw error+ }let transactionStarted = falselet commitSent = falselet releaseError: Error | boolean | undefined@@ -6732,15 +6746,56 @@ export async function withTransaction<T>(}client.on('error', onClientError)+ let termination: Promise<void> | undefined+ const onAbort = (): void => {+ if (termination) return+ releaseError = signal?.reason instanceof Error ? signal.reason : true+ termination = client.end().catch(error => {+ releaseError = error instanceof Error ? error : true+ })+ // pg Client.end() destroys an active query's socket, but an idle client+ // waits for the peer's EOF. Explicit cancellation must also end a half-open+ // idle checkout. Client.connection.stream is the pinned driver's typed+ // transport; Duplex.destroy() is idempotent and its socket close drives the+ // driver's connection end event awaited by Client.end(). Do not evict or+ // finish the caller ahead of that event.+ client.connection.stream.destroy()+ }+ signal?.addEventListener('abort', onAbort, { once: true })+ if (signal?.aborted) onAbort()+try {+ signal?.throwIfAborted()await client.query('BEGIN')transactionStarted = true+ if (signal) {+ signal.throwIfAborted()+ // PostgreSQL 16 on the service's Linux runtime supports socket polling+ // during queries. Its default (0) can leave a disconnected writer running+ // until statement_timeout; this scoped setting detects our transport stop+ // promptly. Unsupported servers fail this transaction instead of silently+ // omitting cancellation. The existing statement_timeout stays in force.+ await client.query("SET LOCAL client_connection_check_interval = '100ms'")+ }// The brand is nominal-only (a declared unique symbol); the checked-out// client IS the transaction session, so this cast is the single blessed// point where the brand is minted (issue #375 M3).- const result = await work(client as unknown as DbTransactionClient)+ signal?.throwIfAborted()+ const transaction = signal+ ? ({+ query: async (text: string, values?: unknown[]) => {+ signal.throwIfAborted()+ const result = await client.query(text, values)+ signal.throwIfAborted()+ return result+ },+ } as unknown as DbTransactionClient)+ : (client as unknown as DbTransactionClient)+ const result = await work(transaction)+ signal?.throwIfAborted()commitSent = trueawait client.query('COMMIT')+ signal?.throwIfAborted()return result} catch (error) {if (commitSent || !transactionStarted) {@@ -6752,8 +6807,15 @@ export async function withTransaction<T>(releaseError = rollbackError instanceof Error ? rollbackError : true}}+ // Once COMMIT was sent, a missing reply is an unknown durable outcome.+ // Cancellation does not establish rollback, no spend or no dispatch.+ signal?.throwIfAborted()throw error} finally {+ // Work (including its own finally) and any rollback have already unwound.+ // Keep both listeners and the checkout until physical termination completes.+ if (termination) await termination+ signal?.removeEventListener('abort', onAbort)client.removeListener('error', onClientError)// A client that emitted 'error' mid-checkout is poisoned; destroy it. Keep an// existing releaseError (the primary failure from work/commit/rollback) and@@ -6762,6 +6824,9 @@ export async function withTransaction<T>(if (asyncClientError && !releaseError) {releaseError = asyncClientError}+ if (signal?.aborted) {+ releaseError = signal.reason instanceof Error ? signal.reason : true+ }client.release(releaseError)}}diff --git a/control-api/src/k8s.ts b/control-api/src/k8s.tsindex fbda2f6f7..6d3fd6c21 100644--- a/control-api/src/k8s.ts+++ b/control-api/src/k8s.ts@@ -234,9 +234,10 @@ export class K8sGateway {async getResource(plural: ClerumResourceType,name: string,- namespace?: string+ namespace?: string,+ signal?: AbortSignal): Promise<unknown> {- return this.resources.getResource(plural, name, namespace)+ return this.resources.getResource(plural, name, namespace, signal)}async createResource(diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.tsnew file mode 100644index 000000000..986211881--- /dev/null+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts@@ -0,0 +1,13 @@+import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmission.js'++// Experimental candidate for the authorized PR #806 cgroup calibration.+// These values are not certified for deployment. Keep this candidate local+// and unpublished until the combined authorize/GFS workload selects and+// records a count, deadlines, heap policy and non-heap margin below 768 MiB.+// The policy is process-wide; every Node worker needs its own measured budget.+export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicy = Object.freeze({+ maxInFlight: 1,+ readDeadlineMs: 10_000,+ workDeadlineMs: 30_000,+ closeGraceMs: 250,+})diff --git a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.tsindex 9ff0ae9ff..79dcec580 100644--- a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts+++ b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts@@ -11,6 +11,11 @@ import {import { config } from '../../config.js'import { asyncHandler } from '../../http/asyncHandler.js'import type { K8sGateway } from '../../k8s.js'+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../../middleware/llmProviderAttemptAdmissionLimits.js'+import {+ AuthorizeBodyAdmission,+ AuthorizeWorkInterrupted,+} from '../../middleware/llmProviderAttemptBodyAdmission.js'import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'import { rootLogger } from '../../observability/logger.js'import {@@ -30,6 +35,10 @@ import { llmProviderAttemptAuthorizeRateLimits } from '../workflows/shared/rateLconst LOG_MODULE = 'mcp-host-llm-provider-attempts'const log = rootLogger.child({ module: LOG_MODULE })+// Retained bodies belong to the Node process, including when multiple apps or+// routers are constructed. A per-router owner would multiply the allowance.+const authorizeBodyAdmission = new AuthorizeBodyAdmission(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY)+// JSON.parse allocates one heap object per container, so a body under// the byte limit can exhaust the heap before either authorizer runs. The// parser scans the raw bytes first. Every bound below is derived key-by-key@@ -93,8 +102,10 @@ export type LiveBrokerAssignment = {export async function resolveHostAssignedAssignment(gateway: Pick<K8sGateway, 'getResource'>,- hostRef: string+ hostRef: string,+ signal?: AbortSignal): Promise<LiveBrokerAssignment> {+ signal?.throwIfAborted()if (hostRef.includes('/')) {const [recipeNamespace, recipeName, ...rest] = hostRef.split('/')if (!recipeNamespace || !recipeName || rest.length > 0) {@@ -107,8 +118,10 @@ export async function resolveHostAssignedAssignment(const recipe = (await gateway.getResource('workflowrecipes',recipeName,- recipeNamespace+ recipeNamespace,+ signal)) as { metadata?: { annotations?: Record<string, string> }; spec?: Record<string, unknown> }+ signal?.throwIfAborted()const spec = recipe?.spec && typeof recipe.spec === 'object' ? recipe.spec : {}const liveBrokerProviders = collectRecipeOauthBrokerProviders(spec)return {@@ -117,6 +130,7 @@ export async function resolveHostAssignedAssignment(annotations: recipe?.metadata?.annotations,}} catch (err) {+ signal?.throwIfAborted()if (err instanceof LlmProviderAttemptAuthorizeError) throw errthrow new LlmProviderAttemptAuthorizeError('host_binding_mismatch',@@ -125,9 +139,10 @@ export async function resolveHostAssignedAssignment(}}try {- const host = (await gateway.getResource('hosts', hostRef, config.hostsNamespace)) as {+ const host = (await gateway.getResource('hosts', hostRef, config.hostsNamespace, signal)) as {spec?: Record<string, unknown>}+ signal?.throwIfAborted()const spec = host?.spec && typeof host.spec === 'object' ? host.spec : {}const model = spec.modelconst connectionRef =@@ -139,6 +154,7 @@ export async function resolveHostAssignedAssignment(liveConnectionRef: readHostCodexConnectionRef(connectionRef),}} catch (err) {+ signal?.throwIfAborted()if (err instanceof LlmProviderAttemptAuthorizeError) throw errthrow new LlmProviderAttemptAuthorizeError('host_binding_mismatch',@@ -149,31 +165,44 @@ export async function resolveHostAssignedAssignment(export function createMcpHostLlmProviderAttemptRoutes(gateway: K8sGateway): Router {const router = Router()+ // Shared by both providers: preserve the larger visual envelope (Codex+ // #660, Grok #784) and each authorizer's provider-specific limit. Refuse+ // encoded bodies and scan raw structure before JSON.parse allocates objects.+ const authorizeBodyParser = express.json({+ limit: Math.max(LIMITS.maxVisualRequestBodyBytes, GROK_LIMITS.maxVisualRequestBodyBytes),+ inflate: false,+ verify: verifyBodyStructure,+ })router.post('/mcp-host/llm/provider-attempts/authorize',...llmProviderAttemptAuthorizeRateLimits(),requireMcpHostJwt,- // Shared by both providers: admit the larger visual envelope (Codex #660,- // Grok #784). Each authorizer then applies its own provider's limit.- // `inflate: false` refuses an encoded body: 35 KiB of gzip inflates to- // 35 MiB.- express.json({- limit: Math.max(LIMITS.maxVisualRequestBodyBytes, GROK_LIMITS.maxVisualRequestBodyBytes),- inflate: false,- verify: verifyBodyStructure,- }),asyncHandler(async (req: Request, res: Response) => {- const claims = req.mcpHostJwt- if (!claims) {- res.status(401).json({ error: 'Unauthorized' })- return- }try {- const result = await authorizeLlmProviderAttempt(claims, req.body, {- resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef),+ await authorizeBodyAdmission.run(req, res, authorizeBodyParser, async signal => {+ signal.throwIfAborted()+ const claims = req.mcpHostJwt+ if (!claims) {+ res.status(401).json({ error: 'Unauthorized' })+ return+ }+ const result = await authorizeLlmProviderAttempt(claims, req.body, {+ signal,+ resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef, signal),+ })+ signal.throwIfAborted()+ res.status(200).json(result)})- res.status(200).json(result)} catch (err) {+ if (err instanceof AuthorizeWorkInterrupted) {+ // A closed transport only requests cancellation. run() has awaited+ // dependency/transaction cleanup before ownership reaches this catch.+ if (req.aborted || res.destroyed || res.writableEnded) return+ const requestLog = req.log?.child({ module: LOG_MODULE }) ?? log+ requestLog.warn({ event: 'llm_provider_attempt_authorize_interrupted', code: err.code })+ res.status(503).json({ error: 'authorize_timeout' })+ return+ }sendAuthorizeError(res, err)}})diff --git a/control-api/src/services/llmProviderAttemptAuthorizer.ts b/control-api/src/services/llmProviderAttemptAuthorizer.tsindex 6a7751e7c..e2d82b4e8 100644--- a/control-api/src/services/llmProviderAttemptAuthorizer.ts+++ b/control-api/src/services/llmProviderAttemptAuthorizer.ts@@ -99,6 +99,8 @@ export type LlmProviderAttemptAuthorizeErrorCode =| 'stale_generation'| 'idempotency_conflict'| 'provider_unavailable'+ | 'authorize_timeout'+ | 'authorize_aborted'export class LlmProviderAttemptAuthorizeError extends Error {constructor(@@ -119,6 +121,7 @@ export type AuthorizeAttemptSuccess = {export type LlmProviderAttemptAuthorizerDeps = {enabled: boolean+ signal?: AbortSignaldb: DbClientwithTransaction: typeof withTransactiongetConnection: typeof getSafeCodexSubscriptionConnection@@ -127,7 +130,10 @@ export type LlmProviderAttemptAuthorizerDeps = {* Live oauth-broker targets on the Host/recipe. Required: a missing* assignment must not fall back to `request.provider` (that made D6 a no-op).*/- resolveAssignment: (hostRef: string) => Promise<{+ resolveAssignment: (+ hostRef: string,+ signal?: AbortSignal+ ) => Promise<{liveBrokerProviders: string[]liveConnectionRef: stringannotations?: Record<string, string>@@ -355,7 +361,13 @@ async function authorizeGrokProviderAttempt('requestHash does not match the canonical request')}- const assignment = await resolvedDeps.resolveAssignment(caller.hostRef)+ const assignment = await resolvedDeps+ .resolveAssignment(caller.hostRef, resolvedDeps.signal)+ .catch(error => {+ resolvedDeps.signal?.throwIfAborted()+ throw error+ })+ resolvedDeps.signal?.throwIfAborted()const liveTarget = attestLiveBrokerTarget({requestedProvider: GROK_PROVIDER,liveBrokerProviders: assignment.liveBrokerProviders,@@ -393,273 +405,278 @@ async function authorizeGrokProviderAttempt('Host has no Grok subscription assigned')}- return resolvedDeps.withTransaction(async tx => {- const db: DbClient = tx- const connection = await getSafeGrokSubscriptionConnection(db, connectionKey)- if (- !connection ||- connection.revokedAt ||- connection.status === 'revoked' ||- connection.status === 'reauth_required' ||- connection.catalogStatus === 'auth-rejected'- ) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'Grok subscription grant is missing, revoked, or requires re-authentication'- )- }- if (- connection.status !== 'connected' ||- connection.catalogStatus === 'unavailable' ||- connection.catalogStatus === 'never_synced'- ) {- throw new LlmProviderAttemptAuthorizeError(- 'connection_unavailable',- 'Grok subscription catalog is not ready'- )- }- const modelState = await getGrokCatalogModelState(db, connection.id, request.model)- if (!modelState || !modelState.enabled || modelState.stale) {- throw new LlmProviderAttemptAuthorizeError(- 'model_not_allowed',- 'model is disabled, stale, or absent from the Grok catalog'- )- }- const expectedPolicyHash = computeGrokPolicyHash({- model: request.model,- catalogRevision: connection.catalogRevision,- credentialRevision: connection.credentialRevision,- connectionKey: connection.connectionKey,- })- if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'policy revision or hash does not match the current Grok catalog'- )- }- const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)- if (attemptGeneration < maxGeneration) {- throw new LlmProviderAttemptAuthorizeError(- 'stale_generation',- 'attemptGeneration is older than the recorded invocation'- )- }- const pluginWorkloadSdkProviderAttemptId =- typeof body.pluginWorkloadSdkProviderAttemptId === 'string'- ? body.pluginWorkloadSdkProviderAttemptId.trim()- : ''- let sdkLinkRecipe: { namespace: string; name: string } | null = null- if (pluginWorkloadSdkProviderAttemptId) {- if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {+ return resolvedDeps.withTransaction(+ async tx => {+ resolvedDeps.signal?.throwIfAborted()+ const db: DbClient = tx+ const connection = await getSafeGrokSubscriptionConnection(db, connectionKey)+ if (+ !connection ||+ connection.revokedAt ||+ connection.status === 'revoked' ||+ connection.status === 'reauth_required' ||+ connection.catalogStatus === 'auth-rejected'+ ) {throw new LlmProviderAttemptAuthorizeError(- 'invalid_request',- 'pluginWorkloadSdkProviderAttemptId must be a UUID'+ 'no_grant',+ 'Grok subscription grant is missing, revoked, or requires re-authentication')}- if (caller.callerKind === 'host') {+ if (+ connection.status !== 'connected' ||+ connection.catalogStatus === 'unavailable' ||+ connection.catalogStatus === 'never_synced'+ ) {throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'host Grok chat cannot bind a Plugin Workload SDK provider attempt'+ 'connection_unavailable',+ 'Grok subscription catalog is not ready')}- if (!caller.recipeNamespace || !caller.recipeName) {+ const modelState = await getGrokCatalogModelState(db, connection.id, request.model)+ if (!modelState || !modelState.enabled || modelState.stale) {throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'+ 'model_not_allowed',+ 'model is disabled, stale, or absent from the Grok catalog')}- sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }- }- if (sdkLinkRecipe) {- await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)- }- const presentedReservationId =- typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''- let budgetReservationId = presentedReservationId || 'unbudgeted'- if (presentedReservationId) {- const active = await resolvedDeps.getActiveReservation(db, {- reservationId: presentedReservationId,- hostRef: caller.hostRef,+ const expectedPolicyHash = computeGrokPolicyHash({+ model: request.model,+ catalogRevision: connection.catalogRevision,+ credentialRevision: connection.credentialRevision,+ connectionKey: connection.connectionKey,})- if (!active) {+ if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {throw new LlmProviderAttemptAuthorizeError(- 'budget_denied',- 'budget reservation is missing or expired'+ 'no_grant',+ 'policy revision or hash does not match the current Grok catalog')}- } else {- const budget = await resolvedDeps.evaluateBudget(- {- host_ref: caller.hostRef,- context_ref: null,- team_id: null,- user_id: null,- provider: GROK_PROVIDER,- model: request.model,- llm_secret_name: null,- source_kind: 'channel',- recipe_name: caller.recipeName,- cron_job_id: null,- task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,- },- db,- { connect: async () => tx as never },- {- requiredUnit: 'tokens',- transactionClient: tx,- reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,- }- )- if (!budget.allowed) {+ const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)+ if (attemptGeneration < maxGeneration) {throw new LlmProviderAttemptAuthorizeError(- 'budget_denied',- 'token budget denied this attempt'+ 'stale_generation',+ 'attemptGeneration is older than the recorded invocation')}- budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'- }- const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''- let reservedSdkAttemptToPromote: {- id: string- invocationId: string- recipeNamespace: string- recipeName: string- attemptGeneration: number- attemptIndex: number- model: string- targetRef: string- } | null = null- if (pluginWorkloadSdkProviderAttemptId) {- const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(- pluginWorkloadSdkProviderAttemptId,- db- )- const spendExists = sdkAttempt- ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)- : false- if (- !sdkAttempt ||- spendExists ||- sdkAttempt.invocationId !== invocationId ||- sdkAttempt.attemptGeneration !== attemptGeneration ||- sdkAttempt.attemptIndex !== providerAttemptIndex ||- sdkAttempt.recipeNamespace !== caller.recipeNamespace ||- sdkAttempt.recipeName !== caller.recipeName ||- sdkAttempt.provider !== GROK_PROVIDER ||- sdkAttempt.model !== request.model ||- !sdkAttempt.targetRef.trim() ||- (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||- !['reserved', 'in_progress'].includes(sdkAttempt.status)- ) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'- )+ const pluginWorkloadSdkProviderAttemptId =+ typeof body.pluginWorkloadSdkProviderAttemptId === 'string'+ ? body.pluginWorkloadSdkProviderAttemptId.trim()+ : ''+ let sdkLinkRecipe: { namespace: string; name: string } | null = null+ if (pluginWorkloadSdkProviderAttemptId) {+ if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {+ throw new LlmProviderAttemptAuthorizeError(+ 'invalid_request',+ 'pluginWorkloadSdkProviderAttemptId must be a UUID'+ )+ }+ if (caller.callerKind === 'host') {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'host Grok chat cannot bind a Plugin Workload SDK provider attempt'+ )+ }+ if (!caller.recipeNamespace || !caller.recipeName) {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'+ )+ }+ sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }+ }+ if (sdkLinkRecipe) {+ await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)}- if (sdkAttempt.status === 'reserved') {- reservedSdkAttemptToPromote = {- id: sdkAttempt.id,- invocationId: sdkAttempt.invocationId,- recipeNamespace: sdkAttempt.recipeNamespace,- recipeName: sdkAttempt.recipeName,- attemptGeneration: sdkAttempt.attemptGeneration,- attemptIndex: sdkAttempt.attemptIndex,- model: sdkAttempt.model,- targetRef: sdkAttempt.targetRef,+ const presentedReservationId =+ typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''+ let budgetReservationId = presentedReservationId || 'unbudgeted'+ if (presentedReservationId) {+ const active = await resolvedDeps.getActiveReservation(db, {+ reservationId: presentedReservationId,+ hostRef: caller.hostRef,+ })+ if (!active) {+ throw new LlmProviderAttemptAuthorizeError(+ 'budget_denied',+ 'budget reservation is missing or expired'+ )}+ } else {+ const budget = await resolvedDeps.evaluateBudget(+ {+ host_ref: caller.hostRef,+ context_ref: null,+ team_id: null,+ user_id: null,+ provider: GROK_PROVIDER,+ model: request.model,+ llm_secret_name: null,+ source_kind: 'channel',+ recipe_name: caller.recipeName,+ cron_job_id: null,+ task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,+ },+ db,+ { connect: async () => tx as never },+ {+ requiredUnit: 'tokens',+ transactionClient: tx,+ reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,+ }+ )+ if (!budget.allowed) {+ throw new LlmProviderAttemptAuthorizeError(+ 'budget_denied',+ 'token budget denied this attempt'+ )+ }+ budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'}- }- try {- const attempt = await resolvedDeps.insertAttempt(db, {- callerKind: caller.callerKind,- hostRef: caller.hostRef,- recipeNamespace: caller.recipeNamespace,- recipeName: caller.recipeName,- invocationId,- attemptGeneration,- providerAttemptIndex,- provider: GROK_PROVIDER,- model: request.model,- requestHash,- policyRevision,- policyHash,- budgetReservationId,- connectionRevision: connection.credentialRevision,- connectionId: connection.id,- ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),- })- if (reservedSdkAttemptToPromote) {- const promoted = await promoteReservedOauthBrokerProviderAttempt(- { ...reservedSdkAttemptToPromote, provider: GROK_PROVIDER },+ const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''+ let reservedSdkAttemptToPromote: {+ id: string+ invocationId: string+ recipeNamespace: string+ recipeName: string+ attemptGeneration: number+ attemptIndex: number+ model: string+ targetRef: string+ } | null = null+ if (pluginWorkloadSdkProviderAttemptId) {+ const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(+ pluginWorkloadSdkProviderAttemptId,db)- if (!promoted) {+ const spendExists = sdkAttempt+ ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)+ : false+ if (+ !sdkAttempt ||+ spendExists ||+ sdkAttempt.invocationId !== invocationId ||+ sdkAttempt.attemptGeneration !== attemptGeneration ||+ sdkAttempt.attemptIndex !== providerAttemptIndex ||+ sdkAttempt.recipeNamespace !== caller.recipeNamespace ||+ sdkAttempt.recipeName !== caller.recipeName ||+ sdkAttempt.provider !== GROK_PROVIDER ||+ sdkAttempt.model !== request.model ||+ !sdkAttempt.targetRef.trim() ||+ (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||+ !['reserved', 'in_progress'].includes(sdkAttempt.status)+ ) {throw new LlmProviderAttemptAuthorizeError('no_grant',- 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Grok authorize'+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt')}+ if (sdkAttempt.status === 'reserved') {+ reservedSdkAttemptToPromote = {+ id: sdkAttempt.id,+ invocationId: sdkAttempt.invocationId,+ recipeNamespace: sdkAttempt.recipeNamespace,+ recipeName: sdkAttempt.recipeName,+ attemptGeneration: sdkAttempt.attemptGeneration,+ attemptIndex: sdkAttempt.attemptIndex,+ model: sdkAttempt.model,+ targetRef: sdkAttempt.targetRef,+ }+ }}- const issued = await issueRegisteredGrokExecutionTicket(db, {- sub: claims.sub,- hostRef: caller.hostRef,- recipeNamespace: caller.recipeNamespace ?? undefined,- recipeName: caller.recipeName ?? undefined,- invocationId,- attemptGeneration,- providerAttemptId: attempt.id,- providerAttemptIndex,- model: request.model,- requestHash,- policyRevision,- policyHash,- budgetReservationId,- connectionRevision: connection.credentialRevision,- connectionId: connection.id,- })- if (request.schemaVersion === 'grok-completion-request.v2') {- // Measure the exact proxy envelope while the attempt, ticket and any- // new reservation are still transactional, as the Codex path does.- const envelope = buildGrokProxyEnvelope({- executionTicket: issued.executionTicket,+ try {+ const attempt = await resolvedDeps.insertAttempt(db, {+ callerKind: caller.callerKind,+ hostRef: caller.hostRef,+ recipeNamespace: caller.recipeNamespace,+ recipeName: caller.recipeName,+ invocationId,+ attemptGeneration,+ providerAttemptIndex,+ provider: GROK_PROVIDER,+ model: request.model,requestHash,- request,+ policyRevision,+ policyHash,+ budgetReservationId,+ connectionRevision: connection.credentialRevision,+ connectionId: connection.id,+ ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),})- if (!envelope.ok) {- throw new LlmProviderAttemptAuthorizeError(- envelope.code === 'limit' && envelope.kind === 'size'- ? 'payload_too_large'- : 'invalid_request',- envelope.message+ if (reservedSdkAttemptToPromote) {+ const promoted = await promoteReservedOauthBrokerProviderAttempt(+ { ...reservedSdkAttemptToPromote, provider: GROK_PROVIDER },+ db)+ if (!promoted) {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Grok authorize'+ )+ }}- }- log.info(- {- event: 'grok_attempt_authorized',- providerAttemptId: attempt.id,+ const issued = await issueRegisteredGrokExecutionTicket(db, {+ sub: claims.sub,hostRef: caller.hostRef,+ recipeNamespace: caller.recipeNamespace ?? undefined,+ recipeName: caller.recipeName ?? undefined,+ invocationId,+ attemptGeneration,+ providerAttemptId: attempt.id,+ providerAttemptIndex,model: request.model,- },- 'authorized Grok provider attempt'- )- return {- providerAttemptId: attempt.id,- requestHash,- executionTicket: issued.executionTicket,- expiresAt: issued.expiresAt.toISOString(),- }- } catch (err) {- const code = (err as { code?: string }).code- if (code === '23505') {- throw new LlmProviderAttemptAuthorizeError(- 'idempotency_conflict',- 'an attempt with this invocation binding already exists'+ requestHash,+ policyRevision,+ policyHash,+ budgetReservationId,+ connectionRevision: connection.credentialRevision,+ connectionId: connection.id,+ })+ if (request.schemaVersion === 'grok-completion-request.v2') {+ // Measure the exact proxy envelope while the attempt, ticket and any+ // new reservation are still transactional, as the Codex path does.+ const envelope = buildGrokProxyEnvelope({+ executionTicket: issued.executionTicket,+ requestHash,+ request,+ })+ if (!envelope.ok) {+ throw new LlmProviderAttemptAuthorizeError(+ envelope.code === 'limit' && envelope.kind === 'size'+ ? 'payload_too_large'+ : 'invalid_request',+ envelope.message+ )+ }+ }+ log.info(+ {+ event: 'grok_attempt_authorized',+ providerAttemptId: attempt.id,+ hostRef: caller.hostRef,+ model: request.model,+ },+ 'authorized Grok provider attempt')+ return {+ providerAttemptId: attempt.id,+ requestHash,+ executionTicket: issued.executionTicket,+ expiresAt: issued.expiresAt.toISOString(),+ }+ } catch (err) {+ const code = (err as { code?: string }).code+ if (code === '23505') {+ throw new LlmProviderAttemptAuthorizeError(+ 'idempotency_conflict',+ 'an attempt with this invocation binding already exists'+ )+ }+ throw err}- throw err- }- })+ },+ undefined,+ { signal: resolvedDeps.signal }+ )}export async function authorizeLlmProviderAttempt(@@ -668,6 +685,7 @@ export async function authorizeLlmProviderAttempt(deps: LlmProviderAttemptAuthorizerDeps | Partial<LlmProviderAttemptAuthorizerDeps> = defaultDeps()): Promise<AuthorizeAttemptSuccess> {const resolvedDeps: LlmProviderAttemptAuthorizerDeps = { ...defaultDeps(), ...deps }+ resolvedDeps.signal?.throwIfAborted()if (!isPlainObject(body)) {throw new LlmProviderAttemptAuthorizeError('invalid_request', 'body must be an object')}@@ -772,7 +790,13 @@ export async function authorizeLlmProviderAttempt()}- const assignment = await resolvedDeps.resolveAssignment(caller.hostRef)+ const assignment = await resolvedDeps+ .resolveAssignment(caller.hostRef, resolvedDeps.signal)+ .catch(error => {+ resolvedDeps.signal?.throwIfAborted()+ throw error+ })+ resolvedDeps.signal?.throwIfAborted()const liveTarget = attestLiveBrokerTarget({requestedProvider: request.provider,liveBrokerProviders: assignment.liveBrokerProviders,@@ -806,304 +830,309 @@ export async function authorizeLlmProviderAttempt('Host has no ChatGPT subscription assigned')}- return resolvedDeps.withTransaction(async tx => {- const db: DbClient = tx- const connection = await resolvedDeps.getConnection(db, connectionKey)- // Terminal grant failures must not look like a retryable proxy outage.- // Missing, revoked, and auth-rejected rows require a new grant or- // operator re-auth — failover onto another provider is the wrong recovery.- if (- !connection ||- connection.revokedAt ||- connection.status === 'revoked' ||- connection.status === 'reauth_required' ||- connection.catalogStatus === 'auth-rejected'- ) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'Codex subscription grant is missing, revoked, or requires re-authentication'- )- }- if (- connection.status !== 'connected' ||- connection.catalogStatus === 'unavailable' ||- connection.catalogStatus === 'never_synced'- ) {- throw new LlmProviderAttemptAuthorizeError(- 'connection_unavailable',- 'Codex subscription catalog is not ready'- )- }-- const modelState = await resolvedDeps.getModelState(db, connection.id, request.model)- if (!modelState || !modelState.enabled || modelState.stale) {- throw new LlmProviderAttemptAuthorizeError(- 'model_not_allowed',- 'model is disabled, stale, or absent from the Codex catalog'- )- }-- const expectedPolicyHash = computeCodexPolicyHash({- model: request.model,- catalogRevision: connection.catalogRevision,- credentialRevision: connection.credentialRevision,- connectionKey: connection.connectionKey,- })- if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'policy revision or hash does not match the current Codex catalog'- )- }-- const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)- if (attemptGeneration < maxGeneration) {- throw new LlmProviderAttemptAuthorizeError(- 'stale_generation',- 'attemptGeneration is older than the recorded invocation'- )- }-- const pluginWorkloadSdkProviderAttemptId =- typeof body.pluginWorkloadSdkProviderAttemptId === 'string'- ? body.pluginWorkloadSdkProviderAttemptId.trim()- : ''- let sdkLinkRecipe: { namespace: string; name: string } | null = null- if (pluginWorkloadSdkProviderAttemptId) {- if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {+ return resolvedDeps.withTransaction(+ async tx => {+ resolvedDeps.signal?.throwIfAborted()+ const db: DbClient = tx+ const connection = await resolvedDeps.getConnection(db, connectionKey)+ // Terminal grant failures must not look like a retryable proxy outage.+ // Missing, revoked, and auth-rejected rows require a new grant or+ // operator re-auth — failover onto another provider is the wrong recovery.+ if (+ !connection ||+ connection.revokedAt ||+ connection.status === 'revoked' ||+ connection.status === 'reauth_required' ||+ connection.catalogStatus === 'auth-rejected'+ ) {throw new LlmProviderAttemptAuthorizeError(- 'invalid_request',- 'pluginWorkloadSdkProviderAttemptId must be a UUID'+ 'no_grant',+ 'Codex subscription grant is missing, revoked, or requires re-authentication')}- if (caller.callerKind === 'host') {+ if (+ connection.status !== 'connected' ||+ connection.catalogStatus === 'unavailable' ||+ connection.catalogStatus === 'never_synced'+ ) {throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'host Codex chat cannot bind a Plugin Workload SDK provider attempt'+ 'connection_unavailable',+ 'Codex subscription catalog is not ready')}- if (!caller.recipeNamespace || !caller.recipeName) {++ const modelState = await resolvedDeps.getModelState(db, connection.id, request.model)+ if (!modelState || !modelState.enabled || modelState.stale) {throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'+ 'model_not_allowed',+ 'model is disabled, stale, or absent from the Codex catalog')}- sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }- }- // Recipe authorize and promptBridge finalize share this advisory. Take it- // before budget reservation work so concurrent Codex + fallback cannot- // deadlock (budget rows then advisory vs advisory then spend/invocation).- //- // Only an SDK-linked authorize goes on to take plugin_workload_sdk_*- // rows FOR UPDATE, which is the order finalize follows after this advisory.- // The workflow lane touches neither, so taking it for every recipe caller- // serialized that lane per recipe for no ordering benefit.- if (sdkLinkRecipe) {- await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)- }-- const presentedReservationId =- typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''- let budgetReservationId = presentedReservationId || 'unbudgeted'- if (presentedReservationId) {- const active = await resolvedDeps.getActiveReservation(db, {- reservationId: presentedReservationId,- hostRef: caller.hostRef,+ const expectedPolicyHash = computeCodexPolicyHash({+ model: request.model,+ catalogRevision: connection.catalogRevision,+ credentialRevision: connection.credentialRevision,+ connectionKey: connection.connectionKey,})- if (!active) {+ if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {throw new LlmProviderAttemptAuthorizeError(- 'budget_denied',- 'budget reservation is missing or expired'+ 'no_grant',+ 'policy revision or hash does not match the current Codex catalog')}- } else {- const budget = await resolvedDeps.evaluateBudget(- {- host_ref: caller.hostRef,- context_ref: null,- team_id: null,- user_id: null,- provider: PROVIDER,- model: request.model,- llm_secret_name: null,- source_kind: 'channel',- recipe_name: caller.recipeName,- cron_job_id: null,- task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,- },- db,- { connect: async () => tx as never },- {- requiredUnit: 'tokens',- transactionClient: tx,- reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,- }- )- if (!budget.allowed) {++ const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)+ if (attemptGeneration < maxGeneration) {throw new LlmProviderAttemptAuthorizeError(- 'budget_denied',- budget.reason === 'cost_unit_rejected'- ? 'Codex attempts reject cost-unit budgets'- : 'token budget denied this attempt'+ 'stale_generation',+ 'attemptGeneration is older than the recorded invocation')}- budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'- }- const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''- if (presentedTargetRef !== '' && !isBoundedId(presentedTargetRef)) {- throw new LlmProviderAttemptAuthorizeError('invalid_request', 'targetRef is invalid')- }- let reservedSdkAttemptToPromote: {- id: string- invocationId: string- recipeNamespace: string- recipeName: string- attemptGeneration: number- attemptIndex: number- model: string- targetRef: string- } | null = null- if (pluginWorkloadSdkProviderAttemptId) {- const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(- pluginWorkloadSdkProviderAttemptId,- db- )- const spendExists = sdkAttempt- ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)- : false- if (- !sdkAttempt ||- spendExists ||- sdkAttempt.invocationId !== invocationId ||- sdkAttempt.attemptGeneration !== attemptGeneration ||- sdkAttempt.attemptIndex !== providerAttemptIndex ||- sdkAttempt.recipeNamespace !== caller.recipeNamespace ||- sdkAttempt.recipeName !== caller.recipeName ||- sdkAttempt.provider !== PROVIDER ||- sdkAttempt.model !== request.model ||- !sdkAttempt.targetRef.trim() ||- (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||- !['reserved', 'in_progress'].includes(sdkAttempt.status)- ) {- throw new LlmProviderAttemptAuthorizeError(- 'no_grant',- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'- )+ const pluginWorkloadSdkProviderAttemptId =+ typeof body.pluginWorkloadSdkProviderAttemptId === 'string'+ ? body.pluginWorkloadSdkProviderAttemptId.trim()+ : ''+ let sdkLinkRecipe: { namespace: string; name: string } | null = null+ if (pluginWorkloadSdkProviderAttemptId) {+ if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {+ throw new LlmProviderAttemptAuthorizeError(+ 'invalid_request',+ 'pluginWorkloadSdkProviderAttemptId must be a UUID'+ )+ }+ if (caller.callerKind === 'host') {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'host Codex chat cannot bind a Plugin Workload SDK provider attempt'+ )+ }+ if (!caller.recipeNamespace || !caller.recipeName) {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'+ )+ }+ sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }+ }++ // Recipe authorize and promptBridge finalize share this advisory. Take it+ // before budget reservation work so concurrent Codex + fallback cannot+ // deadlock (budget rows then advisory vs advisory then spend/invocation).+ //+ // Only an SDK-linked authorize goes on to take plugin_workload_sdk_*+ // rows FOR UPDATE, which is the order finalize follows after this advisory.+ // The workflow lane touches neither, so taking it for every recipe caller+ // serialized that lane per recipe for no ordering benefit.+ if (sdkLinkRecipe) {+ await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)}- if (sdkAttempt.status === 'reserved') {- reservedSdkAttemptToPromote = {- id: sdkAttempt.id,- invocationId: sdkAttempt.invocationId,- recipeNamespace: sdkAttempt.recipeNamespace,- recipeName: sdkAttempt.recipeName,- attemptGeneration: sdkAttempt.attemptGeneration,- attemptIndex: sdkAttempt.attemptIndex,- model: sdkAttempt.model,- targetRef: sdkAttempt.targetRef,++ const presentedReservationId =+ typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''+ let budgetReservationId = presentedReservationId || 'unbudgeted'+ if (presentedReservationId) {+ const active = await resolvedDeps.getActiveReservation(db, {+ reservationId: presentedReservationId,+ hostRef: caller.hostRef,+ })+ if (!active) {+ throw new LlmProviderAttemptAuthorizeError(+ 'budget_denied',+ 'budget reservation is missing or expired'+ )}+ } else {+ const budget = await resolvedDeps.evaluateBudget(+ {+ host_ref: caller.hostRef,+ context_ref: null,+ team_id: null,+ user_id: null,+ provider: PROVIDER,+ model: request.model,+ llm_secret_name: null,+ source_kind: 'channel',+ recipe_name: caller.recipeName,+ cron_job_id: null,+ task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,+ },+ db,+ { connect: async () => tx as never },+ {+ requiredUnit: 'tokens',+ transactionClient: tx,+ reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ }+ )+ if (!budget.allowed) {+ throw new LlmProviderAttemptAuthorizeError(+ 'budget_denied',+ budget.reason === 'cost_unit_rejected'+ ? 'Codex attempts reject cost-unit budgets'+ : 'token budget denied this attempt'+ )+ }+ budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'}- }- try {- const attempt = await resolvedDeps.insertAttempt(db, {- callerKind: caller.callerKind,- hostRef: caller.hostRef,- recipeNamespace: caller.recipeNamespace,- recipeName: caller.recipeName,- invocationId,- attemptGeneration,- providerAttemptIndex,- model: request.model,- requestHash,- policyRevision,- policyHash,- budgetReservationId,- connectionRevision: connection.credentialRevision,- connectionId: connection.id,- ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),- })- if (reservedSdkAttemptToPromote) {- const promoted = await promoteReservedOauthBrokerProviderAttempt(- { ...reservedSdkAttemptToPromote, provider: PROVIDER },+ const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''+ if (presentedTargetRef !== '' && !isBoundedId(presentedTargetRef)) {+ throw new LlmProviderAttemptAuthorizeError('invalid_request', 'targetRef is invalid')+ }+ let reservedSdkAttemptToPromote: {+ id: string+ invocationId: string+ recipeNamespace: string+ recipeName: string+ attemptGeneration: number+ attemptIndex: number+ model: string+ targetRef: string+ } | null = null+ if (pluginWorkloadSdkProviderAttemptId) {+ const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(+ pluginWorkloadSdkProviderAttemptId,db)- if (!promoted) {+ const spendExists = sdkAttempt+ ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)+ : false+ if (+ !sdkAttempt ||+ spendExists ||+ sdkAttempt.invocationId !== invocationId ||+ sdkAttempt.attemptGeneration !== attemptGeneration ||+ sdkAttempt.attemptIndex !== providerAttemptIndex ||+ sdkAttempt.recipeNamespace !== caller.recipeNamespace ||+ sdkAttempt.recipeName !== caller.recipeName ||+ sdkAttempt.provider !== PROVIDER ||+ sdkAttempt.model !== request.model ||+ !sdkAttempt.targetRef.trim() ||+ (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||+ !['reserved', 'in_progress'].includes(sdkAttempt.status)+ ) {throw new LlmProviderAttemptAuthorizeError('no_grant',- 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Codex authorize'+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'+ )+ }+ if (sdkAttempt.status === 'reserved') {+ reservedSdkAttemptToPromote = {+ id: sdkAttempt.id,+ invocationId: sdkAttempt.invocationId,+ recipeNamespace: sdkAttempt.recipeNamespace,+ recipeName: sdkAttempt.recipeName,+ attemptGeneration: sdkAttempt.attemptGeneration,+ attemptIndex: sdkAttempt.attemptIndex,+ model: sdkAttempt.model,+ targetRef: sdkAttempt.targetRef,+ }+ }+ }++ try {+ const attempt = await resolvedDeps.insertAttempt(db, {+ callerKind: caller.callerKind,+ hostRef: caller.hostRef,+ recipeNamespace: caller.recipeNamespace,+ recipeName: caller.recipeName,+ invocationId,+ attemptGeneration,+ providerAttemptIndex,+ model: request.model,+ requestHash,+ policyRevision,+ policyHash,+ budgetReservationId,+ connectionRevision: connection.credentialRevision,+ connectionId: connection.id,+ ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),+ })+ if (reservedSdkAttemptToPromote) {+ const promoted = await promoteReservedOauthBrokerProviderAttempt(+ { ...reservedSdkAttemptToPromote, provider: PROVIDER },+ db+ )+ if (!promoted) {+ throw new LlmProviderAttemptAuthorizeError(+ 'no_grant',+ 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Codex authorize'+ )+ }+ log.info(+ {+ event: 'sdk_oauth_attempt_promoted',+ pluginWorkloadSdkProviderAttemptId: reservedSdkAttemptToPromote.id,+ providerAttemptId: attempt.id,+ credentialJtiPresent: false,+ },+ 'promoted reserved SDK oauth attempt after Codex authorize-link')}+ const issued = await resolvedDeps.issueTicket(db, {+ sub: claims.sub,+ hostRef: caller.hostRef,+ recipeNamespace: caller.recipeNamespace ?? undefined,+ recipeName: caller.recipeName ?? undefined,+ invocationId,+ attemptGeneration,+ providerAttemptId: attempt.id,+ providerAttemptIndex,+ model: request.model,+ requestHash,+ policyRevision,+ policyHash,+ budgetReservationId,+ connectionRevision: connection.credentialRevision,+ connectionId: connection.id,+ })+ if (request.schemaVersion === 'codex-completion-request.v2') {+ // Measure the exact proxy envelope while the attempt, ticket and any+ // new reservation are still transactional. Pre-redeem failures cannot+ // use the ordinary receipt-based finalizer.+ const envelope = buildCodexProxyEnvelope({+ executionTicket: issued.executionTicket,+ requestHash,+ request,+ })+ if (!envelope.ok) {+ throw new LlmProviderAttemptAuthorizeError(+ envelope.code === 'limit' && envelope.kind === 'size'+ ? 'payload_too_large'+ : 'invalid_request',+ envelope.message+ )+ }+ }log.info({- event: 'sdk_oauth_attempt_promoted',- pluginWorkloadSdkProviderAttemptId: reservedSdkAttemptToPromote.id,+ event: 'codex_attempt_authorized',providerAttemptId: attempt.id,- credentialJtiPresent: false,+ hostRef: caller.hostRef,+ model: request.model,},- 'promoted reserved SDK oauth attempt after Codex authorize-link'+ 'authorized Codex provider attempt')- }- const issued = await resolvedDeps.issueTicket(db, {- sub: claims.sub,- hostRef: caller.hostRef,- recipeNamespace: caller.recipeNamespace ?? undefined,- recipeName: caller.recipeName ?? undefined,- invocationId,- attemptGeneration,- providerAttemptId: attempt.id,- providerAttemptIndex,- model: request.model,- requestHash,- policyRevision,- policyHash,- budgetReservationId,- connectionRevision: connection.credentialRevision,- connectionId: connection.id,- })- if (request.schemaVersion === 'codex-completion-request.v2') {- // Measure the exact proxy envelope while the attempt, ticket and any- // new reservation are still transactional. Pre-redeem failures cannot- // use the ordinary receipt-based finalizer.- const envelope = buildCodexProxyEnvelope({- executionTicket: issued.executionTicket,+ return {+ providerAttemptId: attempt.id,requestHash,- request,- })- if (!envelope.ok) {+ executionTicket: issued.executionTicket,+ expiresAt: issued.expiresAt.toISOString(),+ }+ } catch (err) {+ const code = (err as { code?: string }).code+ if (code === '23505') {throw new LlmProviderAttemptAuthorizeError(- envelope.code === 'limit' && envelope.kind === 'size'- ? 'payload_too_large'- : 'invalid_request',- envelope.message+ 'idempotency_conflict',+ 'an attempt with this invocation binding already exists')}+ throw err}- log.info(- {- event: 'codex_attempt_authorized',- providerAttemptId: attempt.id,- hostRef: caller.hostRef,- model: request.model,- },- 'authorized Codex provider attempt'- )- return {- providerAttemptId: attempt.id,- requestHash,- executionTicket: issued.executionTicket,- expiresAt: issued.expiresAt.toISOString(),- }- } catch (err) {- const code = (err as { code?: string }).code- if (code === '23505') {- throw new LlmProviderAttemptAuthorizeError(- 'idempotency_conflict',- 'an attempt with this invocation binding already exists'- )- }- throw err- }- })+ },+ undefined,+ { signal: resolvedDeps.signal }+ )}diff --git a/control-api/src/services/resourceService.ts b/control-api/src/services/resourceService.tsindex 4d29fd161..882907f1d 100644--- a/control-api/src/services/resourceService.ts+++ b/control-api/src/services/resourceService.ts@@ -137,6 +137,25 @@ export function mergeAnnotationsForReplace(return merged}+/** Preserve the kubeconfig client's server, auth and TLS middleware per call. */+function cancellationOptions(signal?: AbortSignal): k8s.ConfigurationOptions | undefined {+ signal?.throwIfAborted()+ if (!signal) return undefined+ const cancellation = k8s.createConfiguration({+ promiseMiddleware: [+ {+ pre: async context => {+ signal.throwIfAborted()+ context.setSignal(signal)+ return context+ },+ post: async response => response,+ },+ ],+ })+ return { middleware: cancellation.middleware, middlewareMergeStrategy: 'append' }+}+export class ResourceService {// Allowed namespaces derived from config at construction time.private readonly allowedNamespaces: Set<string>@@ -177,19 +196,29 @@ export class ResourceService {return Array.from(namespaces)}- async listResource(plural: ClerumResourceType, namespace?: string): Promise<unknown[]> {+ async listResource(+ plural: ClerumResourceType,+ namespace?: string,+ signal?: AbortSignal+ ): Promise<unknown[]> {+ signal?.throwIfAborted()if (namespace === '*') {const items: unknown[] = []for (const nsName of this.listNamespacesForPlural(plural)) {try {- const namespaced = (await this.customApi.listNamespacedCustomObject({- group: CLERUM_GROUP,- version: CLERUM_VERSION,- namespace: nsName,- plural,- })) as unknown as ResourceListResponse+ const namespaced = (await this.customApi.listNamespacedCustomObject(+ {+ group: CLERUM_GROUP,+ version: CLERUM_VERSION,+ namespace: nsName,+ plural,+ },+ cancellationOptions(signal)+ )) as unknown as ResourceListResponse+ signal?.throwIfAborted()items.push(...(namespaced.items || []))} catch {+ signal?.throwIfAborted()// Ignore namespaces where the resource is absent/inaccessible.}}@@ -197,30 +226,43 @@ export class ResourceService {}const resolvedNamespace = this.resolveNamespace(plural, namespace)- const res = (await this.customApi.listNamespacedCustomObject({- group: CLERUM_GROUP,- version: CLERUM_VERSION,- namespace: resolvedNamespace,- plural,- })) as unknown as ResourceListResponse+ const res = (await this.customApi.listNamespacedCustomObject(+ {+ group: CLERUM_GROUP,+ version: CLERUM_VERSION,+ namespace: resolvedNamespace,+ plural,+ },+ cancellationOptions(signal)+ )) as unknown as ResourceListResponse+ signal?.throwIfAborted()return res.items || []}async getResource(plural: ClerumResourceType,name: string,- namespace?: string+ namespace?: string,+ signal?: AbortSignal): Promise<unknown> {+ signal?.throwIfAborted()const ns = this.resolveNamespace(plural, namespace)try {- return await this.customApi.getNamespacedCustomObject({- group: CLERUM_GROUP,- version: CLERUM_VERSION,- namespace: ns,- plural,- name,- })+ const resource = await this.customApi.getNamespacedCustomObject(+ {+ group: CLERUM_GROUP,+ version: CLERUM_VERSION,+ namespace: ns,+ plural,+ name,+ },+ cancellationOptions(signal)+ )+ signal?.throwIfAborted()+ return resource} catch (err) {+ // node-fetch reports AbortError; retain the owner's typed reason instead.+ signal?.throwIfAborted()const status = extractK8sStatus(err)if (namespace && namespace.trim()) {if (status === 404) {@@ -231,7 +273,7 @@ export class ResourceService {if (status !== 404 && status !== null) {throw err}- const items = (await this.listResource(plural, '*')) as Array<{+ const items = (await this.listResource(plural, '*', signal)) as Array<{metadata?: { name?: string }}>const match = items.find(i => i.metadata?.name === name)diff --git a/control-api/test/helpers.realPostgresCancellation.test.ts b/control-api/test/helpers.realPostgresCancellation.test.tsnew file mode 100644index 000000000..a59539726--- /dev/null+++ b/control-api/test/helpers.realPostgresCancellation.test.ts@@ -0,0 +1,92 @@+import { expect, it } from 'vitest'+import { once } from 'node:events'+import { type Socket, createConnection, createServer } from 'node:net'+import { setImmediate as nextTurn } from 'node:timers/promises'+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'++async function observed(check: () => boolean): Promise<void> {+ const deadline = Date.now() + 1_000+ while (!check()) {+ if (Date.now() >= deadline) throw new Error('proxy lifecycle observation did not arrive')+ await nextTurn()+ }+}++function query(sql: string): Buffer {+ const body = Buffer.from(`${sql}\0`)+ const header = Buffer.alloc(5)+ header[0] = 'Q'.charCodeAt(0)+ header.writeInt32BE(body.length + 4, 1)+ return Buffer.concat([header, body])+}++it('forwards fragmented COMMIT bytes, discards the reply and recovers later replies', async () => {+ const sockets = new Set<Socket>()+ let received = Buffer.alloc(0)+ let repliedToCommit = false+ const backend = createServer(socket => {+ sockets.add(socket)+ socket.on('data', chunk => {+ received = Buffer.concat([received, chunk])+ if (!repliedToCommit && received.length === 8 + query('COMMIT').length) {+ repliedToCommit = true+ socket.write('commit-reply')+ } else if (+ repliedToCommit &&+ received.length === 8 + query('COMMIT').length + query('SELECT 1').length+ ) {+ socket.write('later-reply')+ }+ })+ socket.on('end', () => socket.end())+ socket.on('error', () => {})+ })+ backend.listen(0, '127.0.0.1')+ await once(backend, 'listening')+ const address = backend.address()+ if (!address || typeof address === 'string') throw new Error('missing fixture backend address')+ const proxy = await createPostgresCommitReplyBlackhole(+ ['postgresql:', `//[email protected]:${address.port}/fixture`].join('')+ )+ const target = new URL(proxy.connectionString)+ const client = createConnection({ host: target.hostname, port: Number(target.port) })+ const replies: Buffer[] = []+ client.on('data', chunk => replies.push(chunk))+ client.on('error', () => {})+ try {+ await once(client, 'connect')+ const startup = Buffer.alloc(8)+ startup.writeInt32BE(8, 0)+ startup.writeInt32BE(196608, 4)+ client.write(startup)+ await observed(() => received.length === 8)+ const commit = query('COMMIT')+ client.write(commit.subarray(0, 2))+ await observed(() => received.length === 10)+ expect(proxy.commands).toEqual([])+ client.write(commit.subarray(2))+ await proxy.commitForwarded+ await observed(() => proxy.discardedReplyBytes > 0)+ expect(received.subarray(8)).toEqual(commit)+ expect(proxy.commands).toEqual(['COMMIT'])+ expect(replies).toEqual([])+ proxy.allowReplies()+ client.write(query('SELECT 1'))+ await observed(() => Buffer.concat(replies).toString() === 'later-reply')+ } finally {+ client.destroy()+ await proxy.close()+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ backend.close(error => (error ? reject(error) : resolve()))+ )+ }+})++it('refuses an encrypted protocol prerequisite instead of pretending to observe COMMIT', async () => {+ await expect(+ createPostgresCommitReplyBlackhole(+ ['postgresql:', '//[email protected]/fixture?sslmode=require'].join('')+ )+ ).rejects.toThrow('plaintext PostgreSQL connection')+})diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.tsnew file mode 100644index 000000000..a2aa3c6cf--- /dev/null+++ b/control-api/test/helpers/realPostgresCancellation.ts@@ -0,0 +1,83 @@+import { once } from 'node:events'+import { type Socket, createConnection, createServer } from 'node:net'++/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */+export async function createPostgresCommitReplyBlackhole(connectionString: string) {+ const target = new URL(connectionString)+ const sslMode = target.searchParams.get('sslmode')+ if (sslMode && sslMode !== 'disable') {+ throw new Error('COMMIT reply observation requires the harness plaintext PostgreSQL connection')+ }+ const sockets = new Set<Socket>()+ const commands: string[] = []+ let commit!: () => void+ const commitForwarded = new Promise<void>(resolve => {+ commit = resolve+ })+ let armed = true+ let dropping = false+ let discardedReplyBytes = 0+ const server = createServer(front => {+ const host = target.hostname.replace(/^\[|\]$/g, '')+ const back = createConnection({ host, port: Number(target.port || 5432) })+ for (const socket of [front, back]) {+ sockets.add(socket)+ socket.once('close', () => sockets.delete(socket))+ }+ front.on('error', () => back.destroy())+ back.on('error', () => front.destroy())+ front.on('end', () => back.end())+ back.on('end', () => front.end())+ let startup = true+ let buffered = Buffer.alloc(0)+ front.on('data', chunk => {+ buffered = Buffer.concat([buffered, chunk])+ if (startup && buffered.length >= 4 && buffered.length >= buffered.readInt32BE(0)) {+ buffered = buffered.subarray(buffered.readInt32BE(0))+ startup = false+ }+ while (!startup && buffered.length >= 5 && buffered.length >= buffered.readInt32BE(1) + 1) {+ const length = buffered.readInt32BE(1)+ if (buffered[0] === 'Q'.charCodeAt(0)) {+ const sql = buffered.toString('utf8', 5, length)+ if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') commands.push(sql)+ if (sql === 'COMMIT' && armed) {+ armed = false+ dropping = true+ commit()+ }+ }+ buffered = buffered.subarray(length + 1)+ }+ back.write(chunk)+ })+ back.on('data', chunk => {+ if (dropping) discardedReplyBytes += chunk.length+ else front.write(chunk)+ })+ })+ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('missing COMMIT observer address')+ const proxied = new URL(connectionString)+ proxied.hostname = '127.0.0.1'+ proxied.port = String(address.port)+ return {+ connectionString: proxied.toString(),+ commitForwarded,+ commands,+ get discardedReplyBytes() {+ return discardedReplyBytes+ },+ allowReplies: () => {+ dropping = false+ },+ close: async () => {+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ },+ }+}diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.tsnew file mode 100644index 000000000..acc57ea8c--- /dev/null+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts@@ -0,0 +1,248 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { once } from 'node:events'+import { setImmediate as nextTurn } from 'node:timers/promises'+import { Pool } from 'pg'+import { type DbClient, initDb, withTransaction } from '../src/db.js'+import { reserveInDangerZone } from '../src/services/budgets/reservations.js'+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+if (process.env.CONTROL_API_REAL_PG_REQUIRED === '1' && !adminUrl) {+ throw new Error('physical cancellation tests require CONTROL_API_REAL_PG_ADMIN_URL')+}+const realPostgres = adminUrl ? describe : describe.skip++function deferred<T = void>() {+ let resolve!: (value: T) => void+ const promise = new Promise<T>(res => {+ resolve = res+ })+ return { promise, resolve }+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++async function observe(check: () => Promise<boolean>, timeoutMs = 2_000): Promise<void> {+ const deadline = Date.now() + timeoutMs+ while (!(await check())) {+ if (Date.now() >= deadline) throw new Error('physical PostgreSQL cleanup did not complete')+ await nextTurn()+ }+}++// Mechanism proof uses real schema/helpers but bypasses authorize and signing.+// Full authorization cancellation lives in services.llmProviderAttemptAuthorization.+realPostgres('scoped transaction cancellation mechanism and durable outcome', () => {+ const database = `authorize_cancel_${randomBytes(6).toString('hex')}`+ let admin: Pool+ let observer: Pool+ let pool: Pool+ let connectionString: string+ let budgetId: string++ beforeAll(async () => {+ admin = new Pool({+ connectionString: adminUrl,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 15_000,+ })+ await admin.query(`CREATE DATABASE ${quoteIdent(database)}`)+ const url = new URL(adminUrl!)+ url.pathname = `/${database}`+ connectionString = url.toString()+ pool = new Pool({+ connectionString,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 15_000,+ idleTimeoutMillis: 1_000,+ })+ observer = new Pool({+ connectionString,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 2_000,+ idleTimeoutMillis: 1_000,+ })+ const runtime = await pool.query<{ version: string; version_num: string }>(+ "SELECT version(), current_setting('server_version_num') AS version_num"+ )+ expect(Number(runtime.rows[0].version_num)).toBeGreaterThanOrEqual(160000)+ expect(Number(runtime.rows[0].version_num)).toBeLessThan(170000)+ expect(runtime.rows[0].version).toMatch(/linux|darwin|bsd|illumos/i)+ await initDb({ connect: () => pool.connect() })+ const budget = await pool.query<{ id: string }>(+ `INSERT INTO token_budgets (name, unit, limit_amount, period, max_task_amount)+ VALUES ('cancellation fixture', 'tokens', 100, 'daily', 4) RETURNING id`+ )+ budgetId = budget.rows[0].id+ }, 60_000)++ afterAll(async () => {+ await endPoolAndWaitForClients(pool)+ await endPoolAndWaitForClients(observer)+ if (admin) {+ await admin.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await endPoolAndWaitForClients(admin)+ }+ })++ async function writeAuthorizeRows(db: DbClient, invocationId: string) {+ const reservation = await reserveInDangerZone(+ {+ budgetId,+ limit: 100,+ spent: 95,+ minStart: 1,+ estAmount: 4,+ taskRef: invocationId,+ hostRef: 'cancel-host',+ },+ undefined,+ db+ )+ expect(reservation.decision).toBe('allow')+ if (reservation.decision !== 'allow') throw new Error('fixture reservation denied')+ const attempt = await insertLlmProviderAttempt(db, {+ callerKind: 'host',+ hostRef: 'cancel-host',+ recipeNamespace: null,+ recipeName: null,+ invocationId,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ model: 'gpt-5.1',+ requestHash: 'a'.repeat(64),+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ budgetReservationId: reservation.reservationId,+ connectionRevision: 1,+ connectionId: null,+ })+ await db.query(+ `INSERT INTO llm_provider_attempt_tickets (jti, provider_attempt_id, status, expires_at)+ VALUES ($1, $2, 'issued', NOW() + INTERVAL '1 minute')`,+ [randomUUID(), attempt.id]+ )+ }++ async function rowCounts(invocationId: string) {+ const result = await observer.query<{+ attempts: number+ tickets: number+ reservations: number+ }>(+ `SELECT+ (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,+ (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a+ ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,+ (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE task_ref = $1) AS reservations`,+ [invocationId]+ )+ return result.rows[0]+ }++ async function backendGone(pid: number) {+ await observe(async () => {+ const result = await observer.query<{ count: number }>(+ 'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1 AND datname = $2',+ [pid, database]+ )+ return result.rows[0].count === 0+ })+ }++ async function lockAvailable(lock: number) {+ await withTransaction(async db => {+ const result = await db.query('SELECT pg_try_advisory_xact_lock($1) AS available', [lock])+ expect((result.rows[0] as { available: boolean }).available).toBe(true)+ }, observer)+ }++ it.each(['active', 'idle'] as const)(+ 'ends the %s backend, releases writer locks and rolls attempt/ticket/reservation back before COMMIT',+ async mode => {+ const invocationId = randomUUID()+ const lock = mode === 'active' ? 806301 : 806302+ const entered = deferred<number>()+ const controller = new AbortController()+ const reason = Object.assign(new Error('authorize_aborted'), { code: 'authorize_aborted' })+ const outcome = withTransaction(+ async db => {+ const identity = await db.query('SELECT pg_backend_pid() AS pid')+ await writeAuthorizeRows(db, invocationId)+ await db.query('SELECT pg_advisory_xact_lock($1)', [lock])+ entered.resolve((identity.rows[0] as { pid: number }).pid)+ if (mode === 'active') await db.query('SELECT pg_sleep(30)')+ else await once(controller.signal, 'abort')+ },+ pool,+ { signal: controller.signal }+ ).then(+ value => ({ value }),+ error => ({ error })+ )+ const pid = await entered.promise+ await observe(async () => {+ const result = await observer.query<{ state: string; query: string }>(+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1',+ [pid]+ )+ return mode === 'active'+ ? result.rows[0]?.state === 'active' && result.rows[0]?.query === 'SELECT pg_sleep(30)'+ : result.rows[0]?.state === 'idle in transaction'+ })+ controller.abort(reason)+ expect(((await outcome) as { error: unknown }).error).toBe(reason)+ expect(pool.totalCount).toBe(0)+ await backendGone(pid)+ await lockAvailable(lock)+ expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })++ // The same physical writes still commit normally after cancellation.+ const later = randomUUID()+ await withTransaction(db => writeAuthorizeRows(db, later), pool)+ expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })+ await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])+ },+ 10_000+ )++ it('retains a committed attempt/reservation when a real COMMIT reply is lost; cancellation is not rollback proof', async () => {+ const proxy = await createPostgresCommitReplyBlackhole(connectionString)+ const uncertainPool = new Pool({+ connectionString: proxy.connectionString,+ ssl: false,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 15_000,+ })+ const invocationId = randomUUID()+ const controller = new AbortController()+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })+ try {+ const outcome = withTransaction(db => writeAuthorizeRows(db, invocationId), uncertainPool, {+ signal: controller.signal,+ }).then(+ value => ({ value }),+ error => ({ error })+ )+ await proxy.commitForwarded+ await observe(async () => (await rowCounts(invocationId)).attempts === 1)+ controller.abort(reason)+ expect(((await outcome) as { error: unknown }).error).toBe(reason)+ expect(await rowCounts(invocationId)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })+ expect(uncertainPool.totalCount).toBe(0)+ } finally {+ controller.abort(reason)+ proxy.allowReplies()+ await endPoolAndWaitForClients(uncertainPool)+ await proxy.close()+ }+ }, 10_000)+})diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.tsnew file mode 100644index 000000000..55b72e154--- /dev/null+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts@@ -0,0 +1,595 @@+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'+import * as k8s from '@kubernetes/client-node'+import { execFileSync } from 'node:child_process'+import { randomUUID } from 'node:crypto'+import { once } from 'node:events'+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'+import { createServer as createHttpsServer } from 'node:https'+import { type Socket, createServer as createTcpServer } from 'node:net'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import type { Duplex } from 'node:stream'+import { setImmediate as nextTurn } from 'node:timers/promises'+import { Pool, type PoolClient } from 'pg'+import { config } from '../src/config.js'+import { withTransaction } from '../src/db.js'+import { K8sGateway } from '../src/k8s.js'+import {+ LlmProviderAttemptAuthorizeError,+ authorizeLlmProviderAttempt,+} from '../src/services/llmProviderAttemptAuthorizer.js'+import { ResourceService } from '../src/services/resourceService.js'+import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'++function deferred<T = void>() {+ let resolve!: (value: T) => void+ const promise = new Promise<T>(res => {+ resolve = res+ })+ return { promise, resolve }+}++async function within<T>(promise: Promise<T>, milliseconds = 1_000): Promise<T> {+ let timer: NodeJS.Timeout | undefined+ try {+ return await Promise.race([+ promise,+ new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error('dependency did not terminate')), milliseconds)+ }),+ ])+ } finally {+ clearTimeout(timer)+ }+}++async function observed(check: () => boolean): Promise<void> {+ const deadline = Date.now() + 1_000+ while (!check()) {+ if (Date.now() >= deadline) throw new Error('lifecycle observation did not arrive')+ await nextTurn()+ }+}++function interruption(code: 'authorize_timeout' | 'authorize_aborted') {+ return Object.assign(new Error(code), { code })+}++const cleanup: Array<() => Promise<void>> = []+afterEach(async () => {+ vi.restoreAllMocks()+ await Promise.all(cleanup.splice(0).map(close => close()))+})++describe('authorizer Kubernetes cancellation on the configured native client', () => {+ let directory: string+ let certificate: Buffer+ let key: Buffer+ beforeAll(() => {+ // Generate disposable TLS material locally. No application credentials or+ // checked-in private key are needed to prove the configured CA/auth path.+ directory = mkdtempSync(join(tmpdir(), 'authorize-cancel-tls-'))+ execFileSync(+ 'openssl',+ [+ 'req',+ '-x509',+ '-newkey',+ 'rsa:2048',+ '-nodes',+ '-days',+ '1',+ '-subj',+ '/CN=127.0.0.1',+ '-addext',+ 'subjectAltName=IP:127.0.0.1',+ '-keyout',+ join(directory, 'fixture-key.pem'),+ '-out',+ join(directory, 'fixture-cert.pem'),+ ],+ { stdio: 'ignore', timeout: 10_000 }+ )+ certificate = readFileSync(join(directory, 'fixture-cert.pem'))+ key = readFileSync(join(directory, 'fixture-key.pem'))+ })+ afterAll(() => rmSync(directory, { recursive: true, force: true }))++ async function peer(mode: 'no-headers' | 'no-eof') {+ const arrived = deferred()+ const disconnected = deferred()+ const sockets = new Set<Duplex>()+ const requests: Array<{ authorization?: string; url?: string }> = []+ let complete = false+ const server = createHttpsServer({ cert: certificate, key }, (req, res) => {+ requests.push({ authorization: req.headers.authorization, url: req.url })+ req.socket.once('close', () => disconnected.resolve())+ arrived.resolve()+ if (complete) {+ res.setHeader('Content-Type', 'application/json')+ res.end(JSON.stringify({ metadata: { name: 'cancel-host' } }))+ } else if (mode === 'no-eof') {+ res.writeHead(200, { 'Content-Type': 'application/json' })+ res.write('{"metadata":')+ }+ })+ server.on('connection', socket => {+ sockets.add(socket)+ socket.once('close', () => sockets.delete(socket))+ })+ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('missing TLS address')+ const bearer = randomUUID()+ const options: Parameters<k8s.KubeConfig['loadFromOptions']>[0] = {+ clusters: [+ {+ name: 'fixture',+ server: `https://127.0.0.1:${address.port}`,+ caData: certificate.toString('base64'),+ skipTLSVerify: false,+ },+ ],+ users: [{ name: 'fixture', token: bearer }],+ contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],+ currentContext: 'fixture',+ }+ const kubeconfig = new k8s.KubeConfig()+ kubeconfig.loadFromOptions(options)+ const resources = new ResourceService(+ kubeconfig.makeApiClient(k8s.CustomObjectsApi),+ 'default',+ { hosts: 'default' }+ )+ cleanup.push(async () => {+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ })+ return {+ arrived,+ disconnected,+ requests,+ bearer,+ resources,+ options,+ recover: () => {+ complete = true+ },+ }+ }++ it.each(['no-headers', 'no-eof'] as const)(+ 'terminates %s with the configured CA, bearer and server intact, then serves a later lookup',+ async mode => {+ const fixture = await peer(mode)+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ const outcome = fixture.resources+ .getResource('hosts', 'cancel-host', 'default', controller.signal)+ .then(+ value => ({ value }),+ error => ({ error })+ )+ await fixture.arrived.promise+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ await within(fixture.disconnected.promise)+ expect(fixture.requests).toHaveLength(1)+ expect(fixture.requests[0].authorization).toBe(`Bearer ${fixture.bearer}`)+ expect(fixture.requests[0].url).toContain('/namespaces/default/hosts/cancel-host')+ fixture.recover()+ await expect(+ fixture.resources.getResource('hosts', 'cancel-host', 'default')+ ).resolves.toMatchObject({ metadata: { name: 'cancel-host' } })+ expect(fixture.requests).toHaveLength(2)+ }+ )++ it('does not enter the cross-namespace fallback after cancellation without an explicit namespace', async () => {+ const fixture = await peer('no-headers')+ const controller = new AbortController()+ const reason = interruption('authorize_aborted')+ const outcome = fixture.resources+ .getResource('hosts', 'cancel-host', undefined, controller.signal)+ .then(+ value => ({ value }),+ error => ({ error })+ )+ await fixture.arrived.promise+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(fixture.requests).toHaveLength(1)+ })++ it('forwards the optional signal through K8sGateway without reconfiguring its client', async () => {+ const fixture = await peer('no-headers')+ vi.spyOn(k8s.KubeConfig.prototype, 'loadFromDefault').mockImplementation(function (+ this: k8s.KubeConfig+ ) {+ this.loadFromOptions(fixture.options)+ })+ const gateway = new K8sGateway('default')+ const controller = new AbortController()+ const reason = interruption('authorize_aborted')+ const outcome = gateway+ .getResource('hosts', 'cancel-host', config.hostsNamespace, controller.signal)+ .then(+ value => ({ value }),+ error => ({ error })+ )+ await fixture.arrived.promise+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(fixture.requests[0].authorization).toBe(`Bearer ${fixture.bearer}`)+ })++ it.each(['codex-subscription', 'grok-subscription'] as const)(+ 'propagates %s cancellation into assignment and never enters a transaction',+ async provider => {+ const fixture = await peer('no-headers')+ const previous = config.grokSubscriptionEnabled+ config.grokSubscriptionEnabled = true+ try {+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ const transaction = vi.fn()+ const caller: McpHostAccessClaims = {+ sub: 'default/cancel-host',+ hostRefs: ['cancel-host'],+ recipeNamespace: 'default',+ recipeName: 'cancel-host',+ scope: 'workflow:approval:request',+ mcpCapabilities: [],+ workflowControlScopes: ['llm:codex:execute', 'llm:grok:execute'],+ iss: 'control-api',+ aud: 'workflow-approvals',+ jti: randomUUID(),+ exp: Math.floor(Date.now() / 1_000) + 60,+ }+ const outcome = authorizeLlmProviderAttempt(+ caller,+ {+ invocationId: randomUUID(),+ attemptGeneration: 1,+ policyRevision: 1,+ policyHash: 'a'.repeat(64),+ request: {+ schemaVersion:+ provider === 'codex-subscription'+ ? 'codex-completion-request.v1'+ : 'grok-completion-request.v1',+ requestId: randomUUID(),+ idempotencyKey: randomUUID(),+ provider,+ model: provider === 'codex-subscription' ? 'gpt-5.1' : 'grok-4',+ messages: [{ role: 'user', content: 'cancel this lookup' }],+ },+ },+ {+ enabled: true,+ signal: controller.signal,+ withTransaction: transaction,+ resolveAssignment: async (_hostRef, signal) => {+ try {+ await fixture.resources.getResource('hosts', 'cancel-host', 'default', signal)+ } catch {+ // Existing assignment adapters translate lookup errors. The+ // authorizer must recover the typed cancellation through them.+ throw new LlmProviderAttemptAuthorizeError(+ 'host_binding_mismatch',+ 'assignment lookup failed'+ )+ }+ return { liveBrokerProviders: [provider], liveConnectionRef: 'connection-fixture' }+ },+ }+ ).then(+ value => ({ value }),+ error => ({ error })+ )+ await fixture.arrived.promise+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ await within(fixture.disconnected.promise)+ expect(transaction).not.toHaveBeenCalled()+ } finally {+ config.grokSubscriptionEnabled = previous+ }+ }+ )+})++function protocolMessage(type: string, body: Buffer): Buffer {+ const header = Buffer.alloc(5)+ header.write(type, 0)+ header.writeInt32BE(body.length + 4, 1)+ return Buffer.concat([header, body])+}++async function postgresPeer(handshake = true) {+ const connected = deferred()+ const queries: string[] = []+ const sockets = new Set<Socket>()+ let stalledSql = ''+ let failedSql = ''+ const server = createTcpServer({ allowHalfOpen: true }, socket => {+ sockets.add(socket)+ socket.on('error', () => {})+ socket.once('close', () => sockets.delete(socket))+ connected.resolve()+ let startup = true+ let buffered = Buffer.alloc(0)+ socket.on('data', chunk => {+ buffered = Buffer.concat([buffered, chunk])+ if (startup) {+ if (buffered.length < 4 || buffered.length < buffered.readInt32BE(0)) return+ buffered = buffered.subarray(buffered.readInt32BE(0))+ startup = false+ if (handshake)+ socket.write(+ Buffer.concat([+ protocolMessage('R', Buffer.alloc(4)),+ protocolMessage('Z', Buffer.from('I')),+ ])+ )+ }+ while (buffered.length >= 5 && buffered.length >= buffered.readInt32BE(1) + 1) {+ const type = buffered.toString('utf8', 0, 1)+ const length = buffered.readInt32BE(1)+ const body = buffered.subarray(5, length + 1)+ buffered = buffered.subarray(length + 1)+ if (type !== 'Q') continue+ const sql = body.toString('utf8', 0, body.length - 1)+ queries.push(sql)+ if (sql === failedSql) {+ socket.write(+ Buffer.concat([+ protocolMessage(+ 'E',+ Buffer.from('SERROR\0C42704\0Munsupported connection check parameter\0\0')+ ),+ protocolMessage('Z', Buffer.from('E')),+ ])+ )+ continue+ }+ if (sql === stalledSql) continue+ socket.write(+ Buffer.concat([+ protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)),+ protocolMessage('Z', Buffer.from(sql === 'BEGIN' ? 'T' : 'I')),+ ])+ )+ }+ })+ // This deliberate half-open peer never sends EOF in response to Terminate.+ // pg's idle Client.end() alone cannot finish; the scoped transport stop must.+ })+ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('missing PG address')+ const pool = new Pool({+ host: '127.0.0.1',+ port: address.port,+ user: 'cancel_fixture',+ database: 'cancel_fixture',+ max: 1,+ connectionTimeoutMillis: 150,+ idleTimeoutMillis: 1_000,+ })+ pool.on('error', () => {})+ cleanup.push(async () => {+ for (const socket of sockets) socket.destroy()+ await pool.end()+ await new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ })+ return {+ connected,+ queries,+ pool,+ stall: (sql: string) => {+ stalledSql = sql+ },+ fail: (sql: string) => {+ failedSql = sql+ },+ }+}++describe('scoped transaction cancellation with the native pg driver and sockets', () => {+ it('never borrows a client for an already aborted request', async () => {+ const fixture = await postgresPeer()+ const borrow = vi.spyOn(fixture.pool, 'connect')+ const controller = new AbortController()+ const reason = interruption('authorize_aborted')+ controller.abort(reason)+ await expect(+ withTransaction(vi.fn(), fixture.pool, { signal: controller.signal })+ ).rejects.toBe(reason)+ expect(borrow).not.toHaveBeenCalled()+ expect(fixture.queries).toEqual([])+ })++ it('fails without running work if the server does not support scoped connection polling', async () => {+ const fixture = await postgresPeer()+ fixture.fail("SET LOCAL client_connection_check_interval = '100ms'")+ const work = vi.fn()+ await expect(+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })+ ).rejects.toMatchObject({ code: '42704' })+ expect(work).not.toHaveBeenCalled()+ expect(fixture.queries).toEqual([+ 'BEGIN',+ "SET LOCAL client_connection_check_interval = '100ms'",+ 'ROLLBACK',+ ])+ })++ it('terminates an in-flight rollback before final release and preserves the typed owner reason', async () => {+ const fixture = await postgresPeer()+ fixture.stall('ROLLBACK')+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ const outcome = withTransaction(+ async () => {+ throw new Error('work failed')+ },+ fixture.pool,+ { signal: controller.signal }+ ).then(+ value => ({ value }),+ error => ({ error })+ )+ await observed(() => fixture.queries.includes('ROLLBACK'))+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]+ const release = vi.spyOn(borrowed, 'release')+ expect(release).not.toHaveBeenCalled()+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(release).toHaveBeenCalledOnce()+ expect(release).toHaveBeenCalledWith(reason)+ expect(fixture.pool.totalCount).toBe(0)+ expect(fixture.queries).not.toContain('COMMIT')+ })++ it('terminates an active native query and evicts the client exactly once before recovery', async () => {+ const fixture = await postgresPeer()+ fixture.stall('SELECT stalled')+ const removed = vi.fn()+ fixture.pool.on('remove', removed)+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ const outcome = withTransaction(async db => db.query('SELECT stalled'), fixture.pool, {+ signal: controller.signal,+ }).then(+ value => ({ value }),+ error => ({ error })+ )+ await observed(() => fixture.queries.includes('SELECT stalled'))+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(fixture.pool.totalCount).toBe(0)+ expect(removed).toHaveBeenCalledOnce()+ expect(fixture.queries).not.toContain('COMMIT')+ await expect(withTransaction(async () => 'later', fixture.pool)).resolves.toBe('later')+ expect(fixture.pool.idleCount).toBe(1)+ })++ it('terminates an idle half-open client but remains owned through callback cleanup', async () => {+ const fixture = await postgresPeer()+ const entered = deferred()+ const callbackCleanup = deferred()+ const controller = new AbortController()+ const reason = interruption('authorize_aborted')+ let settled = false+ const outcome = withTransaction(+ async () => {+ entered.resolve()+ await once(controller.signal, 'abort')+ // This barrier tests ordering only. Physical cancellation above and the+ // native half-open client end event below are independent of its release.+ await callbackCleanup.promise+ return 'late callback'+ },+ fixture.pool,+ { signal: controller.signal }+ )+ .then(+ value => ({ value }),+ error => ({ error })+ )+ .finally(() => {+ settled = true+ })+ await entered.promise+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]+ const clientEnded = once(borrowed, 'end')+ const release = vi.spyOn(borrowed, 'release')+ controller.abort(reason)+ try {+ await within(clientEnded)+ expect(settled).toBe(false)+ expect(release).not.toHaveBeenCalled()+ expect(fixture.pool.totalCount).toBe(1)+ } finally {+ callbackCleanup.resolve()+ }+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(release).toHaveBeenCalledOnce()+ expect(release).toHaveBeenCalledWith(reason)+ expect(fixture.pool.totalCount).toBe(0)+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')+ })++ it('awaits bounded connect failure and preserves the cancellation reason without sending BEGIN', async () => {+ const fixture = await postgresPeer(false)+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ let settled = false+ const work = vi.fn()+ const outcome = withTransaction(work, fixture.pool, { signal: controller.signal })+ .then(+ value => ({ value }),+ error => ({ error })+ )+ .finally(() => {+ settled = true+ })+ await fixture.connected.promise+ controller.abort(reason)+ await nextTurn()+ expect(settled).toBe(false)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(work).not.toHaveBeenCalled()+ expect(fixture.queries).toEqual([])+ expect(fixture.pool.totalCount).toBe(0)+ })++ it('disposes a borrow that arrives after abort before any new statement or healthy return', async () => {+ const fixture = await postgresPeer()+ const first = await fixture.pool.connect()+ const controller = new AbortController()+ const reason = interruption('authorize_aborted')+ const work = vi.fn()+ const outcome = withTransaction(work, fixture.pool, { signal: controller.signal }).then(+ value => ({ value }),+ error => ({ error })+ )+ await observed(() => fixture.pool.waitingCount === 1)+ controller.abort(reason)+ first.release()+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(work).not.toHaveBeenCalled()+ expect(fixture.queries).toEqual([])+ expect(fixture.pool.totalCount).toBe(0)+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')+ })++ it('keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy', async () => {+ const fixture = await postgresPeer()+ fixture.stall('COMMIT')+ const controller = new AbortController()+ const reason = interruption('authorize_timeout')+ const outcome = withTransaction(async () => 'possibly committed', fixture.pool, {+ signal: controller.signal,+ }).then(+ value => ({ value }),+ error => ({ error })+ )+ await observed(() => fixture.queries.includes('COMMIT'))+ controller.abort(reason)+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)+ expect(fixture.queries).toEqual([+ 'BEGIN',+ "SET LOCAL client_connection_check_interval = '100ms'",+ 'COMMIT',+ ])+ expect(fixture.pool.totalCount).toBe(0)+ })+})diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.tsnew file mode 100644index 000000000..9c35ce4c2--- /dev/null+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts@@ -0,0 +1,503 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import type { Request } from 'express'+import { type IncomingMessage, createServer, request as httpRequest } from 'node:http'+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'+import {+ AuthorizeBodyAdmission,+ AuthorizeWorkInterrupted,+} from '../src/middleware/llmProviderAttemptBodyAdmission.js'+import * as authorizer from '../src/services/llmProviderAttemptAuthorizer.js'+import * as rateLimiter from '../src/services/rateLimiterService.js'+import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'+import { MockGateway } from './mockGateway.js'++vi.mock('../src/config.js', async importOriginal => {+ const actual = await importOriginal<typeof import('../src/config.js')>()+ return { ...actual, config: { ...actual.config, jsonBodyLimit: '1mb' } }+})++vi.mock('../src/db.js', () => ({+ pool: { query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), connect: vi.fn() },+ withTransaction: vi.fn(),+}))++vi.mock('../src/services/notificationEmitter.js', () => ({+ emitNotification: vi.fn().mockResolvedValue(undefined),+ enqueueApprovalRequestedNotification: vi.fn().mockResolvedValue(undefined),+ enqueueApprovalUpdatedNotification: vi.fn().mockResolvedValue(undefined),+}))++vi.mock('../src/services/rateLimiterService.js', () => ({ checkAndIncrement: vi.fn() }))++vi.mock('../src/observability/metrics.js', async importOriginal => {+ const actual = await importOriginal<typeof import('../src/observability/metrics.js')>()+ return { ...actual, rateLimitHitsTotal: { inc: vi.fn() } }+})++vi.mock('../src/services/llmProviderAttemptAuthorizer.js', async importOriginal => {+ const actual =+ await importOriginal<typeof import('../src/services/llmProviderAttemptAuthorizer.js')>()+ return { ...actual, authorizeLlmProviderAttempt: vi.fn() }+})++const AUTHORIZE_PATH = '/api/v1/mcp-host/llm/provider-attempts/authorize'+// Synthetic ticket content is only an authorizer response fixture. No ticket+// creation, grant lookup, budget reservation or provider dispatch is mocked as+// successful by these HTTP ownership tests.+const SUCCESS = {+ providerAttemptId: '33333333-3333-4333-8333-333333333333',+ requestHash: 'a'.repeat(64),+ executionTicket: 'unit-test-execution-ticket',+ expiresAt: '2026-10-02T12:00:00.000Z',+}++function deferred<T = void>() {+ let resolve!: (value: T | PromiseLike<T>) => void+ const promise = new Promise<T>(done => {+ resolve = done+ })+ return { promise, resolve }+}++type Observation = {+ request: IncomingMessage+ bodyDataSubscriptions: number+ readBytesBeforeResponse: number+ responseWritesAfterClose: number+}++type Reply = { status: number; headers: IncomingMessage['headers']; body: string }++function headers(host: string, provider: 'codex' | 'grok' = 'codex') {+ const issued = issueMcpHostAccessJwt('default', host, [host], {+ workflowControlScopes: [provider === 'codex' ? 'llm:codex:execute' : 'llm:grok:execute'],+ })+ return { Authorization: `Bearer ${issued.token}`, 'content-type': 'application/json' }+}++function send(url: string, requestHeaders: Record<string, string>, body?: string) {+ let client!: ReturnType<typeof httpRequest>+ const response = new Promise<Reply>((resolve, reject) => {+ client = httpRequest(url, { method: 'POST', headers: requestHeaders }, res => {+ const chunks: string[] = []+ res.setEncoding('utf8')+ res.on('data', (chunk: string) => chunks.push(chunk))+ res.on('error', reject)+ res.on('end', () =>+ resolve({ status: res.statusCode ?? 0, headers: res.headers, body: chunks.join('') })+ )+ })+ client.on('error', reject)+ client.setTimeout(5_000, () => client.destroy(new Error('HTTP ownership fixture deadline')))+ if (body === undefined) client.flushHeaders()+ else client.end(body)+ })+ return { client, response }+}++async function withApps(+ count: number,+ run: (apps: Array<{ url: string; observations: Observation[] }>) => Promise<void>+) {+ const { createApp } = await import('../src/app.js')+ const servers = Array.from({ length: count }, () => {+ const observations: Observation[] = []+ const app = createApp(new MockGateway() as never)+ const listener = createServer((req, res) => {+ const observed: Observation = {+ request: req,+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ responseWritesAfterClose: 0,+ }+ observations.push(observed)+ const end = res.end+ res.end = function (...args: Parameters<typeof res.end>) {+ if (this.destroyed) observed.responseWritesAfterClose += 1+ return end.apply(this, args)+ } as typeof res.end+ // Observe read calls and listener installation without adding a data+ // listener or switching the IncomingMessage into flowing mode.+ const read = req.read+ req.read = function (size?: number) {+ const value = read.call(this, size)+ // Node may drain already buffered bytes after response handoff. That+ // native _dump is separate from admitting an application body reader.+ if (Buffer.isBuffer(value) && !res.writableEnded)+ observed.readBytesBeforeResponse += value.length+ return value+ }+ const on = req.on+ req.on = function (event: string, listener: (...args: any[]) => void) {+ if (event === 'data') {+ observed.bodyDataSubscriptions += 1+ }+ return on.call(this, event, listener)+ } as typeof req.on+ app(req, res)+ }).listen(0)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')+ return { listener, url: `http://127.0.0.1:${address.port}${AUTHORIZE_PATH}`, observations }+ })+ try {+ await run(servers)+ } finally {+ await Promise.all(+ servers.map(+ ({ listener }) =>+ new Promise<void>((resolve, reject) => {+ listener.closeAllConnections()+ listener.close(err => (err ? reject(err) : resolve()))+ })+ )+ )+ }+}++function observeHeldOwner() {+ const settled = deferred()+ const original = AuthorizeBodyAdmission.prototype.run+ const spy = vi.spyOn(AuthorizeBodyAdmission.prototype, 'run').mockImplementation(async function (+ this: AuthorizeBodyAdmission,+ ...args: Parameters<AuthorizeBodyAdmission['run']>+ ) {+ try {+ return await original.apply(this, args)+ } finally {+ if (args[0].headers['x-admission-case'] === 'held') settled.resolve()+ }+ })+ return { settled: settled.promise, spy }+}++beforeEach(() => {+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockReset().mockResolvedValue(SUCCESS)+ vi.mocked(rateLimiter.checkAndIncrement)+ .mockReset()+ .mockResolvedValue({+ allowed: true,+ backendAvailable: true,+ remaining: 59,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count: 1,+ })+})++describe('createApp retained authorize-body ownership', () => {+ it('shares the actual production cap across apps and principals and retains disconnected work until unwind', async () => {+ const started = deferred()+ const abortObserved = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ const parse = vi.spyOn(JSON, 'parse')+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ deps?.signal?.addEventListener('abort', () => abortObserved.resolve(), { once: true })+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(2, async ([firstApp, otherApp]) => {+ const first = send(+ firstApp.url,+ { ...headers('admission-first-host'), 'x-admission-case': 'held' },+ '{"held":true}'+ )+ const firstOutcome = first.response.catch(error => error)+ try {+ await started.promise+ const rejectedBody = '{"mustNotParse":true}'+ const blocked = await send(+ otherApp.url,+ {+ ...headers('admission-other-host', 'grok'),+ },+ rejectedBody+ ).response+ expect(blocked.status).toBe(503)+ expect(JSON.parse(blocked.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(blocked.headers.connection).toBe('close')+ expect(otherApp.observations[0]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()++ // The same refusal must also arrive from headers alone. An owner+ // that first buffers the next body would stall this request.+ const headersOnly = await send(otherApp.url, {+ ...headers('admission-headers-only-host'),+ 'content-length': '4096',+ }).response+ expect(headersOnly.status).toBe(503)+ expect(otherApp.observations[1]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })++ const malformed = '{unauthenticated while full'+ const unauthenticated = await send(+ otherApp.url,+ { 'content-type': 'application/json' },+ malformed+ ).response+ expect(unauthenticated.status).toBe(401)+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(malformed)+ expect(otherApp.observations[2]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })++ first.client.destroy()+ await abortObserved.promise+ const stillHeld = await send(otherApp.url, headers('admission-third-host'), rejectedBody)+ .response+ expect(stillHeld.status).toBe(503)+ expect(JSON.parse(stillHeld.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)+ expect(otherApp.observations[3]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()++ permitUnwind.resolve()+ await owner.settled+ expect((firstApp.observations[0].request as Request).body).toBeUndefined()+ const recovered = await send(+ otherApp.url,+ headers('admission-other-host', 'grok'),+ '{"recovered":true}'+ ).response+ expect(recovered.status).toBe(200)+ expect(JSON.parse(recovered.body)).toEqual(SUCCESS)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ expect(firstApp.observations[0].responseWritesAfterClose).toBe(0)+ } finally {+ first.client.destroy()+ permitUnwind.resolve()+ await firstOutcome+ }+ })+ } finally {+ parse.mockRestore()+ owner.spy.mockRestore()+ }+ })++ it('runs both rate limits and JWT before taking any body owner', async () => {+ const acquire = vi.spyOn(AuthorizeBodyAdmission.prototype, 'tryAcquire')+ try {+ await withApps(1, async ([app]) => {+ const invalid = await send(app.url, { 'content-type': 'application/json' }, '{malformed')+ .response+ expect(invalid.status).toBe(401)+ expect(acquire).not.toHaveBeenCalled()+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()+ expect(app.observations[0]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })++ vi.mocked(rateLimiter.checkAndIncrement).mockResolvedValueOnce({+ allowed: false,+ backendAvailable: true,+ remaining: 0,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count: 61,+ })+ const rateDenied = await send(app.url, headers('rate-denied-host'), '{malformed').response+ expect(rateDenied.status).toBe(429)+ expect(acquire).not.toHaveBeenCalled()+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()+ expect(app.observations[1]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })++ const recovered = await send(app.url, headers('rate-recovered-host'), '{}').response+ expect(recovered.status).toBe(200)+ expect(acquire).toHaveBeenCalledOnce()+ })+ } finally {+ acquire.mockRestore()+ }+ })++ for (const provider of ['codex', 'grok'] as const) {+ it(`preserves chunked ${provider} JSON through the actual app and releases the body`, async () => {+ await withApps(1, async ([app]) => {+ const body = JSON.stringify({+ request: { provider: `${provider}-subscription`, chunked: true },+ })+ const sent = send(app.url, {+ ...headers(`chunked-${provider}-host`, provider),+ 'transfer-encoding': 'chunked',+ })+ sent.client.write(body.slice(0, 20))+ sent.client.end(body.slice(20))+ const response = await sent.response+ expect(response.status).toBe(200)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ hostRefs: [`chunked-${provider}-host`] }),+ JSON.parse(body),+ expect.objectContaining({+ signal: expect.any(AbortSignal),+ resolveAssignment: expect.any(Function),+ })+ )+ expect((app.observations[0].request as Request).body).toBeUndefined()+ const next = await send(app.url, headers(`next-${provider}-host`, provider), '{}').response+ expect(next.status).toBe(200)+ })+ })+ }++ for (const [name, error, status, code] of [+ [+ 'authorize denial',+ new authorizer.LlmProviderAttemptAuthorizeError('budget_denied', 'fixture budget refusal'),+ 403,+ 'budget_denied',+ ],+ ['unexpected authorizer error', new Error('fixture authorizer failure'), 500, undefined],+ ] as const) {+ it(`releases after ${name} and admits a subsequent request`, async () => {+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockRejectedValueOnce(error)+ await withApps(1, async ([app]) => {+ const response = await send(app.url, headers(`error-${status}-host`), '{}').response+ expect(response.status).toBe(status)+ if (code) expect(JSON.parse(response.body)).toEqual({ error: code })+ expect((app.observations[0].request as Request).body).toBeUndefined()+ const next = await send(app.url, headers(`recovered-${status}-host`), '{}').response+ expect(next.status).toBe(200)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ })+ })+ }++ it('returns terminal authorize_timeout after the real work clock aborts, without publishing late success', async () => {+ const started = deferred()+ const abortObserved = deferred<AbortSignal>()+ const nativeSetTimeout = globalThis.setTimeout+ // Only shorten the owner clock for this socket fixture. The production+ // policy remains unchanged; this test makes no numeric safety claim.+ const clock = vi+ .spyOn(globalThis, 'setTimeout')+ .mockImplementation(((fn, ms, ...args) =>+ nativeSetTimeout(+ fn,+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.workDeadlineMs ? 40 : ms,+ ...args+ )) as typeof setTimeout)+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ const signal = deps?.signal+ if (!signal) throw new Error('authorize work did not receive its owner signal')+ started.resolve()+ await new Promise<void>(resolve =>+ signal.addEventListener(+ 'abort',+ () => {+ abortObserved.resolve(signal)+ resolve()+ },+ { once: true }+ )+ )+ // Deliberately produce a late result: the route must check the signal+ // immediately before publishing an execution ticket.+ return SUCCESS+ }+ )+ try {+ await withApps(1, async ([app]) => {+ const sent = send(app.url, headers('work-timeout-host'), '{}')+ const outcome = sent.response+ await started.promise+ const signal = await abortObserved.promise+ expect(signal.reason).toBeInstanceOf(AuthorizeWorkInterrupted)+ expect(signal.reason.code).toBe('authorize_timeout')+ const response = await outcome+ expect(response.status).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ error: 'authorize_timeout' })+ expect((app.observations[0].request as Request).body).toBeUndefined()+ const recovered = await send(app.url, headers('after-timeout-host'), '{}').response+ expect(recovered.status).toBe(200)+ })+ } finally {+ clock.mockRestore()+ }+ })++ it('stops a stalled body reader on the real read clock before releasing and admits the next request', async () => {+ const owner = observeHeldOwner()+ const nativeSetTimeout = globalThis.setTimeout+ // Contracted clocks are explicit HTTP fixture timing, not runtime policy.+ const clock = vi+ .spyOn(globalThis, 'setTimeout')+ .mockImplementation(((fn, ms, ...args) =>+ nativeSetTimeout(+ fn,+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.readDeadlineMs+ ? 40+ : ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.closeGraceMs+ ? 10+ : ms,+ ...args+ )) as typeof setTimeout)+ try {+ await withApps(1, async ([app]) => {+ const sent = send(app.url, {+ ...headers('read-timeout-host'),+ 'content-length': '128',+ 'x-admission-case': 'held',+ })+ const response = await sent.response+ expect(response.status).toBe(408)+ expect(JSON.parse(response.body)).toEqual({ error: 'request_timeout' })+ expect(response.headers.connection).toBe('close')+ await owner.settled+ expect(app.observations[0].request.destroyed).toBe(true)+ expect((app.observations[0].request as Request).body).toBeUndefined()+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()+ const next = await send(app.url, headers('after-read-timeout-host'), '{}').response+ expect(next.status).toBe(200)+ })+ } finally {+ clock.mockRestore()+ owner.spy.mockRestore()+ }+ })+})++describe('Host and Recipe assignment owner cancellation', () => {+ for (const hostRef of ['assignment-host', 'default/assignment-recipe']) {+ it(`passes the same signal to ${hostRef.includes('/') ? 'Recipe' : 'Host'} lookup and preserves cancellation`, async () => {+ const { resolveHostAssignedAssignment } =+ await import('../src/routes/mcp-host/llmProviderAttempts.routes.js')+ const controller = new AbortController()+ const interruption = new AuthorizeWorkInterrupted('authorize_timeout')+ const getResource = vi.fn(async (_plural, _name, _namespace, signal?: AbortSignal) => {+ expect(signal).toBe(controller.signal)+ controller.abort(interruption)+ throw new Error('fixture lookup unwound after cancellation')+ })+ await expect(+ resolveHostAssignedAssignment({ getResource } as never, hostRef, controller.signal)+ ).rejects.toBe(interruption)+ expect(getResource).toHaveBeenCalledOnce()+ await expect(+ resolveHostAssignedAssignment({ getResource } as never, hostRef, controller.signal)+ ).rejects.toBe(interruption)+ expect(getResource).toHaveBeenCalledOnce()+ })+ }+})diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.test.tsindex d48852964..5bfe3e8c0 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts@@ -884,7 +884,8 @@ describe('resolveHostAssignedAssignment', () => {expect(gateway.getResource).toHaveBeenCalledWith('workflowrecipes','codex-recipe',- 'sandbox-recipes'+ 'sandbox-recipes',+ undefined)})diff --git a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.tsindex a89234f8b..f11c980ee 100644--- a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts+++ b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts@@ -1,6 +1,11 @@import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import * as k8s from '@kubernetes/client-node'import { randomBytes, randomUUID } from 'node:crypto'+import { once } from 'node:events'import { readFileSync } from 'node:fs'+import { createServer as createHttpServer } from 'node:http'+import type { Socket } from 'node:net'+import { setImmediate as nextTurn } from 'node:timers/promises'import { Pool } from 'pg'import {LIMITS as GROK_LIMITS,@@ -8,7 +13,7 @@ import {} from '@clerum/grok-provider-attempt-contract'import { LIMITS } from '@clerum/llm-provider-attempt-contract'import { config } from '../src/config.js'-import { initDb } from '../src/db.js'+import { type DbClient, initDb, withTransaction as withProductionTransaction } from '../src/db.js'import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'import { getActiveReservation } from '../src/services/budgets/reservations.js'@@ -38,7 +43,10 @@ import {insertLlmProviderAttempt,} from '../src/services/llmProviderAttemptStore.js'import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'+import { ResourceService } from '../src/services/resourceService.js'import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'// The Grok authorizer signs through this module, not through a dependency, so// the Grok envelope test taps the real issuer here: the ticket is registered@@ -79,6 +87,77 @@ function quoteIdent(value: string): string {return `"${value.replace(/"/g, '""')}"`}+async function cancellationDeadline<T>(promise: Promise<T>): Promise<T> {+ let timer: NodeJS.Timeout | undefined+ try {+ return await Promise.race([+ promise,+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error('authorizer cancellation phase did not complete')),+ 2_000+ )+ }),+ ])+ } finally {+ clearTimeout(timer)+ }+}++async function observeCancellation(check: () => Promise<boolean>): Promise<void> {+ const deadline = Date.now() + 2_000+ while (!(await check())) {+ if (Date.now() >= deadline) throw new Error('authorizer physical state did not settle')+ await nextTurn()+ }+}++async function createAssignmentLookupPeer(stall: boolean) {+ let arrived!: () => void+ const arrival = new Promise<void>(resolve => {+ arrived = resolve+ })+ const sockets = new Set<Socket>()+ const server = createHttpServer((req, res) => {+ arrived()+ if (stall) return+ res.setHeader('Content-Type', 'application/json')+ res.end(JSON.stringify({ metadata: { name: 'research-host' } }))+ })+ server.on('connection', socket => {+ sockets.add(socket)+ socket.once('close', () => sockets.delete(socket))+ })+ server.listen(0, '127.0.0.1')+ await once(server, 'listening')+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('missing assignment peer address')+ const kubeconfig = new k8s.KubeConfig()+ kubeconfig.loadFromOptions({+ clusters: [+ { name: 'fixture', server: `http://127.0.0.1:${address.port}`, skipTLSVerify: false },+ ],+ users: [{ name: 'fixture' }],+ contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],+ currentContext: 'fixture',+ })+ return {+ resources: new ResourceService(kubeconfig.makeApiClient(k8s.CustomObjectsApi), 'fixture', {+ hosts: 'fixture',+ }),+ arrived: arrival,+ recover: () => {+ stall = false+ },+ close: async () => {+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ },+ }+}+const REQUEST = {schemaVersion: 'codex-completion-request.v1' as const,requestId: 'req-pg-1',@@ -157,6 +236,315 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',}}+ function registerProductionCancellationCases(fixture: {+ provider: 'codex-subscription' | 'grok-subscription'+ model: string+ connectionKey: string+ caller: () => McpHostAccessClaims+ request: () => Record<string, unknown>+ readConnection: () => Promise<{+ id: string+ catalogRevision: number+ credentialRevision: number+ connectionKey: string+ } | null>+ policyHash: (connection: {+ catalogRevision: number+ credentialRevision: number+ connectionKey: string+ }) => string+ }): void {+ describe(`${fixture.provider} production cancellation`, () => {+ async function run(stage: 'assignment' | 'query' | 'commit'): Promise<void> {+ const invocationId = `actual-cancel-${randomUUID()}`+ const budgetId = randomUUID()+ const current = await fixture.readConnection()+ expect(current).not.toBeNull()+ const payload = {+ request: fixture.request(),+ invocationId,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ policyRevision: current!.catalogRevision,+ policyHash: fixture.policyHash(current!),+ }+ const observer = new Pool({+ connectionString,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 2_000,+ idleTimeoutMillis: 1_000,+ })+ const proxy =+ stage === 'commit'+ ? await createPostgresCommitReplyBlackhole(connectionString)+ : undefined+ const workPool = new Pool({+ connectionString: proxy?.connectionString ?? connectionString,+ ssl: false,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 15_000,+ idleTimeoutMillis: 1_000,+ })+ const lookup = await createAssignmentLookupPeer(stage === 'assignment')+ const controller = new AbortController()+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })+ const phases: string[] = []+ let backendPid: number | undefined+ let attemptId: string | undefined+ let reservationId: string | undefined+ let stallQuery = stage === 'query'+ let settled = false+ const previousTap = grokTicketTap.afterIssue++ async function counts() {+ return (+ await observer.query<{ attempts: number; tickets: number; reservations: number }>(+ `SELECT+ (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,+ (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a+ ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,+ (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE budget_id = $2) AS reservations`,+ [invocationId, budgetId]+ )+ ).rows[0]+ }++ async function afterRealIssue(+ db: DbClient,+ input: { providerAttemptId: string; budgetReservationId: string },+ issued: { claims: { jti: string } }+ ): Promise<void> {+ const registered = await db.query(+ `SELECT a.provider, a.connection_id, a.budget_reservation_id,+ t.jti, t.status AS ticket_status, r.budget_id, r.host_ref+ FROM llm_provider_attempts a+ JOIN llm_provider_attempt_tickets t ON t.provider_attempt_id = a.id+ JOIN budget_pending_reservations r ON r.id::text = a.budget_reservation_id+ WHERE a.id = $1`,+ [input.providerAttemptId]+ )+ expect(registered.rows).toEqual([+ {+ provider: fixture.provider,+ connection_id: current!.id,+ budget_reservation_id: input.budgetReservationId,+ jti: issued.claims.jti,+ ticket_status: 'issued',+ budget_id: budgetId,+ host_ref: fixture.caller().hostRefs[0],+ },+ ])+ attemptId = input.providerAttemptId+ reservationId = input.budgetReservationId+ phases.push('ticket_registered')+ // The real issuer has already signed and registered its ticket. This+ // native statement supplies a deterministic interruptible phase; no+ // test barrier resolves the work after cancellation.+ if (stallQuery) await db.query('SELECT pg_sleep(30)')+ }++ function productionDeps(signal: AbortSignal): LlmProviderAttemptAuthorizerDeps {+ if (fixture.provider === 'grok-subscription') {+ grokTicketTap.afterIssue = async (db, input, issued) => {+ await afterRealIssue(db, input, issued)+ return issued+ }+ }+ return testDeps({+ signal,+ resolveAssignment: async (_hostRef, forwarded) => {+ expect(forwarded).toBe(signal)+ phases.push('assignment')+ await lookup.resources.getResource('hosts', 'research-host', 'fixture', forwarded)+ return {+ liveBrokerProviders: [fixture.provider],+ liveConnectionRef: fixture.connectionKey,+ }+ },+ withTransaction: async (work, _unused, options) => {+ // Assert the authorizer's actual forwarding before binding this+ // fixture database; do not inject a separate cancellation source.+ expect(options?.signal).toBe(signal)+ return withProductionTransaction(+ async tx => {+ phases.push('transaction')+ const identity = await tx.query('SELECT pg_backend_pid() AS pid')+ backendPid = (identity.rows[0] as { pid: number }).pid+ return work(tx)+ },+ workPool,+ options+ )+ },+ evaluateBudget: async (...args) => {+ const result = await evaluateBudgetCheck(...args)+ expect(result.allowed).toBe(true)+ expect(result.reservationIds).toHaveLength(1)+ phases.push('budget_reserved')+ return result+ },+ insertAttempt: async (db, input) => {+ const attempt = await insertLlmProviderAttempt(db, input)+ phases.push('attempt_inserted')+ return attempt+ },+ issueTicket: async (db, input) => {+ const issued = await issueRegisteredCodexExecutionTicket(db, input)+ await afterRealIssue(db, input, issued)+ return issued+ },+ })+ }++ try {+ const runtime = (+ await observer.query<{ version: string; version_num: string }>(+ "SELECT version(), current_setting('server_version_num') AS version_num"+ )+ ).rows[0]+ expect(Number(runtime.version_num)).toBeGreaterThanOrEqual(160000)+ expect(Number(runtime.version_num)).toBeLessThan(170000)+ expect(runtime.version).toMatch(/linux|darwin|bsd|illumos/i)+ await observer.query(+ `INSERT INTO token_budgets+ (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)+ VALUES ($1, 'actual authorize cancellation', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,+ [budgetId, JSON.stringify({ provider: [fixture.provider], model: [fixture.model] })]+ )+ __resetBudgetCheckCache()+ const outcome = authorizeLlmProviderAttempt(+ fixture.caller(),+ payload,+ productionDeps(controller.signal)+ )+ .then(+ value => ({ value }),+ error => ({ error })+ )+ .finally(() => {+ settled = true+ })+ const expectedPhases = [+ 'assignment',+ 'transaction',+ 'budget_reserved',+ 'attempt_inserted',+ 'ticket_registered',+ ]+ if (stage === 'assignment') {+ await cancellationDeadline(lookup.arrived)+ expect(phases).toEqual(['assignment'])+ expect(workPool.totalCount).toBe(0)+ } else if (stage === 'query') {+ await observeCancellation(async () => {+ if (!backendPid) return false+ const result = await observer.query<{ state: string; query: string }>(+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1',+ [backendPid]+ )+ return (+ result.rows[0]?.state === 'active' &&+ result.rows[0]?.query === 'SELECT pg_sleep(30)'+ )+ })+ expect(phases).toEqual(expectedPhases)+ expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })+ } else {+ await cancellationDeadline(proxy!.commitForwarded)+ await observeCancellation(async () => (await counts()).attempts === 1)+ await observeCancellation(async () => proxy!.discardedReplyBytes > 0)+ expect(phases).toEqual(expectedPhases)+ expect(proxy!.commands).toEqual(['BEGIN', 'COMMIT'])+ expect(proxy!.discardedReplyBytes).toBeGreaterThan(0)+ }+ expect(settled).toBe(false)+ controller.abort(reason)+ const interrupted = await cancellationDeadline(outcome)+ expect('error' in interrupted ? interrupted.error : undefined).toBe(reason)+ expect(workPool.totalCount).toBe(0)+ if (backendPid) {+ await observeCancellation(+ async () =>+ (+ await observer.query<{ count: number }>(+ 'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1',+ [backendPid]+ )+ ).rows[0].count === 0+ )+ }+ await withProductionTransaction(async db => {+ const result = await db.query(+ 'SELECT pg_try_advisory_xact_lock(hashtext($1)) AS available',+ [budgetId]+ )+ expect((result.rows[0] as { available: boolean }).available).toBe(true)+ }, observer)+ if (stage === 'commit') {+ // The client never received COMMIT acknowledgement, yet the real+ // business ledger is durable. Do not label this rollback/no spend.+ expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })+ const attempt = await observer.query(+ 'SELECT id, provider, connection_id, budget_reservation_id, status FROM llm_provider_attempts WHERE invocation_id = $1',+ [invocationId]+ )+ expect(attempt.rows).toEqual([+ {+ id: attemptId,+ provider: fixture.provider,+ connection_id: current!.id,+ budget_reservation_id: reservationId,+ status: 'authorized',+ },+ ])+ } else {+ expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })+ stallQuery = false+ lookup.recover()+ phases.length = 0+ const later = await authorizeLlmProviderAttempt(+ fixture.caller(),+ payload,+ productionDeps(new AbortController().signal)+ )+ expect(later.providerAttemptId).toBe(attemptId)+ expect(phases).toEqual(expectedPhases)+ expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })+ expect(workPool.idleCount).toBe(1)+ }+ } finally {+ controller.abort(reason)+ grokTicketTap.afterIssue = previousTap+ proxy?.allowReplies()+ await endPoolAndWaitForClients(workPool)+ await proxy?.close()+ await lookup.close()+ await observer.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])+ __resetBudgetCheckCache()+ await endPoolAndWaitForClients(observer)+ }+ }++ it(+ 'cancels a configured Kubernetes lookup before any production transaction, then reauthorizes the same binding',+ () => run('assignment'),+ 10_000+ )+ it(+ 'cancels a native query after real budget, attempt and signed ticket writes, then rolls back and reauthorizes',+ () => run('query'),+ 10_000+ )+ it(+ 'keeps real authorization ledger rows when COMMIT executes but its reply is lost',+ () => run('commit'),+ 10_000+ )+ })+ }+beforeAll(async () => {adminPool = new Pool({ connectionString: adminUrl })await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)@@ -463,6 +851,27 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',})+ registerProductionCancellationCases({+ provider: 'codex-subscription',+ model: REQUEST.model,+ connectionKey: 'deployment-default',+ caller: claims,+ readConnection: () => getSafeCodexSubscriptionConnection(pool),+ request: () => ({+ ...REQUEST,+ schemaVersion: 'codex-completion-request.v2',+ requestId: randomUUID(),+ idempotencyKey: randomUUID(),+ }),+ policyHash: connection =>+ computeCodexPolicyHash({+ model: REQUEST.model,+ catalogRevision: connection.catalogRevision,+ credentialRevision: connection.credentialRevision,+ connectionKey: connection.connectionKey,+ }),+ })+// #784: the Grok twin of the Codex rollback above, on an image-bearing V2// request. The Grok path checks its proxy envelope after signing and before// commit, so a refusal there must roll back the attempt, the registered@@ -520,6 +929,35 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',grokTicketTap.afterIssue = null})+ registerProductionCancellationCases({+ provider: 'grok-subscription',+ model: GROK_MODEL,+ connectionKey: GROK_CONNECTION_KEY,+ caller: grokClaims,+ readConnection: () => getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY),+ request: () => ({+ schemaVersion: 'grok-completion-request.v2',+ requestId: randomUUID(),+ idempotencyKey: randomUUID(),+ provider: 'grok-subscription',+ model: GROK_MODEL,+ messages: [+ {+ role: 'user',+ content: 'look',+ contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],+ },+ ],+ }),+ policyHash: connection =>+ computeGrokPolicyHash({+ model: GROK_MODEL,+ catalogRevision: connection.catalogRevision,+ credentialRevision: connection.credentialRevision,+ connectionKey: GROK_CONNECTION_KEY,+ }),+ })+it.each([false, true])('rolls back an oversized signed Grok V2 envelope without losing an existing reservation (presented=%s)',async presented => {diff --git a/scripts/ci/pinned-vitest-cases/control-api.json b/scripts/ci/pinned-vitest-cases/control-api.jsonindex 5e0ee624f..e69bd8f49 100644--- a/scripts/ci/pinned-vitest-cases/control-api.json+++ b/scripts/ci/pinned-vitest-cases/control-api.json@@ -66,5 +66,161 @@{"suite": "routes.mcp-host.llmProviderAttempts.createApp.test.ts","fullName": "createApp global JSON encoding refuses compressed JSON before parsing or the public handler and accepts the same uncompressed body"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission policy and idempotence rejects invalid counts and timer values and freezes the accepted policy"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission policy and idempotence an old release cannot free the slot occupied by a newer request"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission policy and idempotence clears retained request bodies and ignores duplicate parser callbacks"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'malformed JSON' parser error after removing its raw body"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'over-deep structure' parser error after removing its raw body"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'compressed body' parser error after removing its raw body"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'payload over limit' parser error after removing its raw body"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle stops a stalled read through closeGrace even when response finish never arrives"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle bounds a never-finishing capacity refusal without reading or starting work"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission parser lifecycle uses close during parsing only to stop the reader and never starts work"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission work cancellation preserves typed authorize_timeout after real signal-aware work settles"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission work cancellation does not free a work owner on client close until signal-aware work unwinds"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission HTTP fixture preserves chunked JSON and releases after work completes"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission HTTP fixture refuses an over-cap request before parsing and keeps a disconnected work owner charged"+ },+ {+ "suite": "llmProviderAttemptBodyAdmission.test.ts",+ "fullName": "AuthorizeBodyAdmission HTTP fixture releases only after the real body-parser callback follows a destroyed stalled request"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership shares the actual production cap across apps and principals and retains disconnected work until unwind"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership runs both rate limits and JWT before taking any body owner"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership preserves chunked codex JSON through the actual app and releases the body"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership preserves chunked grok JSON through the actual app and releases the body"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership releases after authorize denial and admits a subsequent request"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership releases after unexpected authorizer error and admits a subsequent request"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership returns terminal authorize_timeout after the real work clock aborts, without publishing late success"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "createApp retained authorize-body ownership stops a stalled body reader on the real read clock before releasing and admits the next request"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "Host and Recipe assignment owner cancellation passes the same signal to Host lookup and preserves cancellation"+ },+ {+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",+ "fullName": "Host and Recipe assignment owner cancellation passes the same signal to Recipe lookup and preserves cancellation"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client terminates no-headers with the configured CA, bearer and server intact, then serves a later lookup"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client terminates no-eof with the configured CA, bearer and server intact, then serves a later lookup"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client does not enter the cross-namespace fallback after cancellation without an explicit namespace"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client forwards the optional signal through K8sGateway without reconfiguring its client"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client propagates codex-subscription cancellation into assignment and never enters a transaction"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "authorizer Kubernetes cancellation on the configured native client propagates grok-subscription cancellation into assignment and never enters a transaction"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets never borrows a client for an already aborted request"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets fails without running work if the server does not support scoped connection polling"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an in-flight rollback before final release and preserves the typed owner reason"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an active native query and evicts the client exactly once before recovery"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an idle half-open client but remains owned through callback cleanup"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets awaits bounded connect failure and preserves the cancellation reason without sending BEGIN"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets disposes a borrow that arrives after abort before any new statement or healthy return"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "scoped transaction cancellation with the native pg driver and sockets keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy"}]