Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): retain and cancel authorization work

Alfredo Lopez19093dcd1f1a
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index fab665aa2..a74a93fe0 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.704",
+ "version": "0.1.705",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.704",
+ "version": "0.1.705",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 04b010bb3..13c95663f 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.704",
+ "version": "0.1.705",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 389f549f3..3a04e3fb8 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -1,4 +1,4 @@
-import { Pool, type PoolClient } from 'pg'
+import { type Client, Pool, type PoolClient } from 'pg'
import {
type BoundedPoolBudget,
type PoolConstructor,
@@ -6705,13 +6705,27 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {
}
}
+export type DbTransactionOptions = { signal?: AbortSignal }
+
export async function withTransaction<T>(
work: (db: DbTransactionClient) => Promise<T>,
// Injectable only so the carrier's client lifecycle can be unit-tested against
// a fake Pool; production always uses the module-level core pool.
- txPool: Pool = pool
+ txPool: Pool = pool,
+ options: DbTransactionOptions = {}
): Promise<T> {
- const client = (await txPool.connect()) as PoolClient
+ const signal = options.signal
+ signal?.throwIfAborted()
+ // The pool's existing finite connection timeout owns a pending borrow. An
+ // abort cannot release its caller until acquisition has settled, including
+ // disposal of a client delivered after the signal fired.
+ let client: PoolClient & Pick<Client, 'end' | 'connection'>
+ try {
+ client = (await txPool.connect()) as typeof client
+ } catch (error) {
+ signal?.throwIfAborted()
+ throw error
+ }
let transactionStarted = false
let commitSent = false
let releaseError: Error | boolean | undefined
@@ -6732,15 +6746,56 @@ export async function withTransaction<T>(
}
client.on('error', onClientError)
+ let termination: Promise<void> | undefined
+ const onAbort = (): void => {
+ if (termination) return
+ releaseError = signal?.reason instanceof Error ? signal.reason : true
+ termination = client.end().catch(error => {
+ releaseError = error instanceof Error ? error : true
+ })
+ // pg Client.end() destroys an active query's socket, but an idle client
+ // waits for the peer's EOF. Explicit cancellation must also end a half-open
+ // idle checkout. Client.connection.stream is the pinned driver's typed
+ // transport; Duplex.destroy() is idempotent and its socket close drives the
+ // driver's connection end event awaited by Client.end(). Do not evict or
+ // finish the caller ahead of that event.
+ client.connection.stream.destroy()
+ }
+ signal?.addEventListener('abort', onAbort, { once: true })
+ if (signal?.aborted) onAbort()
+
try {
+ signal?.throwIfAborted()
await client.query('BEGIN')
transactionStarted = true
+ if (signal) {
+ signal.throwIfAborted()
+ // PostgreSQL 16 on the service's Linux runtime supports socket polling
+ // during queries. Its default (0) can leave a disconnected writer running
+ // until statement_timeout; this scoped setting detects our transport stop
+ // promptly. Unsupported servers fail this transaction instead of silently
+ // omitting cancellation. The existing statement_timeout stays in force.
+ await client.query("SET LOCAL client_connection_check_interval = '100ms'")
+ }
// The brand is nominal-only (a declared unique symbol); the checked-out
// client IS the transaction session, so this cast is the single blessed
// point where the brand is minted (issue #375 M3).
- const result = await work(client as unknown as DbTransactionClient)
+ signal?.throwIfAborted()
+ const transaction = signal
+ ? ({
+ query: async (text: string, values?: unknown[]) => {
+ signal.throwIfAborted()
+ const result = await client.query(text, values)
+ signal.throwIfAborted()
+ return result
+ },
+ } as unknown as DbTransactionClient)
+ : (client as unknown as DbTransactionClient)
+ const result = await work(transaction)
+ signal?.throwIfAborted()
commitSent = true
await client.query('COMMIT')
+ signal?.throwIfAborted()
return result
} catch (error) {
if (commitSent || !transactionStarted) {
@@ -6752,8 +6807,15 @@ export async function withTransaction<T>(
releaseError = rollbackError instanceof Error ? rollbackError : true
}
}
+ // Once COMMIT was sent, a missing reply is an unknown durable outcome.
+ // Cancellation does not establish rollback, no spend or no dispatch.
+ signal?.throwIfAborted()
throw error
} finally {
+ // Work (including its own finally) and any rollback have already unwound.
+ // Keep both listeners and the checkout until physical termination completes.
+ if (termination) await termination
+ signal?.removeEventListener('abort', onAbort)
client.removeListener('error', onClientError)
// A client that emitted 'error' mid-checkout is poisoned; destroy it. Keep an
// existing releaseError (the primary failure from work/commit/rollback) and
@@ -6762,6 +6824,9 @@ export async function withTransaction<T>(
if (asyncClientError && !releaseError) {
releaseError = asyncClientError
}
+ if (signal?.aborted) {
+ releaseError = signal.reason instanceof Error ? signal.reason : true
+ }
client.release(releaseError)
}
}
diff --git a/control-api/src/k8s.ts b/control-api/src/k8s.ts
index fbda2f6f7..6d3fd6c21 100644
--- a/control-api/src/k8s.ts
+++ b/control-api/src/k8s.ts
@@ -234,9 +234,10 @@ export class K8sGateway {
async getResource(
plural: ClerumResourceType,
name: string,
- namespace?: string
+ namespace?: string,
+ signal?: AbortSignal
): Promise<unknown> {
- return this.resources.getResource(plural, name, namespace)
+ return this.resources.getResource(plural, name, namespace, signal)
}
async createResource(
diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts
new file mode 100644
index 000000000..986211881
--- /dev/null
+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts
@@ -0,0 +1,13 @@
+import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmission.js'
+
+// Experimental candidate for the authorized PR #806 cgroup calibration.
+// These values are not certified for deployment. Keep this candidate local
+// and unpublished until the combined authorize/GFS workload selects and
+// records a count, deadlines, heap policy and non-heap margin below 768 MiB.
+// The policy is process-wide; every Node worker needs its own measured budget.
+export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicy = Object.freeze({
+ maxInFlight: 1,
+ readDeadlineMs: 10_000,
+ workDeadlineMs: 30_000,
+ closeGraceMs: 250,
+})
diff --git a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
index 9ff0ae9ff..79dcec580 100644
--- a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
+++ b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
@@ -11,6 +11,11 @@ import {
import { config } from '../../config.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import type { K8sGateway } from '../../k8s.js'
+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../../middleware/llmProviderAttemptAdmissionLimits.js'
+import {
+ AuthorizeBodyAdmission,
+ AuthorizeWorkInterrupted,
+} from '../../middleware/llmProviderAttemptBodyAdmission.js'
import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'
import { rootLogger } from '../../observability/logger.js'
import {
@@ -30,6 +35,10 @@ import { llmProviderAttemptAuthorizeRateLimits } from '../workflows/shared/rateL
const LOG_MODULE = 'mcp-host-llm-provider-attempts'
const log = rootLogger.child({ module: LOG_MODULE })
+// Retained bodies belong to the Node process, including when multiple apps or
+// routers are constructed. A per-router owner would multiply the allowance.
+const authorizeBodyAdmission = new AuthorizeBodyAdmission(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY)
+
// JSON.parse allocates one heap object per container, so a body under
// the byte limit can exhaust the heap before either authorizer runs. The
// parser scans the raw bytes first. Every bound below is derived key-by-key
@@ -93,8 +102,10 @@ export type LiveBrokerAssignment = {
export async function resolveHostAssignedAssignment(
gateway: Pick<K8sGateway, 'getResource'>,
- hostRef: string
+ hostRef: string,
+ signal?: AbortSignal
): Promise<LiveBrokerAssignment> {
+ signal?.throwIfAborted()
if (hostRef.includes('/')) {
const [recipeNamespace, recipeName, ...rest] = hostRef.split('/')
if (!recipeNamespace || !recipeName || rest.length > 0) {
@@ -107,8 +118,10 @@ export async function resolveHostAssignedAssignment(
const recipe = (await gateway.getResource(
'workflowrecipes',
recipeName,
- recipeNamespace
+ recipeNamespace,
+ signal
)) as { metadata?: { annotations?: Record<string, string> }; spec?: Record<string, unknown> }
+ signal?.throwIfAborted()
const spec = recipe?.spec && typeof recipe.spec === 'object' ? recipe.spec : {}
const liveBrokerProviders = collectRecipeOauthBrokerProviders(spec)
return {
@@ -117,6 +130,7 @@ export async function resolveHostAssignedAssignment(
annotations: recipe?.metadata?.annotations,
}
} catch (err) {
+ signal?.throwIfAborted()
if (err instanceof LlmProviderAttemptAuthorizeError) throw err
throw new LlmProviderAttemptAuthorizeError(
'host_binding_mismatch',
@@ -125,9 +139,10 @@ export async function resolveHostAssignedAssignment(
}
}
try {
- const host = (await gateway.getResource('hosts', hostRef, config.hostsNamespace)) as {
+ const host = (await gateway.getResource('hosts', hostRef, config.hostsNamespace, signal)) as {
spec?: Record<string, unknown>
}
+ signal?.throwIfAborted()
const spec = host?.spec && typeof host.spec === 'object' ? host.spec : {}
const model = spec.model
const connectionRef =
@@ -139,6 +154,7 @@ export async function resolveHostAssignedAssignment(
liveConnectionRef: readHostCodexConnectionRef(connectionRef),
}
} catch (err) {
+ signal?.throwIfAborted()
if (err instanceof LlmProviderAttemptAuthorizeError) throw err
throw new LlmProviderAttemptAuthorizeError(
'host_binding_mismatch',
@@ -149,31 +165,44 @@ export async function resolveHostAssignedAssignment(
export function createMcpHostLlmProviderAttemptRoutes(gateway: K8sGateway): Router {
const router = Router()
+ // Shared by both providers: preserve the larger visual envelope (Codex
+ // #660, Grok #784) and each authorizer's provider-specific limit. Refuse
+ // encoded bodies and scan raw structure before JSON.parse allocates objects.
+ const authorizeBodyParser = express.json({
+ limit: Math.max(LIMITS.maxVisualRequestBodyBytes, GROK_LIMITS.maxVisualRequestBodyBytes),
+ inflate: false,
+ verify: verifyBodyStructure,
+ })
router.post(
'/mcp-host/llm/provider-attempts/authorize',
...llmProviderAttemptAuthorizeRateLimits(),
requireMcpHostJwt,
- // Shared by both providers: admit the larger visual envelope (Codex #660,
- // Grok #784). Each authorizer then applies its own provider's limit.
- // `inflate: false` refuses an encoded body: 35 KiB of gzip inflates to
- // 35 MiB.
- express.json({
- limit: Math.max(LIMITS.maxVisualRequestBodyBytes, GROK_LIMITS.maxVisualRequestBodyBytes),
- inflate: false,
- verify: verifyBodyStructure,
- }),
asyncHandler(async (req: Request, res: Response) => {
- const claims = req.mcpHostJwt
- if (!claims) {
- res.status(401).json({ error: 'Unauthorized' })
- return
- }
try {
- const result = await authorizeLlmProviderAttempt(claims, req.body, {
- resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef),
+ await authorizeBodyAdmission.run(req, res, authorizeBodyParser, async signal => {
+ signal.throwIfAborted()
+ const claims = req.mcpHostJwt
+ if (!claims) {
+ res.status(401).json({ error: 'Unauthorized' })
+ return
+ }
+ const result = await authorizeLlmProviderAttempt(claims, req.body, {
+ signal,
+ resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef, signal),
+ })
+ signal.throwIfAborted()
+ res.status(200).json(result)
})
- res.status(200).json(result)
} catch (err) {
+ if (err instanceof AuthorizeWorkInterrupted) {
+ // A closed transport only requests cancellation. run() has awaited
+ // dependency/transaction cleanup before ownership reaches this catch.
+ if (req.aborted || res.destroyed || res.writableEnded) return
+ const requestLog = req.log?.child({ module: LOG_MODULE }) ?? log
+ requestLog.warn({ event: 'llm_provider_attempt_authorize_interrupted', code: err.code })
+ res.status(503).json({ error: 'authorize_timeout' })
+ return
+ }
sendAuthorizeError(res, err)
}
})
diff --git a/control-api/src/services/llmProviderAttemptAuthorizer.ts b/control-api/src/services/llmProviderAttemptAuthorizer.ts
index 6a7751e7c..e2d82b4e8 100644
--- a/control-api/src/services/llmProviderAttemptAuthorizer.ts
+++ b/control-api/src/services/llmProviderAttemptAuthorizer.ts
@@ -99,6 +99,8 @@ export type LlmProviderAttemptAuthorizeErrorCode =
| 'stale_generation'
| 'idempotency_conflict'
| 'provider_unavailable'
+ | 'authorize_timeout'
+ | 'authorize_aborted'
export class LlmProviderAttemptAuthorizeError extends Error {
constructor(
@@ -119,6 +121,7 @@ export type AuthorizeAttemptSuccess = {
export type LlmProviderAttemptAuthorizerDeps = {
enabled: boolean
+ signal?: AbortSignal
db: DbClient
withTransaction: typeof withTransaction
getConnection: typeof getSafeCodexSubscriptionConnection
@@ -127,7 +130,10 @@ export type LlmProviderAttemptAuthorizerDeps = {
* Live oauth-broker targets on the Host/recipe. Required: a missing
* assignment must not fall back to `request.provider` (that made D6 a no-op).
*/
- resolveAssignment: (hostRef: string) => Promise<{
+ resolveAssignment: (
+ hostRef: string,
+ signal?: AbortSignal
+ ) => Promise<{
liveBrokerProviders: string[]
liveConnectionRef: string
annotations?: Record<string, string>
@@ -355,7 +361,13 @@ async function authorizeGrokProviderAttempt(
'requestHash does not match the canonical request'
)
}
- const assignment = await resolvedDeps.resolveAssignment(caller.hostRef)
+ const assignment = await resolvedDeps
+ .resolveAssignment(caller.hostRef, resolvedDeps.signal)
+ .catch(error => {
+ resolvedDeps.signal?.throwIfAborted()
+ throw error
+ })
+ resolvedDeps.signal?.throwIfAborted()
const liveTarget = attestLiveBrokerTarget({
requestedProvider: GROK_PROVIDER,
liveBrokerProviders: assignment.liveBrokerProviders,
@@ -393,273 +405,278 @@ async function authorizeGrokProviderAttempt(
'Host has no Grok subscription assigned'
)
}
- return resolvedDeps.withTransaction(async tx => {
- const db: DbClient = tx
- const connection = await getSafeGrokSubscriptionConnection(db, connectionKey)
- if (
- !connection ||
- connection.revokedAt ||
- connection.status === 'revoked' ||
- connection.status === 'reauth_required' ||
- connection.catalogStatus === 'auth-rejected'
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'Grok subscription grant is missing, revoked, or requires re-authentication'
- )
- }
- if (
- connection.status !== 'connected' ||
- connection.catalogStatus === 'unavailable' ||
- connection.catalogStatus === 'never_synced'
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'connection_unavailable',
- 'Grok subscription catalog is not ready'
- )
- }
- const modelState = await getGrokCatalogModelState(db, connection.id, request.model)
- if (!modelState || !modelState.enabled || modelState.stale) {
- throw new LlmProviderAttemptAuthorizeError(
- 'model_not_allowed',
- 'model is disabled, stale, or absent from the Grok catalog'
- )
- }
- const expectedPolicyHash = computeGrokPolicyHash({
- model: request.model,
- catalogRevision: connection.catalogRevision,
- credentialRevision: connection.credentialRevision,
- connectionKey: connection.connectionKey,
- })
- if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'policy revision or hash does not match the current Grok catalog'
- )
- }
- const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)
- if (attemptGeneration < maxGeneration) {
- throw new LlmProviderAttemptAuthorizeError(
- 'stale_generation',
- 'attemptGeneration is older than the recorded invocation'
- )
- }
- const pluginWorkloadSdkProviderAttemptId =
- typeof body.pluginWorkloadSdkProviderAttemptId === 'string'
- ? body.pluginWorkloadSdkProviderAttemptId.trim()
- : ''
- let sdkLinkRecipe: { namespace: string; name: string } | null = null
- if (pluginWorkloadSdkProviderAttemptId) {
- if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {
+ return resolvedDeps.withTransaction(
+ async tx => {
+ resolvedDeps.signal?.throwIfAborted()
+ const db: DbClient = tx
+ const connection = await getSafeGrokSubscriptionConnection(db, connectionKey)
+ if (
+ !connection ||
+ connection.revokedAt ||
+ connection.status === 'revoked' ||
+ connection.status === 'reauth_required' ||
+ connection.catalogStatus === 'auth-rejected'
+ ) {
throw new LlmProviderAttemptAuthorizeError(
- 'invalid_request',
- 'pluginWorkloadSdkProviderAttemptId must be a UUID'
+ 'no_grant',
+ 'Grok subscription grant is missing, revoked, or requires re-authentication'
)
}
- if (caller.callerKind === 'host') {
+ if (
+ connection.status !== 'connected' ||
+ connection.catalogStatus === 'unavailable' ||
+ connection.catalogStatus === 'never_synced'
+ ) {
throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'host Grok chat cannot bind a Plugin Workload SDK provider attempt'
+ 'connection_unavailable',
+ 'Grok subscription catalog is not ready'
)
}
- if (!caller.recipeNamespace || !caller.recipeName) {
+ const modelState = await getGrokCatalogModelState(db, connection.id, request.model)
+ if (!modelState || !modelState.enabled || modelState.stale) {
throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
+ 'model_not_allowed',
+ 'model is disabled, stale, or absent from the Grok catalog'
)
}
- sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }
- }
- if (sdkLinkRecipe) {
- await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)
- }
- const presentedReservationId =
- typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''
- let budgetReservationId = presentedReservationId || 'unbudgeted'
- if (presentedReservationId) {
- const active = await resolvedDeps.getActiveReservation(db, {
- reservationId: presentedReservationId,
- hostRef: caller.hostRef,
+ const expectedPolicyHash = computeGrokPolicyHash({
+ model: request.model,
+ catalogRevision: connection.catalogRevision,
+ credentialRevision: connection.credentialRevision,
+ connectionKey: connection.connectionKey,
})
- if (!active) {
+ if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {
throw new LlmProviderAttemptAuthorizeError(
- 'budget_denied',
- 'budget reservation is missing or expired'
+ 'no_grant',
+ 'policy revision or hash does not match the current Grok catalog'
)
}
- } else {
- const budget = await resolvedDeps.evaluateBudget(
- {
- host_ref: caller.hostRef,
- context_ref: null,
- team_id: null,
- user_id: null,
- provider: GROK_PROVIDER,
- model: request.model,
- llm_secret_name: null,
- source_kind: 'channel',
- recipe_name: caller.recipeName,
- cron_job_id: null,
- task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,
- },
- db,
- { connect: async () => tx as never },
- {
- requiredUnit: 'tokens',
- transactionClient: tx,
- reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,
- }
- )
- if (!budget.allowed) {
+ const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)
+ if (attemptGeneration < maxGeneration) {
throw new LlmProviderAttemptAuthorizeError(
- 'budget_denied',
- 'token budget denied this attempt'
+ 'stale_generation',
+ 'attemptGeneration is older than the recorded invocation'
)
}
- budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'
- }
- const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''
- let reservedSdkAttemptToPromote: {
- id: string
- invocationId: string
- recipeNamespace: string
- recipeName: string
- attemptGeneration: number
- attemptIndex: number
- model: string
- targetRef: string
- } | null = null
- if (pluginWorkloadSdkProviderAttemptId) {
- const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(
- pluginWorkloadSdkProviderAttemptId,
- db
- )
- const spendExists = sdkAttempt
- ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)
- : false
- if (
- !sdkAttempt ||
- spendExists ||
- sdkAttempt.invocationId !== invocationId ||
- sdkAttempt.attemptGeneration !== attemptGeneration ||
- sdkAttempt.attemptIndex !== providerAttemptIndex ||
- sdkAttempt.recipeNamespace !== caller.recipeNamespace ||
- sdkAttempt.recipeName !== caller.recipeName ||
- sdkAttempt.provider !== GROK_PROVIDER ||
- sdkAttempt.model !== request.model ||
- !sdkAttempt.targetRef.trim() ||
- (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||
- !['reserved', 'in_progress'].includes(sdkAttempt.status)
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
- )
+ const pluginWorkloadSdkProviderAttemptId =
+ typeof body.pluginWorkloadSdkProviderAttemptId === 'string'
+ ? body.pluginWorkloadSdkProviderAttemptId.trim()
+ : ''
+ let sdkLinkRecipe: { namespace: string; name: string } | null = null
+ if (pluginWorkloadSdkProviderAttemptId) {
+ if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'invalid_request',
+ 'pluginWorkloadSdkProviderAttemptId must be a UUID'
+ )
+ }
+ if (caller.callerKind === 'host') {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'host Grok chat cannot bind a Plugin Workload SDK provider attempt'
+ )
+ }
+ if (!caller.recipeNamespace || !caller.recipeName) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
+ )
+ }
+ sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }
+ }
+ if (sdkLinkRecipe) {
+ await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)
}
- if (sdkAttempt.status === 'reserved') {
- reservedSdkAttemptToPromote = {
- id: sdkAttempt.id,
- invocationId: sdkAttempt.invocationId,
- recipeNamespace: sdkAttempt.recipeNamespace,
- recipeName: sdkAttempt.recipeName,
- attemptGeneration: sdkAttempt.attemptGeneration,
- attemptIndex: sdkAttempt.attemptIndex,
- model: sdkAttempt.model,
- targetRef: sdkAttempt.targetRef,
+ const presentedReservationId =
+ typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''
+ let budgetReservationId = presentedReservationId || 'unbudgeted'
+ if (presentedReservationId) {
+ const active = await resolvedDeps.getActiveReservation(db, {
+ reservationId: presentedReservationId,
+ hostRef: caller.hostRef,
+ })
+ if (!active) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'budget_denied',
+ 'budget reservation is missing or expired'
+ )
}
+ } else {
+ const budget = await resolvedDeps.evaluateBudget(
+ {
+ host_ref: caller.hostRef,
+ context_ref: null,
+ team_id: null,
+ user_id: null,
+ provider: GROK_PROVIDER,
+ model: request.model,
+ llm_secret_name: null,
+ source_kind: 'channel',
+ recipe_name: caller.recipeName,
+ cron_job_id: null,
+ task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,
+ },
+ db,
+ { connect: async () => tx as never },
+ {
+ requiredUnit: 'tokens',
+ transactionClient: tx,
+ reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,
+ }
+ )
+ if (!budget.allowed) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'budget_denied',
+ 'token budget denied this attempt'
+ )
+ }
+ budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'
}
- }
- try {
- const attempt = await resolvedDeps.insertAttempt(db, {
- callerKind: caller.callerKind,
- hostRef: caller.hostRef,
- recipeNamespace: caller.recipeNamespace,
- recipeName: caller.recipeName,
- invocationId,
- attemptGeneration,
- providerAttemptIndex,
- provider: GROK_PROVIDER,
- model: request.model,
- requestHash,
- policyRevision,
- policyHash,
- budgetReservationId,
- connectionRevision: connection.credentialRevision,
- connectionId: connection.id,
- ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),
- })
- if (reservedSdkAttemptToPromote) {
- const promoted = await promoteReservedOauthBrokerProviderAttempt(
- { ...reservedSdkAttemptToPromote, provider: GROK_PROVIDER },
+ const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''
+ let reservedSdkAttemptToPromote: {
+ id: string
+ invocationId: string
+ recipeNamespace: string
+ recipeName: string
+ attemptGeneration: number
+ attemptIndex: number
+ model: string
+ targetRef: string
+ } | null = null
+ if (pluginWorkloadSdkProviderAttemptId) {
+ const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(
+ pluginWorkloadSdkProviderAttemptId,
db
)
- if (!promoted) {
+ const spendExists = sdkAttempt
+ ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)
+ : false
+ if (
+ !sdkAttempt ||
+ spendExists ||
+ sdkAttempt.invocationId !== invocationId ||
+ sdkAttempt.attemptGeneration !== attemptGeneration ||
+ sdkAttempt.attemptIndex !== providerAttemptIndex ||
+ sdkAttempt.recipeNamespace !== caller.recipeNamespace ||
+ sdkAttempt.recipeName !== caller.recipeName ||
+ sdkAttempt.provider !== GROK_PROVIDER ||
+ sdkAttempt.model !== request.model ||
+ !sdkAttempt.targetRef.trim() ||
+ (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||
+ !['reserved', 'in_progress'].includes(sdkAttempt.status)
+ ) {
throw new LlmProviderAttemptAuthorizeError(
'no_grant',
- 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Grok authorize'
+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
)
}
+ if (sdkAttempt.status === 'reserved') {
+ reservedSdkAttemptToPromote = {
+ id: sdkAttempt.id,
+ invocationId: sdkAttempt.invocationId,
+ recipeNamespace: sdkAttempt.recipeNamespace,
+ recipeName: sdkAttempt.recipeName,
+ attemptGeneration: sdkAttempt.attemptGeneration,
+ attemptIndex: sdkAttempt.attemptIndex,
+ model: sdkAttempt.model,
+ targetRef: sdkAttempt.targetRef,
+ }
+ }
}
- const issued = await issueRegisteredGrokExecutionTicket(db, {
- sub: claims.sub,
- hostRef: caller.hostRef,
- recipeNamespace: caller.recipeNamespace ?? undefined,
- recipeName: caller.recipeName ?? undefined,
- invocationId,
- attemptGeneration,
- providerAttemptId: attempt.id,
- providerAttemptIndex,
- model: request.model,
- requestHash,
- policyRevision,
- policyHash,
- budgetReservationId,
- connectionRevision: connection.credentialRevision,
- connectionId: connection.id,
- })
- if (request.schemaVersion === 'grok-completion-request.v2') {
- // Measure the exact proxy envelope while the attempt, ticket and any
- // new reservation are still transactional, as the Codex path does.
- const envelope = buildGrokProxyEnvelope({
- executionTicket: issued.executionTicket,
+ try {
+ const attempt = await resolvedDeps.insertAttempt(db, {
+ callerKind: caller.callerKind,
+ hostRef: caller.hostRef,
+ recipeNamespace: caller.recipeNamespace,
+ recipeName: caller.recipeName,
+ invocationId,
+ attemptGeneration,
+ providerAttemptIndex,
+ provider: GROK_PROVIDER,
+ model: request.model,
requestHash,
- request,
+ policyRevision,
+ policyHash,
+ budgetReservationId,
+ connectionRevision: connection.credentialRevision,
+ connectionId: connection.id,
+ ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),
})
- if (!envelope.ok) {
- throw new LlmProviderAttemptAuthorizeError(
- envelope.code === 'limit' && envelope.kind === 'size'
- ? 'payload_too_large'
- : 'invalid_request',
- envelope.message
+ if (reservedSdkAttemptToPromote) {
+ const promoted = await promoteReservedOauthBrokerProviderAttempt(
+ { ...reservedSdkAttemptToPromote, provider: GROK_PROVIDER },
+ db
)
+ if (!promoted) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Grok authorize'
+ )
+ }
}
- }
- log.info(
- {
- event: 'grok_attempt_authorized',
- providerAttemptId: attempt.id,
+ const issued = await issueRegisteredGrokExecutionTicket(db, {
+ sub: claims.sub,
hostRef: caller.hostRef,
+ recipeNamespace: caller.recipeNamespace ?? undefined,
+ recipeName: caller.recipeName ?? undefined,
+ invocationId,
+ attemptGeneration,
+ providerAttemptId: attempt.id,
+ providerAttemptIndex,
model: request.model,
- },
- 'authorized Grok provider attempt'
- )
- return {
- providerAttemptId: attempt.id,
- requestHash,
- executionTicket: issued.executionTicket,
- expiresAt: issued.expiresAt.toISOString(),
- }
- } catch (err) {
- const code = (err as { code?: string }).code
- if (code === '23505') {
- throw new LlmProviderAttemptAuthorizeError(
- 'idempotency_conflict',
- 'an attempt with this invocation binding already exists'
+ requestHash,
+ policyRevision,
+ policyHash,
+ budgetReservationId,
+ connectionRevision: connection.credentialRevision,
+ connectionId: connection.id,
+ })
+ if (request.schemaVersion === 'grok-completion-request.v2') {
+ // Measure the exact proxy envelope while the attempt, ticket and any
+ // new reservation are still transactional, as the Codex path does.
+ const envelope = buildGrokProxyEnvelope({
+ executionTicket: issued.executionTicket,
+ requestHash,
+ request,
+ })
+ if (!envelope.ok) {
+ throw new LlmProviderAttemptAuthorizeError(
+ envelope.code === 'limit' && envelope.kind === 'size'
+ ? 'payload_too_large'
+ : 'invalid_request',
+ envelope.message
+ )
+ }
+ }
+ log.info(
+ {
+ event: 'grok_attempt_authorized',
+ providerAttemptId: attempt.id,
+ hostRef: caller.hostRef,
+ model: request.model,
+ },
+ 'authorized Grok provider attempt'
)
+ return {
+ providerAttemptId: attempt.id,
+ requestHash,
+ executionTicket: issued.executionTicket,
+ expiresAt: issued.expiresAt.toISOString(),
+ }
+ } catch (err) {
+ const code = (err as { code?: string }).code
+ if (code === '23505') {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'idempotency_conflict',
+ 'an attempt with this invocation binding already exists'
+ )
+ }
+ throw err
}
- throw err
- }
- })
+ },
+ undefined,
+ { signal: resolvedDeps.signal }
+ )
}
export async function authorizeLlmProviderAttempt(
@@ -668,6 +685,7 @@ export async function authorizeLlmProviderAttempt(
deps: LlmProviderAttemptAuthorizerDeps | Partial<LlmProviderAttemptAuthorizerDeps> = defaultDeps()
): Promise<AuthorizeAttemptSuccess> {
const resolvedDeps: LlmProviderAttemptAuthorizerDeps = { ...defaultDeps(), ...deps }
+ resolvedDeps.signal?.throwIfAborted()
if (!isPlainObject(body)) {
throw new LlmProviderAttemptAuthorizeError('invalid_request', 'body must be an object')
}
@@ -772,7 +790,13 @@ export async function authorizeLlmProviderAttempt(
)
}
- const assignment = await resolvedDeps.resolveAssignment(caller.hostRef)
+ const assignment = await resolvedDeps
+ .resolveAssignment(caller.hostRef, resolvedDeps.signal)
+ .catch(error => {
+ resolvedDeps.signal?.throwIfAborted()
+ throw error
+ })
+ resolvedDeps.signal?.throwIfAborted()
const liveTarget = attestLiveBrokerTarget({
requestedProvider: request.provider,
liveBrokerProviders: assignment.liveBrokerProviders,
@@ -806,304 +830,309 @@ export async function authorizeLlmProviderAttempt(
'Host has no ChatGPT subscription assigned'
)
}
- return resolvedDeps.withTransaction(async tx => {
- const db: DbClient = tx
- const connection = await resolvedDeps.getConnection(db, connectionKey)
- // Terminal grant failures must not look like a retryable proxy outage.
- // Missing, revoked, and auth-rejected rows require a new grant or
- // operator re-auth — failover onto another provider is the wrong recovery.
- if (
- !connection ||
- connection.revokedAt ||
- connection.status === 'revoked' ||
- connection.status === 'reauth_required' ||
- connection.catalogStatus === 'auth-rejected'
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'Codex subscription grant is missing, revoked, or requires re-authentication'
- )
- }
- if (
- connection.status !== 'connected' ||
- connection.catalogStatus === 'unavailable' ||
- connection.catalogStatus === 'never_synced'
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'connection_unavailable',
- 'Codex subscription catalog is not ready'
- )
- }
-
- const modelState = await resolvedDeps.getModelState(db, connection.id, request.model)
- if (!modelState || !modelState.enabled || modelState.stale) {
- throw new LlmProviderAttemptAuthorizeError(
- 'model_not_allowed',
- 'model is disabled, stale, or absent from the Codex catalog'
- )
- }
-
- const expectedPolicyHash = computeCodexPolicyHash({
- model: request.model,
- catalogRevision: connection.catalogRevision,
- credentialRevision: connection.credentialRevision,
- connectionKey: connection.connectionKey,
- })
- if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'policy revision or hash does not match the current Codex catalog'
- )
- }
-
- const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)
- if (attemptGeneration < maxGeneration) {
- throw new LlmProviderAttemptAuthorizeError(
- 'stale_generation',
- 'attemptGeneration is older than the recorded invocation'
- )
- }
-
- const pluginWorkloadSdkProviderAttemptId =
- typeof body.pluginWorkloadSdkProviderAttemptId === 'string'
- ? body.pluginWorkloadSdkProviderAttemptId.trim()
- : ''
- let sdkLinkRecipe: { namespace: string; name: string } | null = null
- if (pluginWorkloadSdkProviderAttemptId) {
- if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {
+ return resolvedDeps.withTransaction(
+ async tx => {
+ resolvedDeps.signal?.throwIfAborted()
+ const db: DbClient = tx
+ const connection = await resolvedDeps.getConnection(db, connectionKey)
+ // Terminal grant failures must not look like a retryable proxy outage.
+ // Missing, revoked, and auth-rejected rows require a new grant or
+ // operator re-auth — failover onto another provider is the wrong recovery.
+ if (
+ !connection ||
+ connection.revokedAt ||
+ connection.status === 'revoked' ||
+ connection.status === 'reauth_required' ||
+ connection.catalogStatus === 'auth-rejected'
+ ) {
throw new LlmProviderAttemptAuthorizeError(
- 'invalid_request',
- 'pluginWorkloadSdkProviderAttemptId must be a UUID'
+ 'no_grant',
+ 'Codex subscription grant is missing, revoked, or requires re-authentication'
)
}
- if (caller.callerKind === 'host') {
+ if (
+ connection.status !== 'connected' ||
+ connection.catalogStatus === 'unavailable' ||
+ connection.catalogStatus === 'never_synced'
+ ) {
throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'host Codex chat cannot bind a Plugin Workload SDK provider attempt'
+ 'connection_unavailable',
+ 'Codex subscription catalog is not ready'
)
}
- if (!caller.recipeNamespace || !caller.recipeName) {
+
+ const modelState = await resolvedDeps.getModelState(db, connection.id, request.model)
+ if (!modelState || !modelState.enabled || modelState.stale) {
throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
+ 'model_not_allowed',
+ 'model is disabled, stale, or absent from the Codex catalog'
)
}
- sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }
- }
- // Recipe authorize and promptBridge finalize share this advisory. Take it
- // before budget reservation work so concurrent Codex + fallback cannot
- // deadlock (budget rows then advisory vs advisory then spend/invocation).
- //
- // Only an SDK-linked authorize goes on to take plugin_workload_sdk_*
- // rows FOR UPDATE, which is the order finalize follows after this advisory.
- // The workflow lane touches neither, so taking it for every recipe caller
- // serialized that lane per recipe for no ordering benefit.
- if (sdkLinkRecipe) {
- await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)
- }
-
- const presentedReservationId =
- typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''
- let budgetReservationId = presentedReservationId || 'unbudgeted'
- if (presentedReservationId) {
- const active = await resolvedDeps.getActiveReservation(db, {
- reservationId: presentedReservationId,
- hostRef: caller.hostRef,
+ const expectedPolicyHash = computeCodexPolicyHash({
+ model: request.model,
+ catalogRevision: connection.catalogRevision,
+ credentialRevision: connection.credentialRevision,
+ connectionKey: connection.connectionKey,
})
- if (!active) {
+ if (policyRevision !== connection.catalogRevision || policyHash !== expectedPolicyHash) {
throw new LlmProviderAttemptAuthorizeError(
- 'budget_denied',
- 'budget reservation is missing or expired'
+ 'no_grant',
+ 'policy revision or hash does not match the current Codex catalog'
)
}
- } else {
- const budget = await resolvedDeps.evaluateBudget(
- {
- host_ref: caller.hostRef,
- context_ref: null,
- team_id: null,
- user_id: null,
- provider: PROVIDER,
- model: request.model,
- llm_secret_name: null,
- source_kind: 'channel',
- recipe_name: caller.recipeName,
- cron_job_id: null,
- task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,
- },
- db,
- { connect: async () => tx as never },
- {
- requiredUnit: 'tokens',
- transactionClient: tx,
- reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
- }
- )
- if (!budget.allowed) {
+
+ const maxGeneration = await resolvedDeps.getMaxGeneration(db, invocationId)
+ if (attemptGeneration < maxGeneration) {
throw new LlmProviderAttemptAuthorizeError(
- 'budget_denied',
- budget.reason === 'cost_unit_rejected'
- ? 'Codex attempts reject cost-unit budgets'
- : 'token budget denied this attempt'
+ 'stale_generation',
+ 'attemptGeneration is older than the recorded invocation'
)
}
- budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'
- }
- const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''
- if (presentedTargetRef !== '' && !isBoundedId(presentedTargetRef)) {
- throw new LlmProviderAttemptAuthorizeError('invalid_request', 'targetRef is invalid')
- }
- let reservedSdkAttemptToPromote: {
- id: string
- invocationId: string
- recipeNamespace: string
- recipeName: string
- attemptGeneration: number
- attemptIndex: number
- model: string
- targetRef: string
- } | null = null
- if (pluginWorkloadSdkProviderAttemptId) {
- const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(
- pluginWorkloadSdkProviderAttemptId,
- db
- )
- const spendExists = sdkAttempt
- ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)
- : false
- if (
- !sdkAttempt ||
- spendExists ||
- sdkAttempt.invocationId !== invocationId ||
- sdkAttempt.attemptGeneration !== attemptGeneration ||
- sdkAttempt.attemptIndex !== providerAttemptIndex ||
- sdkAttempt.recipeNamespace !== caller.recipeNamespace ||
- sdkAttempt.recipeName !== caller.recipeName ||
- sdkAttempt.provider !== PROVIDER ||
- sdkAttempt.model !== request.model ||
- !sdkAttempt.targetRef.trim() ||
- (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||
- !['reserved', 'in_progress'].includes(sdkAttempt.status)
- ) {
- throw new LlmProviderAttemptAuthorizeError(
- 'no_grant',
- 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
- )
+ const pluginWorkloadSdkProviderAttemptId =
+ typeof body.pluginWorkloadSdkProviderAttemptId === 'string'
+ ? body.pluginWorkloadSdkProviderAttemptId.trim()
+ : ''
+ let sdkLinkRecipe: { namespace: string; name: string } | null = null
+ if (pluginWorkloadSdkProviderAttemptId) {
+ if (!UUID_RE.test(pluginWorkloadSdkProviderAttemptId)) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'invalid_request',
+ 'pluginWorkloadSdkProviderAttemptId must be a UUID'
+ )
+ }
+ if (caller.callerKind === 'host') {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'host Codex chat cannot bind a Plugin Workload SDK provider attempt'
+ )
+ }
+ if (!caller.recipeNamespace || !caller.recipeName) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
+ )
+ }
+ sdkLinkRecipe = { namespace: caller.recipeNamespace, name: caller.recipeName }
+ }
+
+ // Recipe authorize and promptBridge finalize share this advisory. Take it
+ // before budget reservation work so concurrent Codex + fallback cannot
+ // deadlock (budget rows then advisory vs advisory then spend/invocation).
+ //
+ // Only an SDK-linked authorize goes on to take plugin_workload_sdk_*
+ // rows FOR UPDATE, which is the order finalize follows after this advisory.
+ // The workflow lane touches neither, so taking it for every recipe caller
+ // serialized that lane per recipe for no ordering benefit.
+ if (sdkLinkRecipe) {
+ await lockPluginWorkloadSdkRecipe(db, sdkLinkRecipe.namespace, sdkLinkRecipe.name)
}
- if (sdkAttempt.status === 'reserved') {
- reservedSdkAttemptToPromote = {
- id: sdkAttempt.id,
- invocationId: sdkAttempt.invocationId,
- recipeNamespace: sdkAttempt.recipeNamespace,
- recipeName: sdkAttempt.recipeName,
- attemptGeneration: sdkAttempt.attemptGeneration,
- attemptIndex: sdkAttempt.attemptIndex,
- model: sdkAttempt.model,
- targetRef: sdkAttempt.targetRef,
+
+ const presentedReservationId =
+ typeof body.budgetReservationId === 'string' ? body.budgetReservationId.trim() : ''
+ let budgetReservationId = presentedReservationId || 'unbudgeted'
+ if (presentedReservationId) {
+ const active = await resolvedDeps.getActiveReservation(db, {
+ reservationId: presentedReservationId,
+ hostRef: caller.hostRef,
+ })
+ if (!active) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'budget_denied',
+ 'budget reservation is missing or expired'
+ )
}
+ } else {
+ const budget = await resolvedDeps.evaluateBudget(
+ {
+ host_ref: caller.hostRef,
+ context_ref: null,
+ team_id: null,
+ user_id: null,
+ provider: PROVIDER,
+ model: request.model,
+ llm_secret_name: null,
+ source_kind: 'channel',
+ recipe_name: caller.recipeName,
+ cron_job_id: null,
+ task_ref: `${invocationId}:${attemptGeneration}:${providerAttemptIndex}`,
+ },
+ db,
+ { connect: async () => tx as never },
+ {
+ requiredUnit: 'tokens',
+ transactionClient: tx,
+ reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ }
+ )
+ if (!budget.allowed) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'budget_denied',
+ budget.reason === 'cost_unit_rejected'
+ ? 'Codex attempts reject cost-unit budgets'
+ : 'token budget denied this attempt'
+ )
+ }
+ budgetReservationId = budget.reservationIds?.[0] ?? 'unbudgeted'
}
- }
- try {
- const attempt = await resolvedDeps.insertAttempt(db, {
- callerKind: caller.callerKind,
- hostRef: caller.hostRef,
- recipeNamespace: caller.recipeNamespace,
- recipeName: caller.recipeName,
- invocationId,
- attemptGeneration,
- providerAttemptIndex,
- model: request.model,
- requestHash,
- policyRevision,
- policyHash,
- budgetReservationId,
- connectionRevision: connection.credentialRevision,
- connectionId: connection.id,
- ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),
- })
- if (reservedSdkAttemptToPromote) {
- const promoted = await promoteReservedOauthBrokerProviderAttempt(
- { ...reservedSdkAttemptToPromote, provider: PROVIDER },
+ const presentedTargetRef = typeof body.targetRef === 'string' ? body.targetRef.trim() : ''
+ if (presentedTargetRef !== '' && !isBoundedId(presentedTargetRef)) {
+ throw new LlmProviderAttemptAuthorizeError('invalid_request', 'targetRef is invalid')
+ }
+ let reservedSdkAttemptToPromote: {
+ id: string
+ invocationId: string
+ recipeNamespace: string
+ recipeName: string
+ attemptGeneration: number
+ attemptIndex: number
+ model: string
+ targetRef: string
+ } | null = null
+ if (pluginWorkloadSdkProviderAttemptId) {
+ const sdkAttempt = await getPluginWorkloadSdkProviderAttemptForUpdate(
+ pluginWorkloadSdkProviderAttemptId,
db
)
- if (!promoted) {
+ const spendExists = sdkAttempt
+ ? await pluginWorkloadSdkSpendOutcomeExists(sdkAttempt.id, db)
+ : false
+ if (
+ !sdkAttempt ||
+ spendExists ||
+ sdkAttempt.invocationId !== invocationId ||
+ sdkAttempt.attemptGeneration !== attemptGeneration ||
+ sdkAttempt.attemptIndex !== providerAttemptIndex ||
+ sdkAttempt.recipeNamespace !== caller.recipeNamespace ||
+ sdkAttempt.recipeName !== caller.recipeName ||
+ sdkAttempt.provider !== PROVIDER ||
+ sdkAttempt.model !== request.model ||
+ !sdkAttempt.targetRef.trim() ||
+ (presentedTargetRef !== '' && presentedTargetRef !== sdkAttempt.targetRef) ||
+ !['reserved', 'in_progress'].includes(sdkAttempt.status)
+ ) {
throw new LlmProviderAttemptAuthorizeError(
'no_grant',
- 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Codex authorize'
+ 'pluginWorkloadSdkProviderAttemptId does not match the reserved SDK attempt'
+ )
+ }
+ if (sdkAttempt.status === 'reserved') {
+ reservedSdkAttemptToPromote = {
+ id: sdkAttempt.id,
+ invocationId: sdkAttempt.invocationId,
+ recipeNamespace: sdkAttempt.recipeNamespace,
+ recipeName: sdkAttempt.recipeName,
+ attemptGeneration: sdkAttempt.attemptGeneration,
+ attemptIndex: sdkAttempt.attemptIndex,
+ model: sdkAttempt.model,
+ targetRef: sdkAttempt.targetRef,
+ }
+ }
+ }
+
+ try {
+ const attempt = await resolvedDeps.insertAttempt(db, {
+ callerKind: caller.callerKind,
+ hostRef: caller.hostRef,
+ recipeNamespace: caller.recipeNamespace,
+ recipeName: caller.recipeName,
+ invocationId,
+ attemptGeneration,
+ providerAttemptIndex,
+ model: request.model,
+ requestHash,
+ policyRevision,
+ policyHash,
+ budgetReservationId,
+ connectionRevision: connection.credentialRevision,
+ connectionId: connection.id,
+ ...(pluginWorkloadSdkProviderAttemptId ? { pluginWorkloadSdkProviderAttemptId } : {}),
+ })
+ if (reservedSdkAttemptToPromote) {
+ const promoted = await promoteReservedOauthBrokerProviderAttempt(
+ { ...reservedSdkAttemptToPromote, provider: PROVIDER },
+ db
+ )
+ if (!promoted) {
+ throw new LlmProviderAttemptAuthorizeError(
+ 'no_grant',
+ 'pluginWorkloadSdkProviderAttemptId is no longer reserved for Codex authorize'
+ )
+ }
+ log.info(
+ {
+ event: 'sdk_oauth_attempt_promoted',
+ pluginWorkloadSdkProviderAttemptId: reservedSdkAttemptToPromote.id,
+ providerAttemptId: attempt.id,
+ credentialJtiPresent: false,
+ },
+ 'promoted reserved SDK oauth attempt after Codex authorize-link'
)
}
+ const issued = await resolvedDeps.issueTicket(db, {
+ sub: claims.sub,
+ hostRef: caller.hostRef,
+ recipeNamespace: caller.recipeNamespace ?? undefined,
+ recipeName: caller.recipeName ?? undefined,
+ invocationId,
+ attemptGeneration,
+ providerAttemptId: attempt.id,
+ providerAttemptIndex,
+ model: request.model,
+ requestHash,
+ policyRevision,
+ policyHash,
+ budgetReservationId,
+ connectionRevision: connection.credentialRevision,
+ connectionId: connection.id,
+ })
+ if (request.schemaVersion === 'codex-completion-request.v2') {
+ // Measure the exact proxy envelope while the attempt, ticket and any
+ // new reservation are still transactional. Pre-redeem failures cannot
+ // use the ordinary receipt-based finalizer.
+ const envelope = buildCodexProxyEnvelope({
+ executionTicket: issued.executionTicket,
+ requestHash,
+ request,
+ })
+ if (!envelope.ok) {
+ throw new LlmProviderAttemptAuthorizeError(
+ envelope.code === 'limit' && envelope.kind === 'size'
+ ? 'payload_too_large'
+ : 'invalid_request',
+ envelope.message
+ )
+ }
+ }
log.info(
{
- event: 'sdk_oauth_attempt_promoted',
- pluginWorkloadSdkProviderAttemptId: reservedSdkAttemptToPromote.id,
+ event: 'codex_attempt_authorized',
providerAttemptId: attempt.id,
- credentialJtiPresent: false,
+ hostRef: caller.hostRef,
+ model: request.model,
},
- 'promoted reserved SDK oauth attempt after Codex authorize-link'
+ 'authorized Codex provider attempt'
)
- }
- const issued = await resolvedDeps.issueTicket(db, {
- sub: claims.sub,
- hostRef: caller.hostRef,
- recipeNamespace: caller.recipeNamespace ?? undefined,
- recipeName: caller.recipeName ?? undefined,
- invocationId,
- attemptGeneration,
- providerAttemptId: attempt.id,
- providerAttemptIndex,
- model: request.model,
- requestHash,
- policyRevision,
- policyHash,
- budgetReservationId,
- connectionRevision: connection.credentialRevision,
- connectionId: connection.id,
- })
- if (request.schemaVersion === 'codex-completion-request.v2') {
- // Measure the exact proxy envelope while the attempt, ticket and any
- // new reservation are still transactional. Pre-redeem failures cannot
- // use the ordinary receipt-based finalizer.
- const envelope = buildCodexProxyEnvelope({
- executionTicket: issued.executionTicket,
+ return {
+ providerAttemptId: attempt.id,
requestHash,
- request,
- })
- if (!envelope.ok) {
+ executionTicket: issued.executionTicket,
+ expiresAt: issued.expiresAt.toISOString(),
+ }
+ } catch (err) {
+ const code = (err as { code?: string }).code
+ if (code === '23505') {
throw new LlmProviderAttemptAuthorizeError(
- envelope.code === 'limit' && envelope.kind === 'size'
- ? 'payload_too_large'
- : 'invalid_request',
- envelope.message
+ 'idempotency_conflict',
+ 'an attempt with this invocation binding already exists'
)
}
+ throw err
}
- log.info(
- {
- event: 'codex_attempt_authorized',
- providerAttemptId: attempt.id,
- hostRef: caller.hostRef,
- model: request.model,
- },
- 'authorized Codex provider attempt'
- )
- return {
- providerAttemptId: attempt.id,
- requestHash,
- executionTicket: issued.executionTicket,
- expiresAt: issued.expiresAt.toISOString(),
- }
- } catch (err) {
- const code = (err as { code?: string }).code
- if (code === '23505') {
- throw new LlmProviderAttemptAuthorizeError(
- 'idempotency_conflict',
- 'an attempt with this invocation binding already exists'
- )
- }
- throw err
- }
- })
+ },
+ undefined,
+ { signal: resolvedDeps.signal }
+ )
}
diff --git a/control-api/src/services/resourceService.ts b/control-api/src/services/resourceService.ts
index 4d29fd161..882907f1d 100644
--- a/control-api/src/services/resourceService.ts
+++ b/control-api/src/services/resourceService.ts
@@ -137,6 +137,25 @@ export function mergeAnnotationsForReplace(
return merged
}
+/** Preserve the kubeconfig client's server, auth and TLS middleware per call. */
+function cancellationOptions(signal?: AbortSignal): k8s.ConfigurationOptions | undefined {
+ signal?.throwIfAborted()
+ if (!signal) return undefined
+ const cancellation = k8s.createConfiguration({
+ promiseMiddleware: [
+ {
+ pre: async context => {
+ signal.throwIfAborted()
+ context.setSignal(signal)
+ return context
+ },
+ post: async response => response,
+ },
+ ],
+ })
+ return { middleware: cancellation.middleware, middlewareMergeStrategy: 'append' }
+}
+
export class ResourceService {
// Allowed namespaces derived from config at construction time.
private readonly allowedNamespaces: Set<string>
@@ -177,19 +196,29 @@ export class ResourceService {
return Array.from(namespaces)
}
- async listResource(plural: ClerumResourceType, namespace?: string): Promise<unknown[]> {
+ async listResource(
+ plural: ClerumResourceType,
+ namespace?: string,
+ signal?: AbortSignal
+ ): Promise<unknown[]> {
+ signal?.throwIfAborted()
if (namespace === '*') {
const items: unknown[] = []
for (const nsName of this.listNamespacesForPlural(plural)) {
try {
- const namespaced = (await this.customApi.listNamespacedCustomObject({
- group: CLERUM_GROUP,
- version: CLERUM_VERSION,
- namespace: nsName,
- plural,
- })) as unknown as ResourceListResponse
+ const namespaced = (await this.customApi.listNamespacedCustomObject(
+ {
+ group: CLERUM_GROUP,
+ version: CLERUM_VERSION,
+ namespace: nsName,
+ plural,
+ },
+ cancellationOptions(signal)
+ )) as unknown as ResourceListResponse
+ signal?.throwIfAborted()
items.push(...(namespaced.items || []))
} catch {
+ signal?.throwIfAborted()
// Ignore namespaces where the resource is absent/inaccessible.
}
}
@@ -197,30 +226,43 @@ export class ResourceService {
}
const resolvedNamespace = this.resolveNamespace(plural, namespace)
- const res = (await this.customApi.listNamespacedCustomObject({
- group: CLERUM_GROUP,
- version: CLERUM_VERSION,
- namespace: resolvedNamespace,
- plural,
- })) as unknown as ResourceListResponse
+ const res = (await this.customApi.listNamespacedCustomObject(
+ {
+ group: CLERUM_GROUP,
+ version: CLERUM_VERSION,
+ namespace: resolvedNamespace,
+ plural,
+ },
+ cancellationOptions(signal)
+ )) as unknown as ResourceListResponse
+ signal?.throwIfAborted()
return res.items || []
}
async getResource(
plural: ClerumResourceType,
name: string,
- namespace?: string
+ namespace?: string,
+ signal?: AbortSignal
): Promise<unknown> {
+ signal?.throwIfAborted()
const ns = this.resolveNamespace(plural, namespace)
try {
- return await this.customApi.getNamespacedCustomObject({
- group: CLERUM_GROUP,
- version: CLERUM_VERSION,
- namespace: ns,
- plural,
- name,
- })
+ const resource = await this.customApi.getNamespacedCustomObject(
+ {
+ group: CLERUM_GROUP,
+ version: CLERUM_VERSION,
+ namespace: ns,
+ plural,
+ name,
+ },
+ cancellationOptions(signal)
+ )
+ signal?.throwIfAborted()
+ return resource
} catch (err) {
+ // node-fetch reports AbortError; retain the owner's typed reason instead.
+ signal?.throwIfAborted()
const status = extractK8sStatus(err)
if (namespace && namespace.trim()) {
if (status === 404) {
@@ -231,7 +273,7 @@ export class ResourceService {
if (status !== 404 && status !== null) {
throw err
}
- const items = (await this.listResource(plural, '*')) as Array<{
+ const items = (await this.listResource(plural, '*', signal)) as Array<{
metadata?: { name?: string }
}>
const match = items.find(i => i.metadata?.name === name)
diff --git a/control-api/test/helpers.realPostgresCancellation.test.ts b/control-api/test/helpers.realPostgresCancellation.test.ts
new file mode 100644
index 000000000..a59539726
--- /dev/null
+++ b/control-api/test/helpers.realPostgresCancellation.test.ts
@@ -0,0 +1,92 @@
+import { expect, it } from 'vitest'
+import { once } from 'node:events'
+import { type Socket, createConnection, createServer } from 'node:net'
+import { setImmediate as nextTurn } from 'node:timers/promises'
+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+
+async function observed(check: () => boolean): Promise<void> {
+ const deadline = Date.now() + 1_000
+ while (!check()) {
+ if (Date.now() >= deadline) throw new Error('proxy lifecycle observation did not arrive')
+ await nextTurn()
+ }
+}
+
+function query(sql: string): Buffer {
+ const body = Buffer.from(`${sql}\0`)
+ const header = Buffer.alloc(5)
+ header[0] = 'Q'.charCodeAt(0)
+ header.writeInt32BE(body.length + 4, 1)
+ return Buffer.concat([header, body])
+}
+
+it('forwards fragmented COMMIT bytes, discards the reply and recovers later replies', async () => {
+ const sockets = new Set<Socket>()
+ let received = Buffer.alloc(0)
+ let repliedToCommit = false
+ const backend = createServer(socket => {
+ sockets.add(socket)
+ socket.on('data', chunk => {
+ received = Buffer.concat([received, chunk])
+ if (!repliedToCommit && received.length === 8 + query('COMMIT').length) {
+ repliedToCommit = true
+ socket.write('commit-reply')
+ } else if (
+ repliedToCommit &&
+ received.length === 8 + query('COMMIT').length + query('SELECT 1').length
+ ) {
+ socket.write('later-reply')
+ }
+ })
+ socket.on('end', () => socket.end())
+ socket.on('error', () => {})
+ })
+ backend.listen(0, '127.0.0.1')
+ await once(backend, 'listening')
+ const address = backend.address()
+ if (!address || typeof address === 'string') throw new Error('missing fixture backend address')
+ const proxy = await createPostgresCommitReplyBlackhole(
+ ['postgresql:', `//[email protected]:${address.port}/fixture`].join('')
+ )
+ const target = new URL(proxy.connectionString)
+ const client = createConnection({ host: target.hostname, port: Number(target.port) })
+ const replies: Buffer[] = []
+ client.on('data', chunk => replies.push(chunk))
+ client.on('error', () => {})
+ try {
+ await once(client, 'connect')
+ const startup = Buffer.alloc(8)
+ startup.writeInt32BE(8, 0)
+ startup.writeInt32BE(196608, 4)
+ client.write(startup)
+ await observed(() => received.length === 8)
+ const commit = query('COMMIT')
+ client.write(commit.subarray(0, 2))
+ await observed(() => received.length === 10)
+ expect(proxy.commands).toEqual([])
+ client.write(commit.subarray(2))
+ await proxy.commitForwarded
+ await observed(() => proxy.discardedReplyBytes > 0)
+ expect(received.subarray(8)).toEqual(commit)
+ expect(proxy.commands).toEqual(['COMMIT'])
+ expect(replies).toEqual([])
+ proxy.allowReplies()
+ client.write(query('SELECT 1'))
+ await observed(() => Buffer.concat(replies).toString() === 'later-reply')
+ } finally {
+ client.destroy()
+ await proxy.close()
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ backend.close(error => (error ? reject(error) : resolve()))
+ )
+ }
+})
+
+it('refuses an encrypted protocol prerequisite instead of pretending to observe COMMIT', async () => {
+ await expect(
+ createPostgresCommitReplyBlackhole(
+ ['postgresql:', '//[email protected]/fixture?sslmode=require'].join('')
+ )
+ ).rejects.toThrow('plaintext PostgreSQL connection')
+})
diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.ts
new file mode 100644
index 000000000..a2aa3c6cf
--- /dev/null
+++ b/control-api/test/helpers/realPostgresCancellation.ts
@@ -0,0 +1,83 @@
+import { once } from 'node:events'
+import { type Socket, createConnection, createServer } from 'node:net'
+
+/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */
+export async function createPostgresCommitReplyBlackhole(connectionString: string) {
+ const target = new URL(connectionString)
+ const sslMode = target.searchParams.get('sslmode')
+ if (sslMode && sslMode !== 'disable') {
+ throw new Error('COMMIT reply observation requires the harness plaintext PostgreSQL connection')
+ }
+ const sockets = new Set<Socket>()
+ const commands: string[] = []
+ let commit!: () => void
+ const commitForwarded = new Promise<void>(resolve => {
+ commit = resolve
+ })
+ let armed = true
+ let dropping = false
+ let discardedReplyBytes = 0
+ const server = createServer(front => {
+ const host = target.hostname.replace(/^\[|\]$/g, '')
+ const back = createConnection({ host, port: Number(target.port || 5432) })
+ for (const socket of [front, back]) {
+ sockets.add(socket)
+ socket.once('close', () => sockets.delete(socket))
+ }
+ front.on('error', () => back.destroy())
+ back.on('error', () => front.destroy())
+ front.on('end', () => back.end())
+ back.on('end', () => front.end())
+ let startup = true
+ let buffered = Buffer.alloc(0)
+ front.on('data', chunk => {
+ buffered = Buffer.concat([buffered, chunk])
+ if (startup && buffered.length >= 4 && buffered.length >= buffered.readInt32BE(0)) {
+ buffered = buffered.subarray(buffered.readInt32BE(0))
+ startup = false
+ }
+ while (!startup && buffered.length >= 5 && buffered.length >= buffered.readInt32BE(1) + 1) {
+ const length = buffered.readInt32BE(1)
+ if (buffered[0] === 'Q'.charCodeAt(0)) {
+ const sql = buffered.toString('utf8', 5, length)
+ if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') commands.push(sql)
+ if (sql === 'COMMIT' && armed) {
+ armed = false
+ dropping = true
+ commit()
+ }
+ }
+ buffered = buffered.subarray(length + 1)
+ }
+ back.write(chunk)
+ })
+ back.on('data', chunk => {
+ if (dropping) discardedReplyBytes += chunk.length
+ else front.write(chunk)
+ })
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('missing COMMIT observer address')
+ const proxied = new URL(connectionString)
+ proxied.hostname = '127.0.0.1'
+ proxied.port = String(address.port)
+ return {
+ connectionString: proxied.toString(),
+ commitForwarded,
+ commands,
+ get discardedReplyBytes() {
+ return discardedReplyBytes
+ },
+ allowReplies: () => {
+ dropping = false
+ },
+ close: async () => {
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ },
+ }
+}
diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts
new file mode 100644
index 000000000..acc57ea8c
--- /dev/null
+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts
@@ -0,0 +1,248 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { once } from 'node:events'
+import { setImmediate as nextTurn } from 'node:timers/promises'
+import { Pool } from 'pg'
+import { type DbClient, initDb, withTransaction } from '../src/db.js'
+import { reserveInDangerZone } from '../src/services/budgets/reservations.js'
+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+if (process.env.CONTROL_API_REAL_PG_REQUIRED === '1' && !adminUrl) {
+ throw new Error('physical cancellation tests require CONTROL_API_REAL_PG_ADMIN_URL')
+}
+const realPostgres = adminUrl ? describe : describe.skip
+
+function deferred<T = void>() {
+ let resolve!: (value: T) => void
+ const promise = new Promise<T>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+async function observe(check: () => Promise<boolean>, timeoutMs = 2_000): Promise<void> {
+ const deadline = Date.now() + timeoutMs
+ while (!(await check())) {
+ if (Date.now() >= deadline) throw new Error('physical PostgreSQL cleanup did not complete')
+ await nextTurn()
+ }
+}
+
+// Mechanism proof uses real schema/helpers but bypasses authorize and signing.
+// Full authorization cancellation lives in services.llmProviderAttemptAuthorization.
+realPostgres('scoped transaction cancellation mechanism and durable outcome', () => {
+ const database = `authorize_cancel_${randomBytes(6).toString('hex')}`
+ let admin: Pool
+ let observer: Pool
+ let pool: Pool
+ let connectionString: string
+ let budgetId: string
+
+ beforeAll(async () => {
+ admin = new Pool({
+ connectionString: adminUrl,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 15_000,
+ })
+ await admin.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ const url = new URL(adminUrl!)
+ url.pathname = `/${database}`
+ connectionString = url.toString()
+ pool = new Pool({
+ connectionString,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 15_000,
+ idleTimeoutMillis: 1_000,
+ })
+ observer = new Pool({
+ connectionString,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 2_000,
+ idleTimeoutMillis: 1_000,
+ })
+ const runtime = await pool.query<{ version: string; version_num: string }>(
+ "SELECT version(), current_setting('server_version_num') AS version_num"
+ )
+ expect(Number(runtime.rows[0].version_num)).toBeGreaterThanOrEqual(160000)
+ expect(Number(runtime.rows[0].version_num)).toBeLessThan(170000)
+ expect(runtime.rows[0].version).toMatch(/linux|darwin|bsd|illumos/i)
+ await initDb({ connect: () => pool.connect() })
+ const budget = await pool.query<{ id: string }>(
+ `INSERT INTO token_budgets (name, unit, limit_amount, period, max_task_amount)
+ VALUES ('cancellation fixture', 'tokens', 100, 'daily', 4) RETURNING id`
+ )
+ budgetId = budget.rows[0].id
+ }, 60_000)
+
+ afterAll(async () => {
+ await endPoolAndWaitForClients(pool)
+ await endPoolAndWaitForClients(observer)
+ if (admin) {
+ await admin.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await endPoolAndWaitForClients(admin)
+ }
+ })
+
+ async function writeAuthorizeRows(db: DbClient, invocationId: string) {
+ const reservation = await reserveInDangerZone(
+ {
+ budgetId,
+ limit: 100,
+ spent: 95,
+ minStart: 1,
+ estAmount: 4,
+ taskRef: invocationId,
+ hostRef: 'cancel-host',
+ },
+ undefined,
+ db
+ )
+ expect(reservation.decision).toBe('allow')
+ if (reservation.decision !== 'allow') throw new Error('fixture reservation denied')
+ const attempt = await insertLlmProviderAttempt(db, {
+ callerKind: 'host',
+ hostRef: 'cancel-host',
+ recipeNamespace: null,
+ recipeName: null,
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ model: 'gpt-5.1',
+ requestHash: 'a'.repeat(64),
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ budgetReservationId: reservation.reservationId,
+ connectionRevision: 1,
+ connectionId: null,
+ })
+ await db.query(
+ `INSERT INTO llm_provider_attempt_tickets (jti, provider_attempt_id, status, expires_at)
+ VALUES ($1, $2, 'issued', NOW() + INTERVAL '1 minute')`,
+ [randomUUID(), attempt.id]
+ )
+ }
+
+ async function rowCounts(invocationId: string) {
+ const result = await observer.query<{
+ attempts: number
+ tickets: number
+ reservations: number
+ }>(
+ `SELECT
+ (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,
+ (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a
+ ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,
+ (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE task_ref = $1) AS reservations`,
+ [invocationId]
+ )
+ return result.rows[0]
+ }
+
+ async function backendGone(pid: number) {
+ await observe(async () => {
+ const result = await observer.query<{ count: number }>(
+ 'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1 AND datname = $2',
+ [pid, database]
+ )
+ return result.rows[0].count === 0
+ })
+ }
+
+ async function lockAvailable(lock: number) {
+ await withTransaction(async db => {
+ const result = await db.query('SELECT pg_try_advisory_xact_lock($1) AS available', [lock])
+ expect((result.rows[0] as { available: boolean }).available).toBe(true)
+ }, observer)
+ }
+
+ it.each(['active', 'idle'] as const)(
+ 'ends the %s backend, releases writer locks and rolls attempt/ticket/reservation back before COMMIT',
+ async mode => {
+ const invocationId = randomUUID()
+ const lock = mode === 'active' ? 806301 : 806302
+ const entered = deferred<number>()
+ const controller = new AbortController()
+ const reason = Object.assign(new Error('authorize_aborted'), { code: 'authorize_aborted' })
+ const outcome = withTransaction(
+ async db => {
+ const identity = await db.query('SELECT pg_backend_pid() AS pid')
+ await writeAuthorizeRows(db, invocationId)
+ await db.query('SELECT pg_advisory_xact_lock($1)', [lock])
+ entered.resolve((identity.rows[0] as { pid: number }).pid)
+ if (mode === 'active') await db.query('SELECT pg_sleep(30)')
+ else await once(controller.signal, 'abort')
+ },
+ pool,
+ { signal: controller.signal }
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ const pid = await entered.promise
+ await observe(async () => {
+ const result = await observer.query<{ state: string; query: string }>(
+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1',
+ [pid]
+ )
+ return mode === 'active'
+ ? result.rows[0]?.state === 'active' && result.rows[0]?.query === 'SELECT pg_sleep(30)'
+ : result.rows[0]?.state === 'idle in transaction'
+ })
+ controller.abort(reason)
+ expect(((await outcome) as { error: unknown }).error).toBe(reason)
+ expect(pool.totalCount).toBe(0)
+ await backendGone(pid)
+ await lockAvailable(lock)
+ expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
+
+ // The same physical writes still commit normally after cancellation.
+ const later = randomUUID()
+ await withTransaction(db => writeAuthorizeRows(db, later), pool)
+ expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
+ await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])
+ },
+ 10_000
+ )
+
+ it('retains a committed attempt/reservation when a real COMMIT reply is lost; cancellation is not rollback proof', async () => {
+ const proxy = await createPostgresCommitReplyBlackhole(connectionString)
+ const uncertainPool = new Pool({
+ connectionString: proxy.connectionString,
+ ssl: false,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 15_000,
+ })
+ const invocationId = randomUUID()
+ const controller = new AbortController()
+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
+ try {
+ const outcome = withTransaction(db => writeAuthorizeRows(db, invocationId), uncertainPool, {
+ signal: controller.signal,
+ }).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await proxy.commitForwarded
+ await observe(async () => (await rowCounts(invocationId)).attempts === 1)
+ controller.abort(reason)
+ expect(((await outcome) as { error: unknown }).error).toBe(reason)
+ expect(await rowCounts(invocationId)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
+ expect(uncertainPool.totalCount).toBe(0)
+ } finally {
+ controller.abort(reason)
+ proxy.allowReplies()
+ await endPoolAndWaitForClients(uncertainPool)
+ await proxy.close()
+ }
+ }, 10_000)
+})
diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts
new file mode 100644
index 000000000..55b72e154
--- /dev/null
+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts
@@ -0,0 +1,595 @@
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
+import { execFileSync } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+import { once } from 'node:events'
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { createServer as createHttpsServer } from 'node:https'
+import { type Socket, createServer as createTcpServer } from 'node:net'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { Duplex } from 'node:stream'
+import { setImmediate as nextTurn } from 'node:timers/promises'
+import { Pool, type PoolClient } from 'pg'
+import { config } from '../src/config.js'
+import { withTransaction } from '../src/db.js'
+import { K8sGateway } from '../src/k8s.js'
+import {
+ LlmProviderAttemptAuthorizeError,
+ authorizeLlmProviderAttempt,
+} from '../src/services/llmProviderAttemptAuthorizer.js'
+import { ResourceService } from '../src/services/resourceService.js'
+import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
+
+function deferred<T = void>() {
+ let resolve!: (value: T) => void
+ const promise = new Promise<T>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
+async function within<T>(promise: Promise<T>, milliseconds = 1_000): Promise<T> {
+ let timer: NodeJS.Timeout | undefined
+ try {
+ return await Promise.race([
+ promise,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error('dependency did not terminate')), milliseconds)
+ }),
+ ])
+ } finally {
+ clearTimeout(timer)
+ }
+}
+
+async function observed(check: () => boolean): Promise<void> {
+ const deadline = Date.now() + 1_000
+ while (!check()) {
+ if (Date.now() >= deadline) throw new Error('lifecycle observation did not arrive')
+ await nextTurn()
+ }
+}
+
+function interruption(code: 'authorize_timeout' | 'authorize_aborted') {
+ return Object.assign(new Error(code), { code })
+}
+
+const cleanup: Array<() => Promise<void>> = []
+afterEach(async () => {
+ vi.restoreAllMocks()
+ await Promise.all(cleanup.splice(0).map(close => close()))
+})
+
+describe('authorizer Kubernetes cancellation on the configured native client', () => {
+ let directory: string
+ let certificate: Buffer
+ let key: Buffer
+ beforeAll(() => {
+ // Generate disposable TLS material locally. No application credentials or
+ // checked-in private key are needed to prove the configured CA/auth path.
+ directory = mkdtempSync(join(tmpdir(), 'authorize-cancel-tls-'))
+ execFileSync(
+ 'openssl',
+ [
+ 'req',
+ '-x509',
+ '-newkey',
+ 'rsa:2048',
+ '-nodes',
+ '-days',
+ '1',
+ '-subj',
+ '/CN=127.0.0.1',
+ '-addext',
+ 'subjectAltName=IP:127.0.0.1',
+ '-keyout',
+ join(directory, 'fixture-key.pem'),
+ '-out',
+ join(directory, 'fixture-cert.pem'),
+ ],
+ { stdio: 'ignore', timeout: 10_000 }
+ )
+ certificate = readFileSync(join(directory, 'fixture-cert.pem'))
+ key = readFileSync(join(directory, 'fixture-key.pem'))
+ })
+ afterAll(() => rmSync(directory, { recursive: true, force: true }))
+
+ async function peer(mode: 'no-headers' | 'no-eof') {
+ const arrived = deferred()
+ const disconnected = deferred()
+ const sockets = new Set<Duplex>()
+ const requests: Array<{ authorization?: string; url?: string }> = []
+ let complete = false
+ const server = createHttpsServer({ cert: certificate, key }, (req, res) => {
+ requests.push({ authorization: req.headers.authorization, url: req.url })
+ req.socket.once('close', () => disconnected.resolve())
+ arrived.resolve()
+ if (complete) {
+ res.setHeader('Content-Type', 'application/json')
+ res.end(JSON.stringify({ metadata: { name: 'cancel-host' } }))
+ } else if (mode === 'no-eof') {
+ res.writeHead(200, { 'Content-Type': 'application/json' })
+ res.write('{"metadata":')
+ }
+ })
+ server.on('connection', socket => {
+ sockets.add(socket)
+ socket.once('close', () => sockets.delete(socket))
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('missing TLS address')
+ const bearer = randomUUID()
+ const options: Parameters<k8s.KubeConfig['loadFromOptions']>[0] = {
+ clusters: [
+ {
+ name: 'fixture',
+ server: `https://127.0.0.1:${address.port}`,
+ caData: certificate.toString('base64'),
+ skipTLSVerify: false,
+ },
+ ],
+ users: [{ name: 'fixture', token: bearer }],
+ contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],
+ currentContext: 'fixture',
+ }
+ const kubeconfig = new k8s.KubeConfig()
+ kubeconfig.loadFromOptions(options)
+ const resources = new ResourceService(
+ kubeconfig.makeApiClient(k8s.CustomObjectsApi),
+ 'default',
+ { hosts: 'default' }
+ )
+ cleanup.push(async () => {
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ })
+ return {
+ arrived,
+ disconnected,
+ requests,
+ bearer,
+ resources,
+ options,
+ recover: () => {
+ complete = true
+ },
+ }
+ }
+
+ it.each(['no-headers', 'no-eof'] as const)(
+ 'terminates %s with the configured CA, bearer and server intact, then serves a later lookup',
+ async mode => {
+ const fixture = await peer(mode)
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ const outcome = fixture.resources
+ .getResource('hosts', 'cancel-host', 'default', controller.signal)
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await fixture.arrived.promise
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ await within(fixture.disconnected.promise)
+ expect(fixture.requests).toHaveLength(1)
+ expect(fixture.requests[0].authorization).toBe(`Bearer ${fixture.bearer}`)
+ expect(fixture.requests[0].url).toContain('/namespaces/default/hosts/cancel-host')
+ fixture.recover()
+ await expect(
+ fixture.resources.getResource('hosts', 'cancel-host', 'default')
+ ).resolves.toMatchObject({ metadata: { name: 'cancel-host' } })
+ expect(fixture.requests).toHaveLength(2)
+ }
+ )
+
+ it('does not enter the cross-namespace fallback after cancellation without an explicit namespace', async () => {
+ const fixture = await peer('no-headers')
+ const controller = new AbortController()
+ const reason = interruption('authorize_aborted')
+ const outcome = fixture.resources
+ .getResource('hosts', 'cancel-host', undefined, controller.signal)
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await fixture.arrived.promise
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(fixture.requests).toHaveLength(1)
+ })
+
+ it('forwards the optional signal through K8sGateway without reconfiguring its client', async () => {
+ const fixture = await peer('no-headers')
+ vi.spyOn(k8s.KubeConfig.prototype, 'loadFromDefault').mockImplementation(function (
+ this: k8s.KubeConfig
+ ) {
+ this.loadFromOptions(fixture.options)
+ })
+ const gateway = new K8sGateway('default')
+ const controller = new AbortController()
+ const reason = interruption('authorize_aborted')
+ const outcome = gateway
+ .getResource('hosts', 'cancel-host', config.hostsNamespace, controller.signal)
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await fixture.arrived.promise
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(fixture.requests[0].authorization).toBe(`Bearer ${fixture.bearer}`)
+ })
+
+ it.each(['codex-subscription', 'grok-subscription'] as const)(
+ 'propagates %s cancellation into assignment and never enters a transaction',
+ async provider => {
+ const fixture = await peer('no-headers')
+ const previous = config.grokSubscriptionEnabled
+ config.grokSubscriptionEnabled = true
+ try {
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ const transaction = vi.fn()
+ const caller: McpHostAccessClaims = {
+ sub: 'default/cancel-host',
+ hostRefs: ['cancel-host'],
+ recipeNamespace: 'default',
+ recipeName: 'cancel-host',
+ scope: 'workflow:approval:request',
+ mcpCapabilities: [],
+ workflowControlScopes: ['llm:codex:execute', 'llm:grok:execute'],
+ iss: 'control-api',
+ aud: 'workflow-approvals',
+ jti: randomUUID(),
+ exp: Math.floor(Date.now() / 1_000) + 60,
+ }
+ const outcome = authorizeLlmProviderAttempt(
+ caller,
+ {
+ invocationId: randomUUID(),
+ attemptGeneration: 1,
+ policyRevision: 1,
+ policyHash: 'a'.repeat(64),
+ request: {
+ schemaVersion:
+ provider === 'codex-subscription'
+ ? 'codex-completion-request.v1'
+ : 'grok-completion-request.v1',
+ requestId: randomUUID(),
+ idempotencyKey: randomUUID(),
+ provider,
+ model: provider === 'codex-subscription' ? 'gpt-5.1' : 'grok-4',
+ messages: [{ role: 'user', content: 'cancel this lookup' }],
+ },
+ },
+ {
+ enabled: true,
+ signal: controller.signal,
+ withTransaction: transaction,
+ resolveAssignment: async (_hostRef, signal) => {
+ try {
+ await fixture.resources.getResource('hosts', 'cancel-host', 'default', signal)
+ } catch {
+ // Existing assignment adapters translate lookup errors. The
+ // authorizer must recover the typed cancellation through them.
+ throw new LlmProviderAttemptAuthorizeError(
+ 'host_binding_mismatch',
+ 'assignment lookup failed'
+ )
+ }
+ return { liveBrokerProviders: [provider], liveConnectionRef: 'connection-fixture' }
+ },
+ }
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await fixture.arrived.promise
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ await within(fixture.disconnected.promise)
+ expect(transaction).not.toHaveBeenCalled()
+ } finally {
+ config.grokSubscriptionEnabled = previous
+ }
+ }
+ )
+})
+
+function protocolMessage(type: string, body: Buffer): Buffer {
+ const header = Buffer.alloc(5)
+ header.write(type, 0)
+ header.writeInt32BE(body.length + 4, 1)
+ return Buffer.concat([header, body])
+}
+
+async function postgresPeer(handshake = true) {
+ const connected = deferred()
+ const queries: string[] = []
+ const sockets = new Set<Socket>()
+ let stalledSql = ''
+ let failedSql = ''
+ const server = createTcpServer({ allowHalfOpen: true }, socket => {
+ sockets.add(socket)
+ socket.on('error', () => {})
+ socket.once('close', () => sockets.delete(socket))
+ connected.resolve()
+ let startup = true
+ let buffered = Buffer.alloc(0)
+ socket.on('data', chunk => {
+ buffered = Buffer.concat([buffered, chunk])
+ if (startup) {
+ if (buffered.length < 4 || buffered.length < buffered.readInt32BE(0)) return
+ buffered = buffered.subarray(buffered.readInt32BE(0))
+ startup = false
+ if (handshake)
+ socket.write(
+ Buffer.concat([
+ protocolMessage('R', Buffer.alloc(4)),
+ protocolMessage('Z', Buffer.from('I')),
+ ])
+ )
+ }
+ while (buffered.length >= 5 && buffered.length >= buffered.readInt32BE(1) + 1) {
+ const type = buffered.toString('utf8', 0, 1)
+ const length = buffered.readInt32BE(1)
+ const body = buffered.subarray(5, length + 1)
+ buffered = buffered.subarray(length + 1)
+ if (type !== 'Q') continue
+ const sql = body.toString('utf8', 0, body.length - 1)
+ queries.push(sql)
+ if (sql === failedSql) {
+ socket.write(
+ Buffer.concat([
+ protocolMessage(
+ 'E',
+ Buffer.from('SERROR\0C42704\0Munsupported connection check parameter\0\0')
+ ),
+ protocolMessage('Z', Buffer.from('E')),
+ ])
+ )
+ continue
+ }
+ if (sql === stalledSql) continue
+ socket.write(
+ Buffer.concat([
+ protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)),
+ protocolMessage('Z', Buffer.from(sql === 'BEGIN' ? 'T' : 'I')),
+ ])
+ )
+ }
+ })
+ // This deliberate half-open peer never sends EOF in response to Terminate.
+ // pg's idle Client.end() alone cannot finish; the scoped transport stop must.
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('missing PG address')
+ const pool = new Pool({
+ host: '127.0.0.1',
+ port: address.port,
+ user: 'cancel_fixture',
+ database: 'cancel_fixture',
+ max: 1,
+ connectionTimeoutMillis: 150,
+ idleTimeoutMillis: 1_000,
+ })
+ pool.on('error', () => {})
+ cleanup.push(async () => {
+ for (const socket of sockets) socket.destroy()
+ await pool.end()
+ await new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ })
+ return {
+ connected,
+ queries,
+ pool,
+ stall: (sql: string) => {
+ stalledSql = sql
+ },
+ fail: (sql: string) => {
+ failedSql = sql
+ },
+ }
+}
+
+describe('scoped transaction cancellation with the native pg driver and sockets', () => {
+ it('never borrows a client for an already aborted request', async () => {
+ const fixture = await postgresPeer()
+ const borrow = vi.spyOn(fixture.pool, 'connect')
+ const controller = new AbortController()
+ const reason = interruption('authorize_aborted')
+ controller.abort(reason)
+ await expect(
+ withTransaction(vi.fn(), fixture.pool, { signal: controller.signal })
+ ).rejects.toBe(reason)
+ expect(borrow).not.toHaveBeenCalled()
+ expect(fixture.queries).toEqual([])
+ })
+
+ it('fails without running work if the server does not support scoped connection polling', async () => {
+ const fixture = await postgresPeer()
+ fixture.fail("SET LOCAL client_connection_check_interval = '100ms'")
+ const work = vi.fn()
+ await expect(
+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })
+ ).rejects.toMatchObject({ code: '42704' })
+ expect(work).not.toHaveBeenCalled()
+ expect(fixture.queries).toEqual([
+ 'BEGIN',
+ "SET LOCAL client_connection_check_interval = '100ms'",
+ 'ROLLBACK',
+ ])
+ })
+
+ it('terminates an in-flight rollback before final release and preserves the typed owner reason', async () => {
+ const fixture = await postgresPeer()
+ fixture.stall('ROLLBACK')
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ const outcome = withTransaction(
+ async () => {
+ throw new Error('work failed')
+ },
+ fixture.pool,
+ { signal: controller.signal }
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await observed(() => fixture.queries.includes('ROLLBACK'))
+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]
+ const release = vi.spyOn(borrowed, 'release')
+ expect(release).not.toHaveBeenCalled()
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(release).toHaveBeenCalledOnce()
+ expect(release).toHaveBeenCalledWith(reason)
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(fixture.queries).not.toContain('COMMIT')
+ })
+
+ it('terminates an active native query and evicts the client exactly once before recovery', async () => {
+ const fixture = await postgresPeer()
+ fixture.stall('SELECT stalled')
+ const removed = vi.fn()
+ fixture.pool.on('remove', removed)
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ const outcome = withTransaction(async db => db.query('SELECT stalled'), fixture.pool, {
+ signal: controller.signal,
+ }).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await observed(() => fixture.queries.includes('SELECT stalled'))
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(removed).toHaveBeenCalledOnce()
+ expect(fixture.queries).not.toContain('COMMIT')
+ await expect(withTransaction(async () => 'later', fixture.pool)).resolves.toBe('later')
+ expect(fixture.pool.idleCount).toBe(1)
+ })
+
+ it('terminates an idle half-open client but remains owned through callback cleanup', async () => {
+ const fixture = await postgresPeer()
+ const entered = deferred()
+ const callbackCleanup = deferred()
+ const controller = new AbortController()
+ const reason = interruption('authorize_aborted')
+ let settled = false
+ const outcome = withTransaction(
+ async () => {
+ entered.resolve()
+ await once(controller.signal, 'abort')
+ // This barrier tests ordering only. Physical cancellation above and the
+ // native half-open client end event below are independent of its release.
+ await callbackCleanup.promise
+ return 'late callback'
+ },
+ fixture.pool,
+ { signal: controller.signal }
+ )
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ .finally(() => {
+ settled = true
+ })
+ await entered.promise
+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]
+ const clientEnded = once(borrowed, 'end')
+ const release = vi.spyOn(borrowed, 'release')
+ controller.abort(reason)
+ try {
+ await within(clientEnded)
+ expect(settled).toBe(false)
+ expect(release).not.toHaveBeenCalled()
+ expect(fixture.pool.totalCount).toBe(1)
+ } finally {
+ callbackCleanup.resolve()
+ }
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(release).toHaveBeenCalledOnce()
+ expect(release).toHaveBeenCalledWith(reason)
+ expect(fixture.pool.totalCount).toBe(0)
+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')
+ })
+
+ it('awaits bounded connect failure and preserves the cancellation reason without sending BEGIN', async () => {
+ const fixture = await postgresPeer(false)
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ let settled = false
+ const work = vi.fn()
+ const outcome = withTransaction(work, fixture.pool, { signal: controller.signal })
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ .finally(() => {
+ settled = true
+ })
+ await fixture.connected.promise
+ controller.abort(reason)
+ await nextTurn()
+ expect(settled).toBe(false)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(work).not.toHaveBeenCalled()
+ expect(fixture.queries).toEqual([])
+ expect(fixture.pool.totalCount).toBe(0)
+ })
+
+ it('disposes a borrow that arrives after abort before any new statement or healthy return', async () => {
+ const fixture = await postgresPeer()
+ const first = await fixture.pool.connect()
+ const controller = new AbortController()
+ const reason = interruption('authorize_aborted')
+ const work = vi.fn()
+ const outcome = withTransaction(work, fixture.pool, { signal: controller.signal }).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await observed(() => fixture.pool.waitingCount === 1)
+ controller.abort(reason)
+ first.release()
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(work).not.toHaveBeenCalled()
+ expect(fixture.queries).toEqual([])
+ expect(fixture.pool.totalCount).toBe(0)
+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')
+ })
+
+ it('keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy', async () => {
+ const fixture = await postgresPeer()
+ fixture.stall('COMMIT')
+ const controller = new AbortController()
+ const reason = interruption('authorize_timeout')
+ const outcome = withTransaction(async () => 'possibly committed', fixture.pool, {
+ signal: controller.signal,
+ }).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ await observed(() => fixture.queries.includes('COMMIT'))
+ controller.abort(reason)
+ expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
+ expect(fixture.queries).toEqual([
+ 'BEGIN',
+ "SET LOCAL client_connection_check_interval = '100ms'",
+ 'COMMIT',
+ ])
+ expect(fixture.pool.totalCount).toBe(0)
+ })
+})
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
new file mode 100644
index 000000000..9c35ce4c2
--- /dev/null
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
@@ -0,0 +1,503 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { Request } from 'express'
+import { type IncomingMessage, createServer, request as httpRequest } from 'node:http'
+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'
+import {
+ AuthorizeBodyAdmission,
+ AuthorizeWorkInterrupted,
+} from '../src/middleware/llmProviderAttemptBodyAdmission.js'
+import * as authorizer from '../src/services/llmProviderAttemptAuthorizer.js'
+import * as rateLimiter from '../src/services/rateLimiterService.js'
+import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'
+import { MockGateway } from './mockGateway.js'
+
+vi.mock('../src/config.js', async importOriginal => {
+ const actual = await importOriginal<typeof import('../src/config.js')>()
+ return { ...actual, config: { ...actual.config, jsonBodyLimit: '1mb' } }
+})
+
+vi.mock('../src/db.js', () => ({
+ pool: { query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), connect: vi.fn() },
+ withTransaction: vi.fn(),
+}))
+
+vi.mock('../src/services/notificationEmitter.js', () => ({
+ emitNotification: vi.fn().mockResolvedValue(undefined),
+ enqueueApprovalRequestedNotification: vi.fn().mockResolvedValue(undefined),
+ enqueueApprovalUpdatedNotification: vi.fn().mockResolvedValue(undefined),
+}))
+
+vi.mock('../src/services/rateLimiterService.js', () => ({ checkAndIncrement: vi.fn() }))
+
+vi.mock('../src/observability/metrics.js', async importOriginal => {
+ const actual = await importOriginal<typeof import('../src/observability/metrics.js')>()
+ return { ...actual, rateLimitHitsTotal: { inc: vi.fn() } }
+})
+
+vi.mock('../src/services/llmProviderAttemptAuthorizer.js', async importOriginal => {
+ const actual =
+ await importOriginal<typeof import('../src/services/llmProviderAttemptAuthorizer.js')>()
+ return { ...actual, authorizeLlmProviderAttempt: vi.fn() }
+})
+
+const AUTHORIZE_PATH = '/api/v1/mcp-host/llm/provider-attempts/authorize'
+// Synthetic ticket content is only an authorizer response fixture. No ticket
+// creation, grant lookup, budget reservation or provider dispatch is mocked as
+// successful by these HTTP ownership tests.
+const SUCCESS = {
+ providerAttemptId: '33333333-3333-4333-8333-333333333333',
+ requestHash: 'a'.repeat(64),
+ executionTicket: 'unit-test-execution-ticket',
+ expiresAt: '2026-10-02T12:00:00.000Z',
+}
+
+function deferred<T = void>() {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ const promise = new Promise<T>(done => {
+ resolve = done
+ })
+ return { promise, resolve }
+}
+
+type Observation = {
+ request: IncomingMessage
+ bodyDataSubscriptions: number
+ readBytesBeforeResponse: number
+ responseWritesAfterClose: number
+}
+
+type Reply = { status: number; headers: IncomingMessage['headers']; body: string }
+
+function headers(host: string, provider: 'codex' | 'grok' = 'codex') {
+ const issued = issueMcpHostAccessJwt('default', host, [host], {
+ workflowControlScopes: [provider === 'codex' ? 'llm:codex:execute' : 'llm:grok:execute'],
+ })
+ return { Authorization: `Bearer ${issued.token}`, 'content-type': 'application/json' }
+}
+
+function send(url: string, requestHeaders: Record<string, string>, body?: string) {
+ let client!: ReturnType<typeof httpRequest>
+ const response = new Promise<Reply>((resolve, reject) => {
+ client = httpRequest(url, { method: 'POST', headers: requestHeaders }, res => {
+ const chunks: string[] = []
+ res.setEncoding('utf8')
+ res.on('data', (chunk: string) => chunks.push(chunk))
+ res.on('error', reject)
+ res.on('end', () =>
+ resolve({ status: res.statusCode ?? 0, headers: res.headers, body: chunks.join('') })
+ )
+ })
+ client.on('error', reject)
+ client.setTimeout(5_000, () => client.destroy(new Error('HTTP ownership fixture deadline')))
+ if (body === undefined) client.flushHeaders()
+ else client.end(body)
+ })
+ return { client, response }
+}
+
+async function withApps(
+ count: number,
+ run: (apps: Array<{ url: string; observations: Observation[] }>) => Promise<void>
+) {
+ const { createApp } = await import('../src/app.js')
+ const servers = Array.from({ length: count }, () => {
+ const observations: Observation[] = []
+ const app = createApp(new MockGateway() as never)
+ const listener = createServer((req, res) => {
+ const observed: Observation = {
+ request: req,
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ responseWritesAfterClose: 0,
+ }
+ observations.push(observed)
+ const end = res.end
+ res.end = function (...args: Parameters<typeof res.end>) {
+ if (this.destroyed) observed.responseWritesAfterClose += 1
+ return end.apply(this, args)
+ } as typeof res.end
+ // Observe read calls and listener installation without adding a data
+ // listener or switching the IncomingMessage into flowing mode.
+ const read = req.read
+ req.read = function (size?: number) {
+ const value = read.call(this, size)
+ // Node may drain already buffered bytes after response handoff. That
+ // native _dump is separate from admitting an application body reader.
+ if (Buffer.isBuffer(value) && !res.writableEnded)
+ observed.readBytesBeforeResponse += value.length
+ return value
+ }
+ const on = req.on
+ req.on = function (event: string, listener: (...args: any[]) => void) {
+ if (event === 'data') {
+ observed.bodyDataSubscriptions += 1
+ }
+ return on.call(this, event, listener)
+ } as typeof req.on
+ app(req, res)
+ }).listen(0)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+ return { listener, url: `http://127.0.0.1:${address.port}${AUTHORIZE_PATH}`, observations }
+ })
+ try {
+ await run(servers)
+ } finally {
+ await Promise.all(
+ servers.map(
+ ({ listener }) =>
+ new Promise<void>((resolve, reject) => {
+ listener.closeAllConnections()
+ listener.close(err => (err ? reject(err) : resolve()))
+ })
+ )
+ )
+ }
+}
+
+function observeHeldOwner() {
+ const settled = deferred()
+ const original = AuthorizeBodyAdmission.prototype.run
+ const spy = vi.spyOn(AuthorizeBodyAdmission.prototype, 'run').mockImplementation(async function (
+ this: AuthorizeBodyAdmission,
+ ...args: Parameters<AuthorizeBodyAdmission['run']>
+ ) {
+ try {
+ return await original.apply(this, args)
+ } finally {
+ if (args[0].headers['x-admission-case'] === 'held') settled.resolve()
+ }
+ })
+ return { settled: settled.promise, spy }
+}
+
+beforeEach(() => {
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockReset().mockResolvedValue(SUCCESS)
+ vi.mocked(rateLimiter.checkAndIncrement)
+ .mockReset()
+ .mockResolvedValue({
+ allowed: true,
+ backendAvailable: true,
+ remaining: 59,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count: 1,
+ })
+})
+
+describe('createApp retained authorize-body ownership', () => {
+ it('shares the actual production cap across apps and principals and retains disconnected work until unwind', async () => {
+ const started = deferred()
+ const abortObserved = deferred()
+ const permitUnwind = deferred()
+ const owner = observeHeldOwner()
+ const parse = vi.spyOn(JSON, 'parse')
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(
+ async (_claims, _body, deps) => {
+ started.resolve()
+ deps?.signal?.addEventListener('abort', () => abortObserved.resolve(), { once: true })
+ await permitUnwind.promise
+ deps?.signal?.throwIfAborted()
+ return SUCCESS
+ }
+ )
+ try {
+ await withApps(2, async ([firstApp, otherApp]) => {
+ const first = send(
+ firstApp.url,
+ { ...headers('admission-first-host'), 'x-admission-case': 'held' },
+ '{"held":true}'
+ )
+ const firstOutcome = first.response.catch(error => error)
+ try {
+ await started.promise
+ const rejectedBody = '{"mustNotParse":true}'
+ const blocked = await send(
+ otherApp.url,
+ {
+ ...headers('admission-other-host', 'grok'),
+ },
+ rejectedBody
+ ).response
+ expect(blocked.status).toBe(503)
+ expect(JSON.parse(blocked.body)).toEqual({ error: 'authorize_capacity_exceeded' })
+ expect(blocked.headers.connection).toBe('close')
+ expect(otherApp.observations[0]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()
+
+ // The same refusal must also arrive from headers alone. An owner
+ // that first buffers the next body would stall this request.
+ const headersOnly = await send(otherApp.url, {
+ ...headers('admission-headers-only-host'),
+ 'content-length': '4096',
+ }).response
+ expect(headersOnly.status).toBe(503)
+ expect(otherApp.observations[1]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+
+ const malformed = '{unauthenticated while full'
+ const unauthenticated = await send(
+ otherApp.url,
+ { 'content-type': 'application/json' },
+ malformed
+ ).response
+ expect(unauthenticated.status).toBe(401)
+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(malformed)
+ expect(otherApp.observations[2]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+
+ first.client.destroy()
+ await abortObserved.promise
+ const stillHeld = await send(otherApp.url, headers('admission-third-host'), rejectedBody)
+ .response
+ expect(stillHeld.status).toBe(503)
+ expect(JSON.parse(stillHeld.body)).toEqual({ error: 'authorize_capacity_exceeded' })
+ expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)
+ expect(otherApp.observations[3]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()
+
+ permitUnwind.resolve()
+ await owner.settled
+ expect((firstApp.observations[0].request as Request).body).toBeUndefined()
+ const recovered = await send(
+ otherApp.url,
+ headers('admission-other-host', 'grok'),
+ '{"recovered":true}'
+ ).response
+ expect(recovered.status).toBe(200)
+ expect(JSON.parse(recovered.body)).toEqual(SUCCESS)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)
+ expect(firstApp.observations[0].responseWritesAfterClose).toBe(0)
+ } finally {
+ first.client.destroy()
+ permitUnwind.resolve()
+ await firstOutcome
+ }
+ })
+ } finally {
+ parse.mockRestore()
+ owner.spy.mockRestore()
+ }
+ })
+
+ it('runs both rate limits and JWT before taking any body owner', async () => {
+ const acquire = vi.spyOn(AuthorizeBodyAdmission.prototype, 'tryAcquire')
+ try {
+ await withApps(1, async ([app]) => {
+ const invalid = await send(app.url, { 'content-type': 'application/json' }, '{malformed')
+ .response
+ expect(invalid.status).toBe(401)
+ expect(acquire).not.toHaveBeenCalled()
+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()
+ expect(app.observations[0]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+
+ vi.mocked(rateLimiter.checkAndIncrement).mockResolvedValueOnce({
+ allowed: false,
+ backendAvailable: true,
+ remaining: 0,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count: 61,
+ })
+ const rateDenied = await send(app.url, headers('rate-denied-host'), '{malformed').response
+ expect(rateDenied.status).toBe(429)
+ expect(acquire).not.toHaveBeenCalled()
+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()
+ expect(app.observations[1]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
+ })
+
+ const recovered = await send(app.url, headers('rate-recovered-host'), '{}').response
+ expect(recovered.status).toBe(200)
+ expect(acquire).toHaveBeenCalledOnce()
+ })
+ } finally {
+ acquire.mockRestore()
+ }
+ })
+
+ for (const provider of ['codex', 'grok'] as const) {
+ it(`preserves chunked ${provider} JSON through the actual app and releases the body`, async () => {
+ await withApps(1, async ([app]) => {
+ const body = JSON.stringify({
+ request: { provider: `${provider}-subscription`, chunked: true },
+ })
+ const sent = send(app.url, {
+ ...headers(`chunked-${provider}-host`, provider),
+ 'transfer-encoding': 'chunked',
+ })
+ sent.client.write(body.slice(0, 20))
+ sent.client.end(body.slice(20))
+ const response = await sent.response
+ expect(response.status).toBe(200)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ hostRefs: [`chunked-${provider}-host`] }),
+ JSON.parse(body),
+ expect.objectContaining({
+ signal: expect.any(AbortSignal),
+ resolveAssignment: expect.any(Function),
+ })
+ )
+ expect((app.observations[0].request as Request).body).toBeUndefined()
+ const next = await send(app.url, headers(`next-${provider}-host`, provider), '{}').response
+ expect(next.status).toBe(200)
+ })
+ })
+ }
+
+ for (const [name, error, status, code] of [
+ [
+ 'authorize denial',
+ new authorizer.LlmProviderAttemptAuthorizeError('budget_denied', 'fixture budget refusal'),
+ 403,
+ 'budget_denied',
+ ],
+ ['unexpected authorizer error', new Error('fixture authorizer failure'), 500, undefined],
+ ] as const) {
+ it(`releases after ${name} and admits a subsequent request`, async () => {
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockRejectedValueOnce(error)
+ await withApps(1, async ([app]) => {
+ const response = await send(app.url, headers(`error-${status}-host`), '{}').response
+ expect(response.status).toBe(status)
+ if (code) expect(JSON.parse(response.body)).toEqual({ error: code })
+ expect((app.observations[0].request as Request).body).toBeUndefined()
+ const next = await send(app.url, headers(`recovered-${status}-host`), '{}').response
+ expect(next.status).toBe(200)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)
+ })
+ })
+ }
+
+ it('returns terminal authorize_timeout after the real work clock aborts, without publishing late success', async () => {
+ const started = deferred()
+ const abortObserved = deferred<AbortSignal>()
+ const nativeSetTimeout = globalThis.setTimeout
+ // Only shorten the owner clock for this socket fixture. The production
+ // policy remains unchanged; this test makes no numeric safety claim.
+ const clock = vi
+ .spyOn(globalThis, 'setTimeout')
+ .mockImplementation(((fn, ms, ...args) =>
+ nativeSetTimeout(
+ fn,
+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.workDeadlineMs ? 40 : ms,
+ ...args
+ )) as typeof setTimeout)
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(
+ async (_claims, _body, deps) => {
+ const signal = deps?.signal
+ if (!signal) throw new Error('authorize work did not receive its owner signal')
+ started.resolve()
+ await new Promise<void>(resolve =>
+ signal.addEventListener(
+ 'abort',
+ () => {
+ abortObserved.resolve(signal)
+ resolve()
+ },
+ { once: true }
+ )
+ )
+ // Deliberately produce a late result: the route must check the signal
+ // immediately before publishing an execution ticket.
+ return SUCCESS
+ }
+ )
+ try {
+ await withApps(1, async ([app]) => {
+ const sent = send(app.url, headers('work-timeout-host'), '{}')
+ const outcome = sent.response
+ await started.promise
+ const signal = await abortObserved.promise
+ expect(signal.reason).toBeInstanceOf(AuthorizeWorkInterrupted)
+ expect(signal.reason.code).toBe('authorize_timeout')
+ const response = await outcome
+ expect(response.status).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ error: 'authorize_timeout' })
+ expect((app.observations[0].request as Request).body).toBeUndefined()
+ const recovered = await send(app.url, headers('after-timeout-host'), '{}').response
+ expect(recovered.status).toBe(200)
+ })
+ } finally {
+ clock.mockRestore()
+ }
+ })
+
+ it('stops a stalled body reader on the real read clock before releasing and admits the next request', async () => {
+ const owner = observeHeldOwner()
+ const nativeSetTimeout = globalThis.setTimeout
+ // Contracted clocks are explicit HTTP fixture timing, not runtime policy.
+ const clock = vi
+ .spyOn(globalThis, 'setTimeout')
+ .mockImplementation(((fn, ms, ...args) =>
+ nativeSetTimeout(
+ fn,
+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.readDeadlineMs
+ ? 40
+ : ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.closeGraceMs
+ ? 10
+ : ms,
+ ...args
+ )) as typeof setTimeout)
+ try {
+ await withApps(1, async ([app]) => {
+ const sent = send(app.url, {
+ ...headers('read-timeout-host'),
+ 'content-length': '128',
+ 'x-admission-case': 'held',
+ })
+ const response = await sent.response
+ expect(response.status).toBe(408)
+ expect(JSON.parse(response.body)).toEqual({ error: 'request_timeout' })
+ expect(response.headers.connection).toBe('close')
+ await owner.settled
+ expect(app.observations[0].request.destroyed).toBe(true)
+ expect((app.observations[0].request as Request).body).toBeUndefined()
+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()
+ const next = await send(app.url, headers('after-read-timeout-host'), '{}').response
+ expect(next.status).toBe(200)
+ })
+ } finally {
+ clock.mockRestore()
+ owner.spy.mockRestore()
+ }
+ })
+})
+
+describe('Host and Recipe assignment owner cancellation', () => {
+ for (const hostRef of ['assignment-host', 'default/assignment-recipe']) {
+ it(`passes the same signal to ${hostRef.includes('/') ? 'Recipe' : 'Host'} lookup and preserves cancellation`, async () => {
+ const { resolveHostAssignedAssignment } =
+ await import('../src/routes/mcp-host/llmProviderAttempts.routes.js')
+ const controller = new AbortController()
+ const interruption = new AuthorizeWorkInterrupted('authorize_timeout')
+ const getResource = vi.fn(async (_plural, _name, _namespace, signal?: AbortSignal) => {
+ expect(signal).toBe(controller.signal)
+ controller.abort(interruption)
+ throw new Error('fixture lookup unwound after cancellation')
+ })
+ await expect(
+ resolveHostAssignedAssignment({ getResource } as never, hostRef, controller.signal)
+ ).rejects.toBe(interruption)
+ expect(getResource).toHaveBeenCalledOnce()
+ await expect(
+ resolveHostAssignedAssignment({ getResource } as never, hostRef, controller.signal)
+ ).rejects.toBe(interruption)
+ expect(getResource).toHaveBeenCalledOnce()
+ })
+ }
+})
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
index d48852964..5bfe3e8c0 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
@@ -884,7 +884,8 @@ describe('resolveHostAssignedAssignment', () => {
expect(gateway.getResource).toHaveBeenCalledWith(
'workflowrecipes',
'codex-recipe',
- 'sandbox-recipes'
+ 'sandbox-recipes',
+ undefined
)
})
diff --git a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
index a89234f8b..f11c980ee 100644
--- a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
@@ -1,6 +1,11 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
import { randomBytes, randomUUID } from 'node:crypto'
+import { once } from 'node:events'
import { readFileSync } from 'node:fs'
+import { createServer as createHttpServer } from 'node:http'
+import type { Socket } from 'node:net'
+import { setImmediate as nextTurn } from 'node:timers/promises'
import { Pool } from 'pg'
import {
LIMITS as GROK_LIMITS,
@@ -8,7 +13,7 @@ import {
} from '@clerum/grok-provider-attempt-contract'
import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import { config } from '../src/config.js'
-import { initDb } from '../src/db.js'
+import { type DbClient, initDb, withTransaction as withProductionTransaction } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { getActiveReservation } from '../src/services/budgets/reservations.js'
@@ -38,7 +43,10 @@ import {
insertLlmProviderAttempt,
} from '../src/services/llmProviderAttemptStore.js'
import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'
+import { ResourceService } from '../src/services/resourceService.js'
import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
+import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
// The Grok authorizer signs through this module, not through a dependency, so
// the Grok envelope test taps the real issuer here: the ticket is registered
@@ -79,6 +87,77 @@ function quoteIdent(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
+async function cancellationDeadline<T>(promise: Promise<T>): Promise<T> {
+ let timer: NodeJS.Timeout | undefined
+ try {
+ return await Promise.race([
+ promise,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error('authorizer cancellation phase did not complete')),
+ 2_000
+ )
+ }),
+ ])
+ } finally {
+ clearTimeout(timer)
+ }
+}
+
+async function observeCancellation(check: () => Promise<boolean>): Promise<void> {
+ const deadline = Date.now() + 2_000
+ while (!(await check())) {
+ if (Date.now() >= deadline) throw new Error('authorizer physical state did not settle')
+ await nextTurn()
+ }
+}
+
+async function createAssignmentLookupPeer(stall: boolean) {
+ let arrived!: () => void
+ const arrival = new Promise<void>(resolve => {
+ arrived = resolve
+ })
+ const sockets = new Set<Socket>()
+ const server = createHttpServer((req, res) => {
+ arrived()
+ if (stall) return
+ res.setHeader('Content-Type', 'application/json')
+ res.end(JSON.stringify({ metadata: { name: 'research-host' } }))
+ })
+ server.on('connection', socket => {
+ sockets.add(socket)
+ socket.once('close', () => sockets.delete(socket))
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('missing assignment peer address')
+ const kubeconfig = new k8s.KubeConfig()
+ kubeconfig.loadFromOptions({
+ clusters: [
+ { name: 'fixture', server: `http://127.0.0.1:${address.port}`, skipTLSVerify: false },
+ ],
+ users: [{ name: 'fixture' }],
+ contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],
+ currentContext: 'fixture',
+ })
+ return {
+ resources: new ResourceService(kubeconfig.makeApiClient(k8s.CustomObjectsApi), 'fixture', {
+ hosts: 'fixture',
+ }),
+ arrived: arrival,
+ recover: () => {
+ stall = false
+ },
+ close: async () => {
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ },
+ }
+}
+
const REQUEST = {
schemaVersion: 'codex-completion-request.v1' as const,
requestId: 'req-pg-1',
@@ -157,6 +236,315 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',
}
}
+ function registerProductionCancellationCases(fixture: {
+ provider: 'codex-subscription' | 'grok-subscription'
+ model: string
+ connectionKey: string
+ caller: () => McpHostAccessClaims
+ request: () => Record<string, unknown>
+ readConnection: () => Promise<{
+ id: string
+ catalogRevision: number
+ credentialRevision: number
+ connectionKey: string
+ } | null>
+ policyHash: (connection: {
+ catalogRevision: number
+ credentialRevision: number
+ connectionKey: string
+ }) => string
+ }): void {
+ describe(`${fixture.provider} production cancellation`, () => {
+ async function run(stage: 'assignment' | 'query' | 'commit'): Promise<void> {
+ const invocationId = `actual-cancel-${randomUUID()}`
+ const budgetId = randomUUID()
+ const current = await fixture.readConnection()
+ expect(current).not.toBeNull()
+ const payload = {
+ request: fixture.request(),
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: current!.catalogRevision,
+ policyHash: fixture.policyHash(current!),
+ }
+ const observer = new Pool({
+ connectionString,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 2_000,
+ idleTimeoutMillis: 1_000,
+ })
+ const proxy =
+ stage === 'commit'
+ ? await createPostgresCommitReplyBlackhole(connectionString)
+ : undefined
+ const workPool = new Pool({
+ connectionString: proxy?.connectionString ?? connectionString,
+ ssl: false,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 15_000,
+ idleTimeoutMillis: 1_000,
+ })
+ const lookup = await createAssignmentLookupPeer(stage === 'assignment')
+ const controller = new AbortController()
+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
+ const phases: string[] = []
+ let backendPid: number | undefined
+ let attemptId: string | undefined
+ let reservationId: string | undefined
+ let stallQuery = stage === 'query'
+ let settled = false
+ const previousTap = grokTicketTap.afterIssue
+
+ async function counts() {
+ return (
+ await observer.query<{ attempts: number; tickets: number; reservations: number }>(
+ `SELECT
+ (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,
+ (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a
+ ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,
+ (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE budget_id = $2) AS reservations`,
+ [invocationId, budgetId]
+ )
+ ).rows[0]
+ }
+
+ async function afterRealIssue(
+ db: DbClient,
+ input: { providerAttemptId: string; budgetReservationId: string },
+ issued: { claims: { jti: string } }
+ ): Promise<void> {
+ const registered = await db.query(
+ `SELECT a.provider, a.connection_id, a.budget_reservation_id,
+ t.jti, t.status AS ticket_status, r.budget_id, r.host_ref
+ FROM llm_provider_attempts a
+ JOIN llm_provider_attempt_tickets t ON t.provider_attempt_id = a.id
+ JOIN budget_pending_reservations r ON r.id::text = a.budget_reservation_id
+ WHERE a.id = $1`,
+ [input.providerAttemptId]
+ )
+ expect(registered.rows).toEqual([
+ {
+ provider: fixture.provider,
+ connection_id: current!.id,
+ budget_reservation_id: input.budgetReservationId,
+ jti: issued.claims.jti,
+ ticket_status: 'issued',
+ budget_id: budgetId,
+ host_ref: fixture.caller().hostRefs[0],
+ },
+ ])
+ attemptId = input.providerAttemptId
+ reservationId = input.budgetReservationId
+ phases.push('ticket_registered')
+ // The real issuer has already signed and registered its ticket. This
+ // native statement supplies a deterministic interruptible phase; no
+ // test barrier resolves the work after cancellation.
+ if (stallQuery) await db.query('SELECT pg_sleep(30)')
+ }
+
+ function productionDeps(signal: AbortSignal): LlmProviderAttemptAuthorizerDeps {
+ if (fixture.provider === 'grok-subscription') {
+ grokTicketTap.afterIssue = async (db, input, issued) => {
+ await afterRealIssue(db, input, issued)
+ return issued
+ }
+ }
+ return testDeps({
+ signal,
+ resolveAssignment: async (_hostRef, forwarded) => {
+ expect(forwarded).toBe(signal)
+ phases.push('assignment')
+ await lookup.resources.getResource('hosts', 'research-host', 'fixture', forwarded)
+ return {
+ liveBrokerProviders: [fixture.provider],
+ liveConnectionRef: fixture.connectionKey,
+ }
+ },
+ withTransaction: async (work, _unused, options) => {
+ // Assert the authorizer's actual forwarding before binding this
+ // fixture database; do not inject a separate cancellation source.
+ expect(options?.signal).toBe(signal)
+ return withProductionTransaction(
+ async tx => {
+ phases.push('transaction')
+ const identity = await tx.query('SELECT pg_backend_pid() AS pid')
+ backendPid = (identity.rows[0] as { pid: number }).pid
+ return work(tx)
+ },
+ workPool,
+ options
+ )
+ },
+ evaluateBudget: async (...args) => {
+ const result = await evaluateBudgetCheck(...args)
+ expect(result.allowed).toBe(true)
+ expect(result.reservationIds).toHaveLength(1)
+ phases.push('budget_reserved')
+ return result
+ },
+ insertAttempt: async (db, input) => {
+ const attempt = await insertLlmProviderAttempt(db, input)
+ phases.push('attempt_inserted')
+ return attempt
+ },
+ issueTicket: async (db, input) => {
+ const issued = await issueRegisteredCodexExecutionTicket(db, input)
+ await afterRealIssue(db, input, issued)
+ return issued
+ },
+ })
+ }
+
+ try {
+ const runtime = (
+ await observer.query<{ version: string; version_num: string }>(
+ "SELECT version(), current_setting('server_version_num') AS version_num"
+ )
+ ).rows[0]
+ expect(Number(runtime.version_num)).toBeGreaterThanOrEqual(160000)
+ expect(Number(runtime.version_num)).toBeLessThan(170000)
+ expect(runtime.version).toMatch(/linux|darwin|bsd|illumos/i)
+ await observer.query(
+ `INSERT INTO token_budgets
+ (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
+ VALUES ($1, 'actual authorize cancellation', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
+ [budgetId, JSON.stringify({ provider: [fixture.provider], model: [fixture.model] })]
+ )
+ __resetBudgetCheckCache()
+ const outcome = authorizeLlmProviderAttempt(
+ fixture.caller(),
+ payload,
+ productionDeps(controller.signal)
+ )
+ .then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ .finally(() => {
+ settled = true
+ })
+ const expectedPhases = [
+ 'assignment',
+ 'transaction',
+ 'budget_reserved',
+ 'attempt_inserted',
+ 'ticket_registered',
+ ]
+ if (stage === 'assignment') {
+ await cancellationDeadline(lookup.arrived)
+ expect(phases).toEqual(['assignment'])
+ expect(workPool.totalCount).toBe(0)
+ } else if (stage === 'query') {
+ await observeCancellation(async () => {
+ if (!backendPid) return false
+ const result = await observer.query<{ state: string; query: string }>(
+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1',
+ [backendPid]
+ )
+ return (
+ result.rows[0]?.state === 'active' &&
+ result.rows[0]?.query === 'SELECT pg_sleep(30)'
+ )
+ })
+ expect(phases).toEqual(expectedPhases)
+ expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
+ } else {
+ await cancellationDeadline(proxy!.commitForwarded)
+ await observeCancellation(async () => (await counts()).attempts === 1)
+ await observeCancellation(async () => proxy!.discardedReplyBytes > 0)
+ expect(phases).toEqual(expectedPhases)
+ expect(proxy!.commands).toEqual(['BEGIN', 'COMMIT'])
+ expect(proxy!.discardedReplyBytes).toBeGreaterThan(0)
+ }
+ expect(settled).toBe(false)
+ controller.abort(reason)
+ const interrupted = await cancellationDeadline(outcome)
+ expect('error' in interrupted ? interrupted.error : undefined).toBe(reason)
+ expect(workPool.totalCount).toBe(0)
+ if (backendPid) {
+ await observeCancellation(
+ async () =>
+ (
+ await observer.query<{ count: number }>(
+ 'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1',
+ [backendPid]
+ )
+ ).rows[0].count === 0
+ )
+ }
+ await withProductionTransaction(async db => {
+ const result = await db.query(
+ 'SELECT pg_try_advisory_xact_lock(hashtext($1)) AS available',
+ [budgetId]
+ )
+ expect((result.rows[0] as { available: boolean }).available).toBe(true)
+ }, observer)
+ if (stage === 'commit') {
+ // The client never received COMMIT acknowledgement, yet the real
+ // business ledger is durable. Do not label this rollback/no spend.
+ expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
+ const attempt = await observer.query(
+ 'SELECT id, provider, connection_id, budget_reservation_id, status FROM llm_provider_attempts WHERE invocation_id = $1',
+ [invocationId]
+ )
+ expect(attempt.rows).toEqual([
+ {
+ id: attemptId,
+ provider: fixture.provider,
+ connection_id: current!.id,
+ budget_reservation_id: reservationId,
+ status: 'authorized',
+ },
+ ])
+ } else {
+ expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
+ stallQuery = false
+ lookup.recover()
+ phases.length = 0
+ const later = await authorizeLlmProviderAttempt(
+ fixture.caller(),
+ payload,
+ productionDeps(new AbortController().signal)
+ )
+ expect(later.providerAttemptId).toBe(attemptId)
+ expect(phases).toEqual(expectedPhases)
+ expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
+ expect(workPool.idleCount).toBe(1)
+ }
+ } finally {
+ controller.abort(reason)
+ grokTicketTap.afterIssue = previousTap
+ proxy?.allowReplies()
+ await endPoolAndWaitForClients(workPool)
+ await proxy?.close()
+ await lookup.close()
+ await observer.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
+ __resetBudgetCheckCache()
+ await endPoolAndWaitForClients(observer)
+ }
+ }
+
+ it(
+ 'cancels a configured Kubernetes lookup before any production transaction, then reauthorizes the same binding',
+ () => run('assignment'),
+ 10_000
+ )
+ it(
+ 'cancels a native query after real budget, attempt and signed ticket writes, then rolls back and reauthorizes',
+ () => run('query'),
+ 10_000
+ )
+ it(
+ 'keeps real authorization ledger rows when COMMIT executes but its reply is lost',
+ () => run('commit'),
+ 10_000
+ )
+ })
+ }
+
beforeAll(async () => {
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
@@ -463,6 +851,27 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',
}
)
+ registerProductionCancellationCases({
+ provider: 'codex-subscription',
+ model: REQUEST.model,
+ connectionKey: 'deployment-default',
+ caller: claims,
+ readConnection: () => getSafeCodexSubscriptionConnection(pool),
+ request: () => ({
+ ...REQUEST,
+ schemaVersion: 'codex-completion-request.v2',
+ requestId: randomUUID(),
+ idempotencyKey: randomUUID(),
+ }),
+ policyHash: connection =>
+ computeCodexPolicyHash({
+ model: REQUEST.model,
+ catalogRevision: connection.catalogRevision,
+ credentialRevision: connection.credentialRevision,
+ connectionKey: connection.connectionKey,
+ }),
+ })
+
// #784: the Grok twin of the Codex rollback above, on an image-bearing V2
// request. The Grok path checks its proxy envelope after signing and before
// commit, so a refusal there must roll back the attempt, the registered
@@ -520,6 +929,35 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',
grokTicketTap.afterIssue = null
})
+ registerProductionCancellationCases({
+ provider: 'grok-subscription',
+ model: GROK_MODEL,
+ connectionKey: GROK_CONNECTION_KEY,
+ caller: grokClaims,
+ readConnection: () => getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY),
+ request: () => ({
+ schemaVersion: 'grok-completion-request.v2',
+ requestId: randomUUID(),
+ idempotencyKey: randomUUID(),
+ provider: 'grok-subscription',
+ model: GROK_MODEL,
+ messages: [
+ {
+ role: 'user',
+ content: 'look',
+ contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],
+ },
+ ],
+ }),
+ policyHash: connection =>
+ computeGrokPolicyHash({
+ model: GROK_MODEL,
+ catalogRevision: connection.catalogRevision,
+ credentialRevision: connection.credentialRevision,
+ connectionKey: GROK_CONNECTION_KEY,
+ }),
+ })
+
it.each([false, true])(
'rolls back an oversized signed Grok V2 envelope without losing an existing reservation (presented=%s)',
async presented => {
diff --git a/scripts/ci/pinned-vitest-cases/control-api.json b/scripts/ci/pinned-vitest-cases/control-api.json
index 5e0ee624f..e69bd8f49 100644
--- a/scripts/ci/pinned-vitest-cases/control-api.json
+++ b/scripts/ci/pinned-vitest-cases/control-api.json
@@ -66,5 +66,161 @@
{
"suite": "routes.mcp-host.llmProviderAttempts.createApp.test.ts",
"fullName": "createApp global JSON encoding refuses compressed JSON before parsing or the public handler and accepts the same uncompressed body"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission policy and idempotence rejects invalid counts and timer values and freezes the accepted policy"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission policy and idempotence an old release cannot free the slot occupied by a newer request"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission policy and idempotence clears retained request bodies and ignores duplicate parser callbacks"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'malformed JSON' parser error after removing its raw body"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'over-deep structure' parser error after removing its raw body"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'compressed body' parser error after removing its raw body"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle propagates the 'payload over limit' parser error after removing its raw body"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle stops a stalled read through closeGrace even when response finish never arrives"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle bounds a never-finishing capacity refusal without reading or starting work"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission parser lifecycle uses close during parsing only to stop the reader and never starts work"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission work cancellation preserves typed authorize_timeout after real signal-aware work settles"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission work cancellation does not free a work owner on client close until signal-aware work unwinds"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission HTTP fixture preserves chunked JSON and releases after work completes"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission HTTP fixture refuses an over-cap request before parsing and keeps a disconnected work owner charged"
+ },
+ {
+ "suite": "llmProviderAttemptBodyAdmission.test.ts",
+ "fullName": "AuthorizeBodyAdmission HTTP fixture releases only after the real body-parser callback follows a destroyed stalled request"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership shares the actual production cap across apps and principals and retains disconnected work until unwind"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership runs both rate limits and JWT before taking any body owner"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership preserves chunked codex JSON through the actual app and releases the body"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership preserves chunked grok JSON through the actual app and releases the body"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership releases after authorize denial and admits a subsequent request"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership releases after unexpected authorizer error and admits a subsequent request"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership returns terminal authorize_timeout after the real work clock aborts, without publishing late success"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "createApp retained authorize-body ownership stops a stalled body reader on the real read clock before releasing and admits the next request"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "Host and Recipe assignment owner cancellation passes the same signal to Host lookup and preserves cancellation"
+ },
+ {
+ "suite": "routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts",
+ "fullName": "Host and Recipe assignment owner cancellation passes the same signal to Recipe lookup and preserves cancellation"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client terminates no-headers with the configured CA, bearer and server intact, then serves a later lookup"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client terminates no-eof with the configured CA, bearer and server intact, then serves a later lookup"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client does not enter the cross-namespace fallback after cancellation without an explicit namespace"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client forwards the optional signal through K8sGateway without reconfiguring its client"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client propagates codex-subscription cancellation into assignment and never enters a transaction"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "authorizer Kubernetes cancellation on the configured native client propagates grok-subscription cancellation into assignment and never enters a transaction"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets never borrows a client for an already aborted request"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets fails without running work if the server does not support scoped connection polling"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an in-flight rollback before final release and preserves the typed owner reason"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an active native query and evicts the client exactly once before recovery"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets terminates an idle half-open client but remains owned through callback cleanup"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets awaits bounded connect failure and preserves the cancellation reason without sending BEGIN"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets disposes a borrow that arrives after abort before any new statement or healthy return"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "scoped transaction cancellation with the native pg driver and sockets keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy"
}
]