Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { type Response, Router } from 'express'
import {
isCredentialSlotOwnedByProvider,
isLlmProviderId,
isRunnableLlmModelId,
providerDescriptor,
} from '@clerum/llm-providers'
import {
advisoryLockModelNames,
boundCarrierTransactionIdleTimeout,
withTransaction,
} from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import type { K8sGateway } from '../../k8s.js'
import type { UiAuthedRequest } from '../../middleware/controlUIAuth.js'
import { createPluginWorkloadSdkAdminRateLimit } from '../../middleware/pluginWorkloadSdkRateLimits.js'
import { rootLogger } from '../../observability/logger.js'
import { isCodexAssignmentAllowed } from '../../services/codexSubscriptionCatalog.js'
import {
assertCodexConnectionKey,
isCodexUnassignedConnectionKey,
} from '../../services/codexSubscriptionConnection.js'
import { isGrokAssignmentAllowed } from '../../services/grokSubscriptionCatalog.js'
import {
assertGrokConnectionKey,
isGrokUnassignedConnectionKey,
} from '../../services/grokSubscriptionConnection.js'
import { listEnabledModelsWithStaleForProvider } from '../../services/llmAllowedModels.js'
import {
MAX_ALLOWLIST_ENTRY_LENGTH,
MAX_ALLOWLIST_ITEMS,
MAX_PROMPT_BRIDGE_TARGETS_PER_GRANT,
PLUGIN_WORKLOAD_SDK_FAMILIES,
PLUGIN_WORKLOAD_SDK_INVOCATION_STATUSES,
type PluginWorkloadSdkFamily,
type PluginWorkloadSdkInvocationStatus,
type PluginWorkloadSdkModelPolicy,
type PluginWorkloadSdkPromptTarget,
type PluginWorkloadSdkQuotaLimits,
deleteGrant,
getPluginWorkloadSdkLegacyGrantInventory,
getQuotaCounters,
hasUsableClientNotificationRecipients,
listGrants,
listInvocations,
upsertGrant,
} from '../../services/pluginWorkloadSdkDb.js'
import {
RecipeCodexGrantIdentityError,
publishRecipeGrantIdentity,
} from '../../services/recipeCodexGrantIdentity.js'
import { collectRecipeOauthBrokerProviders } from '../../services/subscriptionGrantIdentity.js'
import { isPlainObject } from '../../utils/isPlainObject.js'
import {
type EmitHostSpecIncoherenceTolerated,
type HostSpecIncoherenceToleratedEvent,
emitHostSpecIncoherenceTolerated as emitHostSpecIncoherenceToleratedDefault,
} from './hostWriteGateAudit.js'
import {
type CoverageSet,
isNonWorseningToleration,
offeredKey,
} from './modelAllowlistTolerance.js'
import { STALE_MODEL_ASSIGNED, type StaleModelWarning } from './staleModelWarning.js'
// ─── Plugin Workload SDK — admin routes (plan §2.6) ──────────────────────
// Admin auth is enforced upstream by the control-ui auth gate in app.ts
// (same contract as every other /admin router).
//
// GET /admin/plugin-workload-sdk/grants
// POST /admin/plugin-workload-sdk/grants (upsert by recipe+family)
// DELETE /admin/plugin-workload-sdk/grants/:id
// GET /admin/plugin-workload-sdk/legacy-inventory (read-only migration gate)
// GET /admin/plugin-workload-sdk/quota/:recipeNamespace/:recipeName
// GET /admin/plugin-workload-sdk/invocations (audit search)
const FAMILY_SET = new Set<string>(PLUGIN_WORKLOAD_SDK_FAMILIES)
const STATUS_SET = new Set<string>(PLUGIN_WORKLOAD_SDK_INVOCATION_STATUSES)
// allowedUserRefs are matched verbatim against audience.userId, which every
// delivery path (claim, desktop ACK, SSE) joins to the control-plane user UUID.
// A non-UUID userRef would authorize + enqueue but never claim (no medium-account
// match), leaving the notification to expire silently after 72h. Reject it at
// grant creation so the failure is loud and immediate, not silent data loss.
// (targetRefs are opaque provider identities, so they are intentionally exempt.)
const USER_REF_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
const SECRET_KEY_RE = /^[-._a-zA-Z0-9]+$/
function parseIsoTimestamp(
value: unknown,
field: string,
res: Response
): string | null | undefined {
if (value === undefined) return undefined
if (typeof value !== 'string') {
res.status(400).json({ error: `${field} must be an ISO-8601 timestamp string` })
return null
}
const parsed = Date.parse(value)
if (!Number.isFinite(parsed)) {
res.status(400).json({ error: `${field} must be a valid ISO-8601 timestamp` })
return null
}
return new Date(parsed).toISOString()
}
function parseStringArray(value: unknown, field: string, res: Response): string[] | null {
if (value === undefined) return []
if (!Array.isArray(value) || value.some(v => typeof v !== 'string')) {
res.status(400).json({ error: `${field} must be an array of strings` })
return null
}
if (value.length > MAX_ALLOWLIST_ITEMS) {
res.status(400).json({
error: `${field} must contain at most ${MAX_ALLOWLIST_ITEMS} entries`,
})
return null
}
for (const entry of value) {
if (entry.length > MAX_ALLOWLIST_ENTRY_LENGTH) {
res.status(400).json({
error: `${field} entries must be at most ${MAX_ALLOWLIST_ENTRY_LENGTH} characters`,
})
return null
}
}
return value as string[]
}
function parseModelArray(value: unknown, field: string, res: Response): string[] | null {
const values = parseStringArray(value, field, res)
if (values === null) return null
// Keep wildcard rejection distinct from the runnable-model grammar error.
// Wildcards are a policy-boundary violation, not a malformed model id, and
// callers/tests rely on the stable structured error for that case.
if (values.some(model => model.includes('*'))) {
res.status(400).json({ error: 'wildcard_not_allowed' })
return null
}
const invalid = values.find(
model => !isRunnableLlmModelId(model.trim()) || model.trim() !== model
)
if (invalid !== undefined) {
res.status(400).json({
error: `${field} entries must be valid runnable model ids`,
value: invalid,
})
return null
}
return values
}
// Issue #348: per-run quota keys are deprecated. They remain accepted on the
// wire for compatibility and are validated with the same shape rules, but they
// are stripped before persistence — only per-minute/platform limits are stored.
//
// `maxOutputTokens` was briefly moved here on the premise that nothing
// enforced it. That premise held only for codex-subscription, whose ChatGPT
// wire rejects `max_output_tokens` and whose proxy therefore omits it. Every
// API-key provider does send it (`openai.ts`, `bedrockConverse.ts`,
// `googleGenerative.ts`), so for them it was a working per-grant billing
// ceiling. The key is active again, with its real scope stated wherever an
// operator can read it, rather than a ceiling silently ignored everywhere.
const DEPRECATED_QUOTA_LIMIT_KEYS: ReadonlyArray<keyof PluginWorkloadSdkQuotaLimits> = [
'maxRequestsPerRun',
'maxNotificationsPerRun',
]
const ACTIVE_QUOTA_LIMIT_KEYS: ReadonlyArray<keyof PluginWorkloadSdkQuotaLimits> = [
'maxInvocationsPerMinute',
'maxNotificationsPerMinute',
'maxOutputTokens',
]
function parseQuotaLimits(value: unknown, res: Response): PluginWorkloadSdkQuotaLimits | null {
if (value === undefined) return {}
if (!isPlainObject(value)) {
res.status(400).json({ error: 'quotaLimits must be an object' })
return null
}
const limits: PluginWorkloadSdkQuotaLimits = {}
for (const key of [...DEPRECATED_QUOTA_LIMIT_KEYS, ...ACTIVE_QUOTA_LIMIT_KEYS]) {
const raw = value[key]
if (raw === undefined) continue
if (typeof raw !== 'number' || !Number.isInteger(raw) || raw < 1) {
res.status(400).json({ error: `quotaLimits.${key} must be a positive integer` })
return null
}
// Deprecated keys are validated (malformed input still 400s) but never
// assigned, so they are not persisted (issue #348).
if ((ACTIVE_QUOTA_LIMIT_KEYS as readonly string[]).includes(key)) {
limits[key] = raw
}
}
return limits
}
function parseModelPolicies(
value: unknown,
res: Response
): Record<string, PluginWorkloadSdkModelPolicy> | null {
if (value === undefined) return {}
if (!isPlainObject(value)) {
res.status(400).json({ error: 'modelPolicies must be an object' })
return null
}
const policies: Record<string, PluginWorkloadSdkModelPolicy> = {}
for (const [ref, raw] of Object.entries(value)) {
if (
!isPlainObject(raw) ||
typeof raw.provider !== 'string' ||
!raw.provider.trim() ||
typeof raw.model !== 'string' ||
!isRunnableLlmModelId(raw.model.trim())
) {
res.status(400).json({
error: `modelPolicies.${ref} must declare provider and model`,
})
return null
}
policies[ref] = {
provider: raw.provider,
model: raw.model,
...(typeof raw.temperature === 'number' ? { temperature: raw.temperature } : {}),
...(typeof raw.maxCostUsd === 'number' ? { maxCostUsd: raw.maxCostUsd } : {}),
}
}
return policies
}
/** Whether a validated provider id resolves credentials through the OAuth broker. */
function isBrokerBackedProvider(provider: string): boolean {
return isLlmProviderId(provider) && providerDescriptor(provider).authMode === 'oauth-broker'
}
function credentialSlotBelongsToProvider(provider: string, credentialSlot: string): boolean {
// This shares the exact slot-ownership rule with the runtime broker. A
// target only carries the key identity; this route never reads the Secret or
// returns a value.
return isCredentialSlotOwnedByProvider(provider, credentialSlot)
}
function parsePromptTargets(value: unknown, res: Response): PluginWorkloadSdkPromptTarget[] | null {
if (!Array.isArray(value) || value.length === 0) {
res
.status(400)
.json({ error: 'promptTargets must be a non-empty array for promptBridge grants' })
return null
}
if (value.length > MAX_PROMPT_BRIDGE_TARGETS_PER_GRANT) {
res.status(400).json({
error: `promptTargets must contain at most ${MAX_PROMPT_BRIDGE_TARGETS_PER_GRANT} entries`,
})
return null
}
const targets: PluginWorkloadSdkPromptTarget[] = []
const targetRefs = new Set<string>()
// Provider + model must be unique even when two extra slots exist. Otherwise
// the documented exact `{provider, model}` selector and modelPolicyRef would
// choose a credential slot implicitly, which is a policy bypass.
const providerModels = new Set<string>()
for (let index = 0; index < value.length; index++) {
const raw = value[index]
if (!isPlainObject(raw)) {
res.status(400).json({ error: `promptTargets[${index}] must be an object` })
return null
}
const targetRef = typeof raw.targetRef === 'string' ? raw.targetRef.trim() : ''
const provider = typeof raw.provider === 'string' ? raw.provider.trim() : ''
const model = typeof raw.model === 'string' ? raw.model.trim() : ''
const credentialSlot = typeof raw.credentialSlot === 'string' ? raw.credentialSlot.trim() : ''
let connectionRef = typeof raw.connectionRef === 'string' ? raw.connectionRef.trim() : ''
if (!targetRef || targetRef.length > MAX_ALLOWLIST_ENTRY_LENGTH || targetRef.includes('*')) {
res
.status(400)
.json({ error: `promptTargets[${index}].targetRef must be a non-wildcard identifier` })
return null
}
if (!isLlmProviderId(provider)) {
res.status(400).json({ error: `promptTargets[${index}].provider is invalid` })
return null
}
if (!isRunnableLlmModelId(model) || model.includes('*')) {
res
.status(400)
.json({ error: `promptTargets[${index}].model must be a non-wildcard model id` })
return null
}
const brokerBacked = providerDescriptor(provider).authMode === 'oauth-broker'
if (brokerBacked) {
// oauth-broker targets (Codex) carry no static Secret data key: the
// credential is brokered per attempt. An empty slot is the only correct
// value — a non-empty slot would pretend a ChatGPT refresh token lives
// in an API-key slot, which is exactly the shape this gate forbids.
if (credentialSlot) {
res.status(400).json({
error: `promptTargets[${index}].credentialSlot must be empty for oauth-broker provider "${provider}"`,
})
return null
}
// Named keys must be well-formed. Missing or `unassigned` is allowed so
// a previously stored Codex target can be edited without inventing a
// grant. Spend still fail-closes at authorize until the operator picks
// a live connection. Malformed keys stay rejected.
const unassigned =
provider === 'grok-subscription'
? isGrokUnassignedConnectionKey(connectionRef)
: isCodexUnassignedConnectionKey(connectionRef)
if (connectionRef && !unassigned) {
try {
if (provider === 'grok-subscription') assertGrokConnectionKey(connectionRef)
else assertCodexConnectionKey(connectionRef)
} catch {
res.status(400).json({
error:
provider === 'grok-subscription'
? `promptTargets[${index}].connectionRef must name an existing Grok subscription connection`
: `promptTargets[${index}].connectionRef must name an existing Codex subscription connection`,
})
return null
}
} else {
connectionRef = ''
}
} else {
if (connectionRef) {
res.status(400).json({
error: `promptTargets[${index}].connectionRef is only valid for oauth-broker providers`,
})
return null
}
if (
!credentialSlot ||
credentialSlot.length > 253 ||
!SECRET_KEY_RE.test(credentialSlot) ||
!credentialSlotBelongsToProvider(provider, credentialSlot)
) {
res.status(400).json({
error: `promptTargets[${index}].credentialSlot must be a valid slot owned by provider "${provider}"`,
})
return null
}
}
const providerModel = `${provider}\u0000${model}`
if (targetRefs.has(targetRef) || providerModels.has(providerModel)) {
res.status(400).json({ error: `promptTargets[${index}] duplicates an existing target` })
return null
}
targetRefs.add(targetRef)
providerModels.add(providerModel)
targets.push({
targetRef,
provider,
model,
credentialSlot,
...(brokerBacked ? { connectionRef } : {}),
})
}
const brokerProviders = new Set(
targets.filter(target => isBrokerBackedProvider(target.provider)).map(target => target.provider)
)
if (brokerProviders.size > 1) {
res.status(400).json({ error: 'oauth_broker_provider_conflict' })
return null
}
const brokerKeys = new Set(
targets
.filter(target => isBrokerBackedProvider(target.provider))
.map(target => target.connectionRef ?? '')
)
if (brokerKeys.size > 1) {
res.status(400).json({ error: 'codex_connection_ref_conflict' })
return null
}
return targets
}
// A rejection produced by the enabled-ness write-gate WHILE it runs inside the
// carrier transaction that holds the per-model advisory locks (R1-H3 fase 2).
// It is THROWN (not `res.json`-ed inline) so the transaction ROLLS BACK — the
// upsert never persists and no Pieza D audit is emitted — then mapped back to
// the SAME 400 body the gate has always returned. `body` is byte-stable with
// the pre-fase-2 responses (`model_not_allowed`, `modelPolicies.*` drift).
class GrantModelGateError extends Error {
constructor(readonly body: Record<string, unknown>) {
super('grant_model_gate_rejected')
this.name = 'GrantModelGateError'
}
}
export interface AdminPluginWorkloadSdkRouterDeps {
/**
* Emit the audit signal for a tolerated pre-existing model_not_allowed
* incoherence on a grant write (Pieza D). Injected so the "never silent"
* guarantee is testable through the seam; defaults to the shared emitter.
*/
emitIncoherenceTolerated?: EmitHostSpecIncoherenceTolerated
/**
* Publishes `clerum.io/codex-connection-ref` on the WorkflowRecipe when an
* SDK Codex choose changes. Authorize attests that annotation.
*/
gateway?: Pick<K8sGateway, 'getResource' | 'updateResource'>
}
const log = rootLogger.child({ module: 'admin-plugin-workload-sdk' })
async function publishPromptBridgeGrantIdentity(input: {
gateway: Pick<K8sGateway, 'getResource' | 'updateResource'>
recipeNamespace: string
recipeName: string
nextRef: string
provider?: string
/**
* Upsert rejects a broker target whose provider differs from the recipe's
* agent / step-agent broker. Delete skips the publish instead: the grant row
* is already gone and the agent owns the annotation.
*/
onProviderConflict: 'reject' | 'skip'
}): Promise<{ error?: { status: number; error: string } }> {
let recipeBrokers: string[]
try {
const recipe = (await input.gateway.getResource(
'workflowrecipes',
input.recipeName,
input.recipeNamespace
)) as { spec?: unknown } | null
recipeBrokers = collectRecipeOauthBrokerProviders(
isPlainObject(recipe?.spec) ? recipe.spec : {}
)
} catch (err) {
log.error(
{ err, recipeNamespace: input.recipeNamespace, recipeName: input.recipeName },
'failed to read WorkflowRecipe before publishing grant identity'
)
return { error: { status: 503, error: 'recipe_annotation_publish_failed' } }
}
// A broker SDK target is spendable only when the recipe's agent / step agents
// name that same broker: authorize attests live targets from agent/steps, and
// the promptBridge default target must match the bootstrap agent. A static-
// agent recipe (RP-009) therefore fails closed here instead of publishing an
// identity no SDK call can ever redeem. An unassigned target on such a recipe
// only clears the identity, so legacy grant re-saves keep working.
if (
input.provider &&
!recipeBrokers.includes(input.provider) &&
(recipeBrokers.length > 0 || input.nextRef !== '')
) {
if (input.onProviderConflict === 'skip') return {}
return { error: { status: 400, error: 'oauth_broker_provider_conflict' } }
}
let next = input.nextRef
if (!next) {
// Do not unassign a recipe whose agent (or any step agent) owns the grant.
if (recipeBrokers.length > 0) return {}
next = 'unassigned'
}
try {
await publishRecipeGrantIdentity({
gateway: input.gateway,
namespace: input.recipeNamespace,
name: input.recipeName,
next,
...(input.provider ? { provider: input.provider } : {}),
})
} catch (err) {
if (err instanceof RecipeCodexGrantIdentityError) {
return { error: { status: err.status, error: err.error } }
}
throw err
}
return {}
}
export function createAdminPluginWorkloadSdkRouter(
deps: AdminPluginWorkloadSdkRouterDeps = {}
): Router {
const emitIncoherenceTolerated =
deps.emitIncoherenceTolerated ?? emitHostSpecIncoherenceToleratedDefault
const router = Router()
router.use('/admin/plugin-workload-sdk', createPluginWorkloadSdkAdminRateLimit())
router.get(
'/admin/plugin-workload-sdk/grants',
asyncHandler(async (req, res) => {
const recipeNamespace =
typeof req.query.recipeNamespace === 'string' ? req.query.recipeNamespace : undefined
const recipeName = typeof req.query.recipeName === 'string' ? req.query.recipeName : undefined
const items = await listGrants({ recipeNamespace, recipeName })
res.status(200).json({ items })
})
)
router.post(
'/admin/plugin-workload-sdk/grants',
asyncHandler(async (req, res) => {
const body = isPlainObject(req.body) ? req.body : {}
const recipeNamespace =
typeof body.recipeNamespace === 'string' ? body.recipeNamespace.trim() : ''
const recipeName = typeof body.recipeName === 'string' ? body.recipeName.trim() : ''
if (!recipeNamespace || !recipeName) {
res.status(400).json({ error: 'recipeNamespace and recipeName are required' })
return
}
const capabilityFamily =
typeof body.capabilityFamily === 'string' ? body.capabilityFamily : ''
if (!FAMILY_SET.has(capabilityFamily)) {
res.status(400).json({
error: `capabilityFamily must be one of: ${PLUGIN_WORKLOAD_SDK_FAMILIES.join(', ')}`,
})
return
}
const allowedModels = parseModelArray(body.allowedModels, 'allowedModels', res)
if (allowedModels === null) return
const allowedEventTypes = parseStringArray(body.allowedEventTypes, 'allowedEventTypes', res)
if (allowedEventTypes === null) return
const allowedTargetRefs = parseStringArray(body.allowedTargetRefs, 'allowedTargetRefs', res)
if (allowedTargetRefs === null) return
const allowedUserRefs = parseStringArray(body.allowedUserRefs, 'allowedUserRefs', res)
if (allowedUserRefs === null) return
if (allowedUserRefs.some(ref => !USER_REF_UUID_RE.test(ref))) {
res.status(400).json({
error: 'allowedUserRefs entries must be control-plane user UUIDs',
})
return
}
const allowedCallers = parseStringArray(body.allowedCallers, 'allowedCallers', res)
if (allowedCallers === null) return
const quotaLimits = parseQuotaLimits(body.quotaLimits, res)
if (quotaLimits === null) return
const modelPolicies = parseModelPolicies(body.modelPolicies, res)
if (modelPolicies === null) return
const hasWildcard = [
...allowedModels,
...allowedEventTypes,
...allowedTargetRefs,
...allowedUserRefs,
...allowedCallers,
].some(entry => entry.includes('*'))
if (hasWildcard) {
res.status(400).json({ error: 'wildcard_not_allowed' })
return
}
if (capabilityFamily === 'clientNotifications' && allowedEventTypes.length === 0) {
res
.status(400)
.json({ error: 'allowedEventTypes must be non-empty for clientNotifications grants' })
return
}
if (
capabilityFamily === 'clientNotifications' &&
allowedTargetRefs.length + allowedUserRefs.length === 0
) {
res.status(400).json({
error:
'clientNotifications grants require at least one allowedTargetRefs or allowedUserRefs entry',
})
return
}
if (
capabilityFamily === 'clientNotifications' &&
!(await hasUsableClientNotificationRecipients({ allowedTargetRefs, allowedUserRefs }))
) {
res.status(400).json({
error:
'clientNotifications grants require at least one existing user or verified notification target',
})
return
}
const promptTargets =
capabilityFamily === 'promptBridge' ? parsePromptTargets(body.promptTargets, res) : []
if (promptTargets === null) return
const defaultTargetRef =
capabilityFamily === 'promptBridge' && typeof body.defaultTargetRef === 'string'
? body.defaultTargetRef.trim()
: ''
if (capabilityFamily === 'promptBridge') {
if (!defaultTargetRef || promptTargets[0]?.targetRef !== defaultTargetRef) {
res.status(400).json({
error: 'defaultTargetRef is required and must be the first promptTargets entry',
})
return
}
}
// R1: promptBridge credentials resolve per-provider, so the grant stores an
// explicit provider (it is no longer inferred from the model list). It is
// required for promptBridge and irrelevant for clientNotifications.
// R4: `provider` is now validated against the canonical provider set from
// the shared @clerum/llm-providers package (the guard is prototype-safe).
// provider↔allowedModels consistency IS also enforced (R3): after the
// provider is resolved, every allowedModels entry must exist enabled under
// that provider in the `llm_allowed_models` allowlist (checked below).
let provider: string | undefined
// Pieza D — queued grant tolerations, flushed only once the write is
// ACCEPTED (right before upsert). A grant rejected by a later 400 leaves no
// audit record.
const pendingGrantTolerations: HostSpecIncoherenceToleratedEvent[] = []
// Fase 6 soft-quarantine warnings for NEW assignments of enabled-but-stale
// models. Returned in the 200 body only after the upsert lands (below); a
// grant rejected by a later 400 returns before the response and carries none.
const grantWarnings: StaleModelWarning[] = []
const grantWarnedKeys = new Set<string>()
if (capabilityFamily === 'promptBridge') {
const rawProvider = typeof body.provider === 'string' ? body.provider.trim() : ''
if (!rawProvider) {
res.status(400).json({ error: 'provider is required for promptBridge grants' })
return
}
if (!isLlmProviderId(rawProvider)) {
res.status(400).json({ error: 'Invalid provider' })
return
}
provider = rawProvider
}
if (capabilityFamily === 'promptBridge') {
const defaultTarget = promptTargets[0]!
// `provider` remains the bootstrap host binding during the compatibility
// window. It cannot authorize routes by itself and must agree with the
// policy default instead of being silently reconciled. Pure check, kept
// OUTSIDE the carrier transaction (no model read, no lock needed).
if (provider !== defaultTarget.provider) {
res.status(400).json({ error: 'policy_bootstrap_mismatch' })
return
}
}
// R1-H3 fase 2 — carrier transaction that serializes this grant upsert
// against a concurrent llm-model disable/delete (the reductor, fase 1).
// The reductor enumerates grants by NAME in `allowed_models` under
// `advisoryLockModelName`; so the upsert takes the SAME per-name locks over
// ALL incoming `allowed_models` names, HOLDS them across the enabled-ness
// re-validation, and commits the write atomically. Ordering is the global
// invariant (adenda A5): every `llm-model:*` lock is taken BEFORE the
// recipe lock (`plugin_workload_sdk:*`, taken inside `upsertGrant`). If the
// reductor commits first, the gate below re-reads `llm_allowed_models`
// under the lock and rejects (400); if the grant commits first, the
// reductor sees it and answers 409 — neither can strand a dangling grant.
// Enabled-ness is validated only for promptBridge targets, EXACTLY as
// before (serialization only, no accept/reject change): a disabled model
// reachable solely through `allowed_models` (no promptTarget) is a
// PRE-EXISTING gap, out of R1-H3 scope.
const brokerTargets = promptTargets.filter(target => isBrokerBackedProvider(target.provider))
const brokerProviders = [...new Set(brokerTargets.map(target => target.provider))]
const brokerTarget = brokerTargets.find(
target =>
target.connectionRef &&
!(target.provider === 'grok-subscription'
? isGrokUnassignedConnectionKey(target.connectionRef)
: isCodexUnassignedConnectionKey(target.connectionRef))
)
const brokerConnectionRef = brokerTarget?.connectionRef ?? ''
const hasBrokerTarget = brokerTargets.length > 0
if (capabilityFamily === 'promptBridge' && deps.gateway && hasBrokerTarget) {
const published = await publishPromptBridgeGrantIdentity({
gateway: deps.gateway,
recipeNamespace,
recipeName,
nextRef: brokerConnectionRef,
provider: brokerTarget?.provider ?? brokerProviders[0],
onProviderConflict: 'reject',
})
if (published.error) {
res.status(published.error.status).json({ error: published.error.error })
return
}
}
let grant: Awaited<ReturnType<typeof upsertGrant>>
try {
grant = await withTransaction(async db => {
// Bound idle-in-transaction tenancy for parity with the reductor
// (fase 1). This seam is PG-only (no K8s write under the lock), so the
// lock is never held across an external round-trip — cheap, defensive.
await boundCarrierTransactionIdleTimeout(db)
// Model-name locks FIRST, deduped + totally ordered, over EVERY incoming
// `allowed_models` name (the reductor matches by name across families).
// Empty list = no-op. Taken before the recipe lock (global order A5).
await advisoryLockModelNames(db, allowedModels)
if (capabilityFamily === 'promptBridge') {
// Codex (oauth-broker) targets are validated against the chosen
// grant's per-connection catalog, not the flat `llm_allowed_models`
// union: only a model offered AND enabled (non-stale) on that
// exact connected grant may be authorized. Missing/unassigned is
// allowed so a stored target remains editable; spend still
// fail-closes at authorize. Named unknown/revoked/not-offered
// keys fail closed. No Pieza D silent keep.
for (const target of promptTargets) {
if (!isBrokerBackedProvider(target.provider)) continue
const connectionRef = target.connectionRef ?? ''
const unassigned =
target.provider === 'grok-subscription'
? isGrokUnassignedConnectionKey(connectionRef)
: isCodexUnassignedConnectionKey(connectionRef)
if (!connectionRef || unassigned) {
continue
}
const allowed =
target.provider === 'grok-subscription'
? await isGrokAssignmentAllowed(db, connectionRef, target.model)
: await isCodexAssignmentAllowed(db, connectionRef, target.model)
if (!allowed) {
throw new GrantModelGateError({
error:
target.provider === 'grok-subscription'
? 'grok_connection_not_allowed'
: 'codex_connection_not_allowed',
connectionRef: target.connectionRef ?? '',
model: target.model,
})
}
}
const modelsByProvider = new Map<string, string[]>()
for (const target of promptTargets) {
if (isBrokerBackedProvider(target.provider)) continue
modelsByProvider.set(target.provider, [
...(modelsByProvider.get(target.provider) ?? []),
target.model,
])
}
// Pieza D — role-scoped no-worsening tolerance (editability trap,
// mini-spec 01). A grant that references a model later disabled
// globally would otherwise be uneditable (every save 400s
// `model_not_allowed`). Tolerate a disabled `(provider, model)` iff
// (b) the incoming `allowed_models` does NOT shrink the stored
// coverage AND it kept its ROLE: the incoming DEFAULT target
// (`promptTargets[0]`) is tolerated only if it was the stored default
// target; a NON-default target only if it was any stored target
// (demotion is not worsening). Promoting a disabled non-default target
// to the default slot is NOT tolerated. A brand-new disabled model is
// never tolerated. The stored grant is read lazily — only when a
// candidate rejection appears — so the all-enabled happy path keeps
// its single-query cost.
const incomingDefaultKey = offeredKey(
promptTargets[0]!.provider,
promptTargets[0]!.model
)
let storedLoaded = false
let storedDefaultTarget = new Set<string>()
let storedAnyTarget = new Set<string>()
// Grant `allowed_models` is a flat model-name list; treat it as a
// literal finite coverage set (NOT the Host UNIVERSAL-on-empty
// semantics) so an emptied list is a strict reduction, not a widening.
const incomingCoverage: CoverageSet = new Set(allowedModels)
let storedCoverage: CoverageSet = new Set<string>()
const loadStoredGrant = async (): Promise<void> => {
if (storedLoaded) return
storedLoaded = true
// Read the stored grant on the CARRIER connection (holds the model
// locks) so no extra pool checkout happens under the lock (A3).
const storedGrant = (await listGrants({ recipeNamespace, recipeName }, db))?.find(
existing => existing.capabilityFamily === capabilityFamily
)
const storedTargets = storedGrant?.promptTargets ?? []
storedDefaultTarget = new Set(
storedTargets[0]
? [offeredKey(storedTargets[0].provider, storedTargets[0].model)]
: []
)
storedAnyTarget = new Set(storedTargets.map(t => offeredKey(t.provider, t.model)))
storedCoverage = new Set(storedGrant?.allowedModels ?? [])
}
for (const [targetProvider, targetModels] of modelsByProvider) {
// Enabled-ness re-read UNDER the model locks, on the carrier `db`
// (NOT the global pool) — this is the read the reductor races.
const enabledRows = await listEnabledModelsWithStaleForProvider(targetProvider, db)
const enabled = new Set(enabledRows.map(row => row.model))
const staleEnabled = new Set(
enabledRows.filter(row => row.stale).map(row => row.model)
)
// Fase 6 soft quarantine: warn (never block) when a NEW target
// assigns an ENABLED but `stale` model. A live reference already in
// the stored grant is not revalidated (spec §3.3). The stored grant
// is loaded lazily — only when a stale-enabled candidate is actually
// present — so the all-fresh / no-stale happy path keeps its
// single-query cost.
for (const model of targetModels) {
if (!enabled.has(model) || !staleEnabled.has(model)) continue
const pairKey = offeredKey(targetProvider, model)
if (grantWarnedKeys.has(pairKey)) continue
grantWarnedKeys.add(pairKey)
await loadStoredGrant()
if (storedAnyTarget.has(pairKey)) continue
const i = promptTargets.findIndex(
target => target.provider === targetProvider && target.model === model
)
grantWarnings.push({
code: STALE_MODEL_ASSIGNED,
provider: targetProvider,
model,
field: i >= 0 ? `promptTargets[${i}].model` : 'promptTargets',
})
}
const rawOffenders = targetModels.filter(model => !enabled.has(model))
if (rawOffenders.length === 0) continue
await loadStoredGrant()
const offenders: string[] = []
for (const model of rawOffenders) {
const pairKey = offeredKey(targetProvider, model)
// Role of the offending pair in the INCOMING grant: the default
// slot (`promptTargets[0]`) must have been the stored default; any
// other target may have been any stored target.
const storedRoleSet =
pairKey === incomingDefaultKey ? storedDefaultTarget : storedAnyTarget
const tolerated = isNonWorseningToleration({
pairKey,
storedReferencedPairKeys: storedRoleSet,
incomingCoverage,
storedCoverage,
})
if (tolerated) {
pendingGrantTolerations.push({
resourceKind: 'grant',
namespace: recipeNamespace,
name: recipeName,
provider: targetProvider,
model,
gate: 'grant',
offeredBefore: storedCoverage,
offeredAfter: incomingCoverage,
})
} else {
offenders.push(model)
}
}
if (offenders.length > 0) {
// THROW (not res.json) so the transaction rolls back — the upsert
// never runs and no Pieza D audit fires — mapped to the byte-stable
// 400 `model_not_allowed` body outside the transaction.
throw new GrantModelGateError({
error: 'model_not_allowed',
provider: targetProvider,
models: offenders,
})
}
}
// A modelPolicyRef names an already-authorized target, not a second
// routing authority. Keep the older record shape but reject drift.
// Runs AFTER the enabled-ness gate to preserve error precedence.
for (const [ref, policy] of Object.entries(modelPolicies)) {
if (
!promptTargets.some(
target => target.provider === policy.provider && target.model === policy.model
)
) {
throw new GrantModelGateError({
error: `modelPolicies.${ref} does not match an authorized target`,
})
}
}
}
if (allowedCallers.length === 0) {
throw new GrantModelGateError({ error: 'allowedCallers must be non-empty' })
}
// Reuse THIS transaction: `upsertGrant` takes the recipe lock as its
// first statement — AFTER the model locks above (global order A5) — and
// commits the write in the same critical section as the re-validation.
return upsertGrant(
{
recipeNamespace,
recipeName,
capabilityFamily: capabilityFamily as PluginWorkloadSdkFamily,
provider,
allowedModels,
allowedEventTypes,
allowedTargetRefs,
allowedUserRefs,
allowedCallers,
quotaLimits,
modelPolicies,
promptTargets,
defaultTargetRef: defaultTargetRef || undefined,
},
(req as UiAuthedRequest).adminAuth!.sub,
db
)
})
} catch (err) {
if (err instanceof GrantModelGateError) {
res.status(400).json(err.body)
return
}
throw err
}
// Grant PERSISTED: NOW emit any Pieza D tolerations (never before the upsert
// lands, so a failed upsert leaves no audit record — mini-spec 01).
for (const event of pendingGrantTolerations) emitIncoherenceTolerated(event)
// Fase 6: attach any soft-quarantine warnings additively (older clients
// ignore the field). Absent when there is nothing to warn about.
res
.status(200)
.json(grantWarnings.length > 0 ? { grant, warnings: grantWarnings } : { grant })
})
)
router.delete(
'/admin/plugin-workload-sdk/grants/:id',
asyncHandler(async (req, res) => {
// Scope the delete to the grant's recipe binding (defense in depth): a
// UUID alone must not delete a grant belonging to another recipe.
const recipeNamespace =
typeof req.query.recipeNamespace === 'string' ? req.query.recipeNamespace.trim() : ''
const recipeName = typeof req.query.recipeName === 'string' ? req.query.recipeName.trim() : ''
if (!recipeNamespace || !recipeName) {
res
.status(400)
.json({ error: 'recipeNamespace and recipeName query parameters are required' })
return
}
const deleted = await deleteGrant(
req.params.id,
recipeNamespace,
recipeName,
(req as UiAuthedRequest).adminAuth!.sub
)
if (!deleted) {
res.status(404).json({ error: 'grant not found' })
return
}
if (deps.gateway) {
const remaining = await listGrants({ recipeNamespace, recipeName })
const remainingTarget = remaining
.flatMap(grant => grant.promptTargets)
.find(
target =>
isBrokerBackedProvider(target.provider) &&
target.connectionRef &&
!(target.provider === 'grok-subscription'
? isGrokUnassignedConnectionKey(target.connectionRef)
: isCodexUnassignedConnectionKey(target.connectionRef))
)
const published = await publishPromptBridgeGrantIdentity({
gateway: deps.gateway,
recipeNamespace,
recipeName,
nextRef: remainingTarget?.connectionRef ?? '',
provider: remainingTarget?.provider,
onProviderConflict: 'skip',
})
if (published.error) {
res.status(published.error.status).json({ error: published.error.error })
return
}
}
res.status(200).json({ deleted: true })
})
)
router.get(
'/admin/plugin-workload-sdk/legacy-inventory',
asyncHandler(async (req, res) => {
const recipeNamespace =
typeof req.query.recipeNamespace === 'string' ? req.query.recipeNamespace.trim() : undefined
const recipeName =
typeof req.query.recipeName === 'string' ? req.query.recipeName.trim() : undefined
const inventory = await getPluginWorkloadSdkLegacyGrantInventory({
...(recipeNamespace ? { recipeNamespace } : {}),
...(recipeName ? { recipeName } : {}),
})
res.status(200).json(inventory)
})
)
router.get(
'/admin/plugin-workload-sdk/quota/:recipeNamespace/:recipeName',
asyncHandler(async (req, res) => {
const items = await getQuotaCounters(req.params.recipeNamespace, req.params.recipeName)
res.status(200).json({ items })
})
)
router.get(
'/admin/plugin-workload-sdk/invocations',
asyncHandler(async (req, res) => {
const q = req.query
const method = typeof q.method === 'string' && FAMILY_SET.has(q.method) ? q.method : undefined
const status = typeof q.status === 'string' && STATUS_SET.has(q.status) ? q.status : undefined
const limitRaw = typeof q.limit === 'string' ? Number.parseInt(q.limit, 10) : NaN
const since = parseIsoTimestamp(q.since, 'since', res)
if (since === null) return
const until = parseIsoTimestamp(q.until, 'until', res)
if (until === null) return
const items = await listInvocations({
recipeNamespace: typeof q.recipeNamespace === 'string' ? q.recipeNamespace : undefined,
recipeName: typeof q.recipeName === 'string' ? q.recipeName : undefined,
method: method as PluginWorkloadSdkFamily | undefined,
status: status as PluginWorkloadSdkInvocationStatus | undefined,
since,
until,
limit: Number.isFinite(limitRaw) ? limitRaw : undefined,
})
res.status(200).json({ items })
})
)
return router
}