Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes,control-api): promptly publish Plugin Workload SDK grant re-evaluation (#375)

Alfredo Lopez32e4c285300e
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 305db4444..d6708a2cb 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 5af12984c..0a440c422 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.ts
index d584c15d9..29925e075 100644
--- a/control-api/src/services/pluginWorkloadSdkDb.ts
+++ b/control-api/src/services/pluginWorkloadSdkDb.ts
@@ -18,6 +18,57 @@ import {
export const PLUGIN_WORKLOAD_SDK_FAMILIES = ['promptBridge', 'clientNotifications'] as const
export type PluginWorkloadSdkFamily = (typeof PLUGIN_WORKLOAD_SDK_FAMILIES)[number]
+// ─── Grant-update NOTIFY (issue #375, P3) ────────────────────────────────
+// CROSS-SERVICE CONTRACT: a transactional `pg_notify` fired inside the
+// grant-mutation transactions (upsert / delete / revoke) so the Workflow
+// Recipes Controller re-reconciles the affected recipe immediately (~1–5s)
+// instead of waiting for its ≤30s level-triggered watchdog. The WRC LISTEN side
+// lives in `workflow-recipes/src/reconciler/grantUpdateListener.ts` — the
+// channel name and payload shape below MUST stay in sync with it.
+//
+// This is an app-level NOTIFY (not a DDL trigger) so the payload stays typed and
+// the change is fully revertible. Being inside the transaction, it is delivered
+// on COMMIT and discarded on ROLLBACK. Semantics are best-effort: a dropped
+// notification degrades to the existing polling backstop, never worse.
+export const PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL = 'plugin_workload_sdk_grant_update'
+
+export interface GrantUpdateNotifyInput {
+ recipeNamespace: string
+ recipeName: string
+ /** Omitted for a whole-recipe revoke that spans multiple families. */
+ capabilityFamily?: string
+}
+
+/**
+ * Build the JSON payload for a grant-update NOTIFY. Pure and exported so the
+ * exact wire shape can be unit-tested without a live Postgres.
+ */
+export function buildGrantUpdateNotifyPayload(input: GrantUpdateNotifyInput): string {
+ return JSON.stringify({
+ recipeNamespace: input.recipeNamespace,
+ recipeName: input.recipeName,
+ ...(input.capabilityFamily ? { capabilityFamily: input.capabilityFamily } : {}),
+ })
+}
+
+/**
+ * Emit the grant-update NOTIFY on the shared channel. MUST be called with the
+ * in-transaction `db` client so the signal is delivered on COMMIT (and dropped
+ * on ROLLBACK), never as a separate connection.
+ *
+ * A `pg_notify` inside the transaction would roll the whole grant mutation back
+ * if it threw — but the only failure mode is a payload over Postgres' 8000-byte
+ * NOTIFY limit, and this payload is just a recipe namespace/name (+ optional
+ * short family), which cannot approach that bound. So the mutation is never at
+ * risk from the notify, and the transactional coupling is intentional.
+ */
+async function notifyGrantUpdate(db: DbClient, input: GrantUpdateNotifyInput): Promise<void> {
+ await db.query('SELECT pg_notify($1, $2)', [
+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ buildGrantUpdateNotifyPayload(input),
+ ])
+}
+
export const PLUGIN_WORKLOAD_SDK_INVOCATION_STATUSES = [
'in_progress',
'complete',
@@ -614,6 +665,12 @@ export async function upsertGrant(
})),
]
await appendControlApiPermissionEventsInTransaction(db, { operatorSub, changes })
+ // issue #375 (P3): nudge the WRC to re-reconcile now that the grant changed.
+ await notifyGrantUpdate(db, {
+ recipeNamespace: params.recipeNamespace,
+ recipeName: params.recipeName,
+ capabilityFamily: params.capabilityFamily,
+ })
return grant
})
}
@@ -880,6 +937,17 @@ export async function revokePluginWorkloadSdkForRecipe(
changes,
})
}
+ // issue #375 (P3): a whole-recipe revoke spans every family, so omit the
+ // capabilityFamily — the WRC only needs the recipe coordinates to re-reconcile.
+ // issue #375 (B2): notify ONLY on an actual transition (rows moved to
+ // 'revoking', or permission changes emitted). A no-op revoke — every grant
+ // already revoking/disabled — must stay silent, otherwise the WRC would
+ // force-reconcile the still-cached recipe, which can re-enter revoke and
+ // NOTIFY again: a reconcile+pg_notify spin. Matches upsert/delete "notify
+ // only on a real mutation" semantics.
+ if ((revoked.rowCount ?? 0) > 0 || changes.length > 0) {
+ await notifyGrantUpdate(db, { recipeNamespace, recipeName })
+ }
return {
state:
hasActivePolicy || hasRevokingPolicy || (revoked.rowCount ?? 0) > 0
@@ -1044,6 +1112,13 @@ export async function deleteGrant(
})),
]
await appendControlApiPermissionEventsInTransaction(db, { operatorSub, changes })
+ // issue #375 (P3): nudge the WRC to re-reconcile after the grant is removed
+ // so the capability projection transitions back to awaiting_policy/degraded.
+ await notifyGrantUpdate(db, {
+ recipeNamespace: grant.recipeNamespace,
+ recipeName: grant.recipeName,
+ capabilityFamily: grant.capabilityFamily,
+ })
return true
})
}
diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
new file mode 100644
index 000000000..db9654f17
--- /dev/null
+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
@@ -0,0 +1,178 @@
+import { afterAll, beforeAll, expect, describe as vitestDescribe } from 'vitest'
+import { it } from 'vitest'
+import { randomBytes } from 'node:crypto'
+import { Pool, type PoolClient } from 'pg'
+
+/**
+ * issue #375 (P3) — grant-update NOTIFY on real PostgreSQL.
+ *
+ * Verifies the property the WRC LISTEN side depends on: the grant-mutation
+ * transactions emit a `pg_notify` on `plugin_workload_sdk_grant_update` that is
+ * delivered on COMMIT and discarded on ROLLBACK.
+ *
+ * Real database, no mocks. The service module's module-level pool (src/db.ts) is
+ * pointed at a throwaway database by setting CONTROL_API_PG_CONNECTION_STRING
+ * BEFORE the dynamic src imports in beforeAll — nothing from src/ is value-
+ * imported statically here, matching the other *.realPostgres suites.
+ */
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+// Repo convention (matches the other *.realPostgres.integration.test.ts suites):
+// gate on the DB env so the DB-less unit lane stays green; the Postgres-backed
+// CI lane sets CONTROL_API_REAL_PG_ADMIN_URL and runs it non-skipped.
+const describeRealPostgres = adminUrl ? vitestDescribe : vitestDescribe.skip
+
+const database = `control_api_sdk_notify_${randomBytes(6).toString('hex')}`
+
+function databaseUrl(baseUrl: string, databaseName: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${databaseName}`
+ return url.toString()
+}
+
+const previousPgConnectionString = process.env.CONTROL_API_PG_CONNECTION_STRING
+const throwawayUrl = adminUrl ? databaseUrl(adminUrl, database) : undefined
+if (adminUrl && throwawayUrl) {
+ process.env.CONTROL_API_PG_CONNECTION_STRING = throwawayUrl
+}
+
+const NS = 'sandbox-recipes'
+const RECIPE = 'notify-recipe'
+
+describeRealPostgres(
+ 'Plugin Workload SDK grant-update NOTIFY on real PostgreSQL (issue #375)',
+ () => {
+ let adminPool: Pool
+ let db: typeof import('../src/db.js')
+ let sdk: typeof import('../src/services/pluginWorkloadSdkDb.js')
+ let listenPool: Pool
+ let listenClient: PoolClient
+ let received: Array<{ channel: string; payload?: string }>
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)
+ db = await import('../src/db.js')
+ await db.initDb()
+ sdk = await import('../src/services/pluginWorkloadSdkDb.js')
+
+ received = []
+ listenPool = new Pool({ connectionString: throwawayUrl })
+ listenClient = await listenPool.connect()
+ listenClient.on('notification', msg => {
+ received.push({ channel: msg.channel, payload: msg.payload })
+ })
+ await listenClient.query(`LISTEN ${sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
+ })
+
+ afterAll(async () => {
+ try {
+ await listenClient?.query(`UNLISTEN ${sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
+ } catch {
+ /* teardown */
+ }
+ listenClient?.release()
+ await listenPool?.end()
+ await db?.pool.end()
+ if (previousPgConnectionString === undefined) {
+ delete process.env.CONTROL_API_PG_CONNECTION_STRING
+ } else {
+ process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgConnectionString
+ }
+ if (adminPool) {
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid)
+ FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
+ await adminPool.end()
+ }
+ })
+
+ /** Poll the LISTEN client until a notification arrives on our channel or timeout. */
+ async function waitForGrantNotification(timeoutMs = 3_000): Promise<string | undefined> {
+ const deadline = Date.now() + timeoutMs
+ while (Date.now() < deadline) {
+ // Round-trip a trivial query so pg flushes any pending notifications.
+ await listenClient.query('SELECT 1')
+ const hit = received.find(m => m.channel === sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
+ if (hit) return hit.payload
+ await new Promise(resolve => setTimeout(resolve, 50))
+ }
+ return undefined
+ }
+
+ it('delivers the NOTIFY after upsertGrant COMMITs', async () => {
+ received.length = 0
+ await sdk.upsertGrant(
+ {
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ capabilityFamily: 'promptBridge',
+ provider: 'openai',
+ allowedModels: ['gpt-5'],
+ allowedCallers: ['api'],
+ },
+ 'operator-1'
+ )
+
+ const payload = await waitForGrantNotification()
+ expect(payload, 'expected a grant-update NOTIFY after commit').toBeDefined()
+ expect(JSON.parse(String(payload))).toEqual({
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ capabilityFamily: 'promptBridge',
+ })
+ })
+
+ it('discards BOTH the mutation and the NOTIFY when a real mutation transaction ROLLBACKs', async () => {
+ // This proves the grant-update NOTIFY is transactionally coupled to the grant
+ // mutation (delivered on COMMIT, discarded on ROLLBACK).
+ //
+ // WHY not fail a real service call (upsert/delete/revoke) after its notify:
+ // `notifyGrantUpdate` is the TERMINAL statement of each of those functions —
+ // nothing in them can throw after it — so a "production mutation that fails
+ // after the notify" cannot be constructed. Instead we reproduce the exact
+ // production shape: a REAL mutation (UPDATE of the committed grant) plus the
+ // REAL notify (same channel + payload builder) inside the production
+ // `withTransaction` wrapper, then abort it — and assert BOTH are discarded.
+ received.length = 0
+ const before = await db.pool.query<{ default_target_ref: string | null }>(
+ `SELECT default_target_ref FROM plugin_workload_sdk_grants
+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,
+ [NS, RECIPE]
+ )
+
+ await expect(
+ db.withTransaction(async client => {
+ await client.query(
+ `UPDATE plugin_workload_sdk_grants SET default_target_ref = 'ROLLED-BACK-SENTINEL'
+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,
+ [NS, RECIPE]
+ )
+ await client.query('SELECT pg_notify($1, $2)', [
+ sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ sdk.buildGrantUpdateNotifyPayload({ recipeNamespace: NS, recipeName: RECIPE }),
+ ])
+ throw new Error('force rollback')
+ })
+ ).rejects.toThrow('force rollback')
+
+ // The NOTIFY was discarded on rollback...
+ const delivered = await waitForGrantNotification(1_000)
+ expect(delivered, 'a rolled-back NOTIFY must never be delivered').toBeUndefined()
+
+ // ...and so was the mutation — the grant is unchanged, proving the notify
+ // shared the mutation's transaction.
+ const after = await db.pool.query<{ default_target_ref: string | null }>(
+ `SELECT default_target_ref FROM plugin_workload_sdk_grants
+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,
+ [NS, RECIPE]
+ )
+ expect(after.rows[0]?.default_target_ref).toBe(before.rows[0]?.default_target_ref)
+ expect(after.rows[0]?.default_target_ref).not.toBe('ROLLED-BACK-SENTINEL')
+ })
+ }
+)
diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.ts
index 476997afa..bad27b537 100644
--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts
@@ -1,6 +1,8 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { pool } from '../src/db.js'
import {
+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ buildGrantUpdateNotifyPayload,
deleteGrant,
failStaleInvocations,
finalizePluginWorkloadSdkRevocation,
@@ -864,3 +866,292 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {
expect(permissionEvents.append).not.toHaveBeenCalled()
})
})
+
+// issue #375 (P3): the grant→WRC event-driven nudge. control-api emits a
+// transactional pg_notify on the grant channel inside upsert/delete/revoke so
+// the WRC re-reconciles the recipe immediately instead of waiting for the ≤30s
+// watchdog. Delivered on COMMIT (transactional NOTIFY), discarded on rollback.
+describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
+ it('builds a typed, JSON payload carrying the recipe coordinates and family', () => {
+ expect(
+ JSON.parse(
+ buildGrantUpdateNotifyPayload({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ )
+ ).toEqual({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ })
+
+ it('omits capabilityFamily when it is not supplied (whole-recipe revoke)', () => {
+ expect(
+ JSON.parse(
+ buildGrantUpdateNotifyPayload({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ })
+ )
+ ).toEqual({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })
+ })
+
+ it('exposes the shared channel name used by the WRC LISTEN side', () => {
+ expect(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL).toBe('plugin_workload_sdk_grant_update')
+ })
+
+ it('upsertGrant emits a transactional pg_notify on the grant-update channel', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ permissionEvents.append.mockResolvedValue('operation-id')
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // family FOR UPDATE
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // kill-switch guard
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: '33333333-3333-4333-8333-333333333333',
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'research',
+ capability_family: 'promptBridge',
+ provider: 'openai',
+ allowed_models: ['gpt-5'],
+ allowed_event_types: [],
+ allowed_target_refs: [],
+ allowed_user_refs: [],
+ allowed_callers: ['coordinator'],
+ quota_limits: {},
+ model_policies: {},
+ policy_revision: 2,
+ created_at: '2026-08-17T00:00:00.000Z',
+ updated_at: '2026-08-17T00:00:00.000Z',
+ },
+ ],
+ rowCount: 1,
+ } as never) // upsert RETURNING
+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // notify + any trailing
+
+ await upsertGrant(
+ {
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ provider: 'openai',
+ allowedModels: ['gpt-5'],
+ allowedCallers: ['coordinator'],
+ },
+ 'operator-1'
+ )
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
+ | [string, unknown[]]
+ | undefined
+ expect(notifyCall).toBeDefined()
+ const [, params] = notifyCall as [string, unknown[]]
+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
+ expect(JSON.parse(String(params[1]))).toEqual({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ })
+
+ it('deleteGrant emits a transactional pg_notify on the grant-update channel', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ permissionEvents.append.mockResolvedValue('operation-id')
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: '33333333-3333-4333-8333-333333333333',
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'research',
+ capability_family: 'clientNotifications',
+ allowed_user_refs: [],
+ allowed_models: [],
+ allowed_event_types: [],
+ allowed_target_refs: [],
+ allowed_callers: [],
+ quota_limits: {},
+ model_policies: {},
+ created_at: '2026-08-17T00:00:00.000Z',
+ updated_at: '2026-08-17T00:00:00.000Z',
+ },
+ ],
+ rowCount: 1,
+ } as never) // DELETE RETURNING
+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // notify + trailing
+
+ await deleteGrant(
+ '33333333-3333-4333-8333-333333333333',
+ 'sandbox-recipes',
+ 'research',
+ 'operator-1'
+ )
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
+ | [string, unknown[]]
+ | undefined
+ expect(notifyCall).toBeDefined()
+ const [, params] = notifyCall as [string, unknown[]]
+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
+ expect(JSON.parse(String(params[1]))).toMatchObject({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ })
+ })
+
+ it('revokePluginWorkloadSdkForRecipe emits a transactional pg_notify on the grant-update channel', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ permissionEvents.append.mockResolvedValue('operation-id')
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: '33333333-3333-4333-8333-333333333333',
+ capability_family: 'promptBridge',
+ policy_state: 'active',
+ revocation_id: null,
+ },
+ ],
+ rowCount: 1,
+ } as never) // grants FOR UPDATE
+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // updates + notify
+
+ await revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', {
+ operatorSub: 'operator-1',
+ })
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
+ | [string, unknown[]]
+ | undefined
+ expect(notifyCall).toBeDefined()
+ const [, params] = notifyCall as [string, unknown[]]
+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
+ expect(JSON.parse(String(params[1]))).toMatchObject({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ })
+ })
+
+ it('does NOT emit pg_notify on a NO-OP revoke (issue #375 B2 — no self-sustaining spin)', async () => {
+ // Every grant already revoking/disabled: nothing transitions (revoked
+ // rowCount 0, zero permission changes). Emitting a NOTIFY here would make the
+ // WRC force-reconcile the still-cached recipe, which can re-enter revoke and
+ // NOTIFY again — a reconcile+pg_notify spin. Match upsert/delete: notify only
+ // on a real mutation.
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ permissionEvents.append.mockResolvedValue('operation-id')
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: '33333333-3333-4333-8333-333333333333',
+ capability_family: 'promptBridge',
+ policy_state: 'disabled',
+ revocation_id: '55555555-5555-4555-8555-555555555555',
+ },
+ ],
+ rowCount: 1,
+ } as never) // grants FOR UPDATE (all already disabled)
+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // UPDATEs affect 0 rows
+
+ await revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', {
+ operatorSub: 'operator-1',
+ })
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))
+ expect(notifyCall, 'a no-op revoke must not emit a grant-update NOTIFY').toBeUndefined()
+ expect(permissionEvents.append).not.toHaveBeenCalled()
+ })
+
+ it('does NOT emit pg_notify when deleteGrant matches no row (rowCount 0)', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // DELETE RETURNING (no match)
+
+ await expect(
+ deleteGrant(
+ '33333333-3333-4333-8333-333333333333',
+ 'sandbox-recipes',
+ 'research',
+ 'operator-1'
+ )
+ ).resolves.toBe(false)
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))
+ expect(notifyCall, 'a delete that removed nothing must not NOTIFY').toBeUndefined()
+ })
+
+ it('does NOT emit pg_notify on a revoke that finds no grants (state=missing)', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // grants FOR UPDATE (none)
+
+ await expect(
+ revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', { operatorSub: 'operator-1' })
+ ).resolves.toMatchObject({ state: 'missing' })
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))
+ expect(notifyCall, 'a missing revoke must not NOTIFY').toBeUndefined()
+ })
+
+ it('does NOT emit pg_notify on a revoke that conflicts on revocation epoch (state=conflict)', async () => {
+ vi.mocked(pool.query).mockReset()
+ permissionEvents.append.mockReset()
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: '11111111-1111-4111-8111-111111111111',
+ capability_family: 'promptBridge',
+ policy_state: 'revoking',
+ revocation_id: '55555555-5555-4555-8555-555555555555',
+ },
+ {
+ id: '22222222-2222-4222-8222-222222222222',
+ capability_family: 'clientNotifications',
+ policy_state: 'revoking',
+ revocation_id: '66666666-6666-4666-8666-666666666666',
+ },
+ ],
+ rowCount: 2,
+ } as never) // grants FOR UPDATE — two distinct revocation epochs
+
+ await expect(
+ revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', { operatorSub: 'operator-1' })
+ ).resolves.toMatchObject({ state: 'conflict' })
+
+ const notifyCall = vi
+ .mocked(pool.query)
+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))
+ expect(notifyCall, 'a conflicting revoke must not NOTIFY').toBeUndefined()
+ })
+})
diff --git a/scripts/e2e/e2e-plugin-workload-sdk.sh b/scripts/e2e/e2e-plugin-workload-sdk.sh
index 241d989ab..36c66937e 100755
--- a/scripts/e2e/e2e-plugin-workload-sdk.sh
+++ b/scripts/e2e/e2e-plugin-workload-sdk.sh
@@ -173,6 +173,10 @@ E2E_WAIT_STATUS_VALIDATED="${E2E_WAIT_STATUS_VALIDATED:-120}"
E2E_WAIT_CALLER_MARKER="${E2E_WAIT_CALLER_MARKER:-180}"
E2E_WAIT_CALLER_DONE="${E2E_WAIT_CALLER_DONE:-180}"
E2E_WAIT_NOTIFICATION_ROW="${E2E_WAIT_NOTIFICATION_ROW:-60}"
+# issue #375 (P3): event-driven grant→WRC nudge target latency. A grant change
+# must re-reconcile the recipe via Postgres LISTEN/NOTIFY in ≤10s WITHOUT a WRC
+# restart (the ≤30s watchdog is the backstop, not the mechanism under test).
+E2E_WAIT_NUDGE_VALIDATED="${E2E_WAIT_NUDGE_VALIDATED:-10}"
# Fail closed before creating any fixture if a caller or inherited shell
# environment asks this gate to wait for an unbounded or human-scale interval.
@@ -197,6 +201,7 @@ validate_bounded_seconds E2E_WAIT_STATUS_VALIDATED "$E2E_WAIT_STATUS_VALIDATED"
validate_bounded_seconds E2E_WAIT_CALLER_MARKER "$E2E_WAIT_CALLER_MARKER" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_CALLER_DONE "$E2E_WAIT_CALLER_DONE" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_NOTIFICATION_ROW "$E2E_WAIT_NOTIFICATION_ROW" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
+validate_bounded_seconds E2E_WAIT_NUDGE_VALIDATED "$E2E_WAIT_NUDGE_VALIDATED" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds POLL_INTERVAL "$POLL_INTERVAL" "$E2E_HARD_MAX_POLL_INTERVAL_SECONDS"
require_write_confirmation() {
@@ -1385,5 +1390,75 @@ else
exit 1
fi
+# ─── issue #375 (P3): event-driven grant→WRC nudge ───────────────────────
+# The recipe is validated by now. Prove the grant→WRC nudge accelerates
+# convergence: a grant change must re-reconcile the recipe (bump the published
+# policyRevision, state stays validated) within ≤10s via Postgres LISTEN/NOTIFY,
+# WITHOUT restarting the WRC (the ≤30s watchdog is the backstop, not the path
+# under test). This is an ADDITIVE phase; it does not touch #351's phases.
+header "Plugin Workload SDK grant→WRC event-driven nudge (issue #375 P3)"
+
+# Capture the WRC pod set (by name) so we can assert NO restart drove convergence.
+wrc_pods_before="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \
+ | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"
+if [ -z "$wrc_pods_before" ]; then
+ fail "could not enumerate workflow-recipes pods before the nudge (selector empty)"
+ exit 1
+fi
+
+nudge_rev_before="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
+ -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"
+nudge_rev_before="${nudge_rev_before:-0}"
+
+# Re-upsert the promptBridge grant. The admin upsert increments policy_revision
+# and (issue #375 P3) emits the transactional grant-update NOTIFY on COMMIT.
+nudge_started_at="$SECONDS"
+# issue #375 (R5): fail LOUD on a grant re-upsert failure. `|| true` here only
+# guards `set -e` so we can read ADMIN_CURL_HTTP_STATUS; the explicit assert
+# below is the real gate. Without it, an admin-API failure (expired token,
+# control-api 5xx) would fall through to the poll and be misattributed to the
+# nudge under test ("did not accelerate convergence"), hiding the real cause.
+create_grant promptBridge >/dev/null 2>&1 || true
+if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then
+ fail "grant re-upsert failed (admin API HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}) — cannot test the event-driven nudge"
+ exit 1
+fi
+
+nudge_found=0
+nudge_state=""
+nudge_rev_after=""
+nudge_deadline=$((SECONDS + E2E_WAIT_NUDGE_VALIDATED))
+while [ "$SECONDS" -lt "$nudge_deadline" ]; do
+ gate_assert_deadline "waiting for event-driven policyRevision bump (issue #375 P3)"
+ nudge_state="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
+ -o jsonpath='{.status.pluginWorkloadSdk.state}' 2>/dev/null || true)"
+ nudge_rev_after="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
+ -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"
+ if [ "$nudge_state" = "validated" ] && [ -n "$nudge_rev_after" ] &&
+ [ "$nudge_rev_after" -gt "$nudge_rev_before" ] 2>/dev/null; then
+ nudge_found=1
+ break
+ fi
+ sleep "$POLL_INTERVAL"
+done
+
+nudge_elapsed=$((SECONDS - nudge_started_at))
+if [ "$nudge_found" -eq 1 ]; then
+ ok "grant change re-published (policyRevision ${nudge_rev_before}→${nudge_rev_after}, state=validated) in ~${nudge_elapsed}s via the event-driven nudge (≤${E2E_WAIT_NUDGE_VALIDATED}s)"
+else
+ fail "grant change was NOT re-published within ${E2E_WAIT_NUDGE_VALIDATED}s (state='${nudge_state:-<empty>}', policyRevision '${nudge_rev_before}'→'${nudge_rev_after:-<empty>}'); the LISTEN/NOTIFY nudge did not accelerate convergence"
+ exit 1
+fi
+
+# The acceleration must come from the NOTIFY-driven reconcile, NOT a WRC restart.
+wrc_pods_after="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \
+ | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"
+if [ "$wrc_pods_after" = "$wrc_pods_before" ]; then
+ ok "WRC was NOT restarted during convergence (pod set unchanged) — convergence was event-driven"
+else
+ fail "WRC pod set changed during the nudge phase (before='${wrc_pods_before}' after='${wrc_pods_after}'); convergence must not depend on a restart"
+ exit 1
+fi
+
print_results
exit 0
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 0516c688b..e00f27dd1 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 1057fcbed..71b9682c9 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 0e9c1496e..81fa3aab4 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -383,6 +383,221 @@ describe('shouldPatchRecipeStatus', () => {
)
).toBe(false)
})
+
+ // ── issue #375: publish computed Plugin Workload SDK state transitions ──
+ // The dominant failure mode: the reconciler computes awaiting_policy→validated
+ // but no OTHER observable diff (phase, top-level message, workloads, owned
+ // conditions) changes, so the old shouldPatchRecipeStatus returned false and
+ // the transition was never published — status stayed awaiting_policy for ~40
+ // min. shouldPatchRecipeStatus must now compare the SDK capability projection
+ // carried on result.pluginWorkloadSdkProjection.
+ function validatedProjection() {
+ return {
+ conditions: [],
+ capability: {
+ state: 'validated' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ validatedAt: new Date().toISOString(),
+ verifiedAt: new Date().toISOString(),
+ },
+ }
+ }
+
+ function sdkRecipe(persistedSdk: Record<string, unknown>): WorkflowRecipeCRD {
+ return makeWorkflowRecipe({
+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
+ status: {
+ phase: 'active',
+ message: 'All workloads deployed',
+ // No PluginWorkloadSdkCapability condition, so the pre-existing
+ // message-vs-condition guard stays inert and this asserts ONLY the new
+ // projection comparison.
+ pluginWorkloadSdk: persistedSdk as never,
+ },
+ })
+ }
+
+ it('patches a computed awaiting_policy → validated transition (issue #375 incident replica)', () => {
+ // The exact incident shape: phase and top-level message are unchanged, the
+ // ONLY change is the SDK capability state. Before the fix this returned
+ // false and the validated state was never published.
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'awaiting_policy',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',
+ verifiedAt: '2026-08-17T17:24:50.000Z',
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: validatedProjection(),
+ }
+ )
+ ).toBe(true)
+ })
+
+ it('refreshes a stale verifiedAt on an otherwise-identical validated projection (throttle open)', () => {
+ const staleVerifiedAt = new Date(Date.now() - 6 * 60_000).toISOString()
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: staleVerifiedAt,
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: validatedProjection(),
+ }
+ )
+ ).toBe(true)
+ })
+
+ it('does not patch a fresh verifiedAt on an identical validated projection (throttle closed / anti-loop)', () => {
+ const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: freshVerifiedAt,
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: validatedProjection(),
+ }
+ )
+ ).toBe(false)
+ })
+
+ it('patches a computed validated → awaiting_policy transition (revocation symmetry)', () => {
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: {
+ conditions: [],
+ capability: {
+ state: 'awaiting_policy' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',
+ bootstrapPodUid: 'pod-abc',
+ verifiedAt: new Date().toISOString(),
+ },
+ },
+ }
+ )
+ ).toBe(true)
+ })
+
+ it('does not patch when the SDK projection is semantically unchanged and verifiedAt is fresh', () => {
+ // Anti-loop guard: two back-to-back reconciles with the same computed state
+ // and a still-fresh verifiedAt must NOT keep emitting patches.
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: new Date(Date.now() - 5_000).toISOString(),
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: validatedProjection(),
+ }
+ )
+ ).toBe(false)
+ })
+
+ it('patches an in-place re-bootstrap that only changes bootstrapModel/provider/contractVersion (issue #375 B1)', () => {
+ // A validated promptBridge recipe re-bootstraps in place (SAME pod, same
+ // policyRevision) against a DIFFERENT model/provider — e.g. the operator
+ // corrects the target model. state stays 'validated', message is
+ // families-only, pod/policyRevision unchanged, verifiedAt is fresh (throttle
+ // closed). Only bootstrapModel/provider/contractVersion move — persisted
+ // semantic fields read by control-api at runtime. They MUST force a patch.
+ const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()
+ expect(
+ shouldPatchRecipeStatus(
+ sdkRecipe({
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ bootstrapContractVersion: 2,
+ bootstrapProvider: 'openai',
+ bootstrapModel: 'gpt-5.4-mini',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: freshVerifiedAt,
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: {
+ conditions: [],
+ capability: {
+ state: 'validated' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ bootstrapPodUid: 'pod-abc',
+ bootstrapContractVersion: 2,
+ bootstrapProvider: 'anthropic',
+ bootstrapModel: 'claude-sonnet-4-6',
+ policyRevision: 3,
+ defaultTargetRef: 'primary-openai',
+ verifiedAt: freshVerifiedAt,
+ },
+ },
+ }
+ )
+ ).toBe(true)
+ })
})
describe('infrastructure telemetry projections', () => {
@@ -1718,6 +1933,293 @@ describe('transient-result requeue (scheduleTransientRetry)', () => {
})
})
+describe('reconcile failure re-schedules the transient retry chain (issue #375 P2)', () => {
+ // A transient (non-404) throw inside handleRecipeEvent used to log the failure
+ // and return WITHOUT re-arming the per-recipe retry chain, so the 5s self-heal
+ // died on the first error and only the 30s watchdog could recover it. The fix
+ // keeps the loud error log AND re-schedules the bounded backoff retry; the 404
+ // "already gone" branch must still clear and return without a retry.
+ type InternalWatcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
+ reconciler: {
+ reconcile: ReturnType<typeof vi.fn>
+ isRecipeStillActive: ReturnType<typeof vi.fn>
+ ensureFinalizer: ReturnType<typeof vi.fn>
+ patchStatus: ReturnType<typeof vi.fn>
+ }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+
+ function makeInternal(recipe: WorkflowRecipeCRD, reconcileError: unknown): InternalWatcher {
+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
+ internal.recipes = new Map([[recipe.metadata.name, recipe]])
+ internal.transientRetries = new Map()
+ internal.stopped = false
+ internal.traceReporter = null
+ internal.dbRunProcessor = null
+ internal.eventQueue = { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) }
+ internal.reconciler = {
+ reconcile: vi.fn().mockRejectedValue(reconcileError),
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus: vi.fn().mockResolvedValue(undefined),
+ }
+ return internal
+ }
+
+ function clearRetries(internal: InternalWatcher): void {
+ for (const { timer } of internal.transientRetries.values()) clearTimeout(timer)
+ }
+
+ it('re-arms the transient retry after a non-404 reconcile failure (keeps the loud log)', async () => {
+ const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const recipe = makeWorkflowRecipe({ metadata: { name: 'flaky', namespace: 'sandbox-recipes' } })
+ const internal = makeInternal(recipe, new Error('control-api unreachable'))
+ try {
+ await internal.handleRecipeEvent('ADDED', recipe)
+
+ // fail-loud preserved: the reconciliation-failed error is still logged.
+ expect(errSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Reconciliation failed for "flaky"'),
+ expect.anything()
+ )
+ // NEW: the 5s self-heal chain is re-armed instead of dying on first error.
+ expect(internal.transientRetries.has('flaky')).toBe(true)
+ } finally {
+ clearRetries(internal)
+ errSpy.mockRestore()
+ }
+ })
+
+ it('does not re-arm a retry when the recipe is already gone (404)', async () => {
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const recipe = makeWorkflowRecipe({ metadata: { name: 'gone', namespace: 'sandbox-recipes' } })
+ const internal = makeInternal(recipe, { code: 404 })
+ try {
+ await internal.handleRecipeEvent('ADDED', recipe)
+
+ expect(internal.transientRetries.has('gone')).toBe(false)
+ expect(internal.recipes.has('gone')).toBe(false)
+ } finally {
+ clearRetries(internal)
+ warnSpy.mockRestore()
+ }
+ })
+})
+
+describe('Plugin Workload SDK computed-vs-published state divergence log (issue #375 P4)', () => {
+ // Forensic observability: when the reconciler COMPUTES an SDK capability state
+ // that differs from what is currently published, log old→new and whether a
+ // patch was emitted. This one line would have discriminated H1 (transition
+ // computed but not published) from H2 in minutes during the incident.
+ type InternalWatcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ reconciler: {
+ reconcile: ReturnType<typeof vi.fn>
+ isRecipeStillActive: ReturnType<typeof vi.fn>
+ ensureFinalizer: ReturnType<typeof vi.fn>
+ patchStatus: ReturnType<typeof vi.fn>
+ }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+
+ function makeInternal(recipe: WorkflowRecipeCRD, result: unknown): InternalWatcher {
+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
+ internal.recipes = new Map([[recipe.metadata.name, recipe]])
+ internal.transientRetries = new Map()
+ internal.stopped = false
+ internal.traceReporter = null
+ internal.dbRunProcessor = null
+ internal.reconciler = {
+ reconcile: vi.fn().mockResolvedValue(result),
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus: vi.fn().mockResolvedValue(undefined),
+ }
+ return internal
+ }
+
+ function sdkRecipe(persistedState: 'awaiting_policy' | 'validated'): WorkflowRecipeCRD {
+ return makeWorkflowRecipe({
+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
+ status: {
+ phase: 'active',
+ message: 'All workloads deployed',
+ pluginWorkloadSdk: {
+ state: persistedState,
+ promptBridge: true,
+ clientNotifications: false,
+ },
+ },
+ })
+ }
+
+ it('logs the old→new SDK state and patchEmitted when the computed state diverges', async () => {
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const recipe = sdkRecipe('awaiting_policy')
+ const internal = makeInternal(recipe, {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: {
+ conditions: [],
+ capability: {
+ state: 'validated' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ verifiedAt: new Date().toISOString(),
+ },
+ },
+ })
+ try {
+ await internal.handleRecipeEvent('ADDED', recipe)
+ expect(logSpy).toHaveBeenCalledWith(
+ expect.stringMatching(
+ /Plugin Workload SDK state.*sdk-recipe.*awaiting_policy -> validated.*patchEmitted=true/
+ )
+ )
+ } finally {
+ logSpy.mockRestore()
+ }
+ })
+
+ it('does not log a transition when the computed SDK state is unchanged', async () => {
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const recipe = sdkRecipe('validated')
+ const internal = makeInternal(recipe, {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: {
+ conditions: [],
+ capability: {
+ state: 'validated' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ verifiedAt: new Date().toISOString(),
+ },
+ },
+ })
+ try {
+ await internal.handleRecipeEvent('ADDED', recipe)
+ const transitionLogs = logSpy.mock.calls.filter(([msg]) =>
+ String(msg).includes('Plugin Workload SDK state')
+ )
+ expect(transitionLogs).toHaveLength(0)
+ } finally {
+ logSpy.mockRestore()
+ }
+ })
+})
+
+describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {
+ type InternalWatcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ eventQueue: { enqueue: ReturnType<typeof vi.fn> }
+ handleRecipeEvent: ReturnType<typeof vi.fn>
+ handleGrantUpdateNotification: (recipeNamespace: string, recipeName: string) => void
+ }
+
+ function makeInternal(recipes: WorkflowRecipeCRD[]): {
+ internal: InternalWatcher
+ tasks: Array<() => Promise<void>>
+ } {
+ const tasks: Array<() => Promise<void>> = []
+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
+ internal.recipes = new Map(recipes.map(r => [r.metadata.name, r]))
+ internal.eventQueue = {
+ enqueue: vi.fn((_key: string, task: () => Promise<void>) => {
+ tasks.push(task)
+ return Promise.resolve()
+ }),
+ }
+ internal.handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
+ return { internal, tasks }
+ }
+
+ it('enqueues a forceReconcile for a matching recipe and reconciles the CURRENT cached recipe (H5)', async () => {
+ const recipe = makeWorkflowRecipe({
+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 1 },
+ })
+ const { internal, tasks } = makeInternal([recipe])
+
+ internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')
+
+ expect(internal.eventQueue.enqueue).toHaveBeenCalledWith('sdk-recipe', expect.any(Function))
+ expect(tasks).toHaveLength(1)
+
+ // H5: a newer MODIFIED lands before the queued job runs. The job must
+ // reconcile the FRESH cached recipe, not the snapshot captured at NOTIFY time.
+ const fresher = makeWorkflowRecipe({
+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 2 },
+ })
+ internal.recipes.set('sdk-recipe', fresher)
+
+ await tasks[0]()
+
+ expect(internal.handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', fresher, {
+ forceReconcile: true,
+ })
+ expect(internal.handleRecipeEvent).not.toHaveBeenCalledWith('MODIFIED', recipe, {
+ forceReconcile: true,
+ })
+ })
+
+ it('discards and warns on a namespace mismatch — no enqueue (H4)', () => {
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const recipe = makeWorkflowRecipe({
+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
+ })
+ const { internal } = makeInternal([recipe])
+ try {
+ internal.handleGrantUpdateNotification('other-namespace', 'sdk-recipe')
+ expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
+ expect(warnSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Discarding grant-update NOTIFY for "other-namespace/sdk-recipe"')
+ )
+ } finally {
+ warnSpy.mockRestore()
+ }
+ })
+
+ it('discards and warns on an unknown recipe — no enqueue (H4)', () => {
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const { internal } = makeInternal([])
+ try {
+ internal.handleGrantUpdateNotification('sandbox-recipes', 'nope')
+ expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('unknown recipe'))
+ } finally {
+ warnSpy.mockRestore()
+ }
+ })
+
+ it('the queued job re-checks the guard and does nothing if the recipe vanished while queued (H5)', async () => {
+ const recipe = makeWorkflowRecipe({
+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
+ })
+ const { internal, tasks } = makeInternal([recipe])
+
+ internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')
+ // Recipe deleted between NOTIFY and job execution.
+ internal.recipes.delete('sdk-recipe')
+
+ await tasks[0]()
+
+ expect(internal.handleRecipeEvent).not.toHaveBeenCalled()
+ })
+})
+
// issue #299 §3.2/§5 — the periodic external-egress requeue timer. GitHub rotates
// its api.github.com A record (~15s TTL over 140.82.112.0/20) WITHOUT bumping the
// WorkflowRecipe CRD generation, so a generation-only reconcile trigger never
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index 6f7019923..ee22128ed 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -29,6 +29,10 @@ import {
type DbRunRow,
createDbRunProcessor,
} from './reconciler/dbRunProcessor'
+import {
+ type GrantUpdateListener,
+ createGrantUpdateListener,
+} from './reconciler/grantUpdateListener'
import { getErrorCode } from './reconciler/k8sErrors'
import { RecipeEventQueue } from './reconciler/recipeEventQueue'
import {
@@ -39,6 +43,7 @@ import { SecretReverseIndex } from './reconciler/secretReverseIndex'
import { SecretWatcher } from './reconciler/secretWatcher'
import {
type ReconcileResult,
+ TRANSIENT_REQUEUE_BASE_MS,
WorkflowRecipeReconciler,
} from './reconciler/workflowRecipeReconciler'
import {
@@ -55,6 +60,13 @@ const MIN_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 5_000
const MAX_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 30_000
const WORKLOAD_STATUS_REFRESH_INTERVAL_MS = 30_000
const WORKFLOW_RUN_ID_LABEL = 'clerum.io/workflow-run-id'
+// issue #375: how long a published Plugin Workload SDK `verifiedAt` may age
+// before a steady-state reconcile refreshes it, even when every semantic field
+// is unchanged. `verifiedAt` is recomputed on every bootstrap-proof parse, so
+// without a throttle a fresh timestamp each pass would patch on every reconcile
+// (≤30s) and spin a MODIFIED→reconcile loop. 5 min keeps the liveness signal
+// current while capping steady-state churn at ≤1 patch / 5 min per recipe.
+const VERIFIED_AT_REFRESH_MS = 5 * 60_000
// Bound the fan-out of the periodic external-egress requeue: the per-recipe
// eventQueue already serializes each recipe, but without a global cap a refresh
// pass over N recipes would enqueue N reconciles at once. This caps the number of
@@ -117,6 +129,12 @@ export function shouldPatchRecipeStatus(
// `state=validated` + `message=bootstrap not ready` record behind.
if (pluginWorkloadSdkStatusMessageChanged(recipe)) return true
+ // issue #375: publish a computed Plugin Workload SDK capability transition
+ // (e.g. awaiting_policy→validated) even when nothing else observable changed.
+ // Without this the reconciler computes the new state but no patch is emitted
+ // and the CRD keeps proclaiming the stale state until an incidental diff.
+ if (pluginWorkloadSdkProjectionChanged(recipe, result)) return true
+
if (
ownedConditionsChanged(recipe.status?.conditions, result.internalDependencyConditions, [
'InternalDependenciesReady',
@@ -132,6 +150,86 @@ export function shouldPatchRecipeStatus(
return false
}
+/**
+ * issue #375: decide whether the freshly-computed Plugin Workload SDK capability
+ * projection (carried on `result.pluginWorkloadSdkProjection`, built once by the
+ * watcher via the reconciler) differs from what is currently persisted in
+ * `status.pluginWorkloadSdk` in a way that must be published.
+ *
+ * Semantic fields are compared explicitly (NEVER a deep-equal that would drag in
+ * timestamps): `state`, `message`, `policyRevision`, `policyHash`,
+ * `defaultTargetRef`, `bootstrapPodUid`, `bootstrapContractVersion`,
+ * `bootstrapProvider`, `bootstrapModel`, and the `promptBridge`/
+ * `clientNotifications` family flags. A change in any of them (e.g.
+ * awaiting_policy→validated on grant creation, validated→awaiting_policy on
+ * revocation, or an in-place re-bootstrap against a corrected model/provider
+ * with the same pod) forces a patch immediately — `bootstrapProvider`/
+ * `bootstrapModel`/`bootstrapContractVersion` are persisted values control-api
+ * reads at runtime, NOT timestamps, so they must not hide behind the
+ * `verifiedAt` throttle.
+ *
+ * `verifiedAt` and `validatedAt` are timestamp-derived and EXCLUDED from
+ * semantic equality — otherwise every reconcile would look "changed". Instead,
+ * when the projection is otherwise identical, `verifiedAt` is refreshed on a
+ * throttle: patch only when the newly-computed `verifiedAt` is strictly NEWER
+ * than the persisted one AND the persisted one is older than
+ * `VERIFIED_AT_REFRESH_MS` (see the inline INVARIANT). Loop-safety rests on that
+ * strictly-newer guard alone: an equal-or-older projected timestamp can never
+ * trigger a patch, and after any refresh the persisted value equals the
+ * projected one, so the next pass is not strictly newer and returns false.
+ *
+ * A `undefined` projection (paths that do not run the SDK lane, e.g. the
+ * workload-status-only refresh) short-circuits to false — that pass has no SDK
+ * opinion and must not force a patch.
+ */
+function pluginWorkloadSdkProjectionChanged(
+ recipe: WorkflowRecipeCRD,
+ result: ReconcileResult,
+ nowMs: number = Date.now()
+): boolean {
+ const projected = result.pluginWorkloadSdkProjection?.capability
+ if (projected === undefined) return false
+ const current = recipe.status?.pluginWorkloadSdk
+ // `null` means the projection wants to CLEAR status.pluginWorkloadSdk
+ // (capability removed from spec). Patch only if something is still persisted.
+ if (projected === null) return current !== undefined
+ if (!current) return true
+
+ const nullable = (value: string | number | null | undefined): string | number | null =>
+ value ?? null
+ if (current.state !== projected.state) return true
+ if (nullable(current.message) !== nullable(projected.message)) return true
+ if (nullable(current.policyRevision) !== nullable(projected.policyRevision)) return true
+ if (nullable(current.policyHash) !== nullable(projected.policyHash)) return true
+ if (nullable(current.defaultTargetRef) !== nullable(projected.defaultTargetRef)) return true
+ if (nullable(current.bootstrapPodUid) !== nullable(projected.bootstrapPodUid)) return true
+ if (nullable(current.bootstrapContractVersion) !== nullable(projected.bootstrapContractVersion))
+ return true
+ if (nullable(current.bootstrapProvider) !== nullable(projected.bootstrapProvider)) return true
+ if (nullable(current.bootstrapModel) !== nullable(projected.bootstrapModel)) return true
+ if (current.promptBridge !== projected.promptBridge) return true
+ if (current.clientNotifications !== projected.clientNotifications) return true
+
+ // Semantic equality holds. Refresh verifiedAt on a throttle so the liveness
+ // signal stays current without a per-reconcile patch loop. INVARIANT: only
+ // refresh when the newly-computed verifiedAt is strictly NEWER than the
+ // persisted one AND the persisted one is older than VERIFIED_AT_REFRESH_MS.
+ // Requiring projected > persisted is what makes this loop-proof even if a
+ // future projection ever carried a STALE (equal or older) timestamp: such a
+ // projection can never trigger a patch, so patch→reconcile→patch cannot
+ // self-sustain. After any refresh the persisted value equals the projected
+ // one, so the next pass is not strictly newer and returns false.
+ if (!projected.verifiedAt) return false
+ if (!current.verifiedAt) return true
+ const projectedMs = Date.parse(projected.verifiedAt)
+ // Cannot reason about an unparseable projected timestamp — do not patch.
+ if (Number.isNaN(projectedMs)) return false
+ const persistedMs = Date.parse(current.verifiedAt)
+ // A corrupt persisted timestamp is refreshed once from a valid projection.
+ if (Number.isNaN(persistedMs)) return true
+ return projectedMs > persistedMs && nowMs - persistedMs > VERIFIED_AT_REFRESH_MS
+}
+
function pluginWorkloadSdkStatusMessageChanged(recipe: WorkflowRecipeCRD): boolean {
const capability = recipe.status?.pluginWorkloadSdk
if (!capability) return false
@@ -509,6 +607,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
private recipes: Map<string, WorkflowRecipeCRD> = new Map()
private reconciler: WorkflowRecipeReconciler
private dbRunProcessor: DbRunProcessor | null = null
+ // issue #375 (P3): event-driven grant→WRC nudge. Started only when the WRC has
+ // a DB (same gate as dbRunProcessor); without a DB the listener never starts
+ // and the level-triggered reconcile remains the sole convergence path.
+ private grantUpdateListener: GrantUpdateListener | null = null
private config = loadConfig()
private stopped = false
private runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null = null
@@ -573,9 +675,48 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
})
},
})
+ // issue #375 (P3): on a grant-update NOTIFY, force-reconcile the affected
+ // recipe through the SAME path the 30s watchdog uses. The dispatch body is
+ // a named method so it is unit-testable without standing up a DB.
+ this.grantUpdateListener = createGrantUpdateListener({
+ pool,
+ onGrantUpdate: ({ recipeNamespace, recipeName }) =>
+ this.handleGrantUpdateNotification(recipeNamespace, recipeName),
+ })
}
}
+ /**
+ * issue #375 (P3): dispatch a grant-update NOTIFY into a forced reconcile of
+ * the affected recipe — the SAME path the 30s watchdog uses. Kept as a named
+ * method (not an inline closure) so it is unit-testable without a live DB.
+ */
+ private handleGrantUpdateNotification(recipeNamespace: string, recipeName: string): void {
+ const known = this.recipes.get(recipeName)
+ if (!known || known.metadata.namespace !== recipeNamespace) {
+ // issue #375 (H4): make the discard diagnosable. control-api's emitted
+ // recipeNamespace and the WRC's watched sandboxNamespace come from
+ // independent env vars; if they ever diverge, 100% of NOTIFYs would be
+ // dropped and the feature would silently degrade to 30s polling. Log names
+ // only — no secrets.
+ console.warn(
+ `[WR-K8s] Discarding grant-update NOTIFY for "${recipeNamespace}/${recipeName}": ` +
+ (known ? `namespace mismatch (watching "${known.metadata.namespace}")` : 'unknown recipe')
+ )
+ return
+ }
+ void this.eventQueue.enqueue(recipeName, () => {
+ // issue #375 (H5): reconcile the CURRENT recipe. A newer MODIFIED may have
+ // landed between NOTIFY receipt and this queued job running
+ // (handleRecipeEvent updates this.recipes), so re-read the cache instead of
+ // closing over the stale snapshot. Re-check the guard on the fresh read —
+ // the recipe may have been deleted, or its namespace changed, while queued.
+ const cached = this.recipes.get(recipeName)
+ if (!cached || cached.metadata.namespace !== recipeNamespace) return Promise.resolve()
+ return this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true })
+ })
+ }
+
getAllRecipes(): WorkflowRecipeCRD[] {
return [...this.recipes.values()]
}
@@ -639,6 +780,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
if (this.dbRunProcessor) {
await this.dbRunProcessor.start()
}
+ if (this.grantUpdateListener) {
+ await this.grantUpdateListener.start()
+ }
}
// Refreshes opted-in recipes' broker-token Secrets between CRD events.
@@ -870,7 +1014,22 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
const result = await this.reconciler.reconcile(recipe)
// Patch only when observable status changes. This avoids reconcile loops
// while still allowing active parent recipes to clear stale execution state.
- if (shouldPatchRecipeStatus(recipe, result)) {
+ const willPatch = shouldPatchRecipeStatus(recipe, result)
+ // issue #375 (P4): surface a computed-vs-published Plugin Workload SDK
+ // state divergence — old→new plus whether a patch was emitted. With P1
+ // this is rare; its value is forensic (it would have discriminated
+ // "transition computed but not published" from "never computed" in
+ // minutes) and it verifies P1 in the field. Only states are logged — no
+ // tokens/secrets (sec-003).
+ const projectedSdkState = result.pluginWorkloadSdkProjection?.capability?.state
+ const persistedSdkState = recipe.status?.pluginWorkloadSdk?.state
+ if (projectedSdkState !== undefined && projectedSdkState !== persistedSdkState) {
+ console.log(
+ `[WR-K8s] Plugin Workload SDK state for "${recipe.metadata.name}": ` +
+ `${persistedSdkState ?? 'none'} -> ${projectedSdkState} (patchEmitted=${willPatch})`
+ )
+ }
+ if (willPatch) {
await this.reconciler.patchStatus(recipe, result)
enqueueInfrastructureTelemetryBestEffort(
this.traceReporter,
@@ -902,6 +1061,16 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.traceReporter,
controllerErrorTelemetryProjection(recipe, error)
)
+ // issue #375 (P2): a transient (non-404) failure used to return here
+ // WITHOUT re-arming the retry chain, so a thrown reconcile killed the 5s
+ // self-heal and only the 30s watchdog could recover. Re-schedule the
+ // bounded exponential backoff retry so the recipe re-reconciles on its
+ // own; persistent failure still backs off toward TRANSIENT_RETRY_MAX_MS,
+ // with the watchdog as the second net. The 404 branch above already
+ // cleared and returned, so it never reaches here.
+ if (!this.stopped) {
+ this.scheduleTransientRetry(recipe.metadata.name, TRANSIENT_REQUEUE_BASE_MS, false)
+ }
}
} else if (type === 'DELETED') {
// Resource is gone (finalizer already removed) — just clean cache.
@@ -1209,6 +1378,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
if (this.dbRunProcessor) {
await this.dbRunProcessor.stop()
}
+ if (this.grantUpdateListener) {
+ await this.grantUpdateListener.stop()
+ }
if (this.traceReporter) {
const result = await this.traceReporter.stopAndDrain()
if (!result.drained) {
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
new file mode 100644
index 000000000..f1040c079
--- /dev/null
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
@@ -0,0 +1,274 @@
+import { describe, expect, it, vi } from 'vitest'
+import type { Logger } from '../observability/logger'
+import {
+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ createGrantUpdateListener,
+ parseGrantUpdatePayload,
+} from './grantUpdateListener'
+
+function makeLogger(): Logger {
+ const logger = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ withStep: vi.fn(),
+ }
+ logger.withStep.mockReturnValue(logger)
+ return logger as unknown as Logger
+}
+
+type NotificationHandler = (msg: { channel: string; payload?: string }) => void
+
+/**
+ * Minimal fake of a pg PoolClient scoped to LISTEN: records the channel it
+ * LISTENed on and lets the test emit notifications / connection errors into the
+ * registered handlers. `query` can be made to reject to simulate a LISTEN that
+ * fails without an 'error' emission (H3).
+ */
+function makeFakeClient(opts: { failQuery?: boolean } = {}) {
+ const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}
+ const queries: string[] = []
+ const client = {
+ on(event: string, handler: (...args: unknown[]) => void) {
+ ;(handlers[event] ??= []).push(handler)
+ return client
+ },
+ query: vi.fn(async (sql: string) => {
+ queries.push(sql)
+ if (opts.failQuery) throw new Error('LISTEN failed')
+ return { rows: [], rowCount: 0 }
+ }),
+ release: vi.fn(),
+ }
+ return {
+ client,
+ queries,
+ emitNotification(msg: { channel: string; payload?: string }) {
+ for (const handler of handlers.notification ?? []) (handler as NotificationHandler)(msg)
+ },
+ emitError(err: Error) {
+ for (const handler of handlers.error ?? []) handler(err)
+ },
+ }
+}
+
+describe('parseGrantUpdatePayload', () => {
+ it('parses a well-formed payload with a capability family', () => {
+ expect(
+ parseGrantUpdatePayload(
+ JSON.stringify({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ )
+ ).toEqual({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ })
+
+ it('parses a payload without a capability family (whole-recipe revoke)', () => {
+ expect(
+ parseGrantUpdatePayload(
+ JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })
+ )
+ ).toEqual({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })
+ })
+
+ it('returns null for non-JSON, missing fields, or wrong types (data, not command)', () => {
+ expect(parseGrantUpdatePayload('not json')).toBeNull()
+ expect(parseGrantUpdatePayload(JSON.stringify({ recipeName: 'research' }))).toBeNull()
+ expect(
+ parseGrantUpdatePayload(JSON.stringify({ recipeNamespace: 'sandbox-recipes' }))
+ ).toBeNull()
+ expect(
+ parseGrantUpdatePayload(JSON.stringify({ recipeNamespace: 1, recipeName: 2 }))
+ ).toBeNull()
+ expect(parseGrantUpdatePayload(JSON.stringify(['array']))).toBeNull()
+ })
+})
+
+describe('createGrantUpdateListener', () => {
+ it('LISTENs on the grant-update channel on start', async () => {
+ const fake = makeFakeClient()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => fake.client as never,
+ })
+
+ await listener.start()
+
+ expect(fake.queries.some(sql => /LISTEN\s+plugin_workload_sdk_grant_update/.test(sql))).toBe(
+ true
+ )
+ await listener.stop()
+ expect(fake.client.query).toHaveBeenCalledWith(
+ expect.stringMatching(/UNLISTEN\s+plugin_workload_sdk_grant_update/)
+ )
+ expect(fake.client.release).toHaveBeenCalled()
+ })
+
+ it('forwards the parsed notification to onGrantUpdate', async () => {
+ const fake = makeFakeClient()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => fake.client as never,
+ })
+ await listener.start()
+
+ fake.emitNotification({
+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ payload: JSON.stringify({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ }),
+ })
+
+ expect(onGrantUpdate).toHaveBeenCalledWith({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ capabilityFamily: 'promptBridge',
+ })
+ await listener.stop()
+ })
+
+ it('warns and does not dispatch on an unparseable payload (payload is data, not a command)', async () => {
+ const fake = makeFakeClient()
+ const onGrantUpdate = vi.fn()
+ const logger = makeLogger()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger,
+ connect: async () => fake.client as never,
+ })
+ await listener.start()
+
+ fake.emitNotification({
+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ payload: 'garbage-not-json',
+ })
+
+ expect(onGrantUpdate).not.toHaveBeenCalled()
+ expect(logger.warn).toHaveBeenCalled()
+ await listener.stop()
+ })
+
+ it('ignores notifications on other channels', async () => {
+ const fake = makeFakeClient()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => fake.client as never,
+ })
+ await listener.start()
+
+ fake.emitNotification({
+ channel: 'some_other_channel',
+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),
+ })
+
+ expect(onGrantUpdate).not.toHaveBeenCalled()
+ await listener.stop()
+ })
+})
+
+describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {
+ /** A sleep seam that never resolves, so we can count reconnect scheduling without real timers. */
+ function pendingSleep() {
+ const calls: number[] = []
+ const sleep = vi.fn((ms: number) => {
+ calls.push(ms)
+ return new Promise<void>(() => {
+ /* never resolves — the reconnect stays parked in its backoff */
+ })
+ })
+ return { sleep, calls }
+ }
+
+ it('H1: a connection drop schedules exactly ONE reconnect even when error fires repeatedly', async () => {
+ const fake = makeFakeClient()
+ const { sleep, calls } = pendingSleep()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate: vi.fn(),
+ logger: makeLogger(),
+ connect: async () => fake.client as never,
+ sleep,
+ })
+ await listener.start()
+
+ // Both the LISTEN-rejection path and the 'error' event fire on one drop; and
+ // 'error' can fire more than once. All must collapse into a single reconnect.
+ fake.emitError(new Error('connection reset'))
+ fake.emitError(new Error('connection reset again'))
+
+ expect(fake.client.release).toHaveBeenCalled()
+ expect(calls).toHaveLength(1) // exactly one reconnect scheduled, no leaked loops
+ await listener.stop()
+ })
+
+ it('H3: a LISTEN that fails without an error event releases the client and schedules a retry', async () => {
+ const fake = makeFakeClient({ failQuery: true })
+ const { sleep, calls } = pendingSleep()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate: vi.fn(),
+ logger: makeLogger(),
+ connect: async () => fake.client as never,
+ sleep,
+ })
+
+ await listener.start()
+
+ // LISTEN query rejected → attach catch must release the checked-out client
+ // (no leak) and schedule the retry.
+ expect(fake.client.release).toHaveBeenCalled()
+ expect(calls).toHaveLength(1)
+ await listener.stop()
+ })
+
+ it('H2: stop() during an in-flight connect() leaves no live listener and dispatches nothing', async () => {
+ const fake = makeFakeClient()
+ const onGrantUpdate = vi.fn()
+ let resolveConnect!: () => void
+ const connectGate = new Promise<void>(resolve => {
+ resolveConnect = resolve
+ })
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => {
+ await connectGate
+ return fake.client as never
+ },
+ })
+
+ const startPromise = listener.start() // parks awaiting connect()
+ await listener.stop() // shutdown while the attach is mid-connect
+ resolveConnect() // connect resolves AFTER stop
+ await startPromise
+
+ // The attach observed `stopped` after connect() resolved: it released the
+ // client and registered NO handlers, so a late notification dispatches nothing.
+ expect(fake.client.release).toHaveBeenCalled()
+ fake.emitNotification({
+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),
+ })
+ expect(onGrantUpdate).not.toHaveBeenCalled()
+ })
+})
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.ts
new file mode 100644
index 000000000..f4a456d7f
--- /dev/null
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts
@@ -0,0 +1,222 @@
+/**
+ * Plugin Workload SDK grant-update LISTEN/NOTIFY consumer (issue #375, P3).
+ *
+ * control-api emits `pg_notify('plugin_workload_sdk_grant_update', …)` inside
+ * the transaction that upserts, deletes, or revokes a Plugin Workload SDK grant
+ * (delivered on COMMIT). This listener turns that best-effort signal into an
+ * immediate `forceReconcile` of the affected recipe, so a validated grant is
+ * published in ~1–5s instead of waiting for the ≤30s level-triggered watchdog.
+ *
+ * CROSS-SERVICE CONTRACT: the channel name and payload shape are shared with
+ * `control-api/src/services/pluginWorkloadSdkDb.ts`
+ * (`PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL`). Keep them in sync.
+ *
+ * Semantics are deliberately best-effort: a lost NOTIFY (reconnect gap) degrades
+ * to the existing polling backstop, never worse. The LISTEN session runs on its
+ * own dedicated `pool.connect()` client (a pooled query would not keep the
+ * session sticky for notifications), mirroring `dbRunProcessor.ts`. The payload
+ * is untrusted DATA, not a command: an unparseable payload or unknown recipe is
+ * warned and discarded, never acted on blindly.
+ */
+import type { Pool, PoolClient } from 'pg'
+import { type Logger, createLogger } from '../observability/logger.js'
+
+/** Shared with control-api's grant-mutation NOTIFY. Keep both sides in sync. */
+export const PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL = 'plugin_workload_sdk_grant_update'
+
+/** Validated grant-update notification payload. */
+export interface GrantUpdateNotification {
+ recipeNamespace: string
+ recipeName: string
+ /** Absent for a whole-recipe revoke that touched multiple families. */
+ capabilityFamily?: string
+}
+
+export interface GrantUpdateListenerOptions {
+ /** Pool the dedicated LISTEN client is checked out from. */
+ pool: Pool
+ /**
+ * Invoked with a validated notification. The caller decides whether the
+ * recipe is known and how to react (the watcher enqueues a forceReconcile).
+ */
+ onGrantUpdate: (notification: GrantUpdateNotification) => void
+ logger?: Logger
+ /** Test seam: override how the dedicated LISTEN client is acquired. */
+ connect?: () => Promise<PoolClient>
+ /** Backoff before a reconnect attempt (ms). Default 1000. */
+ reconnectDelayMs?: number
+ /** Test seam: override the reconnect backoff wait (default: real setTimeout). */
+ sleep?: (ms: number) => Promise<void>
+}
+
+export interface GrantUpdateListener {
+ start(): Promise<void>
+ stop(): Promise<void>
+}
+
+/**
+ * Parse a NOTIFY payload into a `GrantUpdateNotification`, or `null` when it is
+ * not a well-formed object with string `recipeNamespace`/`recipeName`. Pure and
+ * exported so it can be unit-tested without a live Postgres. Never throws.
+ */
+export function parseGrantUpdatePayload(payload: string): GrantUpdateNotification | null {
+ let parsed: unknown
+ try {
+ parsed = JSON.parse(payload)
+ } catch {
+ return null
+ }
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return null
+ const record = parsed as Record<string, unknown>
+ const recipeNamespace = typeof record.recipeNamespace === 'string' ? record.recipeNamespace : ''
+ const recipeName = typeof record.recipeName === 'string' ? record.recipeName : ''
+ if (!recipeNamespace || !recipeName) return null
+ const capabilityFamily =
+ typeof record.capabilityFamily === 'string' ? record.capabilityFamily : undefined
+ return {
+ recipeNamespace,
+ recipeName,
+ ...(capabilityFamily ? { capabilityFamily } : {}),
+ }
+}
+
+export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): GrantUpdateListener {
+ const log = opts.logger ?? createLogger('wrc', 'grant-update-listener')
+ const connect = opts.connect ?? (() => opts.pool.connect())
+ const reconnectDelayMs = opts.reconnectDelayMs ?? 1_000
+ const sleep =
+ opts.sleep ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)))
+
+ // NOTE (detection-duty, issue #375): the sibling `dbRunProcessor.ts` LISTEN
+ // consumer has the same latent patterns this module hardens against —
+ // (1) a LISTEN rejection AND an 'error' emission on the same drop can start two
+ // reconnect loops, (2) stop() cannot cancel an attach already awaiting
+ // connect(), and (3) a LISTEN failure without an 'error' event leaks the
+ // checked-out client. Fixing dbRunProcessor is out of #375 scope; this listener
+ // intentionally improves on it and the guards below are the reference.
+ let listenClient: PoolClient | null = null
+ let stopped = true
+ // H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.
+ let reconnecting = false
+
+ async function attachListener(): Promise<void> {
+ let client: PoolClient | null = null
+ try {
+ client = await connect()
+ // H2: stop() may have run while we awaited connect(); do NOT bring up a
+ // LISTEN session after shutdown. Release the client and bail before any
+ // handler is registered, so nothing dispatches post-stop.
+ if (stopped) {
+ client.release()
+ return
+ }
+ const attached = client
+ attached.on('notification', msg => {
+ if (msg.channel !== PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL || !msg.payload) return
+ const notification = parseGrantUpdatePayload(msg.payload)
+ if (!notification) {
+ // Untrusted payload — warn and discard, never act on it.
+ log.warn('discarding unparseable grant-update notification', {
+ payload_length: msg.payload.length,
+ })
+ return
+ }
+ try {
+ opts.onGrantUpdate(notification)
+ } catch (err) {
+ log.error('grant-update dispatch failed', {
+ recipe_namespace: notification.recipeNamespace,
+ recipe_name: notification.recipeName,
+ err: err instanceof Error ? err.message : String(err),
+ })
+ }
+ })
+ attached.on('error', (err: Error) => {
+ log.warn('listen session error — will reconnect', { err: err.message })
+ try {
+ attached.release(err)
+ } catch {
+ /* idempotent */
+ }
+ if (listenClient === attached) listenClient = null
+ scheduleReconnect()
+ })
+ await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
+ // H2: re-check after the async LISTEN resolves — stop() may have run
+ // during the round-trip.
+ if (stopped) {
+ try {
+ attached.release()
+ } catch {
+ /* idempotent */
+ }
+ return
+ }
+ listenClient = attached
+ log.info('LISTEN plugin_workload_sdk_grant_update attached')
+ } catch (err) {
+ log.warn('LISTEN attach failed — will retry', {
+ err: err instanceof Error ? err.message : String(err),
+ })
+ // H3: a LISTEN failure may not emit an 'error' event, so release the
+ // checked-out client here (if we got one) before scheduling the retry —
+ // otherwise the connection leaks back to the pool checked out forever.
+ if (client) {
+ try {
+ client.release(err instanceof Error ? err : new Error(String(err)))
+ } catch {
+ /* idempotent */
+ }
+ }
+ scheduleReconnect()
+ }
+ }
+
+ function scheduleReconnect(): void {
+ // H1: the LISTEN-query rejection and the 'error' event both fire on a single
+ // connection drop. Without this guard each would spawn its own reconnect
+ // loop → a leaked client and duplicated dispatch. Only one may be in flight.
+ if (stopped || reconnecting) return
+ reconnecting = true
+ void reconnectLater()
+ }
+
+ async function reconnectLater(): Promise<void> {
+ await sleep(reconnectDelayMs)
+ // Clear the guard BEFORE re-attaching so a failure inside attachListener can
+ // schedule the NEXT reconnect; the sleep window above is what collapses the
+ // concurrent triggers from a single drop.
+ reconnecting = false
+ if (stopped) return
+ await attachListener()
+ }
+
+ return {
+ async start() {
+ if (!stopped) return
+ stopped = false
+ await attachListener()
+ },
+
+ async stop() {
+ stopped = true
+ // issue #375 (R4): clear the in-flight reconnect guard so a later
+ // stop()→start() restart is not blocked by a stale flag left set by a
+ // reconnect that was still mid-sleep when we stopped (belt-and-suspenders).
+ reconnecting = false
+ if (listenClient) {
+ try {
+ await listenClient.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
+ } catch {
+ /* best-effort — the session is being torn down */
+ }
+ try {
+ listenClient.release()
+ } catch {
+ /* idempotent */
+ }
+ listenClient = null
+ }
+ },
+ }
+}
diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
index cdcda8736..52cc2d80d 100644
--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
@@ -351,6 +351,73 @@ describe('buildPluginWorkloadSdkStatus', () => {
})
})
+ it('preserves the original validatedAt across a steady-state throttle refresh (issue #375 R1)', () => {
+ // The verifiedAt throttle rewrites the whole SDK projection every ~5 min in
+ // steady state. validatedAt is the stable "first reached validated" marker
+ // (operators/dashboards read it), so a still-validated recipe must NOT have
+ // it reset to now on each throttle patch — only verifiedAt advances.
+ const EARLIER = '2026-08-17T10:00:00.000Z'
+ const projection = buildPluginWorkloadSdkStatus({
+ spec: baseSpec({ promptBridge: {} }),
+ existingConditions: undefined,
+ phase: 'active',
+ featureFlagEnabled: true,
+ bootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'pod-uid-1',
+ provider: 'zai',
+ model: 'glm-4.7',
+ policyRevision: 1,
+ defaultTargetRef: 'primary-zai',
+ verifiedAt: NOW,
+ },
+ now: NOW,
+ // Already validated on a prior pass, with the stable marker set EARLIER.
+ existingCapability: {
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ validatedAt: EARLIER,
+ verifiedAt: EARLIER,
+ },
+ })
+ expect(projection.capability?.state).toBe('validated')
+ // validatedAt is carried forward (stable), verifiedAt advances to now.
+ expect(projection.capability?.validatedAt).toBe(EARLIER)
+ expect(projection.capability?.verifiedAt).toBe(NOW)
+ })
+
+ it('stamps a fresh validatedAt on a genuine transition INTO validated (issue #375 R1)', () => {
+ const projection = buildPluginWorkloadSdkStatus({
+ spec: baseSpec({ promptBridge: {} }),
+ existingConditions: undefined,
+ phase: 'active',
+ featureFlagEnabled: true,
+ bootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'pod-uid-1',
+ provider: 'zai',
+ model: 'glm-4.7',
+ policyRevision: 1,
+ defaultTargetRef: 'primary-zai',
+ verifiedAt: NOW,
+ },
+ now: NOW,
+ // Prior state was awaiting_policy — this is the real transition.
+ existingCapability: {
+ state: 'awaiting_policy',
+ promptBridge: true,
+ clientNotifications: false,
+ validatedAt: null,
+ verifiedAt: '2026-08-17T10:00:00.000Z',
+ },
+ })
+ expect(projection.capability?.state).toBe('validated')
+ expect(projection.capability?.validatedAt).toBe(NOW)
+ })
+
it('keeps a mixed recipe pending until every declared family has an active proof', () => {
const projection = buildPluginWorkloadSdkStatus({
spec: baseSpec({
diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
index e433d08c1..9c532f1eb 100644
--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
@@ -244,6 +244,12 @@ export function buildPluginWorkloadSdkStatus(args: {
teardownConfirmed?: boolean
bootstrapProof?: PluginWorkloadSdkBootstrapProofInput
now: string
+ /**
+ * The currently-persisted capability (recipe.status.pluginWorkloadSdk), used
+ * to CARRY FORWARD the stable `validatedAt` marker across steady-state
+ * throttle patches (issue #375 R1) instead of resetting it to `now`.
+ */
+ existingCapability?: PluginWorkloadSdkCapabilityStatus | null
}): PluginWorkloadSdkStatusProjection {
const {
spec,
@@ -255,6 +261,7 @@ export function buildPluginWorkloadSdkStatus(args: {
teardownConfirmed,
bootstrapProof,
now,
+ existingCapability,
} = args
const sdk = spec.pluginWorkloadSdk
@@ -408,6 +415,17 @@ export function buildPluginWorkloadSdkStatus(args: {
.filter((f): f is string => f !== undefined)
.join(', ')
+ // issue #375 (R1): validatedAt is the stable "first reached validated" marker.
+ // On a steady-state throttle refresh (already validated, only verifiedAt
+ // advances) it must be CARRIED FORWARD, not reset to `now`. Stamp a fresh
+ // `now` ONLY on the real transition into validated — no prior capability, or a
+ // prior state other than 'validated', or a prior validated record that somehow
+ // lacked the marker.
+ const validatedAt =
+ existingCapability?.state === 'validated' && existingCapability.validatedAt
+ ? existingCapability.validatedAt
+ : now
+
return {
conditions: [
{
@@ -424,7 +442,7 @@ export function buildPluginWorkloadSdkStatus(args: {
clientNotifications,
message: `Capability validated (${families})`,
...CLEARED_PLUGIN_WORKLOAD_SDK_METADATA,
- validatedAt: now,
+ validatedAt,
...(bootstrapProof
? {
bootstrapContractVersion: bootstrapProof.contractVersion,
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index ac3c59229..73dd1d5fe 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -91,6 +91,7 @@ import {
PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,
PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,
+ type PluginWorkloadSdkStatusProjection,
buildPluginWorkloadSdkStatus,
validatePluginWorkloadSdkSpec,
} from './pluginWorkloadSdkValidator'
@@ -508,6 +509,17 @@ export interface ReconcileResult {
/** Cleanup completed successfully while the SDK feature flag was off. */
pluginWorkloadSdkTeardownConfirmed?: boolean
pluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof
+ /**
+ * SDK capability projection (conditions + `status.pluginWorkloadSdk` value)
+ * derived once per reconcile from this result. Populated by the watcher after
+ * `reconcile()` returns (see `projectPluginWorkloadSdk`) so BOTH
+ * `shouldPatchRecipeStatus` (to detect a computed awaiting_policy↔validated
+ * transition that no other diff would surface — issue #375) and `patchStatus`
+ * (to avoid recomputing it) consume the same object. Left undefined on paths
+ * that do not run the SDK lane (e.g. workload-status-only refresh), where the
+ * SDK comparison is intentionally skipped.
+ */
+ pluginWorkloadSdkProjection?: PluginWorkloadSdkStatusProjection
workflowPhase?: import('../workflow/types').WorkflowPhase
clearWorkflowExecution?: boolean
/** When true, the caller should NOT patch the CRD status — the phase hasn't changed. */
@@ -1288,6 +1300,19 @@ export class WorkflowRecipeReconciler {
// ─── Main Pipeline ────────────────────────────────────────────────
async reconcile(recipe: WorkflowRecipeCRD): Promise<ReconcileResult> {
+ // issue #375: compute the Plugin Workload SDK capability projection ONCE per
+ // reconcile and attach it to the result. shouldPatchRecipeStatus reads it to
+ // publish a computed awaiting_policy↔validated transition that no
+ // phase/message/workload diff would otherwise surface; patchStatus reuses
+ // the same object instead of rebuilding it. Paths that bypass this method
+ // (e.g. observeCurrentWorkloadStatus) leave the projection undefined, which
+ // both consumers treat as "no SDK opinion this pass".
+ const result = await this.reconcileInternal(recipe)
+ result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)
+ return result
+ }
+
+ private async reconcileInternal(recipe: WorkflowRecipeCRD): Promise<ReconcileResult> {
const name = recipe.metadata.name
const ns = recipe.metadata.namespace
const currentPhase = recipe.status?.phase ?? 'candidate'
@@ -6031,6 +6056,44 @@ export class WorkflowRecipeReconciler {
// ─── Status Patch ─────────────────────────────────────────────────
/** Persist reconcile result to the CRD status subresource. */
+ /**
+ * Build the Plugin Workload SDK status projection (owned conditions +
+ * `status.pluginWorkloadSdk` value) from a reconcile result. Extracted so it
+ * can be computed ONCE per reconcile by the watcher (issue #375): the watcher
+ * attaches it to `result.pluginWorkloadSdkProjection` after `reconcile()`
+ * returns, `shouldPatchRecipeStatus` reads it to detect an
+ * awaiting_policy↔validated transition that no other diff would surface, and
+ * `patchStatus` reuses it instead of recomputing. `providerUnavailable` is
+ * derived here (not inferred from phase/message) so the SDK-only provider
+ * health bit stays explicit.
+ */
+ projectPluginWorkloadSdk(
+ recipe: WorkflowRecipeCRD,
+ result: ReconcileResult,
+ now: string = new Date().toISOString()
+ ): PluginWorkloadSdkStatusProjection {
+ return buildPluginWorkloadSdkStatus({
+ spec: recipe.spec,
+ existingConditions: recipe.status?.conditions,
+ phase: result.phase,
+ featureFlagEnabled: this.config.pluginWorkloadSdkEnabled,
+ providerUnavailable:
+ result.pluginWorkloadSdkProviderUnavailable === true ||
+ (result.workflowConditions ?? []).some(
+ condition =>
+ condition.type === PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE &&
+ condition.status === 'True'
+ ),
+ teardownConfirmed: result.pluginWorkloadSdkTeardownConfirmed === true,
+ policyPending: result.pluginWorkloadSdkPolicyPending === true,
+ bootstrapProof: result.pluginWorkloadSdkBootstrapProof,
+ now,
+ // issue #375 (R1): carry forward the stable validatedAt marker across
+ // steady-state throttle patches instead of resetting it to `now`.
+ existingCapability: recipe.status?.pluginWorkloadSdk,
+ })
+ }
+
async patchStatus(recipe: WorkflowRecipeCRD, result: ReconcileResult): Promise<void> {
const workloads = result.workloadStatuses.map(ws => {
const def = (recipe.spec.workloads ?? []).find(w => w.id === ws.id)
@@ -6127,27 +6190,16 @@ export class WorkflowRecipeReconciler {
recipe.status?.conditions,
result.workloadConditions
)
- // Plugin Workload SDK conditions are derived here so every status patch
- // carries a consistent projection of spec.pluginWorkloadSdk + feature flag,
- // while the SDK-only provider health bit is propagated explicitly through
- // ReconcileResult rather than inferred from a free-form phase/message.
- const pluginSdkProjection = buildPluginWorkloadSdkStatus({
- spec: recipe.spec,
- existingConditions: recipe.status?.conditions,
- phase: result.phase,
- featureFlagEnabled: this.config.pluginWorkloadSdkEnabled,
- providerUnavailable:
- result.pluginWorkloadSdkProviderUnavailable === true ||
- (result.workflowConditions ?? []).some(
- condition =>
- condition.type === PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE &&
- condition.status === 'True'
- ),
- teardownConfirmed: result.pluginWorkloadSdkTeardownConfirmed === true,
- policyPending: result.pluginWorkloadSdkPolicyPending === true,
- bootstrapProof: result.pluginWorkloadSdkBootstrapProof,
- now,
- })
+ // Plugin Workload SDK conditions are derived so every status patch carries a
+ // consistent projection of spec.pluginWorkloadSdk + feature flag, while the
+ // SDK-only provider health bit is propagated explicitly through
+ // ReconcileResult rather than inferred from a free-form phase/message. Reuse
+ // the projection the watcher already computed for the patch decision
+ // (issue #375) so the exact object that drove shouldPatchRecipeStatus is
+ // what gets written; recompute only on paths that never attached one (e.g.
+ // observeCurrentWorkloadStatus).
+ const pluginSdkProjection =
+ result.pluginWorkloadSdkProjection ?? this.projectPluginWorkloadSdk(recipe, result, now)
const pluginSdkMergedConditions = mergePluginWorkloadSdkConditions(
workloadReconcileMergedConditions ??
secretOwnershipMergedConditions ??