Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes,control-api): promptly publish Plugin Workload SDK grant re-evaluation (#375)
Alfredo Lopez32e4c285300e
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 305db4444..d6708a2cb 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex 5af12984c..0a440c422 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex d584c15d9..29925e075 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -18,6 +18,57 @@ import {export const PLUGIN_WORKLOAD_SDK_FAMILIES = ['promptBridge', 'clientNotifications'] as constexport type PluginWorkloadSdkFamily = (typeof PLUGIN_WORKLOAD_SDK_FAMILIES)[number]+// ─── Grant-update NOTIFY (issue #375, P3) ────────────────────────────────+// CROSS-SERVICE CONTRACT: a transactional `pg_notify` fired inside the+// grant-mutation transactions (upsert / delete / revoke) so the Workflow+// Recipes Controller re-reconciles the affected recipe immediately (~1–5s)+// instead of waiting for its ≤30s level-triggered watchdog. The WRC LISTEN side+// lives in `workflow-recipes/src/reconciler/grantUpdateListener.ts` — the+// channel name and payload shape below MUST stay in sync with it.+//+// This is an app-level NOTIFY (not a DDL trigger) so the payload stays typed and+// the change is fully revertible. Being inside the transaction, it is delivered+// on COMMIT and discarded on ROLLBACK. Semantics are best-effort: a dropped+// notification degrades to the existing polling backstop, never worse.+export const PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL = 'plugin_workload_sdk_grant_update'++export interface GrantUpdateNotifyInput {+ recipeNamespace: string+ recipeName: string+ /** Omitted for a whole-recipe revoke that spans multiple families. */+ capabilityFamily?: string+}++/**+ * Build the JSON payload for a grant-update NOTIFY. Pure and exported so the+ * exact wire shape can be unit-tested without a live Postgres.+ */+export function buildGrantUpdateNotifyPayload(input: GrantUpdateNotifyInput): string {+ return JSON.stringify({+ recipeNamespace: input.recipeNamespace,+ recipeName: input.recipeName,+ ...(input.capabilityFamily ? { capabilityFamily: input.capabilityFamily } : {}),+ })+}++/**+ * Emit the grant-update NOTIFY on the shared channel. MUST be called with the+ * in-transaction `db` client so the signal is delivered on COMMIT (and dropped+ * on ROLLBACK), never as a separate connection.+ *+ * A `pg_notify` inside the transaction would roll the whole grant mutation back+ * if it threw — but the only failure mode is a payload over Postgres' 8000-byte+ * NOTIFY limit, and this payload is just a recipe namespace/name (+ optional+ * short family), which cannot approach that bound. So the mutation is never at+ * risk from the notify, and the transactional coupling is intentional.+ */+async function notifyGrantUpdate(db: DbClient, input: GrantUpdateNotifyInput): Promise<void> {+ await db.query('SELECT pg_notify($1, $2)', [+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ buildGrantUpdateNotifyPayload(input),+ ])+}+export const PLUGIN_WORKLOAD_SDK_INVOCATION_STATUSES = ['in_progress','complete',@@ -614,6 +665,12 @@ export async function upsertGrant(})),]await appendControlApiPermissionEventsInTransaction(db, { operatorSub, changes })+ // issue #375 (P3): nudge the WRC to re-reconcile now that the grant changed.+ await notifyGrantUpdate(db, {+ recipeNamespace: params.recipeNamespace,+ recipeName: params.recipeName,+ capabilityFamily: params.capabilityFamily,+ })return grant})}@@ -880,6 +937,17 @@ export async function revokePluginWorkloadSdkForRecipe(changes,})}+ // issue #375 (P3): a whole-recipe revoke spans every family, so omit the+ // capabilityFamily — the WRC only needs the recipe coordinates to re-reconcile.+ // issue #375 (B2): notify ONLY on an actual transition (rows moved to+ // 'revoking', or permission changes emitted). A no-op revoke — every grant+ // already revoking/disabled — must stay silent, otherwise the WRC would+ // force-reconcile the still-cached recipe, which can re-enter revoke and+ // NOTIFY again: a reconcile+pg_notify spin. Matches upsert/delete "notify+ // only on a real mutation" semantics.+ if ((revoked.rowCount ?? 0) > 0 || changes.length > 0) {+ await notifyGrantUpdate(db, { recipeNamespace, recipeName })+ }return {state:hasActivePolicy || hasRevokingPolicy || (revoked.rowCount ?? 0) > 0@@ -1044,6 +1112,13 @@ export async function deleteGrant(})),]await appendControlApiPermissionEventsInTransaction(db, { operatorSub, changes })+ // issue #375 (P3): nudge the WRC to re-reconcile after the grant is removed+ // so the capability projection transitions back to awaiting_policy/degraded.+ await notifyGrantUpdate(db, {+ recipeNamespace: grant.recipeNamespace,+ recipeName: grant.recipeName,+ capabilityFamily: grant.capabilityFamily,+ })return true})}diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.tsnew file mode 100644index 000000000..db9654f17--- /dev/null+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts@@ -0,0 +1,178 @@+import { afterAll, beforeAll, expect, describe as vitestDescribe } from 'vitest'+import { it } from 'vitest'+import { randomBytes } from 'node:crypto'+import { Pool, type PoolClient } from 'pg'++/**+ * issue #375 (P3) — grant-update NOTIFY on real PostgreSQL.+ *+ * Verifies the property the WRC LISTEN side depends on: the grant-mutation+ * transactions emit a `pg_notify` on `plugin_workload_sdk_grant_update` that is+ * delivered on COMMIT and discarded on ROLLBACK.+ *+ * Real database, no mocks. The service module's module-level pool (src/db.ts) is+ * pointed at a throwaway database by setting CONTROL_API_PG_CONNECTION_STRING+ * BEFORE the dynamic src imports in beforeAll — nothing from src/ is value-+ * imported statically here, matching the other *.realPostgres suites.+ */++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+// Repo convention (matches the other *.realPostgres.integration.test.ts suites):+// gate on the DB env so the DB-less unit lane stays green; the Postgres-backed+// CI lane sets CONTROL_API_REAL_PG_ADMIN_URL and runs it non-skipped.+const describeRealPostgres = adminUrl ? vitestDescribe : vitestDescribe.skip++const database = `control_api_sdk_notify_${randomBytes(6).toString('hex')}`++function databaseUrl(baseUrl: string, databaseName: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${databaseName}`+ return url.toString()+}++const previousPgConnectionString = process.env.CONTROL_API_PG_CONNECTION_STRING+const throwawayUrl = adminUrl ? databaseUrl(adminUrl, database) : undefined+if (adminUrl && throwawayUrl) {+ process.env.CONTROL_API_PG_CONNECTION_STRING = throwawayUrl+}++const NS = 'sandbox-recipes'+const RECIPE = 'notify-recipe'++describeRealPostgres(+ 'Plugin Workload SDK grant-update NOTIFY on real PostgreSQL (issue #375)',+ () => {+ let adminPool: Pool+ let db: typeof import('../src/db.js')+ let sdk: typeof import('../src/services/pluginWorkloadSdkDb.js')+ let listenPool: Pool+ let listenClient: PoolClient+ let received: Array<{ channel: string; payload?: string }>++ beforeAll(async () => {+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)+ db = await import('../src/db.js')+ await db.initDb()+ sdk = await import('../src/services/pluginWorkloadSdkDb.js')++ received = []+ listenPool = new Pool({ connectionString: throwawayUrl })+ listenClient = await listenPool.connect()+ listenClient.on('notification', msg => {+ received.push({ channel: msg.channel, payload: msg.payload })+ })+ await listenClient.query(`LISTEN ${sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)+ })++ afterAll(async () => {+ try {+ await listenClient?.query(`UNLISTEN ${sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)+ } catch {+ /* teardown */+ }+ listenClient?.release()+ await listenPool?.end()+ await db?.pool.end()+ if (previousPgConnectionString === undefined) {+ delete process.env.CONTROL_API_PG_CONNECTION_STRING+ } else {+ process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgConnectionString+ }+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid)+ FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)+ await adminPool.end()+ }+ })++ /** Poll the LISTEN client until a notification arrives on our channel or timeout. */+ async function waitForGrantNotification(timeoutMs = 3_000): Promise<string | undefined> {+ const deadline = Date.now() + timeoutMs+ while (Date.now() < deadline) {+ // Round-trip a trivial query so pg flushes any pending notifications.+ await listenClient.query('SELECT 1')+ const hit = received.find(m => m.channel === sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)+ if (hit) return hit.payload+ await new Promise(resolve => setTimeout(resolve, 50))+ }+ return undefined+ }++ it('delivers the NOTIFY after upsertGrant COMMITs', async () => {+ received.length = 0+ await sdk.upsertGrant(+ {+ recipeNamespace: NS,+ recipeName: RECIPE,+ capabilityFamily: 'promptBridge',+ provider: 'openai',+ allowedModels: ['gpt-5'],+ allowedCallers: ['api'],+ },+ 'operator-1'+ )++ const payload = await waitForGrantNotification()+ expect(payload, 'expected a grant-update NOTIFY after commit').toBeDefined()+ expect(JSON.parse(String(payload))).toEqual({+ recipeNamespace: NS,+ recipeName: RECIPE,+ capabilityFamily: 'promptBridge',+ })+ })++ it('discards BOTH the mutation and the NOTIFY when a real mutation transaction ROLLBACKs', async () => {+ // This proves the grant-update NOTIFY is transactionally coupled to the grant+ // mutation (delivered on COMMIT, discarded on ROLLBACK).+ //+ // WHY not fail a real service call (upsert/delete/revoke) after its notify:+ // `notifyGrantUpdate` is the TERMINAL statement of each of those functions —+ // nothing in them can throw after it — so a "production mutation that fails+ // after the notify" cannot be constructed. Instead we reproduce the exact+ // production shape: a REAL mutation (UPDATE of the committed grant) plus the+ // REAL notify (same channel + payload builder) inside the production+ // `withTransaction` wrapper, then abort it — and assert BOTH are discarded.+ received.length = 0+ const before = await db.pool.query<{ default_target_ref: string | null }>(+ `SELECT default_target_ref FROM plugin_workload_sdk_grants+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,+ [NS, RECIPE]+ )++ await expect(+ db.withTransaction(async client => {+ await client.query(+ `UPDATE plugin_workload_sdk_grants SET default_target_ref = 'ROLLED-BACK-SENTINEL'+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,+ [NS, RECIPE]+ )+ await client.query('SELECT pg_notify($1, $2)', [+ sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ sdk.buildGrantUpdateNotifyPayload({ recipeNamespace: NS, recipeName: RECIPE }),+ ])+ throw new Error('force rollback')+ })+ ).rejects.toThrow('force rollback')++ // The NOTIFY was discarded on rollback...+ const delivered = await waitForGrantNotification(1_000)+ expect(delivered, 'a rolled-back NOTIFY must never be delivered').toBeUndefined()++ // ...and so was the mutation — the grant is unchanged, proving the notify+ // shared the mutation's transaction.+ const after = await db.pool.query<{ default_target_ref: string | null }>(+ `SELECT default_target_ref FROM plugin_workload_sdk_grants+ WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,+ [NS, RECIPE]+ )+ expect(after.rows[0]?.default_target_ref).toBe(before.rows[0]?.default_target_ref)+ expect(after.rows[0]?.default_target_ref).not.toBe('ROLLED-BACK-SENTINEL')+ })+ }+)diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex 476997afa..bad27b537 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -1,6 +1,8 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import { pool } from '../src/db.js'import {+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ buildGrantUpdateNotifyPayload,deleteGrant,failStaleInvocations,finalizePluginWorkloadSdkRevocation,@@ -864,3 +866,292 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {expect(permissionEvents.append).not.toHaveBeenCalled()})})++// issue #375 (P3): the grant→WRC event-driven nudge. control-api emits a+// transactional pg_notify on the grant channel inside upsert/delete/revoke so+// the WRC re-reconciles the recipe immediately instead of waiting for the ≤30s+// watchdog. Delivered on COMMIT (transactional NOTIFY), discarded on rollback.+describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {+ it('builds a typed, JSON payload carrying the recipe coordinates and family', () => {+ expect(+ JSON.parse(+ buildGrantUpdateNotifyPayload({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ )+ ).toEqual({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ })++ it('omits capabilityFamily when it is not supplied (whole-recipe revoke)', () => {+ expect(+ JSON.parse(+ buildGrantUpdateNotifyPayload({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ })+ )+ ).toEqual({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })+ })++ it('exposes the shared channel name used by the WRC LISTEN side', () => {+ expect(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL).toBe('plugin_workload_sdk_grant_update')+ })++ it('upsertGrant emits a transactional pg_notify on the grant-update channel', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ permissionEvents.append.mockResolvedValue('operation-id')+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // family FOR UPDATE+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // kill-switch guard+ .mockResolvedValueOnce({+ rows: [+ {+ id: '33333333-3333-4333-8333-333333333333',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'research',+ capability_family: 'promptBridge',+ provider: 'openai',+ allowed_models: ['gpt-5'],+ allowed_event_types: [],+ allowed_target_refs: [],+ allowed_user_refs: [],+ allowed_callers: ['coordinator'],+ quota_limits: {},+ model_policies: {},+ policy_revision: 2,+ created_at: '2026-08-17T00:00:00.000Z',+ updated_at: '2026-08-17T00:00:00.000Z',+ },+ ],+ rowCount: 1,+ } as never) // upsert RETURNING+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // notify + any trailing++ await upsertGrant(+ {+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ provider: 'openai',+ allowedModels: ['gpt-5'],+ allowedCallers: ['coordinator'],+ },+ 'operator-1'+ )++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as+ | [string, unknown[]]+ | undefined+ expect(notifyCall).toBeDefined()+ const [, params] = notifyCall as [string, unknown[]]+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)+ expect(JSON.parse(String(params[1]))).toEqual({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ })++ it('deleteGrant emits a transactional pg_notify on the grant-update channel', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ permissionEvents.append.mockResolvedValue('operation-id')+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({+ rows: [+ {+ id: '33333333-3333-4333-8333-333333333333',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'research',+ capability_family: 'clientNotifications',+ allowed_user_refs: [],+ allowed_models: [],+ allowed_event_types: [],+ allowed_target_refs: [],+ allowed_callers: [],+ quota_limits: {},+ model_policies: {},+ created_at: '2026-08-17T00:00:00.000Z',+ updated_at: '2026-08-17T00:00:00.000Z',+ },+ ],+ rowCount: 1,+ } as never) // DELETE RETURNING+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // notify + trailing++ await deleteGrant(+ '33333333-3333-4333-8333-333333333333',+ 'sandbox-recipes',+ 'research',+ 'operator-1'+ )++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as+ | [string, unknown[]]+ | undefined+ expect(notifyCall).toBeDefined()+ const [, params] = notifyCall as [string, unknown[]]+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)+ expect(JSON.parse(String(params[1]))).toMatchObject({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ })+ })++ it('revokePluginWorkloadSdkForRecipe emits a transactional pg_notify on the grant-update channel', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ permissionEvents.append.mockResolvedValue('operation-id')+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({+ rows: [+ {+ id: '33333333-3333-4333-8333-333333333333',+ capability_family: 'promptBridge',+ policy_state: 'active',+ revocation_id: null,+ },+ ],+ rowCount: 1,+ } as never) // grants FOR UPDATE+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // updates + notify++ await revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', {+ operatorSub: 'operator-1',+ })++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as+ | [string, unknown[]]+ | undefined+ expect(notifyCall).toBeDefined()+ const [, params] = notifyCall as [string, unknown[]]+ expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)+ expect(JSON.parse(String(params[1]))).toMatchObject({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ })+ })++ it('does NOT emit pg_notify on a NO-OP revoke (issue #375 B2 — no self-sustaining spin)', async () => {+ // Every grant already revoking/disabled: nothing transitions (revoked+ // rowCount 0, zero permission changes). Emitting a NOTIFY here would make the+ // WRC force-reconcile the still-cached recipe, which can re-enter revoke and+ // NOTIFY again — a reconcile+pg_notify spin. Match upsert/delete: notify only+ // on a real mutation.+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ permissionEvents.append.mockResolvedValue('operation-id')+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({+ rows: [+ {+ id: '33333333-3333-4333-8333-333333333333',+ capability_family: 'promptBridge',+ policy_state: 'disabled',+ revocation_id: '55555555-5555-4555-8555-555555555555',+ },+ ],+ rowCount: 1,+ } as never) // grants FOR UPDATE (all already disabled)+ .mockResolvedValue({ rows: [], rowCount: 0 } as never) // UPDATEs affect 0 rows++ await revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', {+ operatorSub: 'operator-1',+ })++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))+ expect(notifyCall, 'a no-op revoke must not emit a grant-update NOTIFY').toBeUndefined()+ expect(permissionEvents.append).not.toHaveBeenCalled()+ })++ it('does NOT emit pg_notify when deleteGrant matches no row (rowCount 0)', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // DELETE RETURNING (no match)++ await expect(+ deleteGrant(+ '33333333-3333-4333-8333-333333333333',+ 'sandbox-recipes',+ 'research',+ 'operator-1'+ )+ ).resolves.toBe(false)++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))+ expect(notifyCall, 'a delete that removed nothing must not NOTIFY').toBeUndefined()+ })++ it('does NOT emit pg_notify on a revoke that finds no grants (state=missing)', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // grants FOR UPDATE (none)++ await expect(+ revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', { operatorSub: 'operator-1' })+ ).resolves.toMatchObject({ state: 'missing' })++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))+ expect(notifyCall, 'a missing revoke must not NOTIFY').toBeUndefined()+ })++ it('does NOT emit pg_notify on a revoke that conflicts on revocation epoch (state=conflict)', async () => {+ vi.mocked(pool.query).mockReset()+ permissionEvents.append.mockReset()+ vi.mocked(pool.query)+ .mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock+ .mockResolvedValueOnce({+ rows: [+ {+ id: '11111111-1111-4111-8111-111111111111',+ capability_family: 'promptBridge',+ policy_state: 'revoking',+ revocation_id: '55555555-5555-4555-8555-555555555555',+ },+ {+ id: '22222222-2222-4222-8222-222222222222',+ capability_family: 'clientNotifications',+ policy_state: 'revoking',+ revocation_id: '66666666-6666-4666-8666-666666666666',+ },+ ],+ rowCount: 2,+ } as never) // grants FOR UPDATE — two distinct revocation epochs++ await expect(+ revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'research', { operatorSub: 'operator-1' })+ ).resolves.toMatchObject({ state: 'conflict' })++ const notifyCall = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]) => String(sql).includes('pg_notify'))+ expect(notifyCall, 'a conflicting revoke must not NOTIFY').toBeUndefined()+ })+})diff --git a/scripts/e2e/e2e-plugin-workload-sdk.sh b/scripts/e2e/e2e-plugin-workload-sdk.shindex 241d989ab..36c66937e 100755--- a/scripts/e2e/e2e-plugin-workload-sdk.sh+++ b/scripts/e2e/e2e-plugin-workload-sdk.sh@@ -173,6 +173,10 @@ E2E_WAIT_STATUS_VALIDATED="${E2E_WAIT_STATUS_VALIDATED:-120}"E2E_WAIT_CALLER_MARKER="${E2E_WAIT_CALLER_MARKER:-180}"E2E_WAIT_CALLER_DONE="${E2E_WAIT_CALLER_DONE:-180}"E2E_WAIT_NOTIFICATION_ROW="${E2E_WAIT_NOTIFICATION_ROW:-60}"+# issue #375 (P3): event-driven grant→WRC nudge target latency. A grant change+# must re-reconcile the recipe via Postgres LISTEN/NOTIFY in ≤10s WITHOUT a WRC+# restart (the ≤30s watchdog is the backstop, not the mechanism under test).+E2E_WAIT_NUDGE_VALIDATED="${E2E_WAIT_NUDGE_VALIDATED:-10}"# Fail closed before creating any fixture if a caller or inherited shell# environment asks this gate to wait for an unbounded or human-scale interval.@@ -197,6 +201,7 @@ validate_bounded_seconds E2E_WAIT_STATUS_VALIDATED "$E2E_WAIT_STATUS_VALIDATED"validate_bounded_seconds E2E_WAIT_CALLER_MARKER "$E2E_WAIT_CALLER_MARKER" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"validate_bounded_seconds E2E_WAIT_CALLER_DONE "$E2E_WAIT_CALLER_DONE" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"validate_bounded_seconds E2E_WAIT_NOTIFICATION_ROW "$E2E_WAIT_NOTIFICATION_ROW" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"+validate_bounded_seconds E2E_WAIT_NUDGE_VALIDATED "$E2E_WAIT_NUDGE_VALIDATED" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"validate_bounded_seconds POLL_INTERVAL "$POLL_INTERVAL" "$E2E_HARD_MAX_POLL_INTERVAL_SECONDS"require_write_confirmation() {@@ -1385,5 +1390,75 @@ elseexit 1fi+# ─── issue #375 (P3): event-driven grant→WRC nudge ───────────────────────+# The recipe is validated by now. Prove the grant→WRC nudge accelerates+# convergence: a grant change must re-reconcile the recipe (bump the published+# policyRevision, state stays validated) within ≤10s via Postgres LISTEN/NOTIFY,+# WITHOUT restarting the WRC (the ≤30s watchdog is the backstop, not the path+# under test). This is an ADDITIVE phase; it does not touch #351's phases.+header "Plugin Workload SDK grant→WRC event-driven nudge (issue #375 P3)"++# Capture the WRC pod set (by name) so we can assert NO restart drove convergence.+wrc_pods_before="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \+ | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"+if [ -z "$wrc_pods_before" ]; then+ fail "could not enumerate workflow-recipes pods before the nudge (selector empty)"+ exit 1+fi++nudge_rev_before="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \+ -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"+nudge_rev_before="${nudge_rev_before:-0}"++# Re-upsert the promptBridge grant. The admin upsert increments policy_revision+# and (issue #375 P3) emits the transactional grant-update NOTIFY on COMMIT.+nudge_started_at="$SECONDS"+# issue #375 (R5): fail LOUD on a grant re-upsert failure. `|| true` here only+# guards `set -e` so we can read ADMIN_CURL_HTTP_STATUS; the explicit assert+# below is the real gate. Without it, an admin-API failure (expired token,+# control-api 5xx) would fall through to the poll and be misattributed to the+# nudge under test ("did not accelerate convergence"), hiding the real cause.+create_grant promptBridge >/dev/null 2>&1 || true+if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then+ fail "grant re-upsert failed (admin API HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}) — cannot test the event-driven nudge"+ exit 1+fi++nudge_found=0+nudge_state=""+nudge_rev_after=""+nudge_deadline=$((SECONDS + E2E_WAIT_NUDGE_VALIDATED))+while [ "$SECONDS" -lt "$nudge_deadline" ]; do+ gate_assert_deadline "waiting for event-driven policyRevision bump (issue #375 P3)"+ nudge_state="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \+ -o jsonpath='{.status.pluginWorkloadSdk.state}' 2>/dev/null || true)"+ nudge_rev_after="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \+ -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"+ if [ "$nudge_state" = "validated" ] && [ -n "$nudge_rev_after" ] &&+ [ "$nudge_rev_after" -gt "$nudge_rev_before" ] 2>/dev/null; then+ nudge_found=1+ break+ fi+ sleep "$POLL_INTERVAL"+done++nudge_elapsed=$((SECONDS - nudge_started_at))+if [ "$nudge_found" -eq 1 ]; then+ ok "grant change re-published (policyRevision ${nudge_rev_before}→${nudge_rev_after}, state=validated) in ~${nudge_elapsed}s via the event-driven nudge (≤${E2E_WAIT_NUDGE_VALIDATED}s)"+else+ fail "grant change was NOT re-published within ${E2E_WAIT_NUDGE_VALIDATED}s (state='${nudge_state:-<empty>}', policyRevision '${nudge_rev_before}'→'${nudge_rev_after:-<empty>}'); the LISTEN/NOTIFY nudge did not accelerate convergence"+ exit 1+fi++# The acceleration must come from the NOTIFY-driven reconcile, NOT a WRC restart.+wrc_pods_after="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \+ | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"+if [ "$wrc_pods_after" = "$wrc_pods_before" ]; then+ ok "WRC was NOT restarted during convergence (pod set unchanged) — convergence was event-driven"+else+ fail "WRC pod set changed during the nudge phase (before='${wrc_pods_before}' after='${wrc_pods_after}'); convergence must not depend on a restart"+ exit 1+fi+print_resultsexit 0diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 0516c688b..e00f27dd1 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.288",+ "version": "0.1.289","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.288",+ "version": "0.1.289","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 1057fcbed..71b9682c9 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.288",+ "version": "0.1.289","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 0e9c1496e..81fa3aab4 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -383,6 +383,221 @@ describe('shouldPatchRecipeStatus', () => {)).toBe(false)})++ // ── issue #375: publish computed Plugin Workload SDK state transitions ──+ // The dominant failure mode: the reconciler computes awaiting_policy→validated+ // but no OTHER observable diff (phase, top-level message, workloads, owned+ // conditions) changes, so the old shouldPatchRecipeStatus returned false and+ // the transition was never published — status stayed awaiting_policy for ~40+ // min. shouldPatchRecipeStatus must now compare the SDK capability projection+ // carried on result.pluginWorkloadSdkProjection.+ function validatedProjection() {+ return {+ conditions: [],+ capability: {+ state: 'validated' as const,+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ validatedAt: new Date().toISOString(),+ verifiedAt: new Date().toISOString(),+ },+ }+ }++ function sdkRecipe(persistedSdk: Record<string, unknown>): WorkflowRecipeCRD {+ return makeWorkflowRecipe({+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },+ status: {+ phase: 'active',+ message: 'All workloads deployed',+ // No PluginWorkloadSdkCapability condition, so the pre-existing+ // message-vs-condition guard stays inert and this asserts ONLY the new+ // projection comparison.+ pluginWorkloadSdk: persistedSdk as never,+ },+ })+ }++ it('patches a computed awaiting_policy → validated transition (issue #375 incident replica)', () => {+ // The exact incident shape: phase and top-level message are unchanged, the+ // ONLY change is the SDK capability state. Before the fix this returned+ // false and the validated state was never published.+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'awaiting_policy',+ promptBridge: true,+ clientNotifications: false,+ message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',+ verifiedAt: '2026-08-17T17:24:50.000Z',+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: validatedProjection(),+ }+ )+ ).toBe(true)+ })++ it('refreshes a stale verifiedAt on an otherwise-identical validated projection (throttle open)', () => {+ const staleVerifiedAt = new Date(Date.now() - 6 * 60_000).toISOString()+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: staleVerifiedAt,+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: validatedProjection(),+ }+ )+ ).toBe(true)+ })++ it('does not patch a fresh verifiedAt on an identical validated projection (throttle closed / anti-loop)', () => {+ const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: freshVerifiedAt,+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: validatedProjection(),+ }+ )+ ).toBe(false)+ })++ it('patches a computed validated → awaiting_policy transition (revocation symmetry)', () => {+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: {+ conditions: [],+ capability: {+ state: 'awaiting_policy' as const,+ promptBridge: true,+ clientNotifications: false,+ message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',+ bootstrapPodUid: 'pod-abc',+ verifiedAt: new Date().toISOString(),+ },+ },+ }+ )+ ).toBe(true)+ })++ it('does not patch when the SDK projection is semantically unchanged and verifiedAt is fresh', () => {+ // Anti-loop guard: two back-to-back reconciles with the same computed state+ // and a still-fresh verifiedAt must NOT keep emitting patches.+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: new Date(Date.now() - 5_000).toISOString(),+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: validatedProjection(),+ }+ )+ ).toBe(false)+ })++ it('patches an in-place re-bootstrap that only changes bootstrapModel/provider/contractVersion (issue #375 B1)', () => {+ // A validated promptBridge recipe re-bootstraps in place (SAME pod, same+ // policyRevision) against a DIFFERENT model/provider — e.g. the operator+ // corrects the target model. state stays 'validated', message is+ // families-only, pod/policyRevision unchanged, verifiedAt is fresh (throttle+ // closed). Only bootstrapModel/provider/contractVersion move — persisted+ // semantic fields read by control-api at runtime. They MUST force a patch.+ const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()+ expect(+ shouldPatchRecipeStatus(+ sdkRecipe({+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ bootstrapContractVersion: 2,+ bootstrapProvider: 'openai',+ bootstrapModel: 'gpt-5.4-mini',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: freshVerifiedAt,+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: {+ conditions: [],+ capability: {+ state: 'validated' as const,+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ bootstrapPodUid: 'pod-abc',+ bootstrapContractVersion: 2,+ bootstrapProvider: 'anthropic',+ bootstrapModel: 'claude-sonnet-4-6',+ policyRevision: 3,+ defaultTargetRef: 'primary-openai',+ verifiedAt: freshVerifiedAt,+ },+ },+ }+ )+ ).toBe(true)+ })})describe('infrastructure telemetry projections', () => {@@ -1718,6 +1933,293 @@ describe('transient-result requeue (scheduleTransientRetry)', () => {})})+describe('reconcile failure re-schedules the transient retry chain (issue #375 P2)', () => {+ // A transient (non-404) throw inside handleRecipeEvent used to log the failure+ // and return WITHOUT re-arming the per-recipe retry chain, so the 5s self-heal+ // died on the first error and only the 30s watchdog could recover it. The fix+ // keeps the loud error log AND re-schedules the bounded backoff retry; the 404+ // "already gone" branch must still clear and return without a retry.+ type InternalWatcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }+ reconciler: {+ reconcile: ReturnType<typeof vi.fn>+ isRecipeStillActive: ReturnType<typeof vi.fn>+ ensureFinalizer: ReturnType<typeof vi.fn>+ patchStatus: ReturnType<typeof vi.fn>+ }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }++ function makeInternal(recipe: WorkflowRecipeCRD, reconcileError: unknown): InternalWatcher {+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher+ internal.recipes = new Map([[recipe.metadata.name, recipe]])+ internal.transientRetries = new Map()+ internal.stopped = false+ internal.traceReporter = null+ internal.dbRunProcessor = null+ internal.eventQueue = { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) }+ internal.reconciler = {+ reconcile: vi.fn().mockRejectedValue(reconcileError),+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus: vi.fn().mockResolvedValue(undefined),+ }+ return internal+ }++ function clearRetries(internal: InternalWatcher): void {+ for (const { timer } of internal.transientRetries.values()) clearTimeout(timer)+ }++ it('re-arms the transient retry after a non-404 reconcile failure (keeps the loud log)', async () => {+ const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ const recipe = makeWorkflowRecipe({ metadata: { name: 'flaky', namespace: 'sandbox-recipes' } })+ const internal = makeInternal(recipe, new Error('control-api unreachable'))+ try {+ await internal.handleRecipeEvent('ADDED', recipe)++ // fail-loud preserved: the reconciliation-failed error is still logged.+ expect(errSpy).toHaveBeenCalledWith(+ expect.stringContaining('Reconciliation failed for "flaky"'),+ expect.anything()+ )+ // NEW: the 5s self-heal chain is re-armed instead of dying on first error.+ expect(internal.transientRetries.has('flaky')).toBe(true)+ } finally {+ clearRetries(internal)+ errSpy.mockRestore()+ }+ })++ it('does not re-arm a retry when the recipe is already gone (404)', async () => {+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const recipe = makeWorkflowRecipe({ metadata: { name: 'gone', namespace: 'sandbox-recipes' } })+ const internal = makeInternal(recipe, { code: 404 })+ try {+ await internal.handleRecipeEvent('ADDED', recipe)++ expect(internal.transientRetries.has('gone')).toBe(false)+ expect(internal.recipes.has('gone')).toBe(false)+ } finally {+ clearRetries(internal)+ warnSpy.mockRestore()+ }+ })+})++describe('Plugin Workload SDK computed-vs-published state divergence log (issue #375 P4)', () => {+ // Forensic observability: when the reconciler COMPUTES an SDK capability state+ // that differs from what is currently published, log old→new and whether a+ // patch was emitted. This one line would have discriminated H1 (transition+ // computed but not published) from H2 in minutes during the incident.+ type InternalWatcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ reconciler: {+ reconcile: ReturnType<typeof vi.fn>+ isRecipeStillActive: ReturnType<typeof vi.fn>+ ensureFinalizer: ReturnType<typeof vi.fn>+ patchStatus: ReturnType<typeof vi.fn>+ }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }++ function makeInternal(recipe: WorkflowRecipeCRD, result: unknown): InternalWatcher {+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher+ internal.recipes = new Map([[recipe.metadata.name, recipe]])+ internal.transientRetries = new Map()+ internal.stopped = false+ internal.traceReporter = null+ internal.dbRunProcessor = null+ internal.reconciler = {+ reconcile: vi.fn().mockResolvedValue(result),+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus: vi.fn().mockResolvedValue(undefined),+ }+ return internal+ }++ function sdkRecipe(persistedState: 'awaiting_policy' | 'validated'): WorkflowRecipeCRD {+ return makeWorkflowRecipe({+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },+ status: {+ phase: 'active',+ message: 'All workloads deployed',+ pluginWorkloadSdk: {+ state: persistedState,+ promptBridge: true,+ clientNotifications: false,+ },+ },+ })+ }++ it('logs the old→new SDK state and patchEmitted when the computed state diverges', async () => {+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)+ const recipe = sdkRecipe('awaiting_policy')+ const internal = makeInternal(recipe, {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: {+ conditions: [],+ capability: {+ state: 'validated' as const,+ promptBridge: true,+ clientNotifications: false,+ verifiedAt: new Date().toISOString(),+ },+ },+ })+ try {+ await internal.handleRecipeEvent('ADDED', recipe)+ expect(logSpy).toHaveBeenCalledWith(+ expect.stringMatching(+ /Plugin Workload SDK state.*sdk-recipe.*awaiting_policy -> validated.*patchEmitted=true/+ )+ )+ } finally {+ logSpy.mockRestore()+ }+ })++ it('does not log a transition when the computed SDK state is unchanged', async () => {+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)+ const recipe = sdkRecipe('validated')+ const internal = makeInternal(recipe, {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: {+ conditions: [],+ capability: {+ state: 'validated' as const,+ promptBridge: true,+ clientNotifications: false,+ verifiedAt: new Date().toISOString(),+ },+ },+ })+ try {+ await internal.handleRecipeEvent('ADDED', recipe)+ const transitionLogs = logSpy.mock.calls.filter(([msg]) =>+ String(msg).includes('Plugin Workload SDK state')+ )+ expect(transitionLogs).toHaveLength(0)+ } finally {+ logSpy.mockRestore()+ }+ })+})++describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {+ type InternalWatcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ eventQueue: { enqueue: ReturnType<typeof vi.fn> }+ handleRecipeEvent: ReturnType<typeof vi.fn>+ handleGrantUpdateNotification: (recipeNamespace: string, recipeName: string) => void+ }++ function makeInternal(recipes: WorkflowRecipeCRD[]): {+ internal: InternalWatcher+ tasks: Array<() => Promise<void>>+ } {+ const tasks: Array<() => Promise<void>> = []+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher+ internal.recipes = new Map(recipes.map(r => [r.metadata.name, r]))+ internal.eventQueue = {+ enqueue: vi.fn((_key: string, task: () => Promise<void>) => {+ tasks.push(task)+ return Promise.resolve()+ }),+ }+ internal.handleRecipeEvent = vi.fn().mockResolvedValue(undefined)+ return { internal, tasks }+ }++ it('enqueues a forceReconcile for a matching recipe and reconciles the CURRENT cached recipe (H5)', async () => {+ const recipe = makeWorkflowRecipe({+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 1 },+ })+ const { internal, tasks } = makeInternal([recipe])++ internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')++ expect(internal.eventQueue.enqueue).toHaveBeenCalledWith('sdk-recipe', expect.any(Function))+ expect(tasks).toHaveLength(1)++ // H5: a newer MODIFIED lands before the queued job runs. The job must+ // reconcile the FRESH cached recipe, not the snapshot captured at NOTIFY time.+ const fresher = makeWorkflowRecipe({+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 2 },+ })+ internal.recipes.set('sdk-recipe', fresher)++ await tasks[0]()++ expect(internal.handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', fresher, {+ forceReconcile: true,+ })+ expect(internal.handleRecipeEvent).not.toHaveBeenCalledWith('MODIFIED', recipe, {+ forceReconcile: true,+ })+ })++ it('discards and warns on a namespace mismatch — no enqueue (H4)', () => {+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const recipe = makeWorkflowRecipe({+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },+ })+ const { internal } = makeInternal([recipe])+ try {+ internal.handleGrantUpdateNotification('other-namespace', 'sdk-recipe')+ expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()+ expect(warnSpy).toHaveBeenCalledWith(+ expect.stringContaining('Discarding grant-update NOTIFY for "other-namespace/sdk-recipe"')+ )+ } finally {+ warnSpy.mockRestore()+ }+ })++ it('discards and warns on an unknown recipe — no enqueue (H4)', () => {+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const { internal } = makeInternal([])+ try {+ internal.handleGrantUpdateNotification('sandbox-recipes', 'nope')+ expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('unknown recipe'))+ } finally {+ warnSpy.mockRestore()+ }+ })++ it('the queued job re-checks the guard and does nothing if the recipe vanished while queued (H5)', async () => {+ const recipe = makeWorkflowRecipe({+ metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },+ })+ const { internal, tasks } = makeInternal([recipe])++ internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')+ // Recipe deleted between NOTIFY and job execution.+ internal.recipes.delete('sdk-recipe')++ await tasks[0]()++ expect(internal.handleRecipeEvent).not.toHaveBeenCalled()+ })+})+// issue #299 §3.2/§5 — the periodic external-egress requeue timer. GitHub rotates// its api.github.com A record (~15s TTL over 140.82.112.0/20) WITHOUT bumping the// WorkflowRecipe CRD generation, so a generation-only reconcile trigger neverdiff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex 6f7019923..ee22128ed 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -29,6 +29,10 @@ import {type DbRunRow,createDbRunProcessor,} from './reconciler/dbRunProcessor'+import {+ type GrantUpdateListener,+ createGrantUpdateListener,+} from './reconciler/grantUpdateListener'import { getErrorCode } from './reconciler/k8sErrors'import { RecipeEventQueue } from './reconciler/recipeEventQueue'import {@@ -39,6 +43,7 @@ import { SecretReverseIndex } from './reconciler/secretReverseIndex'import { SecretWatcher } from './reconciler/secretWatcher'import {type ReconcileResult,+ TRANSIENT_REQUEUE_BASE_MS,WorkflowRecipeReconciler,} from './reconciler/workflowRecipeReconciler'import {@@ -55,6 +60,13 @@ const MIN_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 5_000const MAX_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 30_000const WORKLOAD_STATUS_REFRESH_INTERVAL_MS = 30_000const WORKFLOW_RUN_ID_LABEL = 'clerum.io/workflow-run-id'+// issue #375: how long a published Plugin Workload SDK `verifiedAt` may age+// before a steady-state reconcile refreshes it, even when every semantic field+// is unchanged. `verifiedAt` is recomputed on every bootstrap-proof parse, so+// without a throttle a fresh timestamp each pass would patch on every reconcile+// (≤30s) and spin a MODIFIED→reconcile loop. 5 min keeps the liveness signal+// current while capping steady-state churn at ≤1 patch / 5 min per recipe.+const VERIFIED_AT_REFRESH_MS = 5 * 60_000// Bound the fan-out of the periodic external-egress requeue: the per-recipe// eventQueue already serializes each recipe, but without a global cap a refresh// pass over N recipes would enqueue N reconciles at once. This caps the number of@@ -117,6 +129,12 @@ export function shouldPatchRecipeStatus(// `state=validated` + `message=bootstrap not ready` record behind.if (pluginWorkloadSdkStatusMessageChanged(recipe)) return true+ // issue #375: publish a computed Plugin Workload SDK capability transition+ // (e.g. awaiting_policy→validated) even when nothing else observable changed.+ // Without this the reconciler computes the new state but no patch is emitted+ // and the CRD keeps proclaiming the stale state until an incidental diff.+ if (pluginWorkloadSdkProjectionChanged(recipe, result)) return true+if (ownedConditionsChanged(recipe.status?.conditions, result.internalDependencyConditions, ['InternalDependenciesReady',@@ -132,6 +150,86 @@ export function shouldPatchRecipeStatus(return false}+/**+ * issue #375: decide whether the freshly-computed Plugin Workload SDK capability+ * projection (carried on `result.pluginWorkloadSdkProjection`, built once by the+ * watcher via the reconciler) differs from what is currently persisted in+ * `status.pluginWorkloadSdk` in a way that must be published.+ *+ * Semantic fields are compared explicitly (NEVER a deep-equal that would drag in+ * timestamps): `state`, `message`, `policyRevision`, `policyHash`,+ * `defaultTargetRef`, `bootstrapPodUid`, `bootstrapContractVersion`,+ * `bootstrapProvider`, `bootstrapModel`, and the `promptBridge`/+ * `clientNotifications` family flags. A change in any of them (e.g.+ * awaiting_policy→validated on grant creation, validated→awaiting_policy on+ * revocation, or an in-place re-bootstrap against a corrected model/provider+ * with the same pod) forces a patch immediately — `bootstrapProvider`/+ * `bootstrapModel`/`bootstrapContractVersion` are persisted values control-api+ * reads at runtime, NOT timestamps, so they must not hide behind the+ * `verifiedAt` throttle.+ *+ * `verifiedAt` and `validatedAt` are timestamp-derived and EXCLUDED from+ * semantic equality — otherwise every reconcile would look "changed". Instead,+ * when the projection is otherwise identical, `verifiedAt` is refreshed on a+ * throttle: patch only when the newly-computed `verifiedAt` is strictly NEWER+ * than the persisted one AND the persisted one is older than+ * `VERIFIED_AT_REFRESH_MS` (see the inline INVARIANT). Loop-safety rests on that+ * strictly-newer guard alone: an equal-or-older projected timestamp can never+ * trigger a patch, and after any refresh the persisted value equals the+ * projected one, so the next pass is not strictly newer and returns false.+ *+ * A `undefined` projection (paths that do not run the SDK lane, e.g. the+ * workload-status-only refresh) short-circuits to false — that pass has no SDK+ * opinion and must not force a patch.+ */+function pluginWorkloadSdkProjectionChanged(+ recipe: WorkflowRecipeCRD,+ result: ReconcileResult,+ nowMs: number = Date.now()+): boolean {+ const projected = result.pluginWorkloadSdkProjection?.capability+ if (projected === undefined) return false+ const current = recipe.status?.pluginWorkloadSdk+ // `null` means the projection wants to CLEAR status.pluginWorkloadSdk+ // (capability removed from spec). Patch only if something is still persisted.+ if (projected === null) return current !== undefined+ if (!current) return true++ const nullable = (value: string | number | null | undefined): string | number | null =>+ value ?? null+ if (current.state !== projected.state) return true+ if (nullable(current.message) !== nullable(projected.message)) return true+ if (nullable(current.policyRevision) !== nullable(projected.policyRevision)) return true+ if (nullable(current.policyHash) !== nullable(projected.policyHash)) return true+ if (nullable(current.defaultTargetRef) !== nullable(projected.defaultTargetRef)) return true+ if (nullable(current.bootstrapPodUid) !== nullable(projected.bootstrapPodUid)) return true+ if (nullable(current.bootstrapContractVersion) !== nullable(projected.bootstrapContractVersion))+ return true+ if (nullable(current.bootstrapProvider) !== nullable(projected.bootstrapProvider)) return true+ if (nullable(current.bootstrapModel) !== nullable(projected.bootstrapModel)) return true+ if (current.promptBridge !== projected.promptBridge) return true+ if (current.clientNotifications !== projected.clientNotifications) return true++ // Semantic equality holds. Refresh verifiedAt on a throttle so the liveness+ // signal stays current without a per-reconcile patch loop. INVARIANT: only+ // refresh when the newly-computed verifiedAt is strictly NEWER than the+ // persisted one AND the persisted one is older than VERIFIED_AT_REFRESH_MS.+ // Requiring projected > persisted is what makes this loop-proof even if a+ // future projection ever carried a STALE (equal or older) timestamp: such a+ // projection can never trigger a patch, so patch→reconcile→patch cannot+ // self-sustain. After any refresh the persisted value equals the projected+ // one, so the next pass is not strictly newer and returns false.+ if (!projected.verifiedAt) return false+ if (!current.verifiedAt) return true+ const projectedMs = Date.parse(projected.verifiedAt)+ // Cannot reason about an unparseable projected timestamp — do not patch.+ if (Number.isNaN(projectedMs)) return false+ const persistedMs = Date.parse(current.verifiedAt)+ // A corrupt persisted timestamp is refreshed once from a valid projection.+ if (Number.isNaN(persistedMs)) return true+ return projectedMs > persistedMs && nowMs - persistedMs > VERIFIED_AT_REFRESH_MS+}+function pluginWorkloadSdkStatusMessageChanged(recipe: WorkflowRecipeCRD): boolean {const capability = recipe.status?.pluginWorkloadSdkif (!capability) return false@@ -509,6 +607,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {private recipes: Map<string, WorkflowRecipeCRD> = new Map()private reconciler: WorkflowRecipeReconcilerprivate dbRunProcessor: DbRunProcessor | null = null+ // issue #375 (P3): event-driven grant→WRC nudge. Started only when the WRC has+ // a DB (same gate as dbRunProcessor); without a DB the listener never starts+ // and the level-triggered reconcile remains the sole convergence path.+ private grantUpdateListener: GrantUpdateListener | null = nullprivate config = loadConfig()private stopped = falseprivate runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null = null@@ -573,9 +675,48 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {})},})+ // issue #375 (P3): on a grant-update NOTIFY, force-reconcile the affected+ // recipe through the SAME path the 30s watchdog uses. The dispatch body is+ // a named method so it is unit-testable without standing up a DB.+ this.grantUpdateListener = createGrantUpdateListener({+ pool,+ onGrantUpdate: ({ recipeNamespace, recipeName }) =>+ this.handleGrantUpdateNotification(recipeNamespace, recipeName),+ })}}+ /**+ * issue #375 (P3): dispatch a grant-update NOTIFY into a forced reconcile of+ * the affected recipe — the SAME path the 30s watchdog uses. Kept as a named+ * method (not an inline closure) so it is unit-testable without a live DB.+ */+ private handleGrantUpdateNotification(recipeNamespace: string, recipeName: string): void {+ const known = this.recipes.get(recipeName)+ if (!known || known.metadata.namespace !== recipeNamespace) {+ // issue #375 (H4): make the discard diagnosable. control-api's emitted+ // recipeNamespace and the WRC's watched sandboxNamespace come from+ // independent env vars; if they ever diverge, 100% of NOTIFYs would be+ // dropped and the feature would silently degrade to 30s polling. Log names+ // only — no secrets.+ console.warn(+ `[WR-K8s] Discarding grant-update NOTIFY for "${recipeNamespace}/${recipeName}": ` ++ (known ? `namespace mismatch (watching "${known.metadata.namespace}")` : 'unknown recipe')+ )+ return+ }+ void this.eventQueue.enqueue(recipeName, () => {+ // issue #375 (H5): reconcile the CURRENT recipe. A newer MODIFIED may have+ // landed between NOTIFY receipt and this queued job running+ // (handleRecipeEvent updates this.recipes), so re-read the cache instead of+ // closing over the stale snapshot. Re-check the guard on the fresh read —+ // the recipe may have been deleted, or its namespace changed, while queued.+ const cached = this.recipes.get(recipeName)+ if (!cached || cached.metadata.namespace !== recipeNamespace) return Promise.resolve()+ return this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true })+ })+ }+getAllRecipes(): WorkflowRecipeCRD[] {return [...this.recipes.values()]}@@ -639,6 +780,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {if (this.dbRunProcessor) {await this.dbRunProcessor.start()}+ if (this.grantUpdateListener) {+ await this.grantUpdateListener.start()+ }}// Refreshes opted-in recipes' broker-token Secrets between CRD events.@@ -870,7 +1014,22 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {const result = await this.reconciler.reconcile(recipe)// Patch only when observable status changes. This avoids reconcile loops// while still allowing active parent recipes to clear stale execution state.- if (shouldPatchRecipeStatus(recipe, result)) {+ const willPatch = shouldPatchRecipeStatus(recipe, result)+ // issue #375 (P4): surface a computed-vs-published Plugin Workload SDK+ // state divergence — old→new plus whether a patch was emitted. With P1+ // this is rare; its value is forensic (it would have discriminated+ // "transition computed but not published" from "never computed" in+ // minutes) and it verifies P1 in the field. Only states are logged — no+ // tokens/secrets (sec-003).+ const projectedSdkState = result.pluginWorkloadSdkProjection?.capability?.state+ const persistedSdkState = recipe.status?.pluginWorkloadSdk?.state+ if (projectedSdkState !== undefined && projectedSdkState !== persistedSdkState) {+ console.log(+ `[WR-K8s] Plugin Workload SDK state for "${recipe.metadata.name}": ` ++ `${persistedSdkState ?? 'none'} -> ${projectedSdkState} (patchEmitted=${willPatch})`+ )+ }+ if (willPatch) {await this.reconciler.patchStatus(recipe, result)enqueueInfrastructureTelemetryBestEffort(this.traceReporter,@@ -902,6 +1061,16 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.traceReporter,controllerErrorTelemetryProjection(recipe, error))+ // issue #375 (P2): a transient (non-404) failure used to return here+ // WITHOUT re-arming the retry chain, so a thrown reconcile killed the 5s+ // self-heal and only the 30s watchdog could recover. Re-schedule the+ // bounded exponential backoff retry so the recipe re-reconciles on its+ // own; persistent failure still backs off toward TRANSIENT_RETRY_MAX_MS,+ // with the watchdog as the second net. The 404 branch above already+ // cleared and returned, so it never reaches here.+ if (!this.stopped) {+ this.scheduleTransientRetry(recipe.metadata.name, TRANSIENT_REQUEUE_BASE_MS, false)+ }}} else if (type === 'DELETED') {// Resource is gone (finalizer already removed) — just clean cache.@@ -1209,6 +1378,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {if (this.dbRunProcessor) {await this.dbRunProcessor.stop()}+ if (this.grantUpdateListener) {+ await this.grantUpdateListener.stop()+ }if (this.traceReporter) {const result = await this.traceReporter.stopAndDrain()if (!result.drained) {diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.tsnew file mode 100644index 000000000..f1040c079--- /dev/null+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts@@ -0,0 +1,274 @@+import { describe, expect, it, vi } from 'vitest'+import type { Logger } from '../observability/logger'+import {+ PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ createGrantUpdateListener,+ parseGrantUpdatePayload,+} from './grantUpdateListener'++function makeLogger(): Logger {+ const logger = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ withStep: vi.fn(),+ }+ logger.withStep.mockReturnValue(logger)+ return logger as unknown as Logger+}++type NotificationHandler = (msg: { channel: string; payload?: string }) => void++/**+ * Minimal fake of a pg PoolClient scoped to LISTEN: records the channel it+ * LISTENed on and lets the test emit notifications / connection errors into the+ * registered handlers. `query` can be made to reject to simulate a LISTEN that+ * fails without an 'error' emission (H3).+ */+function makeFakeClient(opts: { failQuery?: boolean } = {}) {+ const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}+ const queries: string[] = []+ const client = {+ on(event: string, handler: (...args: unknown[]) => void) {+ ;(handlers[event] ??= []).push(handler)+ return client+ },+ query: vi.fn(async (sql: string) => {+ queries.push(sql)+ if (opts.failQuery) throw new Error('LISTEN failed')+ return { rows: [], rowCount: 0 }+ }),+ release: vi.fn(),+ }+ return {+ client,+ queries,+ emitNotification(msg: { channel: string; payload?: string }) {+ for (const handler of handlers.notification ?? []) (handler as NotificationHandler)(msg)+ },+ emitError(err: Error) {+ for (const handler of handlers.error ?? []) handler(err)+ },+ }+}++describe('parseGrantUpdatePayload', () => {+ it('parses a well-formed payload with a capability family', () => {+ expect(+ parseGrantUpdatePayload(+ JSON.stringify({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ )+ ).toEqual({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ })++ it('parses a payload without a capability family (whole-recipe revoke)', () => {+ expect(+ parseGrantUpdatePayload(+ JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })+ )+ ).toEqual({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' })+ })++ it('returns null for non-JSON, missing fields, or wrong types (data, not command)', () => {+ expect(parseGrantUpdatePayload('not json')).toBeNull()+ expect(parseGrantUpdatePayload(JSON.stringify({ recipeName: 'research' }))).toBeNull()+ expect(+ parseGrantUpdatePayload(JSON.stringify({ recipeNamespace: 'sandbox-recipes' }))+ ).toBeNull()+ expect(+ parseGrantUpdatePayload(JSON.stringify({ recipeNamespace: 1, recipeName: 2 }))+ ).toBeNull()+ expect(parseGrantUpdatePayload(JSON.stringify(['array']))).toBeNull()+ })+})++describe('createGrantUpdateListener', () => {+ it('LISTENs on the grant-update channel on start', async () => {+ const fake = makeFakeClient()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => fake.client as never,+ })++ await listener.start()++ expect(fake.queries.some(sql => /LISTEN\s+plugin_workload_sdk_grant_update/.test(sql))).toBe(+ true+ )+ await listener.stop()+ expect(fake.client.query).toHaveBeenCalledWith(+ expect.stringMatching(/UNLISTEN\s+plugin_workload_sdk_grant_update/)+ )+ expect(fake.client.release).toHaveBeenCalled()+ })++ it('forwards the parsed notification to onGrantUpdate', async () => {+ const fake = makeFakeClient()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => fake.client as never,+ })+ await listener.start()++ fake.emitNotification({+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ payload: JSON.stringify({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ }),+ })++ expect(onGrantUpdate).toHaveBeenCalledWith({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ capabilityFamily: 'promptBridge',+ })+ await listener.stop()+ })++ it('warns and does not dispatch on an unparseable payload (payload is data, not a command)', async () => {+ const fake = makeFakeClient()+ const onGrantUpdate = vi.fn()+ const logger = makeLogger()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger,+ connect: async () => fake.client as never,+ })+ await listener.start()++ fake.emitNotification({+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ payload: 'garbage-not-json',+ })++ expect(onGrantUpdate).not.toHaveBeenCalled()+ expect(logger.warn).toHaveBeenCalled()+ await listener.stop()+ })++ it('ignores notifications on other channels', async () => {+ const fake = makeFakeClient()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => fake.client as never,+ })+ await listener.start()++ fake.emitNotification({+ channel: 'some_other_channel',+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),+ })++ expect(onGrantUpdate).not.toHaveBeenCalled()+ await listener.stop()+ })+})++describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {+ /** A sleep seam that never resolves, so we can count reconnect scheduling without real timers. */+ function pendingSleep() {+ const calls: number[] = []+ const sleep = vi.fn((ms: number) => {+ calls.push(ms)+ return new Promise<void>(() => {+ /* never resolves — the reconnect stays parked in its backoff */+ })+ })+ return { sleep, calls }+ }++ it('H1: a connection drop schedules exactly ONE reconnect even when error fires repeatedly', async () => {+ const fake = makeFakeClient()+ const { sleep, calls } = pendingSleep()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate: vi.fn(),+ logger: makeLogger(),+ connect: async () => fake.client as never,+ sleep,+ })+ await listener.start()++ // Both the LISTEN-rejection path and the 'error' event fire on one drop; and+ // 'error' can fire more than once. All must collapse into a single reconnect.+ fake.emitError(new Error('connection reset'))+ fake.emitError(new Error('connection reset again'))++ expect(fake.client.release).toHaveBeenCalled()+ expect(calls).toHaveLength(1) // exactly one reconnect scheduled, no leaked loops+ await listener.stop()+ })++ it('H3: a LISTEN that fails without an error event releases the client and schedules a retry', async () => {+ const fake = makeFakeClient({ failQuery: true })+ const { sleep, calls } = pendingSleep()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate: vi.fn(),+ logger: makeLogger(),+ connect: async () => fake.client as never,+ sleep,+ })++ await listener.start()++ // LISTEN query rejected → attach catch must release the checked-out client+ // (no leak) and schedule the retry.+ expect(fake.client.release).toHaveBeenCalled()+ expect(calls).toHaveLength(1)+ await listener.stop()+ })++ it('H2: stop() during an in-flight connect() leaves no live listener and dispatches nothing', async () => {+ const fake = makeFakeClient()+ const onGrantUpdate = vi.fn()+ let resolveConnect!: () => void+ const connectGate = new Promise<void>(resolve => {+ resolveConnect = resolve+ })+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => {+ await connectGate+ return fake.client as never+ },+ })++ const startPromise = listener.start() // parks awaiting connect()+ await listener.stop() // shutdown while the attach is mid-connect+ resolveConnect() // connect resolves AFTER stop+ await startPromise++ // The attach observed `stopped` after connect() resolved: it released the+ // client and registered NO handlers, so a late notification dispatches nothing.+ expect(fake.client.release).toHaveBeenCalled()+ fake.emitNotification({+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),+ })+ expect(onGrantUpdate).not.toHaveBeenCalled()+ })+})diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.tsnew file mode 100644index 000000000..f4a456d7f--- /dev/null+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts@@ -0,0 +1,222 @@+/**+ * Plugin Workload SDK grant-update LISTEN/NOTIFY consumer (issue #375, P3).+ *+ * control-api emits `pg_notify('plugin_workload_sdk_grant_update', …)` inside+ * the transaction that upserts, deletes, or revokes a Plugin Workload SDK grant+ * (delivered on COMMIT). This listener turns that best-effort signal into an+ * immediate `forceReconcile` of the affected recipe, so a validated grant is+ * published in ~1–5s instead of waiting for the ≤30s level-triggered watchdog.+ *+ * CROSS-SERVICE CONTRACT: the channel name and payload shape are shared with+ * `control-api/src/services/pluginWorkloadSdkDb.ts`+ * (`PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL`). Keep them in sync.+ *+ * Semantics are deliberately best-effort: a lost NOTIFY (reconnect gap) degrades+ * to the existing polling backstop, never worse. The LISTEN session runs on its+ * own dedicated `pool.connect()` client (a pooled query would not keep the+ * session sticky for notifications), mirroring `dbRunProcessor.ts`. The payload+ * is untrusted DATA, not a command: an unparseable payload or unknown recipe is+ * warned and discarded, never acted on blindly.+ */+import type { Pool, PoolClient } from 'pg'+import { type Logger, createLogger } from '../observability/logger.js'++/** Shared with control-api's grant-mutation NOTIFY. Keep both sides in sync. */+export const PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL = 'plugin_workload_sdk_grant_update'++/** Validated grant-update notification payload. */+export interface GrantUpdateNotification {+ recipeNamespace: string+ recipeName: string+ /** Absent for a whole-recipe revoke that touched multiple families. */+ capabilityFamily?: string+}++export interface GrantUpdateListenerOptions {+ /** Pool the dedicated LISTEN client is checked out from. */+ pool: Pool+ /**+ * Invoked with a validated notification. The caller decides whether the+ * recipe is known and how to react (the watcher enqueues a forceReconcile).+ */+ onGrantUpdate: (notification: GrantUpdateNotification) => void+ logger?: Logger+ /** Test seam: override how the dedicated LISTEN client is acquired. */+ connect?: () => Promise<PoolClient>+ /** Backoff before a reconnect attempt (ms). Default 1000. */+ reconnectDelayMs?: number+ /** Test seam: override the reconnect backoff wait (default: real setTimeout). */+ sleep?: (ms: number) => Promise<void>+}++export interface GrantUpdateListener {+ start(): Promise<void>+ stop(): Promise<void>+}++/**+ * Parse a NOTIFY payload into a `GrantUpdateNotification`, or `null` when it is+ * not a well-formed object with string `recipeNamespace`/`recipeName`. Pure and+ * exported so it can be unit-tested without a live Postgres. Never throws.+ */+export function parseGrantUpdatePayload(payload: string): GrantUpdateNotification | null {+ let parsed: unknown+ try {+ parsed = JSON.parse(payload)+ } catch {+ return null+ }+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return null+ const record = parsed as Record<string, unknown>+ const recipeNamespace = typeof record.recipeNamespace === 'string' ? record.recipeNamespace : ''+ const recipeName = typeof record.recipeName === 'string' ? record.recipeName : ''+ if (!recipeNamespace || !recipeName) return null+ const capabilityFamily =+ typeof record.capabilityFamily === 'string' ? record.capabilityFamily : undefined+ return {+ recipeNamespace,+ recipeName,+ ...(capabilityFamily ? { capabilityFamily } : {}),+ }+}++export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): GrantUpdateListener {+ const log = opts.logger ?? createLogger('wrc', 'grant-update-listener')+ const connect = opts.connect ?? (() => opts.pool.connect())+ const reconnectDelayMs = opts.reconnectDelayMs ?? 1_000+ const sleep =+ opts.sleep ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)))++ // NOTE (detection-duty, issue #375): the sibling `dbRunProcessor.ts` LISTEN+ // consumer has the same latent patterns this module hardens against —+ // (1) a LISTEN rejection AND an 'error' emission on the same drop can start two+ // reconnect loops, (2) stop() cannot cancel an attach already awaiting+ // connect(), and (3) a LISTEN failure without an 'error' event leaks the+ // checked-out client. Fixing dbRunProcessor is out of #375 scope; this listener+ // intentionally improves on it and the guards below are the reference.+ let listenClient: PoolClient | null = null+ let stopped = true+ // H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.+ let reconnecting = false++ async function attachListener(): Promise<void> {+ let client: PoolClient | null = null+ try {+ client = await connect()+ // H2: stop() may have run while we awaited connect(); do NOT bring up a+ // LISTEN session after shutdown. Release the client and bail before any+ // handler is registered, so nothing dispatches post-stop.+ if (stopped) {+ client.release()+ return+ }+ const attached = client+ attached.on('notification', msg => {+ if (msg.channel !== PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL || !msg.payload) return+ const notification = parseGrantUpdatePayload(msg.payload)+ if (!notification) {+ // Untrusted payload — warn and discard, never act on it.+ log.warn('discarding unparseable grant-update notification', {+ payload_length: msg.payload.length,+ })+ return+ }+ try {+ opts.onGrantUpdate(notification)+ } catch (err) {+ log.error('grant-update dispatch failed', {+ recipe_namespace: notification.recipeNamespace,+ recipe_name: notification.recipeName,+ err: err instanceof Error ? err.message : String(err),+ })+ }+ })+ attached.on('error', (err: Error) => {+ log.warn('listen session error — will reconnect', { err: err.message })+ try {+ attached.release(err)+ } catch {+ /* idempotent */+ }+ if (listenClient === attached) listenClient = null+ scheduleReconnect()+ })+ await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)+ // H2: re-check after the async LISTEN resolves — stop() may have run+ // during the round-trip.+ if (stopped) {+ try {+ attached.release()+ } catch {+ /* idempotent */+ }+ return+ }+ listenClient = attached+ log.info('LISTEN plugin_workload_sdk_grant_update attached')+ } catch (err) {+ log.warn('LISTEN attach failed — will retry', {+ err: err instanceof Error ? err.message : String(err),+ })+ // H3: a LISTEN failure may not emit an 'error' event, so release the+ // checked-out client here (if we got one) before scheduling the retry —+ // otherwise the connection leaks back to the pool checked out forever.+ if (client) {+ try {+ client.release(err instanceof Error ? err : new Error(String(err)))+ } catch {+ /* idempotent */+ }+ }+ scheduleReconnect()+ }+ }++ function scheduleReconnect(): void {+ // H1: the LISTEN-query rejection and the 'error' event both fire on a single+ // connection drop. Without this guard each would spawn its own reconnect+ // loop → a leaked client and duplicated dispatch. Only one may be in flight.+ if (stopped || reconnecting) return+ reconnecting = true+ void reconnectLater()+ }++ async function reconnectLater(): Promise<void> {+ await sleep(reconnectDelayMs)+ // Clear the guard BEFORE re-attaching so a failure inside attachListener can+ // schedule the NEXT reconnect; the sleep window above is what collapses the+ // concurrent triggers from a single drop.+ reconnecting = false+ if (stopped) return+ await attachListener()+ }++ return {+ async start() {+ if (!stopped) return+ stopped = false+ await attachListener()+ },++ async stop() {+ stopped = true+ // issue #375 (R4): clear the in-flight reconnect guard so a later+ // stop()→start() restart is not blocked by a stale flag left set by a+ // reconnect that was still mid-sleep when we stopped (belt-and-suspenders).+ reconnecting = false+ if (listenClient) {+ try {+ await listenClient.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)+ } catch {+ /* best-effort — the session is being torn down */+ }+ try {+ listenClient.release()+ } catch {+ /* idempotent */+ }+ listenClient = null+ }+ },+ }+}diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.tsindex cdcda8736..52cc2d80d 100644--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts@@ -351,6 +351,73 @@ describe('buildPluginWorkloadSdkStatus', () => {})})+ it('preserves the original validatedAt across a steady-state throttle refresh (issue #375 R1)', () => {+ // The verifiedAt throttle rewrites the whole SDK projection every ~5 min in+ // steady state. validatedAt is the stable "first reached validated" marker+ // (operators/dashboards read it), so a still-validated recipe must NOT have+ // it reset to now on each throttle patch — only verifiedAt advances.+ const EARLIER = '2026-08-17T10:00:00.000Z'+ const projection = buildPluginWorkloadSdkStatus({+ spec: baseSpec({ promptBridge: {} }),+ existingConditions: undefined,+ phase: 'active',+ featureFlagEnabled: true,+ bootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'pod-uid-1',+ provider: 'zai',+ model: 'glm-4.7',+ policyRevision: 1,+ defaultTargetRef: 'primary-zai',+ verifiedAt: NOW,+ },+ now: NOW,+ // Already validated on a prior pass, with the stable marker set EARLIER.+ existingCapability: {+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ validatedAt: EARLIER,+ verifiedAt: EARLIER,+ },+ })+ expect(projection.capability?.state).toBe('validated')+ // validatedAt is carried forward (stable), verifiedAt advances to now.+ expect(projection.capability?.validatedAt).toBe(EARLIER)+ expect(projection.capability?.verifiedAt).toBe(NOW)+ })++ it('stamps a fresh validatedAt on a genuine transition INTO validated (issue #375 R1)', () => {+ const projection = buildPluginWorkloadSdkStatus({+ spec: baseSpec({ promptBridge: {} }),+ existingConditions: undefined,+ phase: 'active',+ featureFlagEnabled: true,+ bootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'pod-uid-1',+ provider: 'zai',+ model: 'glm-4.7',+ policyRevision: 1,+ defaultTargetRef: 'primary-zai',+ verifiedAt: NOW,+ },+ now: NOW,+ // Prior state was awaiting_policy — this is the real transition.+ existingCapability: {+ state: 'awaiting_policy',+ promptBridge: true,+ clientNotifications: false,+ validatedAt: null,+ verifiedAt: '2026-08-17T10:00:00.000Z',+ },+ })+ expect(projection.capability?.state).toBe('validated')+ expect(projection.capability?.validatedAt).toBe(NOW)+ })+it('keeps a mixed recipe pending until every declared family has an active proof', () => {const projection = buildPluginWorkloadSdkStatus({spec: baseSpec({diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.tsindex e433d08c1..9c532f1eb 100644--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts@@ -244,6 +244,12 @@ export function buildPluginWorkloadSdkStatus(args: {teardownConfirmed?: booleanbootstrapProof?: PluginWorkloadSdkBootstrapProofInputnow: string+ /**+ * The currently-persisted capability (recipe.status.pluginWorkloadSdk), used+ * to CARRY FORWARD the stable `validatedAt` marker across steady-state+ * throttle patches (issue #375 R1) instead of resetting it to `now`.+ */+ existingCapability?: PluginWorkloadSdkCapabilityStatus | null}): PluginWorkloadSdkStatusProjection {const {spec,@@ -255,6 +261,7 @@ export function buildPluginWorkloadSdkStatus(args: {teardownConfirmed,bootstrapProof,now,+ existingCapability,} = argsconst sdk = spec.pluginWorkloadSdk@@ -408,6 +415,17 @@ export function buildPluginWorkloadSdkStatus(args: {.filter((f): f is string => f !== undefined).join(', ')+ // issue #375 (R1): validatedAt is the stable "first reached validated" marker.+ // On a steady-state throttle refresh (already validated, only verifiedAt+ // advances) it must be CARRIED FORWARD, not reset to `now`. Stamp a fresh+ // `now` ONLY on the real transition into validated — no prior capability, or a+ // prior state other than 'validated', or a prior validated record that somehow+ // lacked the marker.+ const validatedAt =+ existingCapability?.state === 'validated' && existingCapability.validatedAt+ ? existingCapability.validatedAt+ : now+return {conditions: [{@@ -424,7 +442,7 @@ export function buildPluginWorkloadSdkStatus(args: {clientNotifications,message: `Capability validated (${families})`,...CLEARED_PLUGIN_WORKLOAD_SDK_METADATA,- validatedAt: now,+ validatedAt,...(bootstrapProof? {bootstrapContractVersion: bootstrapProof.contractVersion,diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex ac3c59229..73dd1d5fe 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -91,6 +91,7 @@ import {PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,+ type PluginWorkloadSdkStatusProjection,buildPluginWorkloadSdkStatus,validatePluginWorkloadSdkSpec,} from './pluginWorkloadSdkValidator'@@ -508,6 +509,17 @@ export interface ReconcileResult {/** Cleanup completed successfully while the SDK feature flag was off. */pluginWorkloadSdkTeardownConfirmed?: booleanpluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof+ /**+ * SDK capability projection (conditions + `status.pluginWorkloadSdk` value)+ * derived once per reconcile from this result. Populated by the watcher after+ * `reconcile()` returns (see `projectPluginWorkloadSdk`) so BOTH+ * `shouldPatchRecipeStatus` (to detect a computed awaiting_policy↔validated+ * transition that no other diff would surface — issue #375) and `patchStatus`+ * (to avoid recomputing it) consume the same object. Left undefined on paths+ * that do not run the SDK lane (e.g. workload-status-only refresh), where the+ * SDK comparison is intentionally skipped.+ */+ pluginWorkloadSdkProjection?: PluginWorkloadSdkStatusProjectionworkflowPhase?: import('../workflow/types').WorkflowPhaseclearWorkflowExecution?: boolean/** When true, the caller should NOT patch the CRD status — the phase hasn't changed. */@@ -1288,6 +1300,19 @@ export class WorkflowRecipeReconciler {// ─── Main Pipeline ────────────────────────────────────────────────async reconcile(recipe: WorkflowRecipeCRD): Promise<ReconcileResult> {+ // issue #375: compute the Plugin Workload SDK capability projection ONCE per+ // reconcile and attach it to the result. shouldPatchRecipeStatus reads it to+ // publish a computed awaiting_policy↔validated transition that no+ // phase/message/workload diff would otherwise surface; patchStatus reuses+ // the same object instead of rebuilding it. Paths that bypass this method+ // (e.g. observeCurrentWorkloadStatus) leave the projection undefined, which+ // both consumers treat as "no SDK opinion this pass".+ const result = await this.reconcileInternal(recipe)+ result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)+ return result+ }++ private async reconcileInternal(recipe: WorkflowRecipeCRD): Promise<ReconcileResult> {const name = recipe.metadata.nameconst ns = recipe.metadata.namespaceconst currentPhase = recipe.status?.phase ?? 'candidate'@@ -6031,6 +6056,44 @@ export class WorkflowRecipeReconciler {// ─── Status Patch ─────────────────────────────────────────────────/** Persist reconcile result to the CRD status subresource. */+ /**+ * Build the Plugin Workload SDK status projection (owned conditions ++ * `status.pluginWorkloadSdk` value) from a reconcile result. Extracted so it+ * can be computed ONCE per reconcile by the watcher (issue #375): the watcher+ * attaches it to `result.pluginWorkloadSdkProjection` after `reconcile()`+ * returns, `shouldPatchRecipeStatus` reads it to detect an+ * awaiting_policy↔validated transition that no other diff would surface, and+ * `patchStatus` reuses it instead of recomputing. `providerUnavailable` is+ * derived here (not inferred from phase/message) so the SDK-only provider+ * health bit stays explicit.+ */+ projectPluginWorkloadSdk(+ recipe: WorkflowRecipeCRD,+ result: ReconcileResult,+ now: string = new Date().toISOString()+ ): PluginWorkloadSdkStatusProjection {+ return buildPluginWorkloadSdkStatus({+ spec: recipe.spec,+ existingConditions: recipe.status?.conditions,+ phase: result.phase,+ featureFlagEnabled: this.config.pluginWorkloadSdkEnabled,+ providerUnavailable:+ result.pluginWorkloadSdkProviderUnavailable === true ||+ (result.workflowConditions ?? []).some(+ condition =>+ condition.type === PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE &&+ condition.status === 'True'+ ),+ teardownConfirmed: result.pluginWorkloadSdkTeardownConfirmed === true,+ policyPending: result.pluginWorkloadSdkPolicyPending === true,+ bootstrapProof: result.pluginWorkloadSdkBootstrapProof,+ now,+ // issue #375 (R1): carry forward the stable validatedAt marker across+ // steady-state throttle patches instead of resetting it to `now`.+ existingCapability: recipe.status?.pluginWorkloadSdk,+ })+ }+async patchStatus(recipe: WorkflowRecipeCRD, result: ReconcileResult): Promise<void> {const workloads = result.workloadStatuses.map(ws => {const def = (recipe.spec.workloads ?? []).find(w => w.id === ws.id)@@ -6127,27 +6190,16 @@ export class WorkflowRecipeReconciler {recipe.status?.conditions,result.workloadConditions)- // Plugin Workload SDK conditions are derived here so every status patch- // carries a consistent projection of spec.pluginWorkloadSdk + feature flag,- // while the SDK-only provider health bit is propagated explicitly through- // ReconcileResult rather than inferred from a free-form phase/message.- const pluginSdkProjection = buildPluginWorkloadSdkStatus({- spec: recipe.spec,- existingConditions: recipe.status?.conditions,- phase: result.phase,- featureFlagEnabled: this.config.pluginWorkloadSdkEnabled,- providerUnavailable:- result.pluginWorkloadSdkProviderUnavailable === true ||- (result.workflowConditions ?? []).some(- condition =>- condition.type === PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE &&- condition.status === 'True'- ),- teardownConfirmed: result.pluginWorkloadSdkTeardownConfirmed === true,- policyPending: result.pluginWorkloadSdkPolicyPending === true,- bootstrapProof: result.pluginWorkloadSdkBootstrapProof,- now,- })+ // Plugin Workload SDK conditions are derived so every status patch carries a+ // consistent projection of spec.pluginWorkloadSdk + feature flag, while the+ // SDK-only provider health bit is propagated explicitly through+ // ReconcileResult rather than inferred from a free-form phase/message. Reuse+ // the projection the watcher already computed for the patch decision+ // (issue #375) so the exact object that drove shouldPatchRecipeStatus is+ // what gets written; recompute only on paths that never attached one (e.g.+ // observeCurrentWorkloadStatus).+ const pluginSdkProjection =+ result.pluginWorkloadSdkProjection ?? this.projectPluginWorkloadSdk(recipe, result, now)const pluginSdkMergedConditions = mergePluginWorkloadSdkConditions(workloadReconcileMergedConditions ??secretOwnershipMergedConditions ??