Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes,control-api): address jozer-rami review (BLOCKER + M1-M5) (#375)
Alfredo Lopezde3253be93e9
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 12ff07e27..969060244 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.463",+ "version": "0.1.464","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.463",+ "version": "0.1.464","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex 5041ea58d..81399954b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.463",+ "version": "0.1.464","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 632a1498f..9518c82a7 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -36,6 +36,20 @@ export type DbClient = {query: (text: string, values?: unknown[]) => Promise<{ rows: unknown[]; rowCount: number | null }>}+// issue #375 M3 (jozer review): brand for clients that live INSIDE an open+// transaction. `DbClient` is structural (`{ query }`), so `Pool` satisfies it+// and a refactor from the in-transaction `db` to the module-level `pool` still+// typechecks while silently breaking COMMIT/ROLLBACK coupling (e.g. a+// transactional `pg_notify` becoming an autocommitted one). The unique-symbol+// brand cannot be satisfied structurally, so any API that REQUIRES the+// transaction session (like the grant-update NOTIFY) declares+// `DbTransactionClient` and the pool no longer compiles there. It remains+// assignable to `DbClient`, so passing it onward to ordinary helpers is free.+declare const dbTransactionClientBrand: unique symbol+export type DbTransactionClient = DbClient & {+ readonly [dbTransactionClientBrand]: 'transaction'+}+type DbSessionClient = DbClient & {release: () => void}@@ -6076,7 +6090,9 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {}}-export async function withTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {+export async function withTransaction<T>(+ work: (db: DbTransactionClient) => Promise<T>+): Promise<T> {const client = (await pool.connect()) as PoolClientlet transactionStarted = falselet commitSent = false@@ -6084,7 +6100,10 @@ export async function withTransaction<T>(work: (db: DbClient) => Promise<T>): Prtry {await client.query('BEGIN')transactionStarted = true- const result = await work(client)+ // The brand is nominal-only (a declared unique symbol); the checked-out+ // client IS the transaction session, so this cast is the single blessed+ // point where the brand is minted (issue #375 M3).+ const result = await work(client as unknown as DbTransactionClient)commitSent = trueawait client.query('COMMIT')return resultdiff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex 29925e075..63a52170c 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -1,5 +1,5 @@import { createHash, randomUUID } from 'node:crypto'-import { type DbClient, pool, withTransaction } from '../db.js'+import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../db.js'import type { AdministrativeEventSubmitterPrincipalV1 } from '../middleware/tracingSubmitterAuth.js'import { stableStringify } from '../utils/stableStringify.js'import {@@ -62,7 +62,13 @@ export function buildGrantUpdateNotifyPayload(input: GrantUpdateNotifyInput): st* short family), which cannot approach that bound. So the mutation is never at* risk from the notify, and the transactional coupling is intentional.*/-async function notifyGrantUpdate(db: DbClient, input: GrantUpdateNotifyInput): Promise<void> {+async function notifyGrantUpdate(+ // issue #375 M3: REQUIRES the branded transaction session — `pool` no longer+ // structurally satisfies this parameter, so the db→pool refactor that breaks+ // COMMIT/ROLLBACK coupling fails to COMPILE instead of merely failing a test.+ db: DbTransactionClient,+ input: GrantUpdateNotifyInput+): Promise<void> {await db.query('SELECT pg_notify($1, $2)', [PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,buildGrantUpdateNotifyPayload(input),diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.tsindex c848a763a..507405b76 100644--- a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts@@ -1,8 +1,32 @@-import { afterAll, beforeAll, expect, describe as vitestDescribe } from 'vitest'+import { afterAll, beforeAll, expect, vi, describe as vitestDescribe } from 'vitest'import { it } from 'vitest'import { randomBytes, randomUUID } from 'node:crypto'import { Pool, type PoolClient } from 'pg'+// issue #375 M3 (jozer review): controllable PASSTHROUGH seam. Everything stays+// real (the actual append implementation runs) until a test arms `failNext`,+// which makes the NEXT permission-event append inside a grant mutation throw —+// forcing a REAL `upsertGrant` transaction to ROLLBACK so the rollback test+// drives the actual producer instead of hand-writing pg_notify.+const permissionEventsSeam = vi.hoisted(() => ({ failNext: null as Error | null }))+vi.mock('../src/services/tracing/controlApiPermissionEvents.js', async importOriginal => {+ const actual =+ await importOriginal<typeof import('../src/services/tracing/controlApiPermissionEvents.js')>()+ return {+ ...actual,+ appendControlApiPermissionEventsInTransaction: async (+ ...args: Parameters<typeof actual.appendControlApiPermissionEventsInTransaction>+ ) => {+ if (permissionEventsSeam.failNext) {+ const err = permissionEventsSeam.failNext+ permissionEventsSeam.failNext = null+ throw err+ }+ return actual.appendControlApiPermissionEventsInTransaction(...args)+ },+ }+})+/*** issue #375 (P3) — grant-update NOTIFY on real PostgreSQL.*@@ -142,17 +166,18 @@ describeRealPostgres(})})- it('discards BOTH the mutation and the NOTIFY when a real mutation transaction ROLLBACKs', async () => {- // This proves the grant-update NOTIFY is transactionally coupled to the grant- // mutation (delivered on COMMIT, discarded on ROLLBACK).- //- // WHY not fail a real service call (upsert/delete/revoke) after its notify:- // `notifyGrantUpdate` is the TERMINAL statement of each of those functions —- // nothing in them can throw after it — so a "production mutation that fails- // after the notify" cannot be constructed. Instead we reproduce the exact- // production shape: a REAL mutation (UPDATE of the committed grant) plus the- // REAL notify (same channel + payload builder) inside the production- // `withTransaction` wrapper, then abort it — and assert BOTH are discarded.+ it('discards BOTH the mutation and the NOTIFY when the REAL upsertGrant transaction ROLLBACKs', async () => {+ // issue #375 M3 (jozer review): this drives the ACTUAL producer, not a+ // hand-written pg_notify. `appendControlApiPermissionEventsInTransaction`+ // runs inside the upsert transaction (before its terminal notify); the+ // armed seam makes it throw, so the REAL `upsertGrant` — advisory lock,+ // row mutation, audit append — rolls back, and BOTH the row change and+ // any NOTIFY from that transaction must be discarded. This pins the+ // producer's transactional atomicity (e.g. moving the grant mutation or+ // the audit append off the transaction client goes RED here). The+ // notify-specific `db → pool` swap is pinned by the unit-lane+ // transaction-client assertion and, at compile time, by the+ // `DbTransactionClient` brand on `notifyGrantUpdate`.received.length = 0const before = await db.pool.query<{ default_target_ref: string | null }>(`SELECT default_target_ref FROM plugin_workload_sdk_grants@@ -160,27 +185,31 @@ describeRealPostgres([NS, RECIPE])+ permissionEventsSeam.failNext = new Error('injected permission-event failure (issue #375 M3)')await expect(- db.withTransaction(async client => {- await client.query(- `UPDATE plugin_workload_sdk_grants SET default_target_ref = 'ROLLED-BACK-SENTINEL'- WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,- [NS, RECIPE]- )- await client.query('SELECT pg_notify($1, $2)', [- sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,- sdk.buildGrantUpdateNotifyPayload({ recipeNamespace: NS, recipeName: RECIPE }),- ])- throw new Error('force rollback')- })- ).rejects.toThrow('force rollback')-- // The NOTIFY was discarded on rollback...+ sdk.upsertGrant(+ {+ recipeNamespace: NS,+ recipeName: RECIPE,+ capabilityFamily: 'promptBridge',+ provider: 'openai',+ allowedModels: ['gpt-5'],+ allowedCallers: ['api'],+ defaultTargetRef: 'ROLLED-BACK-SENTINEL',+ },+ OPERATOR_ID+ )+ ).rejects.toThrow('injected permission-event failure')+ expect(+ permissionEventsSeam.failNext,+ 'the injected failure must have been consumed by the real upsert path'+ ).toBeNull()++ // The transaction never delivered a NOTIFY...const delivered = await waitForGrantNotification(1_000)expect(delivered, 'a rolled-back NOTIFY must never be delivered').toBeUndefined()- // ...and so was the mutation — the grant is unchanged, proving the notify- // shared the mutation's transaction.+ // ...and the REAL mutation was rolled back — the grant row is unchanged.const after = await db.pool.query<{ default_target_ref: string | null }>(`SELECT default_target_ref FROM plugin_workload_sdk_grantsWHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex bad27b537..6b8669294 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -21,14 +21,27 @@ import {const permissionEvents = vi.hoisted(() => ({ append: vi.fn() }))+// issue #375 M3 (jozer review): the client handed out by the mocked+// withTransaction is DISTINGUISHABLE from the pool. Its query spy DELEGATES to+// pool.query (so the per-test mockResolvedValueOnce queues keep driving row+// responses) but records its own calls — so an out-of-transaction refactor+// (`notifyGrantUpdate(pool, …)` instead of the in-transaction `db`) never+// reaches `txClient` and turns the NOTIFY assertions below RED. The previous+// mock passed `work({ query: pool.query })`, making the transaction client and+// the pool literally the same object and that distinction unrepresentable.+const txClient = vi.hoisted(() => ({ query: vi.fn() }))+// Mock the pg pool so we can inspect the exact SQL + bind parameters the real// db-layer functions produce.vi.mock('../src/db.js', () => ({...(() => {const query = vi.fn().mockResolvedValue({ rows: [{ prompt_bridge_count: 1 }], rowCount: 1 })+ txClient.query.mockImplementation((...args: unknown[]) =>+ (query as (...inner: unknown[]) => unknown)(...args)+ )return {pool: { query, connect: vi.fn() },- withTransaction: (work: (db: { query: typeof query }) => unknown) => work({ query }),+ withTransaction: (work: (db: { query: typeof txClient.query }) => unknown) => work(txClient),}})(),}))@@ -65,6 +78,7 @@ function mockUpsertGrantQueries(row: Record<string, unknown>): void {describe('JIT credential ticket jti registry', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()})it('registers a non-secret, invocation-bound jti and reports insertion conflicts', async () => {@@ -145,6 +159,7 @@ describe('JIT credential ticket jti registry', () => {describe('ordered provider-attempt reservation', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()})it('reserves only the next authorized target after an eligible prior failure', async () => {@@ -245,6 +260,7 @@ describe('ordered provider-attempt reservation', () => {describe('legacy promptBridge inventory', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()})it('reports legacy shape without rewriting policy or exposing secrets', async () => {@@ -314,6 +330,7 @@ describe('legacy promptBridge inventory', () => {describe('invocation retry lifecycle timestamps', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()vi.mocked(pool.query).mockResolvedValue({ rows: [], rowCount: 1 } as never)})@@ -451,6 +468,7 @@ describe('upsertGrant — provider column (R1)', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()})it('persists the explicit provider and maps it back on the returned grant', async () => {@@ -579,6 +597,7 @@ describe('resolveRecipientProfiles', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()})it('returns [] without querying when no ref is a UUID', async () => {@@ -691,6 +710,7 @@ describe('Plugin Workload SDK governed permission events', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')})@@ -716,7 +736,9 @@ describe('Plugin Workload SDK governed permission events', () => {)expect(permissionEvents.append).toHaveBeenCalledWith(- expect.objectContaining({ query: pool.query }),+ // issue #375 M3: the audit append must receive the TRANSACTION client+ // handed out by withTransaction, never the pool.+ expect.objectContaining({ query: txClient.query }),expect.objectContaining({operatorSub: OPERATOR_ID,changes: [@@ -769,6 +791,7 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {beforeEach(() => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')})@@ -796,7 +819,9 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'sdk-recipe', actor)).resolves.toMatchObject({ state: 'revoking', revoked: 1 })expect(permissionEvents.append).toHaveBeenCalledWith(- expect.objectContaining({ query: pool.query }),+ // issue #375 M3: the audit append must receive the TRANSACTION client+ // handed out by withTransaction, never the pool.+ expect.objectContaining({ query: txClient.query }),expect.objectContaining({operatorSub: 'wrc-provisioner',internalPrincipal,@@ -833,7 +858,9 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {finalizePluginWorkloadSdkRevocation('sandbox-recipes', 'sdk-recipe', REVOCATION_ID, actor)).resolves.toMatchObject({ state: 'disabled', disabled: 1 })expect(permissionEvents.append).toHaveBeenCalledWith(- expect.objectContaining({ query: pool.query }),+ // issue #375 M3: the audit append must receive the TRANSACTION client+ // handed out by withTransaction, never the pool.+ expect.objectContaining({ query: txClient.query }),expect.objectContaining({operatorSub: 'wrc-provisioner',internalPrincipal,@@ -905,6 +932,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('upsertGrant emits a transactional pg_notify on the grant-update channel', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')vi.mocked(pool.query)@@ -947,12 +975,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {'operator-1')- const notifyCall = vi- .mocked(pool.query)- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as- | [string, unknown[]]- | undefined- expect(notifyCall).toBeDefined()+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not+ // the pool — searching txClient (not pool.query) is what turns the+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>+ String(sql).includes('pg_notify')+ ) as unknown as [string, unknown[]] | undefined+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()const [, params] = notifyCall as [string, unknown[]]expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)expect(JSON.parse(String(params[1]))).toEqual({@@ -964,6 +994,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('deleteGrant emits a transactional pg_notify on the grant-update channel', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')vi.mocked(pool.query)@@ -997,12 +1028,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {'operator-1')- const notifyCall = vi- .mocked(pool.query)- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as- | [string, unknown[]]- | undefined- expect(notifyCall).toBeDefined()+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not+ // the pool — searching txClient (not pool.query) is what turns the+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>+ String(sql).includes('pg_notify')+ ) as unknown as [string, unknown[]] | undefined+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()const [, params] = notifyCall as [string, unknown[]]expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)expect(JSON.parse(String(params[1]))).toMatchObject({@@ -1013,6 +1046,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('revokePluginWorkloadSdkForRecipe emits a transactional pg_notify on the grant-update channel', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')vi.mocked(pool.query)@@ -1034,12 +1068,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {operatorSub: 'operator-1',})- const notifyCall = vi- .mocked(pool.query)- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as- | [string, unknown[]]- | undefined- expect(notifyCall).toBeDefined()+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not+ // the pool — searching txClient (not pool.query) is what turns the+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>+ String(sql).includes('pg_notify')+ ) as unknown as [string, unknown[]] | undefined+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()const [, params] = notifyCall as [string, unknown[]]expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)expect(JSON.parse(String(params[1]))).toMatchObject({@@ -1055,6 +1091,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {// NOTIFY again — a reconcile+pg_notify spin. Match upsert/delete: notify only// on a real mutation.vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()permissionEvents.append.mockResolvedValue('operation-id')vi.mocked(pool.query)@@ -1085,6 +1122,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('does NOT emit pg_notify when deleteGrant matches no row (rowCount 0)', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()vi.mocked(pool.query).mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock@@ -1107,6 +1145,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('does NOT emit pg_notify on a revoke that finds no grants (state=missing)', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()vi.mocked(pool.query).mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock@@ -1124,6 +1163,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {it('does NOT emit pg_notify on a revoke that conflicts on revocation epoch (state=conflict)', async () => {vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()permissionEvents.append.mockReset()vi.mocked(pool.query).mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lockdiff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex c4ae4b6de..6bbfbc5b1 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.291",+ "version": "0.1.292","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.291",+ "version": "0.1.292","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 29782fa82..3f80d255c 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.291",+ "version": "0.1.292","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/db.ts b/workflow-recipes/src/db.tsindex 77195f12d..c895f1471 100644--- a/workflow-recipes/src/db.ts+++ b/workflow-recipes/src/db.ts@@ -69,6 +69,31 @@ export function getPool(): Pool {return pool}+/**+ * issue #375 (M4, jozer review): dedicated single-connection pool for the+ * grant-update LISTEN session.+ *+ * The shared pool has `poolMax` (production default 4) slots, and TWO consumers+ * pin a client for the process lifetime: dbRunProcessor's LISTEN session+ * (pre-existing) and the grant-update listener (new). Checking the second+ * LISTEN session out of the shared pool cut working headroom from 3 to 2 and+ * made the 30s tick (pollOrphans + checkStuckRuns) plus one CRD MODIFIED+ * enough to exhaust it (5s connectionTimeoutMillis rejections). A LISTEN+ * session should not be a shared-pool checkout at all: this dedicated+ * `max: 1` pool gives the listener its own connection, so it consumes ZERO+ * shared-pool slots while keeping the exact PoolClient checkout/release and+ * reconnect semantics the listener is built (and tested) against.+ */+export function createGrantUpdateListenerPool(cfg: DbConfig): Pool {+ const p = new Pool({ ...createPoolConfig(cfg), max: 1 })+ // Same mandate as initDb: a client error without a pool-level listener+ // crashes the process via 'error' event propagation.+ p.on('error', (err: Error) => {+ log.error('pg grant-update listener pool error', { err: err.message })+ })+ return p+}+/** Test seam — allow unit tests to swap the module-level pool. */export function __setPoolForTests(p: Pool | null): void {pool = pdiff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 3f7e62f1e..d3cff3bf0 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -426,6 +426,10 @@ describe('shouldPatchRecipeStatus', () => {// The exact incident shape: phase and top-level message are unchanged, the// ONLY change is the SDK capability state. Before the fix this returned// false and the validated state was never published.+ // M2 (jozer review): the persisted verifiedAt is FRESH, so the >5-min+ // throttle branch stays CLOSED and this can only pass through the semantic+ // field comparisons — a hardcoded stale timestamp made it pass via the+ // throttle, pinning nothing about the incident's mechanism.expect(shouldPatchRecipeStatus(sdkRecipe({@@ -433,7 +437,7 @@ describe('shouldPatchRecipeStatus', () => {promptBridge: true,clientNotifications: false,message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',- verifiedAt: '2026-08-17T17:24:50.000Z',+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),}),{phase: 'active',@@ -646,6 +650,107 @@ describe('shouldPatchRecipeStatus', () => {)).toBe(false)})++ // ── issue #375 M2 (jozer review): every projection field comparison is pinned+ // INDIVIDUALLY. Each case varies exactly ONE field between the persisted+ // capability and the projection, with a FRESH persisted verifiedAt so the+ // >5-min throttle branch stays closed — removing that single field's+ // comparison in pluginWorkloadSdkProjectionChanged turns that case RED.+ describe('per-field projection comparisons (issue #375 M2)', () => {+ function persistedBase(): Record<string, unknown> {+ return {+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ message: 'Capability validated (promptBridge)',+ policyRevision: 3,+ policyHash: 'sha256:policy-a',+ defaultTargetRef: 'primary-openai',+ bootstrapPodUid: 'pod-abc',+ bootstrapContractVersion: 2,+ bootstrapProvider: 'openai',+ bootstrapModel: 'gpt-5.4-mini',+ // FRESH: keeps the verifiedAt throttle CLOSED so a `true` can only come+ // from the varied field's own comparison.+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),+ }+ }++ function projectionFrom(+ persisted: Record<string, unknown>,+ override: Record<string, unknown>+ ): { conditions: never[]; capability: never } {+ const { verifiedAt: _persistedVerifiedAt, ...semantic } = persisted+ return {+ conditions: [],+ capability: {+ ...semantic,+ verifiedAt: new Date().toISOString(),+ validatedAt: new Date().toISOString(),+ ...override,+ } as never,+ }+ }++ const cases: Array<[field: string, override: Record<string, unknown>]> = [+ ['state', { state: 'awaiting_policy' }],+ ['message', { message: 'Capability validated (promptBridge, clientNotifications)' }],+ ['policyRevision', { policyRevision: 4 }],+ ['policyHash', { policyHash: 'sha256:policy-b' }],+ ['defaultTargetRef', { defaultTargetRef: 'fallback-anthropic' }],+ ['bootstrapPodUid', { bootstrapPodUid: 'pod-def' }],+ ['bootstrapContractVersion', { bootstrapContractVersion: null }],+ ['bootstrapProvider', { bootstrapProvider: 'anthropic' }],+ ['bootstrapModel', { bootstrapModel: 'claude-sonnet-4-6' }],+ ['promptBridge', { promptBridge: false }],+ ['clientNotifications', { clientNotifications: true }],+ ]++ for (const [field, override] of cases) {+ it(`a lone ${field} change forces a patch (throttle closed)`, () => {+ const persisted = persistedBase()+ expect(+ shouldPatchRecipeStatus(sdkRecipe(persisted), {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: projectionFrom(persisted, override),+ })+ ).toBe(true)+ })+ }++ it('control: with NO field varied and a fresh persisted verifiedAt, no patch is emitted', () => {+ const persisted = persistedBase()+ expect(+ shouldPatchRecipeStatus(sdkRecipe(persisted), {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: projectionFrom(persisted, {}),+ })+ ).toBe(false)+ })++ it('first-ever publish: a computed capability with NOTHING persisted forces a patch', () => {+ // Pins the `if (!current) return true` branch — the first publish of+ // status.pluginWorkloadSdk after a recipe declares the capability.+ expect(+ shouldPatchRecipeStatus(+ makeWorkflowRecipe({+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },+ status: { phase: 'active', message: 'All workloads deployed' },+ }),+ {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: projectionFrom(persistedBase(), {}),+ }+ )+ ).toBe(true)+ })+ })})describe('infrastructure telemetry projections', () => {@@ -2672,3 +2777,103 @@ describe('external egress periodic refresh (issue #299 §3.2/§5)', () => {}})})++// ── issue #375 M1/M4 (jozer review): the REAL constructor + start()/stop()+// wiring of the grant-update listener. Every prior watcher test hand-installed+// collaborators onto Object.create(prototype), so removing the constructor+// wiring or the start()/stop() calls left the whole suite green. These tests+// construct the real watcher and go RED if:+// - the constructor stops creating the listener,+// - the listener is handed the SHARED pool instead of its dedicated one (M4),+// - `await this.grantUpdateListener.start()` is removed from start(),+// - `await this.grantUpdateListener.stop()` is removed from stop().+describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issue #375 M1/M4)', () => {+ function makeFakeKubeConfig() {+ return {+ makeApiClient: vi.fn(() => ({})),+ } as unknown as import('@kubernetes/client-node').KubeConfig+ }++ type ListenerStub = { start: () => Promise<void>; stop: () => Promise<void> }+ type InternalWatcher = {+ grantUpdateListener: ListenerStub | null+ grantUpdateListenerPool: { connect?: unknown; end: () => Promise<void> } | null+ dbRunProcessor: { start: () => Promise<void>; stop: () => Promise<void> } | null+ traceReporter: null+ tryInitializeWorkflow: () => Promise<void>+ startWatch: () => Promise<void>+ startSecretWatch: () => Promise<void>+ startBrokerTokenRotationLoop: () => void+ startRuntimeCredentialRefreshLoop: () => void+ startWorkloadStatusRefreshLoop: () => void+ startExternalEgressRefreshLoop: () => void+ }++ it('constructor wires a listener backed by a DEDICATED pool — LISTEN never checks out of the shared pool (M4)', async () => {+ const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)+ const internal = watcher as unknown as InternalWatcher++ expect(internal.grantUpdateListener).not.toBeNull()+ expect(internal.grantUpdateListenerPool).not.toBeNull()+ // M4: a lifetime-pinned LISTEN session must not burn one of the shared+ // pool's 4 slots — the listener owns a dedicated single-connection pool.+ expect(internal.grantUpdateListenerPool).not.toBe(sharedPool)++ // Behavioral pin: starting the listener checks out of the DEDICATED pool+ // and never touches the shared one.+ const fakeClient = {+ on: vi.fn(),+ query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),+ release: vi.fn(),+ removeAllListeners: vi.fn(),+ }+ const dedicatedConnect = vi.fn().mockResolvedValue(fakeClient)+ ;(internal.grantUpdateListenerPool as { connect?: unknown }).connect = dedicatedConnect+ await (internal.grantUpdateListener as ListenerStub).start()+ expect(dedicatedConnect).toHaveBeenCalledTimes(1)+ expect(sharedPool.connect).not.toHaveBeenCalled()+ await (internal.grantUpdateListener as ListenerStub).stop()+ })++ it('does not create the listener (or its pool) without a DB pool', () => {+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), null)+ const internal = watcher as unknown as InternalWatcher+ expect(internal.grantUpdateListener).toBeNull()+ expect(internal.grantUpdateListenerPool).toBeNull()+ })++ it('start() starts and stop() stops the grant-update listener and ends its dedicated pool (M1)', async () => {+ const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)+ const internal = watcher as unknown as InternalWatcher++ // Stub the heavy collaborators. The REAL start()/stop() method bodies stay+ // in play, so neutering the listener wiring inside them goes RED here.+ internal.tryInitializeWorkflow = vi.fn().mockResolvedValue(undefined)+ internal.startWatch = vi.fn().mockResolvedValue(undefined)+ internal.startSecretWatch = vi.fn().mockResolvedValue(undefined)+ internal.startBrokerTokenRotationLoop = vi.fn()+ internal.startRuntimeCredentialRefreshLoop = vi.fn()+ internal.startWorkloadStatusRefreshLoop = vi.fn()+ internal.startExternalEgressRefreshLoop = vi.fn()+ internal.dbRunProcessor = {+ start: vi.fn().mockResolvedValue(undefined),+ stop: vi.fn().mockResolvedValue(undefined),+ }+ internal.traceReporter = null+ const listenerStart = vi.fn().mockResolvedValue(undefined)+ const listenerStop = vi.fn().mockResolvedValue(undefined)+ internal.grantUpdateListener = { start: listenerStart, stop: listenerStop }+ const poolEnd = vi.fn().mockResolvedValue(undefined)+ internal.grantUpdateListenerPool = { end: poolEnd }++ await watcher.start()+ expect(listenerStart).toHaveBeenCalledTimes(1)++ await watcher.stop()+ expect(listenerStop).toHaveBeenCalledTimes(1)+ // M4: the watcher owns the dedicated pool and must close its connection.+ expect(poolEnd).toHaveBeenCalledTimes(1)+ })+})diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex e7866edca..8643b8fd7 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -14,7 +14,7 @@import * as k8s from '@kubernetes/client-node'import type { Pool } from 'pg'import { loadConfig } from './config'-import { getPool, initDb } from './db'+import { createGrantUpdateListenerPool, getPool, initDb } from './db'import {type GovernedTraceReporter,type WorkflowInfrastructureTelemetryProjection,@@ -619,6 +619,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// a DB (same gate as dbRunProcessor); without a DB the listener never starts// and the level-triggered reconcile remains the sole convergence path.private grantUpdateListener: GrantUpdateListener | null = null+ // issue #375 (M4, jozer review): the LISTEN session pins its client for the+ // process lifetime, so it gets its OWN single-connection pool and consumes+ // zero slots of the shared `poolMax` (default 4) pool that dbRunProcessor,+ // leader election, and run-sync all contend for. Owned (and ended) by this+ // watcher because it exists solely for the grant-update listener.+ private grantUpdateListenerPool: Pool | null = nullprivate config = loadConfig()private stopped = falseprivate runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null = null@@ -686,8 +692,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// issue #375 (P3): on a grant-update NOTIFY, force-reconcile the affected// recipe through the SAME path the 30s watchdog uses. The dispatch body is// a named method so it is unit-testable without standing up a DB.+ // issue #375 (M4): the listener LISTENs on a DEDICATED single-connection+ // pool, never the shared one — a lifetime-pinned LISTEN checkout would+ // permanently burn one of the shared pool's 4 slots (dbRunProcessor+ // already pins another).+ this.grantUpdateListenerPool = createGrantUpdateListenerPool(this.config.db)this.grantUpdateListener = createGrantUpdateListener({- pool,+ pool: this.grantUpdateListenerPool,onGrantUpdate: ({ recipeNamespace, recipeName }) =>this.handleGrantUpdateNotification(recipeNamespace, recipeName),})@@ -1389,6 +1400,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {if (this.grantUpdateListener) {await this.grantUpdateListener.stop()}+ if (this.grantUpdateListenerPool) {+ // issue #375 (M4): the watcher owns the dedicated LISTEN pool — end it on+ // shutdown so its single connection is closed with the process.+ await this.grantUpdateListenerPool.end()+ this.grantUpdateListenerPool = null+ }if (this.traceReporter) {const result = await this.traceReporter.stopAndDrain()if (!result.drained) {diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.tsindex a25a60d6c..c678b7571 100644--- a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts@@ -40,6 +40,13 @@ function makeFakeClient(opts: { failQuery?: boolean } = {}) {return { rows: [], rowCount: 0 }}),release: vi.fn(),+ // pg PoolClient is an EventEmitter; the listener strips its own handlers+ // before handing a subscribed client back to the pool (issue #375 M5).+ removeAllListeners: vi.fn((event?: string) => {+ if (event) delete handlers[event]+ else for (const key of Object.keys(handlers)) delete handlers[key]+ return client+ }),}return {client,@@ -194,50 +201,116 @@ describe('createGrantUpdateListener', () => {describe('createGrantUpdateListener — sustained reconnect regime (issue #375 Finding 1)', () => {interface FakeGenClient {released: boolean+ destroyed: boolean+ subscribed: boolean+ handlerCount: (event: string) => numberon: (event: string, handler: (...args: unknown[]) => void) => FakeGenClientquery: ReturnType<typeof vi.fn>release: ReturnType<typeof vi.fn>+ removeAllListeners: (event?: string) => FakeGenClientemitError: (err: Error) => voidemitNotification: (msg: { channel: string; payload?: string }) => void}- /** A fresh fake client per connect(), tracking created + per-client released. */+ /**+ * pg-pool-FAITHFUL fake (issue #375 M5, jozer review): the previous fake+ * modeled a plain-released client as silently non-dispatching and minted a+ * fresh object on every connect() — exactly the two divergences from pg-pool+ * that hid the still-subscribed-client leak. Faithful semantics:+ * - `release()` (no error) re-idles the SAME object; pg-pool `_release`+ * re-attaches only its OWN idle listener — user handlers stay attached and+ * the server-side LISTEN subscription stays live, so the client KEEPS+ * dispatching after a plain release.+ * - `connect()` REUSES an idle client before creating a new one.+ * - `release(err)` DESTROYS the client (removed from the pool; its session+ * — and therefore its subscription — is gone).+ * - `LISTEN`/`UNLISTEN` queries toggle the server-side subscription; only a+ * subscribed, non-destroyed session receives a broadcast NOTIFY.+ */function makeFakeClientFactory() {const created: FakeGenClient[] = []- function connect(): FakeGenClient {+ const idle: FakeGenClient[] = []+ let listenHold: Promise<void> | null = null+ let releaseListenHold: (() => void) | null = null+ function makeClient(): FakeGenClient {const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}const client: FakeGenClient = {released: false,+ destroyed: false,+ subscribed: false,+ handlerCount: (event: string) => (handlers[event] ?? []).length,on(event, handler) {;(handlers[event] ??= []).push(handler)return client},- query: vi.fn(async () => ({ rows: [], rowCount: 0 })),- release: vi.fn(() => {+ query: vi.fn(async (sql: string) => {+ if (/^LISTEN\b/.test(sql)) {+ if (listenHold) await listenHold+ client.subscribed = true+ }+ if (/^UNLISTEN\b/.test(sql)) client.subscribed = false+ return { rows: [], rowCount: 0 }+ }),+ release: vi.fn((err?: Error) => {client.released = true+ if (err) {+ client.destroyed = true+ client.subscribed = false+ } else if (!client.destroyed) {+ idle.push(client)+ }}),+ removeAllListeners(event?: string) {+ if (event) delete handlers[event]+ else for (const key of Object.keys(handlers)) delete handlers[key]+ return client+ },emitError(err: Error) {- for (const h of handlers.error ?? []) h(err)+ for (const h of [...(handlers.error ?? [])]) h(err)},emitNotification(msg) {- // A released client is back in the pool and must not dispatch.- if (client.released) return- for (const h of handlers.notification ?? []) h(msg)+ // Faithful: dispatch depends ONLY on the session being alive and+ // subscribed — NOT on pool checkout state. A plain-released client+ // keeps dispatching (that is the leak the old fake hid).+ if (client.destroyed || !client.subscribed) return+ for (const h of [...(handlers.notification ?? [])]) h(msg)},}+ return client+ }+ function connect(): FakeGenClient {+ const reused = idle.shift()+ if (reused) {+ reused.released = false+ return reused+ }+ const client = makeClient()created.push(client)return client}return {created,+ idle,connect,- /** Live (checked-out, unreleased) sessions right now. */- liveCount: () => created.filter(c => !c.released).length,+ /** Live (checked-out, unreleased, undestroyed) sessions right now. */+ liveCount: () => created.filter(c => !c.released && !c.destroyed).length,current: () => created[created.length - 1],- /** Simulate a real pg NOTIFY broadcast to EVERY live session. */+ /** Simulate a real pg NOTIFY broadcast to EVERY subscribed session. */broadcast(msg: { channel: string; payload?: string }) {for (const c of created) c.emitNotification(msg)},+ /** Park the next LISTEN round-trip until releaseListen() is called. */+ holdListen() {+ listenHold = new Promise<void>(resolve => {+ releaseListenHold = () => {+ listenHold = null+ resolve()+ }+ })+ },+ releaseListen() {+ releaseListenHold?.()+ },}}@@ -360,6 +433,101 @@ describe('createGrantUpdateListener — sustained reconnect regime (issue #375 Fawait listener.stop()expect(factory.liveCount()).toBe(0)})++ // ── issue #375 M5 (jozer review): a client that ran LISTEN and carries this+ // module's handlers must never be plain-release()d back to the shared pool.+ // pg-pool re-idles the SAME object (no UNLISTEN, user handlers kept), so the+ // session stays subscribed: it dispatches after stop() and double-dispatches+ // after a reuse. These three tests drive the exact leak paths with the+ // pg-pool-faithful fake and go RED without the UNLISTEN/removeAllListeners/+ // generation hardening.+ it('M5a: stop() during the in-flight LISTEN round-trip leaves NO subscribed, dispatching session behind', async () => {+ const factory = makeFakeClientFactory()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => factory.connect() as never,+ })++ factory.holdListen()+ const startPromise = listener.start() // parks inside the LISTEN round-trip+ await flush()+ await listener.stop() // stop() lands while LISTEN is in flight+ factory.releaseListen() // LISTEN resolves AFTER stop+ await startPromise+ await flush()++ // The released client went back to the pool; it must NOT still be a live+ // subscriber that dispatches into a stopped listener.+ factory.broadcast(NOTIFY)+ expect(onGrantUpdate, 'post-stop dispatch from a pool-returned client').not.toHaveBeenCalled()+ expect(factory.liveCount(), 'no session may stay checked out').toBe(0)+ expect(+ factory.created[0].handlerCount('notification'),+ 'listener handlers must be stripped before the client re-enters the pool'+ ).toBe(0)+ })++ it('M5b: stop() then start() reusing the pooled client dispatches exactly once (no stacked handler pairs)', async () => {+ const factory = makeFakeClientFactory()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => factory.connect() as never,+ })++ await listener.start()+ await listener.stop()+ await listener.start() // pg-pool hands back the SAME client object++ expect(factory.created, 'pool-faithful fake must reuse the idle client').toHaveLength(1)+ factory.broadcast(NOTIFY)+ // Without stripping handlers on release, each stop→start cycle stacks one+ // handler pair on the same client and one NOTIFY dispatches twice.+ expect(onGrantUpdate).toHaveBeenCalledTimes(1)+ await listener.stop()+ expect(factory.liveCount()).toBe(0)+ })++ it('M5c: a reconnect parked across stop()→start() must not leave the superseded session subscribed (single dispatch)', async () => {+ const factory = makeFakeClientFactory()+ const { sleep, resolvers } = resolverQueueSleep()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => factory.connect() as never,+ sleep,+ })++ await listener.start()+ const clientA = factory.current()+ clientA.emitError(new Error('drop')) // destroys A, parks a reconnect in its backoff sleep+ await flush()+ await listener.stop()+ await listener.start() // session B attaches and becomes current+ await flush()+ const clientB = factory.current()+ expect(clientB.subscribed).toBe(true)++ // The parked reconnect resumes AFTER the restart: session C attaches and the+ // non-overwrite guard releases B. Pre-fix that release was plain — B stayed+ // subscribed with handlers attached → one logical NOTIFY dispatched twice.+ ;(resolvers.shift() as () => void)()+ await flush()++ expect(factory.liveCount(), 'at most one live session').toBeLessThanOrEqual(1)+ onGrantUpdate.mockClear()+ factory.broadcast(NOTIFY)+ expect(onGrantUpdate, 'superseded session must not double-dispatch').toHaveBeenCalledTimes(1)+ await listener.stop()+ expect(factory.liveCount()).toBe(0)+ })})describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.tsindex b3aeb681c..eeb184d97 100644--- a/workflow-recipes/src/reconciler/grantUpdateListener.ts+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts@@ -103,6 +103,45 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gralet stopped = true// H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.let reconnecting = false+ // M5 (jozer review): monotonic session generation. A session's notification+ // handler dispatches ONLY while its own generation is the current one — a+ // superseded-but-not-destroyed session (e.g. one that raced a release) goes+ // mute instead of double-dispatching. The 'error' identity gate below covers+ // reconnect scheduling; this covers the dispatch path it never guarded.+ let generation = 0++ /**+ * M5 (jozer review): a client that has run LISTEN and carries this module's+ * handlers must NEVER be plain-release()d back to the shared pool. pg-pool's+ * `_release` re-idles the SAME Client object and re-attaches only its OWN+ * idle listener — user handlers stay attached and the Postgres session stays+ * subscribed, so the client keeps dispatching after stop() and+ * double-dispatches after a pool reuse (and dbRunProcessor could check out a+ * client with this module's stale 'error' handler still attached). Unsubscribe+ * and strip our handlers first; if the UNLISTEN cannot be delivered the+ * session state is unknowable, so DESTROY the client (release(err)) instead of+ * re-idling it.+ */+ async function releaseSubscribedClient(attached: PoolClient): Promise<void> {+ let unlistened = false+ try {+ await attached.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)+ unlistened = true+ } catch {+ /* session unusable — destroyed below instead of re-idled */+ }+ attached.removeAllListeners('notification')+ attached.removeAllListeners('error')+ try {+ if (unlistened) {+ attached.release()+ } else {+ attached.release(new Error('grant-update LISTEN session UNLISTEN failed — destroying'))+ }+ } catch {+ /* idempotent */+ }+ }async function attachListener(): Promise<void> {let client: PoolClient | null = null@@ -110,13 +149,24 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Graclient = await connect()// H2: stop() may have run while we awaited connect(); do NOT bring up a// LISTEN session after shutdown. Release the client and bail before any- // handler is registered, so nothing dispatches post-stop.+ // handler is registered, so nothing dispatches post-stop. (Plain release+ // is correct HERE: no LISTEN has run and no handler is attached yet.)if (stopped) {client.release()return}const attached = client+ // M5: assigned only when this session is INSTALLED as the current+ // listener; until then — and forever after supersession — the handler is+ // mute (generation is bumped on every install and on stop()).+ let sessionGeneration = -1attached.on('notification', msg => {+ if (sessionGeneration !== generation) {+ log.debug('superseded listen session notification — dropped', {+ channel: msg.channel,+ })+ return+ }if (msg.channel !== PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL || !msg.payload) returnconst notification = parseGrantUpdatePayload(msg.payload)if (!notification) {@@ -157,32 +207,35 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Grareturn}log.warn('listen session error — will reconnect', { err: err.message })+ // M5: mute the errored session's dispatch path immediately (its client+ // was destroyed above, but the generation bump costs nothing and closes+ // the window where release(err) itself failed).+ generation += 1listenClient = nullscheduleReconnect()})await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)// H2: re-check after the async LISTEN resolves — stop() may have run- // during the round-trip.+ // during the round-trip. M5: this client has ALREADY run LISTEN with+ // handlers attached — it must be unsubscribed and stripped, never+ // plain-released into the pool where it would keep dispatching.if (stopped) {- try {- attached.release()- } catch {- /* idempotent */- }+ await releaseSubscribedClient(attached)return}// Finding 1: never overwrite a live `listenClient` without releasing the// prior one. Defense-in-depth alongside the identity gate above — a stray// interleaving that reaches here with a different session still checked out- // must not leak it.+ // must not leak it. M5: the loser ran LISTEN with handlers attached, so it+ // gets the full unsubscribe+strip release, not a plain one.if (listenClient && listenClient !== attached) {- try {- listenClient.release()- } catch {- /* idempotent */- }+ await releaseSubscribedClient(listenClient)}listenClient = attached+ // M5: install-time bump — makes THIS session the sole dispatcher and any+ // surviving predecessor stale.+ generation += 1+ sessionGeneration = generationlog.info('LISTEN plugin_workload_sdk_grant_update attached')} catch (err) {log.warn('LISTEN attach failed — will retry', {@@ -234,18 +287,16 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra// stop()→start() restart is not blocked by a stale flag left set by a// reconnect that was still mid-sleep when we stopped (belt-and-suspenders).reconnecting = false+ // M5: mute ANY session that survives teardown (e.g. one parked in an+ // in-flight LISTEN whose release races this stop).+ generation += 1if (listenClient) {- try {- await listenClient.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)- } catch {- /* best-effort — the session is being torn down */- }- try {- listenClient.release()- } catch {- /* idempotent */- }+ const current = listenClientlistenClient = null+ // M5: unsubscribe + strip handlers before the client re-enters the+ // pool — a plain release would re-idle a still-subscribed client whose+ // handlers dispatch after stop() and stack up on the next start().+ await releaseSubscribedClient(current)}},}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex a8a8c3dfa..a17f325a3 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -5,6 +5,7 @@ import { readFileSync } from 'node:fs'import { resolve } from 'node:path'import { EVENFIRE_REGISTRY_PULL_SECRET_NAME } from '@clerum/workflow-runtime-core'import { loadConfig } from '../config'+import { shouldPatchRecipeStatus } from '../k8sClient'import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'@@ -9960,4 +9961,336 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).not.toBe('failed')})})++ // ── issue #375 BLOCKER (jozer-rami review): clientNotifications-only workflow-+ // lane SDK recipes must FALL THROUGH the two active-phase short-circuits (the+ // awaiting-trigger one and the steady-active one) the same way promptBridge+ // does, so the inner reconcile gathers the bootstrap proof and the capability+ // projects `validated` with NO skipStatusPatch. Before the fix both+ // short-circuits returned skipStatusPatch:true, shouldPatchRecipeStatus bailed+ // at its first line, and the awaiting_policy→validated transition was never+ // published — the original #375 symptom, unchanged, for that family.+ describe('issue #375 BLOCKER — clientNotifications-only workflow-lane fall-through', () => {+ const CN_MESSAGE = 'Workflow trigger infrastructure registered'++ function clientNotificationsOnlyRecipe(labels?: Record<string, string>): WorkflowRecipeCRD {+ return makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ ...(labels ? { labels } : {}),+ },+ spec: {+ steps: [{ id: 'step-1', run: snippetRun() }],+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ pluginWorkloadSdk: {+ clientNotifications: { allowedEventTypes: ['workflow.completed'] },+ allowedCallers: ['app'],+ },+ },+ status: {+ phase: 'active',+ message: CN_MESSAGE,+ pluginWorkloadSdk: {+ state: 'awaiting_policy',+ promptBridge: false,+ clientNotifications: true,+ message: 'Plugin Workload SDK clientNotifications is awaiting an operator grant',+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),+ } as never,+ },+ })+ }++ function stubWorkflowReconciler() {+ const workflowReconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: CN_MESSAGE,+ pluginWorkloadSdkBootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'sdk-pod-uid',+ policyReady: true,+ clientNotificationsPolicyReady: true,+ verifiedAt: new Date().toISOString(),+ },+ })+ const reconcilePluginWorkloadSdkOnly = vi.fn()+ ;(+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }+ ).config.pluginWorkloadSdkEnabled = true+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcile: typeof workflowReconcile+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly+ validateWorkflowSpec: () => undefined+ }+ }+ ).workflowReconciler = {+ reconcile: workflowReconcile,+ reconcilePluginWorkloadSdkOnly,+ validateWorkflowSpec: () => undefined,+ }+ return { workflowReconcile, reconcilePluginWorkloadSdkOnly }+ }++ it('awaiting-trigger lane: recomputes and publishes awaiting_policy→validated (no skipStatusPatch)', async () => {+ // Phase active + mcpHost required (flag ON) + no run label + no+ // workflowExecution ⇒ the awaiting-trigger short-circuit. Pre-fix this+ // returned skipStatusPatch:true without ever calling the inner reconcile.+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()+ const recipe = clientNotificationsOnlyRecipe()++ const result = await reconciler.reconcile(recipe)++ // Fall-through proof: the inner reconcile actually ran (steps present ⇒+ // the stepless SDK-only adapter must NOT be used).+ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()+ // The reconcile pass has an SDK opinion and does not suppress the patch.+ expect(result.skipStatusPatch).toBeFalsy()+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')+ // The full chain the incident depends on: the watcher would publish it.+ expect(shouldPatchRecipeStatus(recipe, result)).toBe(true)+ })++ it('steady-active lane (run-labelled): recomputes and publishes awaiting_policy→validated (no skipStatusPatch)', async () => {+ // Phase active + run-id label ⇒ awaitsTriggeredRun=false ⇒ the steady-+ // active short-circuit, which pre-fix returned skipStatusPatch:true+ // unconditionally (no family carve-out at all).+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()+ const recipe = clientNotificationsOnlyRecipe({ 'clerum.io/workflow-run-id': 'run-e2e-1' })++ const result = await reconciler.reconcile(recipe)++ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()+ expect(result.skipStatusPatch).toBeFalsy()+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')+ expect(shouldPatchRecipeStatus(recipe, result)).toBe(true)+ })++ it('steady-active lane with an IN-PROGRESS execution: SDK recipes still short-circuit (409-window protection)', async () => {+ // The carve-out is bounded: while a workflow execution is in progress the+ // steady-active short-circuit is KEPT even for SDK recipes — that branch+ // protects the coordinator's 409-free windows, and the capability+ // publication is level-triggered so it lands on the next non-running+ // pass. Removing the `|| wfInProgress` term from the carve-out condition+ // turns this RED: the running recipe would fall through to the inner+ // reconcile and drop skipStatusPatch.+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()+ const recipe = clientNotificationsOnlyRecipe({ 'clerum.io/workflow-run-id': 'run-e2e-3' })+ recipe.status = {+ ...recipe.status,+ phase: 'active',+ workflowExecution: { phase: 'running' },+ } as typeof recipe.status++ const result = await reconciler.reconcile(recipe)++ expect(result.skipStatusPatch).toBe(true)+ expect(workflowReconcile).not.toHaveBeenCalled()+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()+ })++ it('non-SDK workflow recipes keep both steady-state short-circuits (no regression)', async () => {+ // The carve-out is SDK-scoped: a plain workflow recipe in the same two+ // shapes must still short-circuit with skipStatusPatch:true and must NOT+ // reach the inner reconcile on every steady tick.+ const { workflowReconcile } = stubWorkflowReconciler()+ const awaitingTrigger = makeRecipe({+ spec: {+ steps: [{ id: 'step-1', instruction: 'run' }],+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ },+ status: { phase: 'active', message: CN_MESSAGE },+ })+ const awaitingResult = await reconciler.reconcile(awaitingTrigger)+ expect(awaitingResult.skipStatusPatch).toBe(true)+ expect(workflowReconcile).not.toHaveBeenCalled()++ const runLabelled = makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ labels: { 'clerum.io/workflow-run-id': 'run-e2e-2' },+ },+ spec: {+ steps: [{ id: 'step-1', run: snippetRun() }],+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ },+ status: { phase: 'active', message: CN_MESSAGE },+ })+ const steadyResult = await reconciler.reconcile(runLabelled)+ expect(steadyResult.skipStatusPatch).toBe(true)+ expect(workflowReconcile).not.toHaveBeenCalled()+ })+ })++ // ── issue #375 M1 (jozer review): the projection WIRING is pinned against the+ // real pipeline, not hand-built ReconcileResults. Each test goes RED if its+ // wiring line is neutered:+ // - reconcile() must ATTACH result.pluginWorkloadSdkProjection (the single+ // line the whole P1 fix hangs on),+ // - projectPluginWorkloadSdk must pass the persisted capability through as+ // existingCapability (the R1 validatedAt carry-forward),+ // - patchStatus must REUSE the attached projection instead of recomputing.+ describe('issue #375 M1 — projection wiring through the real pipeline', () => {+ it('reconcile() attaches a populated pluginWorkloadSdkProjection to every result', async () => {+ // Non-SDK recipe: the projection must still be attached (capability null =+ // "clear the field"), proving production populates it on EVERY reconcile.+ const result = await reconciler.reconcile(makeRecipe())+ expect(result.pluginWorkloadSdkProjection).toBeDefined()+ expect(result.pluginWorkloadSdkProjection?.capability).toBeNull()+ })++ it('reconcile() projects the SDK-only lane bootstrap proof into a validated capability', async () => {+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'Plugin Workload SDK mcp-host registered',+ pluginWorkloadSdkBootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'sdk-pod-uid',+ provider: 'zai',+ model: 'glm-4.7',+ policyReady: true,+ verifiedAt: '2026-08-04T00:00:00.000Z',+ },+ })+ ;(+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }+ ).config.pluginWorkloadSdkEnabled = true+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly+ }+ }+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }++ const result = await reconciler.reconcile(+ makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },+ },+ })+ )++ // The REAL pipeline attached the projection and it carries the computed+ // capability — no test-side hand-building involved.+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')+ expect(result.pluginWorkloadSdkProjection?.capability?.bootstrapPodUid).toBe('sdk-pod-uid')+ })++ it('carries forward the persisted validatedAt on a steady validated capability (R1 wiring)', async () => {+ const FIRST_VALIDATED_AT = '2026-08-01T00:00:00.000Z'+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'Plugin Workload SDK mcp-host registered',+ pluginWorkloadSdkBootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'sdk-pod-uid',+ provider: 'zai',+ model: 'glm-4.7',+ policyReady: true,+ verifiedAt: new Date().toISOString(),+ },+ })+ ;(+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }+ ).config.pluginWorkloadSdkEnabled = true+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly+ }+ }+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }++ const result = await reconciler.reconcile(+ makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },+ },+ status: {+ phase: 'active',+ pluginWorkloadSdk: {+ state: 'validated',+ promptBridge: true,+ clientNotifications: false,+ validatedAt: FIRST_VALIDATED_AT,+ verifiedAt: new Date(Date.now() - 60_000).toISOString(),+ } as never,+ },+ })+ )++ // existingCapability wiring (workflowRecipeReconciler.ts): the stable+ // validatedAt marker must be CARRIED FORWARD, not re-stamped to `now`.+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')+ expect(result.pluginWorkloadSdkProjection?.capability?.validatedAt).toBe(FIRST_VALIDATED_AT)+ })++ it('patchStatus REUSES the attached projection instead of recomputing it', async () => {+ // The attached projection is deliberately DISTINCT from anything+ // patchStatus could recompute from this recipe/result (no bootstrap proof+ // ⇒ a recompute projects degraded/BootstrapNotReady). If patchStatus+ // recomputes instead of reusing, the marker never reaches the patch body.+ const recipe = makeRecipe({+ spec: {+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },+ },+ status: { phase: 'active' },+ })+ ;(+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }+ ).config.pluginWorkloadSdkEnabled = true++ const attachedCapability = {+ state: 'validated' as const,+ promptBridge: true,+ clientNotifications: false,+ message: 'REUSED-PROJECTION-MARKER',+ bootstrapPodUid: 'marker-pod',+ verifiedAt: new Date().toISOString(),+ }+ await reconciler.patchStatus(recipe, {+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ pluginWorkloadSdkProjection: {+ conditions: [+ {+ type: 'PluginWorkloadSdkCapability',+ status: 'True',+ reason: 'Validated',+ message: 'REUSED-PROJECTION-MARKER',+ lastTransitionTime: new Date().toISOString(),+ },+ ],+ capability: attachedCapability as never,+ },+ })++ const patchCall = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)?.[0] as {+ body: { status: { pluginWorkloadSdk?: { message?: string; bootstrapPodUid?: string } } }+ }+ expect(patchCall.body.status.pluginWorkloadSdk?.message).toBe('REUSED-PROJECTION-MARKER')+ expect(patchCall.body.status.pluginWorkloadSdk?.bootstrapPodUid).toBe('marker-pod')+ })+ })})diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 73dd1d5fe..7d8e76466 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1517,13 +1517,17 @@ export class WorkflowRecipeReconciler {}if (currentPhase === 'active' && awaitsTriggeredRun && !wfExecPhase) {- // Plugin Workload SDK promptBridge recipes keep an eager mcp-host whose- // provider must be (re-)configured once it becomes Ready and after any- // pod restart. Fall through to the inner reconcile so ensureEagerSdkMcpHost- // retries the /configure; the short-circuit below would freeze it- // provider_unavailable forever. clientNotifications-only needs no provider,- // so it keeps the cheap short-circuit.- if (!recipe.spec.pluginWorkloadSdk?.promptBridge) {+ // Plugin Workload SDK recipes (BOTH families) keep an eager mcp-host and+ // must fall through to the inner reconcile: promptBridge so+ // ensureEagerSdkMcpHost retries the /configure (the short-circuit below+ // would freeze it provider_unavailable forever), and clientNotifications-+ // only so the reconcile re-gathers the bootstrap proof and recomputes the+ // capability projection (issue #375 jozer BLOCKER: short-circuiting here+ // returned skipStatusPatch:true, so a computed awaiting_policy→validated+ // transition was never published for that family). The Step 8 ownership+ // gate inside the inner reconcile covers Secret-ownership revocation on+ // the fall-through path, exactly as it already did for promptBridge.+ if (!recipe.spec.pluginWorkloadSdk) {if (coordinatorGfsPolicyCanOpen) {await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)}@@ -1649,11 +1653,24 @@ export class WorkflowRecipeReconciler {// operator re-labels the Secret.const activeOwnership = await this.revokeOrRequeueSteadyWorkflow(recipe, 'active')if (activeOwnership) return activeOwnership- return {- phase: 'active' as RecipePhase,- message: wfExecPhase ? `Workflow ${wfExecPhase}` : 'Workflow completed',- workloadStatuses: [],- skipStatusPatch: true,+ // issue #375 (jozer BLOCKER): Plugin Workload SDK recipes fall through to+ // the inner reconcile (mirroring the awaiting-trigger carve-out above) so+ // the bootstrap proof is re-gathered and a computed capability transition+ // (e.g. awaiting_policy→validated) is actually published — this+ // short-circuit's skipStatusPatch:true suppressed it unconditionally.+ // The gfs/credentials/ownership enforcement above has already run.+ // EXCEPTION: while a workflow execution is in progress the short-circuit+ // is kept even for SDK recipes — this branch exists to protect the+ // coordinator's 409-free windows, and the capability publication is+ // level-triggered (requeue + watchdog) so it lands on the next+ // non-running pass.+ if (!recipe.spec.pluginWorkloadSdk || wfInProgress) {+ return {+ phase: 'active' as RecipePhase,+ message: wfExecPhase ? `Workflow ${wfExecPhase}` : 'Workflow completed',+ workloadStatuses: [],+ skipStatusPatch: true,+ }}}