Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes,control-api): address jozer-rami review (BLOCKER + M1-M5) (#375)

Alfredo Lopezde3253be93e9
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 12ff07e27..969060244 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.463",
+ "version": "0.1.464",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.463",
+ "version": "0.1.464",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 5041ea58d..81399954b 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.463",
+ "version": "0.1.464",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 632a1498f..9518c82a7 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -36,6 +36,20 @@ export type DbClient = {
query: (text: string, values?: unknown[]) => Promise<{ rows: unknown[]; rowCount: number | null }>
}
+// issue #375 M3 (jozer review): brand for clients that live INSIDE an open
+// transaction. `DbClient` is structural (`{ query }`), so `Pool` satisfies it
+// and a refactor from the in-transaction `db` to the module-level `pool` still
+// typechecks while silently breaking COMMIT/ROLLBACK coupling (e.g. a
+// transactional `pg_notify` becoming an autocommitted one). The unique-symbol
+// brand cannot be satisfied structurally, so any API that REQUIRES the
+// transaction session (like the grant-update NOTIFY) declares
+// `DbTransactionClient` and the pool no longer compiles there. It remains
+// assignable to `DbClient`, so passing it onward to ordinary helpers is free.
+declare const dbTransactionClientBrand: unique symbol
+export type DbTransactionClient = DbClient & {
+ readonly [dbTransactionClientBrand]: 'transaction'
+}
+
type DbSessionClient = DbClient & {
release: () => void
}
@@ -6076,7 +6090,9 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {
}
}
-export async function withTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {
+export async function withTransaction<T>(
+ work: (db: DbTransactionClient) => Promise<T>
+): Promise<T> {
const client = (await pool.connect()) as PoolClient
let transactionStarted = false
let commitSent = false
@@ -6084,7 +6100,10 @@ export async function withTransaction<T>(work: (db: DbClient) => Promise<T>): Pr
try {
await client.query('BEGIN')
transactionStarted = true
- const result = await work(client)
+ // The brand is nominal-only (a declared unique symbol); the checked-out
+ // client IS the transaction session, so this cast is the single blessed
+ // point where the brand is minted (issue #375 M3).
+ const result = await work(client as unknown as DbTransactionClient)
commitSent = true
await client.query('COMMIT')
return result
diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.ts
index 29925e075..63a52170c 100644
--- a/control-api/src/services/pluginWorkloadSdkDb.ts
+++ b/control-api/src/services/pluginWorkloadSdkDb.ts
@@ -1,5 +1,5 @@
import { createHash, randomUUID } from 'node:crypto'
-import { type DbClient, pool, withTransaction } from '../db.js'
+import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../db.js'
import type { AdministrativeEventSubmitterPrincipalV1 } from '../middleware/tracingSubmitterAuth.js'
import { stableStringify } from '../utils/stableStringify.js'
import {
@@ -62,7 +62,13 @@ export function buildGrantUpdateNotifyPayload(input: GrantUpdateNotifyInput): st
* short family), which cannot approach that bound. So the mutation is never at
* risk from the notify, and the transactional coupling is intentional.
*/
-async function notifyGrantUpdate(db: DbClient, input: GrantUpdateNotifyInput): Promise<void> {
+async function notifyGrantUpdate(
+ // issue #375 M3: REQUIRES the branded transaction session — `pool` no longer
+ // structurally satisfies this parameter, so the db→pool refactor that breaks
+ // COMMIT/ROLLBACK coupling fails to COMPILE instead of merely failing a test.
+ db: DbTransactionClient,
+ input: GrantUpdateNotifyInput
+): Promise<void> {
await db.query('SELECT pg_notify($1, $2)', [
PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
buildGrantUpdateNotifyPayload(input),
diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
index c848a763a..507405b76 100644
--- a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
@@ -1,8 +1,32 @@
-import { afterAll, beforeAll, expect, describe as vitestDescribe } from 'vitest'
+import { afterAll, beforeAll, expect, vi, describe as vitestDescribe } from 'vitest'
import { it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool, type PoolClient } from 'pg'
+// issue #375 M3 (jozer review): controllable PASSTHROUGH seam. Everything stays
+// real (the actual append implementation runs) until a test arms `failNext`,
+// which makes the NEXT permission-event append inside a grant mutation throw —
+// forcing a REAL `upsertGrant` transaction to ROLLBACK so the rollback test
+// drives the actual producer instead of hand-writing pg_notify.
+const permissionEventsSeam = vi.hoisted(() => ({ failNext: null as Error | null }))
+vi.mock('../src/services/tracing/controlApiPermissionEvents.js', async importOriginal => {
+ const actual =
+ await importOriginal<typeof import('../src/services/tracing/controlApiPermissionEvents.js')>()
+ return {
+ ...actual,
+ appendControlApiPermissionEventsInTransaction: async (
+ ...args: Parameters<typeof actual.appendControlApiPermissionEventsInTransaction>
+ ) => {
+ if (permissionEventsSeam.failNext) {
+ const err = permissionEventsSeam.failNext
+ permissionEventsSeam.failNext = null
+ throw err
+ }
+ return actual.appendControlApiPermissionEventsInTransaction(...args)
+ },
+ }
+})
+
/**
* issue #375 (P3) — grant-update NOTIFY on real PostgreSQL.
*
@@ -142,17 +166,18 @@ describeRealPostgres(
})
})
- it('discards BOTH the mutation and the NOTIFY when a real mutation transaction ROLLBACKs', async () => {
- // This proves the grant-update NOTIFY is transactionally coupled to the grant
- // mutation (delivered on COMMIT, discarded on ROLLBACK).
- //
- // WHY not fail a real service call (upsert/delete/revoke) after its notify:
- // `notifyGrantUpdate` is the TERMINAL statement of each of those functions —
- // nothing in them can throw after it — so a "production mutation that fails
- // after the notify" cannot be constructed. Instead we reproduce the exact
- // production shape: a REAL mutation (UPDATE of the committed grant) plus the
- // REAL notify (same channel + payload builder) inside the production
- // `withTransaction` wrapper, then abort it — and assert BOTH are discarded.
+ it('discards BOTH the mutation and the NOTIFY when the REAL upsertGrant transaction ROLLBACKs', async () => {
+ // issue #375 M3 (jozer review): this drives the ACTUAL producer, not a
+ // hand-written pg_notify. `appendControlApiPermissionEventsInTransaction`
+ // runs inside the upsert transaction (before its terminal notify); the
+ // armed seam makes it throw, so the REAL `upsertGrant` — advisory lock,
+ // row mutation, audit append — rolls back, and BOTH the row change and
+ // any NOTIFY from that transaction must be discarded. This pins the
+ // producer's transactional atomicity (e.g. moving the grant mutation or
+ // the audit append off the transaction client goes RED here). The
+ // notify-specific `db → pool` swap is pinned by the unit-lane
+ // transaction-client assertion and, at compile time, by the
+ // `DbTransactionClient` brand on `notifyGrantUpdate`.
received.length = 0
const before = await db.pool.query<{ default_target_ref: string | null }>(
`SELECT default_target_ref FROM plugin_workload_sdk_grants
@@ -160,27 +185,31 @@ describeRealPostgres(
[NS, RECIPE]
)
+ permissionEventsSeam.failNext = new Error('injected permission-event failure (issue #375 M3)')
await expect(
- db.withTransaction(async client => {
- await client.query(
- `UPDATE plugin_workload_sdk_grants SET default_target_ref = 'ROLLED-BACK-SENTINEL'
- WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,
- [NS, RECIPE]
- )
- await client.query('SELECT pg_notify($1, $2)', [
- sdk.PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
- sdk.buildGrantUpdateNotifyPayload({ recipeNamespace: NS, recipeName: RECIPE }),
- ])
- throw new Error('force rollback')
- })
- ).rejects.toThrow('force rollback')
-
- // The NOTIFY was discarded on rollback...
+ sdk.upsertGrant(
+ {
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ capabilityFamily: 'promptBridge',
+ provider: 'openai',
+ allowedModels: ['gpt-5'],
+ allowedCallers: ['api'],
+ defaultTargetRef: 'ROLLED-BACK-SENTINEL',
+ },
+ OPERATOR_ID
+ )
+ ).rejects.toThrow('injected permission-event failure')
+ expect(
+ permissionEventsSeam.failNext,
+ 'the injected failure must have been consumed by the real upsert path'
+ ).toBeNull()
+
+ // The transaction never delivered a NOTIFY...
const delivered = await waitForGrantNotification(1_000)
expect(delivered, 'a rolled-back NOTIFY must never be delivered').toBeUndefined()
- // ...and so was the mutation — the grant is unchanged, proving the notify
- // shared the mutation's transaction.
+ // ...and the REAL mutation was rolled back — the grant row is unchanged.
const after = await db.pool.query<{ default_target_ref: string | null }>(
`SELECT default_target_ref FROM plugin_workload_sdk_grants
WHERE recipe_namespace = $1 AND recipe_name = $2 AND capability_family = 'promptBridge'`,
diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.ts
index bad27b537..6b8669294 100644
--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts
@@ -21,14 +21,27 @@ import {
const permissionEvents = vi.hoisted(() => ({ append: vi.fn() }))
+// issue #375 M3 (jozer review): the client handed out by the mocked
+// withTransaction is DISTINGUISHABLE from the pool. Its query spy DELEGATES to
+// pool.query (so the per-test mockResolvedValueOnce queues keep driving row
+// responses) but records its own calls — so an out-of-transaction refactor
+// (`notifyGrantUpdate(pool, …)` instead of the in-transaction `db`) never
+// reaches `txClient` and turns the NOTIFY assertions below RED. The previous
+// mock passed `work({ query: pool.query })`, making the transaction client and
+// the pool literally the same object and that distinction unrepresentable.
+const txClient = vi.hoisted(() => ({ query: vi.fn() }))
+
// Mock the pg pool so we can inspect the exact SQL + bind parameters the real
// db-layer functions produce.
vi.mock('../src/db.js', () => ({
...(() => {
const query = vi.fn().mockResolvedValue({ rows: [{ prompt_bridge_count: 1 }], rowCount: 1 })
+ txClient.query.mockImplementation((...args: unknown[]) =>
+ (query as (...inner: unknown[]) => unknown)(...args)
+ )
return {
pool: { query, connect: vi.fn() },
- withTransaction: (work: (db: { query: typeof query }) => unknown) => work({ query }),
+ withTransaction: (work: (db: { query: typeof txClient.query }) => unknown) => work(txClient),
}
})(),
}))
@@ -65,6 +78,7 @@ function mockUpsertGrantQueries(row: Record<string, unknown>): void {
describe('JIT credential ticket jti registry', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
})
it('registers a non-secret, invocation-bound jti and reports insertion conflicts', async () => {
@@ -145,6 +159,7 @@ describe('JIT credential ticket jti registry', () => {
describe('ordered provider-attempt reservation', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
})
it('reserves only the next authorized target after an eligible prior failure', async () => {
@@ -245,6 +260,7 @@ describe('ordered provider-attempt reservation', () => {
describe('legacy promptBridge inventory', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
})
it('reports legacy shape without rewriting policy or exposing secrets', async () => {
@@ -314,6 +330,7 @@ describe('legacy promptBridge inventory', () => {
describe('invocation retry lifecycle timestamps', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
vi.mocked(pool.query).mockResolvedValue({ rows: [], rowCount: 1 } as never)
})
@@ -451,6 +468,7 @@ describe('upsertGrant — provider column (R1)', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
})
it('persists the explicit provider and maps it back on the returned grant', async () => {
@@ -579,6 +597,7 @@ describe('resolveRecipientProfiles', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
})
it('returns [] without querying when no ref is a UUID', async () => {
@@ -691,6 +710,7 @@ describe('Plugin Workload SDK governed permission events', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
})
@@ -716,7 +736,9 @@ describe('Plugin Workload SDK governed permission events', () => {
)
expect(permissionEvents.append).toHaveBeenCalledWith(
- expect.objectContaining({ query: pool.query }),
+ // issue #375 M3: the audit append must receive the TRANSACTION client
+ // handed out by withTransaction, never the pool.
+ expect.objectContaining({ query: txClient.query }),
expect.objectContaining({
operatorSub: OPERATOR_ID,
changes: [
@@ -769,6 +791,7 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {
beforeEach(() => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
})
@@ -796,7 +819,9 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {
revokePluginWorkloadSdkForRecipe('sandbox-recipes', 'sdk-recipe', actor)
).resolves.toMatchObject({ state: 'revoking', revoked: 1 })
expect(permissionEvents.append).toHaveBeenCalledWith(
- expect.objectContaining({ query: pool.query }),
+ // issue #375 M3: the audit append must receive the TRANSACTION client
+ // handed out by withTransaction, never the pool.
+ expect.objectContaining({ query: txClient.query }),
expect.objectContaining({
operatorSub: 'wrc-provisioner',
internalPrincipal,
@@ -833,7 +858,9 @@ describe('Plugin Workload SDK internal revocation audit actor', () => {
finalizePluginWorkloadSdkRevocation('sandbox-recipes', 'sdk-recipe', REVOCATION_ID, actor)
).resolves.toMatchObject({ state: 'disabled', disabled: 1 })
expect(permissionEvents.append).toHaveBeenCalledWith(
- expect.objectContaining({ query: pool.query }),
+ // issue #375 M3: the audit append must receive the TRANSACTION client
+ // handed out by withTransaction, never the pool.
+ expect.objectContaining({ query: txClient.query }),
expect.objectContaining({
operatorSub: 'wrc-provisioner',
internalPrincipal,
@@ -905,6 +932,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('upsertGrant emits a transactional pg_notify on the grant-update channel', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
vi.mocked(pool.query)
@@ -947,12 +975,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
'operator-1'
)
- const notifyCall = vi
- .mocked(pool.query)
- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
- | [string, unknown[]]
- | undefined
- expect(notifyCall).toBeDefined()
+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not
+ // the pool — searching txClient (not pool.query) is what turns the
+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is
+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).
+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>
+ String(sql).includes('pg_notify')
+ ) as unknown as [string, unknown[]] | undefined
+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()
const [, params] = notifyCall as [string, unknown[]]
expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
expect(JSON.parse(String(params[1]))).toEqual({
@@ -964,6 +994,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('deleteGrant emits a transactional pg_notify on the grant-update channel', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
vi.mocked(pool.query)
@@ -997,12 +1028,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
'operator-1'
)
- const notifyCall = vi
- .mocked(pool.query)
- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
- | [string, unknown[]]
- | undefined
- expect(notifyCall).toBeDefined()
+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not
+ // the pool — searching txClient (not pool.query) is what turns the
+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is
+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).
+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>
+ String(sql).includes('pg_notify')
+ ) as unknown as [string, unknown[]] | undefined
+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()
const [, params] = notifyCall as [string, unknown[]]
expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
expect(JSON.parse(String(params[1]))).toMatchObject({
@@ -1013,6 +1046,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('revokePluginWorkloadSdkForRecipe emits a transactional pg_notify on the grant-update channel', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
vi.mocked(pool.query)
@@ -1034,12 +1068,14 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
operatorSub: 'operator-1',
})
- const notifyCall = vi
- .mocked(pool.query)
- .mock.calls.find(([sql]) => String(sql).includes('pg_notify')) as unknown as
- | [string, unknown[]]
- | undefined
- expect(notifyCall).toBeDefined()
+ // issue #375 M3: the pg_notify MUST run on the in-transaction client, not
+ // the pool — searching txClient (not pool.query) is what turns the
+ // `notifyGrantUpdate(pool, …)` refactor RED (a pool-issued NOTIFY is
+ // autocommitted and escapes the mutation's COMMIT/ROLLBACK coupling).
+ const notifyCall = txClient.query.mock.calls.find(([sql]) =>
+ String(sql).includes('pg_notify')
+ ) as unknown as [string, unknown[]] | undefined
+ expect(notifyCall, 'pg_notify must be issued on the transaction client').toBeDefined()
const [, params] = notifyCall as [string, unknown[]]
expect(params[0]).toBe(PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL)
expect(JSON.parse(String(params[1]))).toMatchObject({
@@ -1055,6 +1091,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
// NOTIFY again — a reconcile+pg_notify spin. Match upsert/delete: notify only
// on a real mutation.
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
permissionEvents.append.mockResolvedValue('operation-id')
vi.mocked(pool.query)
@@ -1085,6 +1122,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('does NOT emit pg_notify when deleteGrant matches no row (rowCount 0)', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
vi.mocked(pool.query)
.mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
@@ -1107,6 +1145,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('does NOT emit pg_notify on a revoke that finds no grants (state=missing)', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
vi.mocked(pool.query)
.mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
@@ -1124,6 +1163,7 @@ describe('Plugin Workload SDK grant-update NOTIFY (issue #375 P3)', () => {
it('does NOT emit pg_notify on a revoke that conflicts on revocation epoch (state=conflict)', async () => {
vi.mocked(pool.query).mockReset()
+ txClient.query.mockClear()
permissionEvents.append.mockReset()
vi.mocked(pool.query)
.mockResolvedValueOnce({ rows: [], rowCount: 0 } as never) // recipe advisory lock
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index c4ae4b6de..6bbfbc5b1 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.291",
+ "version": "0.1.292",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.291",
+ "version": "0.1.292",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 29782fa82..3f80d255c 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.291",
+ "version": "0.1.292",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/db.ts b/workflow-recipes/src/db.ts
index 77195f12d..c895f1471 100644
--- a/workflow-recipes/src/db.ts
+++ b/workflow-recipes/src/db.ts
@@ -69,6 +69,31 @@ export function getPool(): Pool {
return pool
}
+/**
+ * issue #375 (M4, jozer review): dedicated single-connection pool for the
+ * grant-update LISTEN session.
+ *
+ * The shared pool has `poolMax` (production default 4) slots, and TWO consumers
+ * pin a client for the process lifetime: dbRunProcessor's LISTEN session
+ * (pre-existing) and the grant-update listener (new). Checking the second
+ * LISTEN session out of the shared pool cut working headroom from 3 to 2 and
+ * made the 30s tick (pollOrphans + checkStuckRuns) plus one CRD MODIFIED
+ * enough to exhaust it (5s connectionTimeoutMillis rejections). A LISTEN
+ * session should not be a shared-pool checkout at all: this dedicated
+ * `max: 1` pool gives the listener its own connection, so it consumes ZERO
+ * shared-pool slots while keeping the exact PoolClient checkout/release and
+ * reconnect semantics the listener is built (and tested) against.
+ */
+export function createGrantUpdateListenerPool(cfg: DbConfig): Pool {
+ const p = new Pool({ ...createPoolConfig(cfg), max: 1 })
+ // Same mandate as initDb: a client error without a pool-level listener
+ // crashes the process via 'error' event propagation.
+ p.on('error', (err: Error) => {
+ log.error('pg grant-update listener pool error', { err: err.message })
+ })
+ return p
+}
+
/** Test seam — allow unit tests to swap the module-level pool. */
export function __setPoolForTests(p: Pool | null): void {
pool = p
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 3f7e62f1e..d3cff3bf0 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -426,6 +426,10 @@ describe('shouldPatchRecipeStatus', () => {
// The exact incident shape: phase and top-level message are unchanged, the
// ONLY change is the SDK capability state. Before the fix this returned
// false and the validated state was never published.
+ // M2 (jozer review): the persisted verifiedAt is FRESH, so the >5-min
+ // throttle branch stays CLOSED and this can only pass through the semantic
+ // field comparisons — a hardcoded stale timestamp made it pass via the
+ // throttle, pinning nothing about the incident's mechanism.
expect(
shouldPatchRecipeStatus(
sdkRecipe({
@@ -433,7 +437,7 @@ describe('shouldPatchRecipeStatus', () => {
promptBridge: true,
clientNotifications: false,
message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',
- verifiedAt: '2026-08-17T17:24:50.000Z',
+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),
}),
{
phase: 'active',
@@ -646,6 +650,107 @@ describe('shouldPatchRecipeStatus', () => {
)
).toBe(false)
})
+
+ // ── issue #375 M2 (jozer review): every projection field comparison is pinned
+ // INDIVIDUALLY. Each case varies exactly ONE field between the persisted
+ // capability and the projection, with a FRESH persisted verifiedAt so the
+ // >5-min throttle branch stays closed — removing that single field's
+ // comparison in pluginWorkloadSdkProjectionChanged turns that case RED.
+ describe('per-field projection comparisons (issue #375 M2)', () => {
+ function persistedBase(): Record<string, unknown> {
+ return {
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'Capability validated (promptBridge)',
+ policyRevision: 3,
+ policyHash: 'sha256:policy-a',
+ defaultTargetRef: 'primary-openai',
+ bootstrapPodUid: 'pod-abc',
+ bootstrapContractVersion: 2,
+ bootstrapProvider: 'openai',
+ bootstrapModel: 'gpt-5.4-mini',
+ // FRESH: keeps the verifiedAt throttle CLOSED so a `true` can only come
+ // from the varied field's own comparison.
+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),
+ }
+ }
+
+ function projectionFrom(
+ persisted: Record<string, unknown>,
+ override: Record<string, unknown>
+ ): { conditions: never[]; capability: never } {
+ const { verifiedAt: _persistedVerifiedAt, ...semantic } = persisted
+ return {
+ conditions: [],
+ capability: {
+ ...semantic,
+ verifiedAt: new Date().toISOString(),
+ validatedAt: new Date().toISOString(),
+ ...override,
+ } as never,
+ }
+ }
+
+ const cases: Array<[field: string, override: Record<string, unknown>]> = [
+ ['state', { state: 'awaiting_policy' }],
+ ['message', { message: 'Capability validated (promptBridge, clientNotifications)' }],
+ ['policyRevision', { policyRevision: 4 }],
+ ['policyHash', { policyHash: 'sha256:policy-b' }],
+ ['defaultTargetRef', { defaultTargetRef: 'fallback-anthropic' }],
+ ['bootstrapPodUid', { bootstrapPodUid: 'pod-def' }],
+ ['bootstrapContractVersion', { bootstrapContractVersion: null }],
+ ['bootstrapProvider', { bootstrapProvider: 'anthropic' }],
+ ['bootstrapModel', { bootstrapModel: 'claude-sonnet-4-6' }],
+ ['promptBridge', { promptBridge: false }],
+ ['clientNotifications', { clientNotifications: true }],
+ ]
+
+ for (const [field, override] of cases) {
+ it(`a lone ${field} change forces a patch (throttle closed)`, () => {
+ const persisted = persistedBase()
+ expect(
+ shouldPatchRecipeStatus(sdkRecipe(persisted), {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: projectionFrom(persisted, override),
+ })
+ ).toBe(true)
+ })
+ }
+
+ it('control: with NO field varied and a fresh persisted verifiedAt, no patch is emitted', () => {
+ const persisted = persistedBase()
+ expect(
+ shouldPatchRecipeStatus(sdkRecipe(persisted), {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: projectionFrom(persisted, {}),
+ })
+ ).toBe(false)
+ })
+
+ it('first-ever publish: a computed capability with NOTHING persisted forces a patch', () => {
+ // Pins the `if (!current) return true` branch — the first publish of
+ // status.pluginWorkloadSdk after a recipe declares the capability.
+ expect(
+ shouldPatchRecipeStatus(
+ makeWorkflowRecipe({
+ spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
+ status: { phase: 'active', message: 'All workloads deployed' },
+ }),
+ {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: projectionFrom(persistedBase(), {}),
+ }
+ )
+ ).toBe(true)
+ })
+ })
})
describe('infrastructure telemetry projections', () => {
@@ -2672,3 +2777,103 @@ describe('external egress periodic refresh (issue #299 §3.2/§5)', () => {
}
})
})
+
+// ── issue #375 M1/M4 (jozer review): the REAL constructor + start()/stop()
+// wiring of the grant-update listener. Every prior watcher test hand-installed
+// collaborators onto Object.create(prototype), so removing the constructor
+// wiring or the start()/stop() calls left the whole suite green. These tests
+// construct the real watcher and go RED if:
+// - the constructor stops creating the listener,
+// - the listener is handed the SHARED pool instead of its dedicated one (M4),
+// - `await this.grantUpdateListener.start()` is removed from start(),
+// - `await this.grantUpdateListener.stop()` is removed from stop().
+describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issue #375 M1/M4)', () => {
+ function makeFakeKubeConfig() {
+ return {
+ makeApiClient: vi.fn(() => ({})),
+ } as unknown as import('@kubernetes/client-node').KubeConfig
+ }
+
+ type ListenerStub = { start: () => Promise<void>; stop: () => Promise<void> }
+ type InternalWatcher = {
+ grantUpdateListener: ListenerStub | null
+ grantUpdateListenerPool: { connect?: unknown; end: () => Promise<void> } | null
+ dbRunProcessor: { start: () => Promise<void>; stop: () => Promise<void> } | null
+ traceReporter: null
+ tryInitializeWorkflow: () => Promise<void>
+ startWatch: () => Promise<void>
+ startSecretWatch: () => Promise<void>
+ startBrokerTokenRotationLoop: () => void
+ startRuntimeCredentialRefreshLoop: () => void
+ startWorkloadStatusRefreshLoop: () => void
+ startExternalEgressRefreshLoop: () => void
+ }
+
+ it('constructor wires a listener backed by a DEDICATED pool — LISTEN never checks out of the shared pool (M4)', async () => {
+ const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }
+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)
+ const internal = watcher as unknown as InternalWatcher
+
+ expect(internal.grantUpdateListener).not.toBeNull()
+ expect(internal.grantUpdateListenerPool).not.toBeNull()
+ // M4: a lifetime-pinned LISTEN session must not burn one of the shared
+ // pool's 4 slots — the listener owns a dedicated single-connection pool.
+ expect(internal.grantUpdateListenerPool).not.toBe(sharedPool)
+
+ // Behavioral pin: starting the listener checks out of the DEDICATED pool
+ // and never touches the shared one.
+ const fakeClient = {
+ on: vi.fn(),
+ query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
+ release: vi.fn(),
+ removeAllListeners: vi.fn(),
+ }
+ const dedicatedConnect = vi.fn().mockResolvedValue(fakeClient)
+ ;(internal.grantUpdateListenerPool as { connect?: unknown }).connect = dedicatedConnect
+ await (internal.grantUpdateListener as ListenerStub).start()
+ expect(dedicatedConnect).toHaveBeenCalledTimes(1)
+ expect(sharedPool.connect).not.toHaveBeenCalled()
+ await (internal.grantUpdateListener as ListenerStub).stop()
+ })
+
+ it('does not create the listener (or its pool) without a DB pool', () => {
+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), null)
+ const internal = watcher as unknown as InternalWatcher
+ expect(internal.grantUpdateListener).toBeNull()
+ expect(internal.grantUpdateListenerPool).toBeNull()
+ })
+
+ it('start() starts and stop() stops the grant-update listener and ends its dedicated pool (M1)', async () => {
+ const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }
+ const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)
+ const internal = watcher as unknown as InternalWatcher
+
+ // Stub the heavy collaborators. The REAL start()/stop() method bodies stay
+ // in play, so neutering the listener wiring inside them goes RED here.
+ internal.tryInitializeWorkflow = vi.fn().mockResolvedValue(undefined)
+ internal.startWatch = vi.fn().mockResolvedValue(undefined)
+ internal.startSecretWatch = vi.fn().mockResolvedValue(undefined)
+ internal.startBrokerTokenRotationLoop = vi.fn()
+ internal.startRuntimeCredentialRefreshLoop = vi.fn()
+ internal.startWorkloadStatusRefreshLoop = vi.fn()
+ internal.startExternalEgressRefreshLoop = vi.fn()
+ internal.dbRunProcessor = {
+ start: vi.fn().mockResolvedValue(undefined),
+ stop: vi.fn().mockResolvedValue(undefined),
+ }
+ internal.traceReporter = null
+ const listenerStart = vi.fn().mockResolvedValue(undefined)
+ const listenerStop = vi.fn().mockResolvedValue(undefined)
+ internal.grantUpdateListener = { start: listenerStart, stop: listenerStop }
+ const poolEnd = vi.fn().mockResolvedValue(undefined)
+ internal.grantUpdateListenerPool = { end: poolEnd }
+
+ await watcher.start()
+ expect(listenerStart).toHaveBeenCalledTimes(1)
+
+ await watcher.stop()
+ expect(listenerStop).toHaveBeenCalledTimes(1)
+ // M4: the watcher owns the dedicated pool and must close its connection.
+ expect(poolEnd).toHaveBeenCalledTimes(1)
+ })
+})
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index e7866edca..8643b8fd7 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -14,7 +14,7 @@
import * as k8s from '@kubernetes/client-node'
import type { Pool } from 'pg'
import { loadConfig } from './config'
-import { getPool, initDb } from './db'
+import { createGrantUpdateListenerPool, getPool, initDb } from './db'
import {
type GovernedTraceReporter,
type WorkflowInfrastructureTelemetryProjection,
@@ -619,6 +619,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// a DB (same gate as dbRunProcessor); without a DB the listener never starts
// and the level-triggered reconcile remains the sole convergence path.
private grantUpdateListener: GrantUpdateListener | null = null
+ // issue #375 (M4, jozer review): the LISTEN session pins its client for the
+ // process lifetime, so it gets its OWN single-connection pool and consumes
+ // zero slots of the shared `poolMax` (default 4) pool that dbRunProcessor,
+ // leader election, and run-sync all contend for. Owned (and ended) by this
+ // watcher because it exists solely for the grant-update listener.
+ private grantUpdateListenerPool: Pool | null = null
private config = loadConfig()
private stopped = false
private runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null = null
@@ -686,8 +692,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// issue #375 (P3): on a grant-update NOTIFY, force-reconcile the affected
// recipe through the SAME path the 30s watchdog uses. The dispatch body is
// a named method so it is unit-testable without standing up a DB.
+ // issue #375 (M4): the listener LISTENs on a DEDICATED single-connection
+ // pool, never the shared one — a lifetime-pinned LISTEN checkout would
+ // permanently burn one of the shared pool's 4 slots (dbRunProcessor
+ // already pins another).
+ this.grantUpdateListenerPool = createGrantUpdateListenerPool(this.config.db)
this.grantUpdateListener = createGrantUpdateListener({
- pool,
+ pool: this.grantUpdateListenerPool,
onGrantUpdate: ({ recipeNamespace, recipeName }) =>
this.handleGrantUpdateNotification(recipeNamespace, recipeName),
})
@@ -1389,6 +1400,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
if (this.grantUpdateListener) {
await this.grantUpdateListener.stop()
}
+ if (this.grantUpdateListenerPool) {
+ // issue #375 (M4): the watcher owns the dedicated LISTEN pool — end it on
+ // shutdown so its single connection is closed with the process.
+ await this.grantUpdateListenerPool.end()
+ this.grantUpdateListenerPool = null
+ }
if (this.traceReporter) {
const result = await this.traceReporter.stopAndDrain()
if (!result.drained) {
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
index a25a60d6c..c678b7571 100644
--- a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
@@ -40,6 +40,13 @@ function makeFakeClient(opts: { failQuery?: boolean } = {}) {
return { rows: [], rowCount: 0 }
}),
release: vi.fn(),
+ // pg PoolClient is an EventEmitter; the listener strips its own handlers
+ // before handing a subscribed client back to the pool (issue #375 M5).
+ removeAllListeners: vi.fn((event?: string) => {
+ if (event) delete handlers[event]
+ else for (const key of Object.keys(handlers)) delete handlers[key]
+ return client
+ }),
}
return {
client,
@@ -194,50 +201,116 @@ describe('createGrantUpdateListener', () => {
describe('createGrantUpdateListener — sustained reconnect regime (issue #375 Finding 1)', () => {
interface FakeGenClient {
released: boolean
+ destroyed: boolean
+ subscribed: boolean
+ handlerCount: (event: string) => number
on: (event: string, handler: (...args: unknown[]) => void) => FakeGenClient
query: ReturnType<typeof vi.fn>
release: ReturnType<typeof vi.fn>
+ removeAllListeners: (event?: string) => FakeGenClient
emitError: (err: Error) => void
emitNotification: (msg: { channel: string; payload?: string }) => void
}
- /** A fresh fake client per connect(), tracking created + per-client released. */
+ /**
+ * pg-pool-FAITHFUL fake (issue #375 M5, jozer review): the previous fake
+ * modeled a plain-released client as silently non-dispatching and minted a
+ * fresh object on every connect() — exactly the two divergences from pg-pool
+ * that hid the still-subscribed-client leak. Faithful semantics:
+ * - `release()` (no error) re-idles the SAME object; pg-pool `_release`
+ * re-attaches only its OWN idle listener — user handlers stay attached and
+ * the server-side LISTEN subscription stays live, so the client KEEPS
+ * dispatching after a plain release.
+ * - `connect()` REUSES an idle client before creating a new one.
+ * - `release(err)` DESTROYS the client (removed from the pool; its session
+ * — and therefore its subscription — is gone).
+ * - `LISTEN`/`UNLISTEN` queries toggle the server-side subscription; only a
+ * subscribed, non-destroyed session receives a broadcast NOTIFY.
+ */
function makeFakeClientFactory() {
const created: FakeGenClient[] = []
- function connect(): FakeGenClient {
+ const idle: FakeGenClient[] = []
+ let listenHold: Promise<void> | null = null
+ let releaseListenHold: (() => void) | null = null
+ function makeClient(): FakeGenClient {
const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}
const client: FakeGenClient = {
released: false,
+ destroyed: false,
+ subscribed: false,
+ handlerCount: (event: string) => (handlers[event] ?? []).length,
on(event, handler) {
;(handlers[event] ??= []).push(handler)
return client
},
- query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
- release: vi.fn(() => {
+ query: vi.fn(async (sql: string) => {
+ if (/^LISTEN\b/.test(sql)) {
+ if (listenHold) await listenHold
+ client.subscribed = true
+ }
+ if (/^UNLISTEN\b/.test(sql)) client.subscribed = false
+ return { rows: [], rowCount: 0 }
+ }),
+ release: vi.fn((err?: Error) => {
client.released = true
+ if (err) {
+ client.destroyed = true
+ client.subscribed = false
+ } else if (!client.destroyed) {
+ idle.push(client)
+ }
}),
+ removeAllListeners(event?: string) {
+ if (event) delete handlers[event]
+ else for (const key of Object.keys(handlers)) delete handlers[key]
+ return client
+ },
emitError(err: Error) {
- for (const h of handlers.error ?? []) h(err)
+ for (const h of [...(handlers.error ?? [])]) h(err)
},
emitNotification(msg) {
- // A released client is back in the pool and must not dispatch.
- if (client.released) return
- for (const h of handlers.notification ?? []) h(msg)
+ // Faithful: dispatch depends ONLY on the session being alive and
+ // subscribed — NOT on pool checkout state. A plain-released client
+ // keeps dispatching (that is the leak the old fake hid).
+ if (client.destroyed || !client.subscribed) return
+ for (const h of [...(handlers.notification ?? [])]) h(msg)
},
}
+ return client
+ }
+ function connect(): FakeGenClient {
+ const reused = idle.shift()
+ if (reused) {
+ reused.released = false
+ return reused
+ }
+ const client = makeClient()
created.push(client)
return client
}
return {
created,
+ idle,
connect,
- /** Live (checked-out, unreleased) sessions right now. */
- liveCount: () => created.filter(c => !c.released).length,
+ /** Live (checked-out, unreleased, undestroyed) sessions right now. */
+ liveCount: () => created.filter(c => !c.released && !c.destroyed).length,
current: () => created[created.length - 1],
- /** Simulate a real pg NOTIFY broadcast to EVERY live session. */
+ /** Simulate a real pg NOTIFY broadcast to EVERY subscribed session. */
broadcast(msg: { channel: string; payload?: string }) {
for (const c of created) c.emitNotification(msg)
},
+ /** Park the next LISTEN round-trip until releaseListen() is called. */
+ holdListen() {
+ listenHold = new Promise<void>(resolve => {
+ releaseListenHold = () => {
+ listenHold = null
+ resolve()
+ }
+ })
+ },
+ releaseListen() {
+ releaseListenHold?.()
+ },
}
}
@@ -360,6 +433,101 @@ describe('createGrantUpdateListener — sustained reconnect regime (issue #375 F
await listener.stop()
expect(factory.liveCount()).toBe(0)
})
+
+ // ── issue #375 M5 (jozer review): a client that ran LISTEN and carries this
+ // module's handlers must never be plain-release()d back to the shared pool.
+ // pg-pool re-idles the SAME object (no UNLISTEN, user handlers kept), so the
+ // session stays subscribed: it dispatches after stop() and double-dispatches
+ // after a reuse. These three tests drive the exact leak paths with the
+ // pg-pool-faithful fake and go RED without the UNLISTEN/removeAllListeners/
+ // generation hardening.
+ it('M5a: stop() during the in-flight LISTEN round-trip leaves NO subscribed, dispatching session behind', async () => {
+ const factory = makeFakeClientFactory()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => factory.connect() as never,
+ })
+
+ factory.holdListen()
+ const startPromise = listener.start() // parks inside the LISTEN round-trip
+ await flush()
+ await listener.stop() // stop() lands while LISTEN is in flight
+ factory.releaseListen() // LISTEN resolves AFTER stop
+ await startPromise
+ await flush()
+
+ // The released client went back to the pool; it must NOT still be a live
+ // subscriber that dispatches into a stopped listener.
+ factory.broadcast(NOTIFY)
+ expect(onGrantUpdate, 'post-stop dispatch from a pool-returned client').not.toHaveBeenCalled()
+ expect(factory.liveCount(), 'no session may stay checked out').toBe(0)
+ expect(
+ factory.created[0].handlerCount('notification'),
+ 'listener handlers must be stripped before the client re-enters the pool'
+ ).toBe(0)
+ })
+
+ it('M5b: stop() then start() reusing the pooled client dispatches exactly once (no stacked handler pairs)', async () => {
+ const factory = makeFakeClientFactory()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => factory.connect() as never,
+ })
+
+ await listener.start()
+ await listener.stop()
+ await listener.start() // pg-pool hands back the SAME client object
+
+ expect(factory.created, 'pool-faithful fake must reuse the idle client').toHaveLength(1)
+ factory.broadcast(NOTIFY)
+ // Without stripping handlers on release, each stop→start cycle stacks one
+ // handler pair on the same client and one NOTIFY dispatches twice.
+ expect(onGrantUpdate).toHaveBeenCalledTimes(1)
+ await listener.stop()
+ expect(factory.liveCount()).toBe(0)
+ })
+
+ it('M5c: a reconnect parked across stop()→start() must not leave the superseded session subscribed (single dispatch)', async () => {
+ const factory = makeFakeClientFactory()
+ const { sleep, resolvers } = resolverQueueSleep()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => factory.connect() as never,
+ sleep,
+ })
+
+ await listener.start()
+ const clientA = factory.current()
+ clientA.emitError(new Error('drop')) // destroys A, parks a reconnect in its backoff sleep
+ await flush()
+ await listener.stop()
+ await listener.start() // session B attaches and becomes current
+ await flush()
+ const clientB = factory.current()
+ expect(clientB.subscribed).toBe(true)
+
+ // The parked reconnect resumes AFTER the restart: session C attaches and the
+ // non-overwrite guard releases B. Pre-fix that release was plain — B stayed
+ // subscribed with handlers attached → one logical NOTIFY dispatched twice.
+ ;(resolvers.shift() as () => void)()
+ await flush()
+
+ expect(factory.liveCount(), 'at most one live session').toBeLessThanOrEqual(1)
+ onGrantUpdate.mockClear()
+ factory.broadcast(NOTIFY)
+ expect(onGrantUpdate, 'superseded session must not double-dispatch').toHaveBeenCalledTimes(1)
+ await listener.stop()
+ expect(factory.liveCount()).toBe(0)
+ })
})
describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.ts
index b3aeb681c..eeb184d97 100644
--- a/workflow-recipes/src/reconciler/grantUpdateListener.ts
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts
@@ -103,6 +103,45 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
let stopped = true
// H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.
let reconnecting = false
+ // M5 (jozer review): monotonic session generation. A session's notification
+ // handler dispatches ONLY while its own generation is the current one — a
+ // superseded-but-not-destroyed session (e.g. one that raced a release) goes
+ // mute instead of double-dispatching. The 'error' identity gate below covers
+ // reconnect scheduling; this covers the dispatch path it never guarded.
+ let generation = 0
+
+ /**
+ * M5 (jozer review): a client that has run LISTEN and carries this module's
+ * handlers must NEVER be plain-release()d back to the shared pool. pg-pool's
+ * `_release` re-idles the SAME Client object and re-attaches only its OWN
+ * idle listener — user handlers stay attached and the Postgres session stays
+ * subscribed, so the client keeps dispatching after stop() and
+ * double-dispatches after a pool reuse (and dbRunProcessor could check out a
+ * client with this module's stale 'error' handler still attached). Unsubscribe
+ * and strip our handlers first; if the UNLISTEN cannot be delivered the
+ * session state is unknowable, so DESTROY the client (release(err)) instead of
+ * re-idling it.
+ */
+ async function releaseSubscribedClient(attached: PoolClient): Promise<void> {
+ let unlistened = false
+ try {
+ await attached.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
+ unlistened = true
+ } catch {
+ /* session unusable — destroyed below instead of re-idled */
+ }
+ attached.removeAllListeners('notification')
+ attached.removeAllListeners('error')
+ try {
+ if (unlistened) {
+ attached.release()
+ } else {
+ attached.release(new Error('grant-update LISTEN session UNLISTEN failed — destroying'))
+ }
+ } catch {
+ /* idempotent */
+ }
+ }
async function attachListener(): Promise<void> {
let client: PoolClient | null = null
@@ -110,13 +149,24 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
client = await connect()
// H2: stop() may have run while we awaited connect(); do NOT bring up a
// LISTEN session after shutdown. Release the client and bail before any
- // handler is registered, so nothing dispatches post-stop.
+ // handler is registered, so nothing dispatches post-stop. (Plain release
+ // is correct HERE: no LISTEN has run and no handler is attached yet.)
if (stopped) {
client.release()
return
}
const attached = client
+ // M5: assigned only when this session is INSTALLED as the current
+ // listener; until then — and forever after supersession — the handler is
+ // mute (generation is bumped on every install and on stop()).
+ let sessionGeneration = -1
attached.on('notification', msg => {
+ if (sessionGeneration !== generation) {
+ log.debug('superseded listen session notification — dropped', {
+ channel: msg.channel,
+ })
+ return
+ }
if (msg.channel !== PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL || !msg.payload) return
const notification = parseGrantUpdatePayload(msg.payload)
if (!notification) {
@@ -157,32 +207,35 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
return
}
log.warn('listen session error — will reconnect', { err: err.message })
+ // M5: mute the errored session's dispatch path immediately (its client
+ // was destroyed above, but the generation bump costs nothing and closes
+ // the window where release(err) itself failed).
+ generation += 1
listenClient = null
scheduleReconnect()
})
await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
// H2: re-check after the async LISTEN resolves — stop() may have run
- // during the round-trip.
+ // during the round-trip. M5: this client has ALREADY run LISTEN with
+ // handlers attached — it must be unsubscribed and stripped, never
+ // plain-released into the pool where it would keep dispatching.
if (stopped) {
- try {
- attached.release()
- } catch {
- /* idempotent */
- }
+ await releaseSubscribedClient(attached)
return
}
// Finding 1: never overwrite a live `listenClient` without releasing the
// prior one. Defense-in-depth alongside the identity gate above — a stray
// interleaving that reaches here with a different session still checked out
- // must not leak it.
+ // must not leak it. M5: the loser ran LISTEN with handlers attached, so it
+ // gets the full unsubscribe+strip release, not a plain one.
if (listenClient && listenClient !== attached) {
- try {
- listenClient.release()
- } catch {
- /* idempotent */
- }
+ await releaseSubscribedClient(listenClient)
}
listenClient = attached
+ // M5: install-time bump — makes THIS session the sole dispatcher and any
+ // surviving predecessor stale.
+ generation += 1
+ sessionGeneration = generation
log.info('LISTEN plugin_workload_sdk_grant_update attached')
} catch (err) {
log.warn('LISTEN attach failed — will retry', {
@@ -234,18 +287,16 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
// stop()→start() restart is not blocked by a stale flag left set by a
// reconnect that was still mid-sleep when we stopped (belt-and-suspenders).
reconnecting = false
+ // M5: mute ANY session that survives teardown (e.g. one parked in an
+ // in-flight LISTEN whose release races this stop).
+ generation += 1
if (listenClient) {
- try {
- await listenClient.query(`UNLISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
- } catch {
- /* best-effort — the session is being torn down */
- }
- try {
- listenClient.release()
- } catch {
- /* idempotent */
- }
+ const current = listenClient
listenClient = null
+ // M5: unsubscribe + strip handlers before the client re-enters the
+ // pool — a plain release would re-idle a still-subscribed client whose
+ // handlers dispatch after stop() and stack up on the next start().
+ await releaseSubscribedClient(current)
}
},
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index a8a8c3dfa..a17f325a3 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -5,6 +5,7 @@ import { readFileSync } from 'node:fs'
import { resolve } from 'node:path'
import { EVENFIRE_REGISTRY_PULL_SECRET_NAME } from '@clerum/workflow-runtime-core'
import { loadConfig } from '../config'
+import { shouldPatchRecipeStatus } from '../k8sClient'
import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'
import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'
import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'
@@ -9960,4 +9961,336 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).not.toBe('failed')
})
})
+
+ // ── issue #375 BLOCKER (jozer-rami review): clientNotifications-only workflow-
+ // lane SDK recipes must FALL THROUGH the two active-phase short-circuits (the
+ // awaiting-trigger one and the steady-active one) the same way promptBridge
+ // does, so the inner reconcile gathers the bootstrap proof and the capability
+ // projects `validated` with NO skipStatusPatch. Before the fix both
+ // short-circuits returned skipStatusPatch:true, shouldPatchRecipeStatus bailed
+ // at its first line, and the awaiting_policy→validated transition was never
+ // published — the original #375 symptom, unchanged, for that family.
+ describe('issue #375 BLOCKER — clientNotifications-only workflow-lane fall-through', () => {
+ const CN_MESSAGE = 'Workflow trigger infrastructure registered'
+
+ function clientNotificationsOnlyRecipe(labels?: Record<string, string>): WorkflowRecipeCRD {
+ return makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ ...(labels ? { labels } : {}),
+ },
+ spec: {
+ steps: [{ id: 'step-1', run: snippetRun() }],
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ pluginWorkloadSdk: {
+ clientNotifications: { allowedEventTypes: ['workflow.completed'] },
+ allowedCallers: ['app'],
+ },
+ },
+ status: {
+ phase: 'active',
+ message: CN_MESSAGE,
+ pluginWorkloadSdk: {
+ state: 'awaiting_policy',
+ promptBridge: false,
+ clientNotifications: true,
+ message: 'Plugin Workload SDK clientNotifications is awaiting an operator grant',
+ verifiedAt: new Date(Date.now() - 10_000).toISOString(),
+ } as never,
+ },
+ })
+ }
+
+ function stubWorkflowReconciler() {
+ const workflowReconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: CN_MESSAGE,
+ pluginWorkloadSdkBootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'sdk-pod-uid',
+ policyReady: true,
+ clientNotificationsPolicyReady: true,
+ verifiedAt: new Date().toISOString(),
+ },
+ })
+ const reconcilePluginWorkloadSdkOnly = vi.fn()
+ ;(
+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }
+ ).config.pluginWorkloadSdkEnabled = true
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcile: typeof workflowReconcile
+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly
+ validateWorkflowSpec: () => undefined
+ }
+ }
+ ).workflowReconciler = {
+ reconcile: workflowReconcile,
+ reconcilePluginWorkloadSdkOnly,
+ validateWorkflowSpec: () => undefined,
+ }
+ return { workflowReconcile, reconcilePluginWorkloadSdkOnly }
+ }
+
+ it('awaiting-trigger lane: recomputes and publishes awaiting_policy→validated (no skipStatusPatch)', async () => {
+ // Phase active + mcpHost required (flag ON) + no run label + no
+ // workflowExecution ⇒ the awaiting-trigger short-circuit. Pre-fix this
+ // returned skipStatusPatch:true without ever calling the inner reconcile.
+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()
+ const recipe = clientNotificationsOnlyRecipe()
+
+ const result = await reconciler.reconcile(recipe)
+
+ // Fall-through proof: the inner reconcile actually ran (steps present ⇒
+ // the stepless SDK-only adapter must NOT be used).
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()
+ // The reconcile pass has an SDK opinion and does not suppress the patch.
+ expect(result.skipStatusPatch).toBeFalsy()
+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')
+ // The full chain the incident depends on: the watcher would publish it.
+ expect(shouldPatchRecipeStatus(recipe, result)).toBe(true)
+ })
+
+ it('steady-active lane (run-labelled): recomputes and publishes awaiting_policy→validated (no skipStatusPatch)', async () => {
+ // Phase active + run-id label ⇒ awaitsTriggeredRun=false ⇒ the steady-
+ // active short-circuit, which pre-fix returned skipStatusPatch:true
+ // unconditionally (no family carve-out at all).
+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()
+ const recipe = clientNotificationsOnlyRecipe({ 'clerum.io/workflow-run-id': 'run-e2e-1' })
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()
+ expect(result.skipStatusPatch).toBeFalsy()
+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')
+ expect(shouldPatchRecipeStatus(recipe, result)).toBe(true)
+ })
+
+ it('steady-active lane with an IN-PROGRESS execution: SDK recipes still short-circuit (409-window protection)', async () => {
+ // The carve-out is bounded: while a workflow execution is in progress the
+ // steady-active short-circuit is KEPT even for SDK recipes — that branch
+ // protects the coordinator's 409-free windows, and the capability
+ // publication is level-triggered so it lands on the next non-running
+ // pass. Removing the `|| wfInProgress` term from the carve-out condition
+ // turns this RED: the running recipe would fall through to the inner
+ // reconcile and drop skipStatusPatch.
+ const { workflowReconcile, reconcilePluginWorkloadSdkOnly } = stubWorkflowReconciler()
+ const recipe = clientNotificationsOnlyRecipe({ 'clerum.io/workflow-run-id': 'run-e2e-3' })
+ recipe.status = {
+ ...recipe.status,
+ phase: 'active',
+ workflowExecution: { phase: 'running' },
+ } as typeof recipe.status
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(result.skipStatusPatch).toBe(true)
+ expect(workflowReconcile).not.toHaveBeenCalled()
+ expect(reconcilePluginWorkloadSdkOnly).not.toHaveBeenCalled()
+ })
+
+ it('non-SDK workflow recipes keep both steady-state short-circuits (no regression)', async () => {
+ // The carve-out is SDK-scoped: a plain workflow recipe in the same two
+ // shapes must still short-circuit with skipStatusPatch:true and must NOT
+ // reach the inner reconcile on every steady tick.
+ const { workflowReconcile } = stubWorkflowReconciler()
+ const awaitingTrigger = makeRecipe({
+ spec: {
+ steps: [{ id: 'step-1', instruction: 'run' }],
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ },
+ status: { phase: 'active', message: CN_MESSAGE },
+ })
+ const awaitingResult = await reconciler.reconcile(awaitingTrigger)
+ expect(awaitingResult.skipStatusPatch).toBe(true)
+ expect(workflowReconcile).not.toHaveBeenCalled()
+
+ const runLabelled = makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ labels: { 'clerum.io/workflow-run-id': 'run-e2e-2' },
+ },
+ spec: {
+ steps: [{ id: 'step-1', run: snippetRun() }],
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ },
+ status: { phase: 'active', message: CN_MESSAGE },
+ })
+ const steadyResult = await reconciler.reconcile(runLabelled)
+ expect(steadyResult.skipStatusPatch).toBe(true)
+ expect(workflowReconcile).not.toHaveBeenCalled()
+ })
+ })
+
+ // ── issue #375 M1 (jozer review): the projection WIRING is pinned against the
+ // real pipeline, not hand-built ReconcileResults. Each test goes RED if its
+ // wiring line is neutered:
+ // - reconcile() must ATTACH result.pluginWorkloadSdkProjection (the single
+ // line the whole P1 fix hangs on),
+ // - projectPluginWorkloadSdk must pass the persisted capability through as
+ // existingCapability (the R1 validatedAt carry-forward),
+ // - patchStatus must REUSE the attached projection instead of recomputing.
+ describe('issue #375 M1 — projection wiring through the real pipeline', () => {
+ it('reconcile() attaches a populated pluginWorkloadSdkProjection to every result', async () => {
+ // Non-SDK recipe: the projection must still be attached (capability null =
+ // "clear the field"), proving production populates it on EVERY reconcile.
+ const result = await reconciler.reconcile(makeRecipe())
+ expect(result.pluginWorkloadSdkProjection).toBeDefined()
+ expect(result.pluginWorkloadSdkProjection?.capability).toBeNull()
+ })
+
+ it('reconcile() projects the SDK-only lane bootstrap proof into a validated capability', async () => {
+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'Plugin Workload SDK mcp-host registered',
+ pluginWorkloadSdkBootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'sdk-pod-uid',
+ provider: 'zai',
+ model: 'glm-4.7',
+ policyReady: true,
+ verifiedAt: '2026-08-04T00:00:00.000Z',
+ },
+ })
+ ;(
+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }
+ ).config.pluginWorkloadSdkEnabled = true
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly
+ }
+ }
+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }
+
+ const result = await reconciler.reconcile(
+ makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },
+ },
+ })
+ )
+
+ // The REAL pipeline attached the projection and it carries the computed
+ // capability — no test-side hand-building involved.
+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')
+ expect(result.pluginWorkloadSdkProjection?.capability?.bootstrapPodUid).toBe('sdk-pod-uid')
+ })
+
+ it('carries forward the persisted validatedAt on a steady validated capability (R1 wiring)', async () => {
+ const FIRST_VALIDATED_AT = '2026-08-01T00:00:00.000Z'
+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'Plugin Workload SDK mcp-host registered',
+ pluginWorkloadSdkBootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'sdk-pod-uid',
+ provider: 'zai',
+ model: 'glm-4.7',
+ policyReady: true,
+ verifiedAt: new Date().toISOString(),
+ },
+ })
+ ;(
+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }
+ ).config.pluginWorkloadSdkEnabled = true
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly
+ }
+ }
+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }
+
+ const result = await reconciler.reconcile(
+ makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },
+ },
+ status: {
+ phase: 'active',
+ pluginWorkloadSdk: {
+ state: 'validated',
+ promptBridge: true,
+ clientNotifications: false,
+ validatedAt: FIRST_VALIDATED_AT,
+ verifiedAt: new Date(Date.now() - 60_000).toISOString(),
+ } as never,
+ },
+ })
+ )
+
+ // existingCapability wiring (workflowRecipeReconciler.ts): the stable
+ // validatedAt marker must be CARRIED FORWARD, not re-stamped to `now`.
+ expect(result.pluginWorkloadSdkProjection?.capability?.state).toBe('validated')
+ expect(result.pluginWorkloadSdkProjection?.capability?.validatedAt).toBe(FIRST_VALIDATED_AT)
+ })
+
+ it('patchStatus REUSES the attached projection instead of recomputing it', async () => {
+ // The attached projection is deliberately DISTINCT from anything
+ // patchStatus could recompute from this recipe/result (no bootstrap proof
+ // ⇒ a recompute projects degraded/BootstrapNotReady). If patchStatus
+ // recomputes instead of reusing, the marker never reaches the patch body.
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },
+ },
+ status: { phase: 'active' },
+ })
+ ;(
+ reconciler as unknown as { config: { pluginWorkloadSdkEnabled: boolean } }
+ ).config.pluginWorkloadSdkEnabled = true
+
+ const attachedCapability = {
+ state: 'validated' as const,
+ promptBridge: true,
+ clientNotifications: false,
+ message: 'REUSED-PROJECTION-MARKER',
+ bootstrapPodUid: 'marker-pod',
+ verifiedAt: new Date().toISOString(),
+ }
+ await reconciler.patchStatus(recipe, {
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ pluginWorkloadSdkProjection: {
+ conditions: [
+ {
+ type: 'PluginWorkloadSdkCapability',
+ status: 'True',
+ reason: 'Validated',
+ message: 'REUSED-PROJECTION-MARKER',
+ lastTransitionTime: new Date().toISOString(),
+ },
+ ],
+ capability: attachedCapability as never,
+ },
+ })
+
+ const patchCall = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)?.[0] as {
+ body: { status: { pluginWorkloadSdk?: { message?: string; bootstrapPodUid?: string } } }
+ }
+ expect(patchCall.body.status.pluginWorkloadSdk?.message).toBe('REUSED-PROJECTION-MARKER')
+ expect(patchCall.body.status.pluginWorkloadSdk?.bootstrapPodUid).toBe('marker-pod')
+ })
+ })
})
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 73dd1d5fe..7d8e76466 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1517,13 +1517,17 @@ export class WorkflowRecipeReconciler {
}
if (currentPhase === 'active' && awaitsTriggeredRun && !wfExecPhase) {
- // Plugin Workload SDK promptBridge recipes keep an eager mcp-host whose
- // provider must be (re-)configured once it becomes Ready and after any
- // pod restart. Fall through to the inner reconcile so ensureEagerSdkMcpHost
- // retries the /configure; the short-circuit below would freeze it
- // provider_unavailable forever. clientNotifications-only needs no provider,
- // so it keeps the cheap short-circuit.
- if (!recipe.spec.pluginWorkloadSdk?.promptBridge) {
+ // Plugin Workload SDK recipes (BOTH families) keep an eager mcp-host and
+ // must fall through to the inner reconcile: promptBridge so
+ // ensureEagerSdkMcpHost retries the /configure (the short-circuit below
+ // would freeze it provider_unavailable forever), and clientNotifications-
+ // only so the reconcile re-gathers the bootstrap proof and recomputes the
+ // capability projection (issue #375 jozer BLOCKER: short-circuiting here
+ // returned skipStatusPatch:true, so a computed awaiting_policy→validated
+ // transition was never published for that family). The Step 8 ownership
+ // gate inside the inner reconcile covers Secret-ownership revocation on
+ // the fall-through path, exactly as it already did for promptBridge.
+ if (!recipe.spec.pluginWorkloadSdk) {
if (coordinatorGfsPolicyCanOpen) {
await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)
}
@@ -1649,11 +1653,24 @@ export class WorkflowRecipeReconciler {
// operator re-labels the Secret.
const activeOwnership = await this.revokeOrRequeueSteadyWorkflow(recipe, 'active')
if (activeOwnership) return activeOwnership
- return {
- phase: 'active' as RecipePhase,
- message: wfExecPhase ? `Workflow ${wfExecPhase}` : 'Workflow completed',
- workloadStatuses: [],
- skipStatusPatch: true,
+ // issue #375 (jozer BLOCKER): Plugin Workload SDK recipes fall through to
+ // the inner reconcile (mirroring the awaiting-trigger carve-out above) so
+ // the bootstrap proof is re-gathered and a computed capability transition
+ // (e.g. awaiting_policy→validated) is actually published — this
+ // short-circuit's skipStatusPatch:true suppressed it unconditionally.
+ // The gfs/credentials/ownership enforcement above has already run.
+ // EXCEPTION: while a workflow execution is in progress the short-circuit
+ // is kept even for SDK recipes — this branch exists to protect the
+ // coordinator's 409-free windows, and the capability publication is
+ // level-triggered (requeue + watchdog) so it lands on the next
+ // non-running pass.
+ if (!recipe.spec.pluginWorkloadSdk || wfInProgress) {
+ return {
+ phase: 'active' as RecipePhase,
+ message: wfExecPhase ? `Workflow ${wfExecPhase}` : 'Workflow completed',
+ workloadStatuses: [],
+ skipStatusPatch: true,
+ }
}
}