Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): address PR #855 full-review findings
Alfredo Lopezf07285bf2448
diff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.shindex fd3f533c5..6599fd5ef 100755--- a/scripts/minikube-profiles/branch-profile.sh+++ b/scripts/minikube-profiles/branch-profile.sh@@ -823,7 +823,11 @@ cmd_prepare_shims() {rewrite_shim "${SHIMS_DIR}/generate-keys.sh" 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "generate-keys.sh"rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "seed-test-data.sh REPO_ROOT"- rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh CONTEXT"+ # One call per substitution, so an upstream edit that stops matching one of them+ # fails on its own instead of hiding behind the other. The earlier combined+ # expression carried two alternatives that matched nothing in the current upstream.+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=clerum-test\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-clerum-test}}"#g' "seed-test-data.sh CONTEXT default"+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh ALLOWED_CONTEXTS"rewrite_shim "${SHIMS_DIR}/build-images.sh" 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "build-images.sh"# full-setup.sh resolves BRANCH_PROFILE_DEPLOY_DIR natively (ACTIVE_MINIKUBE_*),# so it needs no path rewrite; rewrite_shim would fail on it as a no-op.@@ -1098,7 +1102,7 @@ cmd_setup() {MINIKUBE_PROFILE="${PROFILE}" \CLERUM_PROJECT_DIR="${REPO_DIR}" \BRANCH_PROFILE_DEPLOY_DIR="${DEPLOY_SHIM_DIR}" \- "${SHIMS_DIR}/full-setup.sh" "${setup_args[@]}"+ "${SHIMS_DIR}/full-setup.sh" ${setup_args[@]+"${setup_args[@]}"}}cmd_delete() {diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.shindex 9d21919b6..aa7d473b1 100755--- a/scripts/tests/test-repo-intake-packet.sh+++ b/scripts/tests/test-repo-intake-packet.sh@@ -272,16 +272,16 @@ assert_contains "${DETACHED_OUTPUT}" "repo_root:" "detached output includes repoassert_contains "${DETACHED_OUTPUT}" "${DETACHED_WT}" "detached output uses current worktree root"assert_contains "${DETACHED_OUTPUT}" "primary_checkout:" "detached output includes primary checkout label"assert_contains "${DETACHED_OUTPUT}" "${MAIN_REPO}" "detached output resolves primary checkout"-assert_contains "${DETACHED_OUTPUT}" "detached:" "detached output includes detached label"-assert_contains "${DETACHED_OUTPUT}" "yes" "detached output reports detached state"-assert_contains "${DETACHED_OUTPUT}" "profile_helper_local:" "detached output includes local helper label"-assert_contains "${DETACHED_OUTPUT}" "no" "detached output tolerates missing local scripts helper"-assert_contains "${DETACHED_OUTPUT}" "profile_helper_primary:" "detached output includes primary helper label"-assert_contains "${DETACHED_OUTPUT}" "yes" "detached output finds primary scripts helper"+# A bare "yes"/"no" matches any line of the packet, so each value is asserted on+# its own label row, formatted exactly as the packet's kv() prints it.+kv_row() { printf '%-30s %s' "$1:" "$2"; }+assert_contains "${DETACHED_OUTPUT}" "$(kv_row detached yes)" "detached output reports detached state"+assert_contains "${DETACHED_OUTPUT}" "$(kv_row profile_helper_local no)" "detached output tolerates missing local scripts helper"+assert_contains "${DETACHED_OUTPUT}" "$(kv_row profile_helper_primary yes)" "detached output finds primary scripts helper"assert_not_contains "${DETACHED_OUTPUT}" "unbound variable" "detached output has no shell unbound-variable warnings"# A run that skipped its assertions must not exit 0.-if (( PASSED < 44 )); then- fail "expected at least 44 passing assertions, got ${PASSED}"+if (( PASSED < 41 )); then+ fail "expected at least 41 passing assertions, got ${PASSED}"fiexit "${FAIL}"diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex f3a166a25..5fb7841e7 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.412",+ "version": "0.1.413","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.412",+ "version": "0.1.413","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 341e136e6..171c101d4 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.412",+ "version": "0.1.413","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 5545401de..271a46756 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -17,8 +17,34 @@ import {workflowRecipeFromWatchObject,} from './k8sClient'import { captureLogger } from './reconciler/__tests__/captureLogger'+import type { SecretEventType, SecretLike } from './reconciler/secretWatcher'import type { WorkflowRecipeCRD } from './types'+// startSecretWatch builds one SecretWatcher per namespace and hands it to a+// K8sSecretWatchLoop. Capturing that hand-off lets a test drive the REAL+// watcher instances the client constructed, so the callback wiring is covered+// end to end instead of being replaced by a hand-injected callback.+const secretLoops = vi.hoisted(+ () =>+ [] as Array<{+ namespace: string+ handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }+ }>+)+vi.mock('./k8sSecretWatchLoop', () => ({+ K8sSecretWatchLoop: class {+ constructor(+ _kc: unknown,+ namespace: string,+ handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }+ ) {+ secretLoops.push({ namespace, handler })+ }+ async start(): Promise<void> {}+ stop(): void {}+ },+}))+function makeWorkflowRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {return {apiVersion: 'clerum.io/v1alpha1',@@ -3254,7 +3280,7 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issudescribe('handleOAuthBrokerTokenAdded', () => {type BrokerAddedInternal = {config: { sandboxNamespace: string }- reconciler: { invalidateOAuthBrokerDeleteLedger: (recipeName: string) => void }+ reconciler: { invalidateOAuthBrokerSecretLedger: (recipeName: string) => void }recipes: Map<string, WorkflowRecipeCRD>eventQueue: { enqueue: (key: string, task: () => Promise<void>) => unknown }handleOAuthBrokerTokenAdded: (namespace: string, recipeName: string) => void@@ -3290,7 +3316,7 @@ describe('handleOAuthBrokerTokenAdded', () => {const internal = watcher as unknown as BrokerAddedInternalconst invalidate = vi.fn()const enqueue = vi.fn()- internal.reconciler = { invalidateOAuthBrokerDeleteLedger: invalidate }+ internal.reconciler = { invalidateOAuthBrokerSecretLedger: invalidate }internal.eventQueue = { enqueue }internal.recipes = new Map([['no-bg', recipeWith('no-bg', false)],@@ -3314,6 +3340,36 @@ describe('handleOAuthBrokerTokenAdded', () => {expect(enqueue).not.toHaveBeenCalled()})+ it('startSecretWatch wires each namespace watcher to the handler with its own namespace', async () => {+ const { internal, invalidate, enqueue } = setup()+ secretLoops.length = 0+ await (internal as unknown as { startSecretWatch: () => Promise<void> }).startSecretWatch()++ const brokerToken = {+ metadata: {+ name: 'wf-no-bg-oauth-broker-token',+ labels: { 'clerum.io/component': 'oauth-broker-token', 'clerum.io/recipe': 'no-bg' },+ },+ data: { token: 'x' },+ }++ // Liveness witness: the sandbox-namespace watcher exists and reaches the handler.+ const sandbox = secretLoops.find(l => l.namespace === internal.config.sandboxNamespace)+ expect(sandbox).toBeDefined()+ sandbox?.handler.handleEvent('ADDED', brokerToken)+ expect(invalidate).toHaveBeenCalledWith('no-bg')+ expect(enqueue).toHaveBeenCalledTimes(1)++ // Every other namespace's watcher is bound to ITS namespace, so the handler ignores it.+ const others = secretLoops.filter(l => l.namespace !== internal.config.sandboxNamespace)+ expect(others.length).toBeGreaterThan(0)+ for (const other of others) {+ other.handler.handleEvent('ADDED', brokerToken)+ }+ expect(invalidate).toHaveBeenCalledTimes(1)+ expect(enqueue).toHaveBeenCalledTimes(1)+ })+it('ignores a same-named Secret outside the sandbox namespace', () => {const { internal, invalidate, enqueue } = setup()// Liveness witness: the same call in the sandbox namespace does act.diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex 9468489be..2299de08e 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -933,7 +933,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {*/private handleOAuthBrokerTokenAdded(namespace: string, recipeName: string): void {if (namespace !== this.config.sandboxNamespace) return- this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)+ this.reconciler.invalidateOAuthBrokerSecretLedger(recipeName)const recipe = this.recipes.get(recipeName)if (recipe && !recipeHasBackgroundAccessClient(recipe)) {this.triggerSecretDrivenReconcile(recipeName)diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.tsnew file mode 100644index 000000000..b3d94a744--- /dev/null+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts@@ -0,0 +1,68 @@+import { describe, expect, it } from 'vitest'+import { OAUTH_BROKER_NP_TTL_MS, OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'++describe('OAuthBrokerDeleteLedger', () => {+ it('remembers a Secret delete for the recorded generation only', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)+ ledger.recordSecretDelete('r', 4)+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)+ expect(ledger.shouldDeleteSecret('r', 5)).toBe(true)+ })++ it('keeps one entry per recipe: a newer generation replaces the older one', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.recordSecretDelete('r', 4)+ ledger.recordSecretDelete('r', 5)+ // Generations only increase, so the older key is dead weight. Not remembering it+ // is what bounds the ledger at one entry per recipe instead of one per spec edit.+ expect(ledger.shouldDeleteSecret('r', 5)).toBe(false)+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)++ ledger.recordPolicyDelete('r', 4, 0)+ ledger.recordPolicyDelete('r', 5, 0)+ expect(ledger.shouldDeletePolicy('r', 5, 1)).toBe(false)+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)+ })++ it('does not let one recipe suppress another', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.recordSecretDelete('a', 1)+ expect(ledger.shouldDeleteSecret('a', 1)).toBe(false)+ expect(ledger.shouldDeleteSecret('a-b', 1)).toBe(true)+ expect(ledger.shouldDeleteSecret('b', 1)).toBe(true)+ })++ it('expires the NetworkPolicy entry after the TTL', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.recordPolicyDelete('r', 4, 1_000)+ expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)+ expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)+ })++ it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.recordSecretDelete('r', 4)+ ledger.recordPolicyDelete('r', 4, 0)+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)++ ledger.invalidateSecret('r')+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)+ })++ it('invalidate drops both sides for the named recipe and no other', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.recordSecretDelete('r', 4)+ ledger.recordPolicyDelete('r', 4, 0)+ ledger.recordSecretDelete('r2', 4)+ ledger.recordPolicyDelete('r2', 4, 0)++ ledger.invalidate('r')+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)+ expect(ledger.shouldDeleteSecret('r2', 4)).toBe(false)+ expect(ledger.shouldDeletePolicy('r2', 4, 1)).toBe(false)+ })+})diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.tsindex 76f3a5d4a..0f1667c0b 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts@@ -3,24 +3,33 @@* Transition is (recipe, metadata.generation). The NetworkPolicy side adds a* 1h TTL so a later pass can reap a leftover without a watch. Indexing only* by recipe name is the E.6 vacuity mutation: a generation bump must miss.+ *+ * One entry per recipe (the last generation wins): generations only increase,+ * so older entries can never match again and keeping them would grow the+ * ledger by one key per spec edit for the life of the process.*/export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000-function ledgerKey(recipeName: string, generation: number | undefined): string {- return `${recipeName}:${generation ?? 0}`+interface PolicyEntry {+ generation: number+ expiresAt: number+}++function normalizeGeneration(generation: number | undefined): number {+ return generation ?? 0}export class OAuthBrokerDeleteLedger {- private readonly secrets = new Set<string>()- private readonly policies = new Map<string, number>()+ private readonly secrets = new Map<string, number>()+ private readonly policies = new Map<string, PolicyEntry>()shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {- return !this.secrets.has(ledgerKey(recipeName, generation))+ return this.secrets.get(recipeName) !== normalizeGeneration(generation)}recordSecretDelete(recipeName: string, generation: number | undefined): void {- this.secrets.add(ledgerKey(recipeName, generation))+ this.secrets.set(recipeName, normalizeGeneration(generation))}shouldDeletePolicy(@@ -28,21 +37,34 @@ export class OAuthBrokerDeleteLedger {generation: number | undefined,nowMs = Date.now()): boolean {- const expiresAt = this.policies.get(ledgerKey(recipeName, generation))- return expiresAt === undefined || nowMs >= expiresAt+ const entry = this.policies.get(recipeName)+ return (+ entry === undefined ||+ entry.generation !== normalizeGeneration(generation) ||+ nowMs >= entry.expiresAt+ )}recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {- this.policies.set(ledgerKey(recipeName, generation), nowMs + OAUTH_BROKER_NP_TTL_MS)+ this.policies.set(recipeName, {+ generation: normalizeGeneration(generation),+ expiresAt: nowMs + OAUTH_BROKER_NP_TTL_MS,+ })+ }++ /**+ * A recreated token Secret re-arms only the Secret side. The NetworkPolicy+ * TTL is not tied to that Secret, and a watch reconnect replays ADDED for+ * every existing token, so clearing the policy side here would cost one+ * extra DELETE per recipe after each reconnect.+ */+ invalidateSecret(recipeName: string): void {+ this.secrets.delete(recipeName)}+ /** Recipe deletion drops both sides. */invalidate(recipeName: string): void {- const prefix = `${recipeName}:`- for (const key of [...this.secrets]) {- if (key.startsWith(prefix)) this.secrets.delete(key)- }- for (const key of [...this.policies.keys()]) {- if (key.startsWith(prefix)) this.policies.delete(key)- }+ this.secrets.delete(recipeName)+ this.policies.delete(recipeName)}}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 095f06493..21fe2ca2f 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -7473,6 +7473,35 @@ describe('WorkflowRecipeReconciler', () => {)})+ // The legacy internet NP ledger is keyed by recipeUid and cleared from+ // cleanupPluginWorkloadSdk only when the uid is passed. The SDK-only delete+ // path is the one that has to hand it over.+ it('reconcileDelete hands the recipe uid to the SDK cleanup on the SDK-only path', async () => {+ const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)+ ;(+ reconciler as unknown as {+ workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }+ }+ ).workflowReconciler = { cleanupPluginWorkloadSdk }+ const recipe = makeRecipe({+ metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },+ spec: {+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ },+ status: {+ phase: 'active',+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },+ },+ } as Partial<WorkflowRecipeCRD>)++ await reconciler.reconcileDelete(recipe)++ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {+ recipeUid: 'uid-sdk-delete',+ })+ })+// ─── Phase 8: Namespace Splitting ─────────────────────────────────it('R.8.1 — non-MCP workload deploys to sandbox-recipes namespace', async () => {@@ -15457,9 +15486,23 @@ describe('WorkflowRecipeReconciler', () => {await reconciler.ensureOAuthBrokerTokenSecret(recipe)expect(secretDeletes()).toBe(1)- reconciler.invalidateOAuthBrokerDeleteLedger('test-recipe')+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)+ })++ it('ADDED invalidation re-arms the Secret delete but keeps the NetworkPolicy TTL', async () => {+ const recipe = reapRecipe(4)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reapPolicy(recipe)+ expect(secretDeletes()).toBe(1)+ expect(policyDeletes()).toBe(1)++ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reapPolicy(recipe)expect(secretDeletes()).toBe(2)+ expect(policyDeletes()).toBe(1)})it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {@@ -15521,8 +15564,10 @@ describe('WorkflowRecipeReconciler', () => {expect(policyDeletes()).toBe(2)})- it('c3: generation 0 on one recipe does not skip another recipe with undefined generation', async () => {+ it('c3: a recorded delete for one recipe does not skip another recipe', async () => {await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))+ // Liveness witness: the first recipe's delete really ran.+ expect(secretDeletes()).toBe(1)const other = makeRecipe({metadata: {name: 'other-recipe',@@ -15621,6 +15666,16 @@ describe('WorkflowRecipeReconciler', () => {labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },},},+ {+ // Liveness witness: an ordinary catalog member that is not desired sits in+ // the same LIST and must be deleted, so "GFS was not deleted" cannot hold+ // just because the prune never iterated.+ metadata: {+ name: `${RECIPE}-mcp-host-to-grok-proxy`,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },],})const inner = installRealInner()@@ -15666,6 +15721,9 @@ describe('WorkflowRecipeReconciler', () => {)expect(result, result.message).toMatchObject({ skipStatusPatch: true })+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })+ )expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: GFS }))@@ -15686,6 +15744,13 @@ describe('WorkflowRecipeReconciler', () => {labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },},},+ {+ metadata: {+ name: `${RECIPE}-mcp-host-to-grok-proxy`,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },],})const inner = installRealInner()@@ -15702,6 +15767,9 @@ describe('WorkflowRecipeReconciler', () => {// absence of a GFS DELETE below is the catalog gate holding, not a retry// that never reached the prune.expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })+ )expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: GFS }))diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex a74afaf3c..894bb5165 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -948,8 +948,13 @@ export class WorkflowRecipeReconciler {WorkflowRecipeReconcilerDeps['verifyWorkflowRunProvenance']>- invalidateOAuthBrokerDeleteLedger(recipeName: string): void {- this.oauthBrokerDeleteLedger.invalidate(recipeName)+ /**+ * Re-arms the Secret delete for a recipe whose token Secret was (re)created.+ * The NetworkPolicy TTL is untouched: a watch reconnect replays ADDED for+ * every existing token and must not reset it.+ */+ invalidateOAuthBrokerSecretLedger(recipeName: string): void {+ this.oauthBrokerDeleteLedger.invalidateSecret(recipeName)}constructor(kc: k8s.KubeConfig, config?: OperatorConfig, deps?: WorkflowRecipeReconcilerDeps) {diff --git a/workflow-recipes/src/workflow/crashRecovery.test.ts b/workflow-recipes/src/workflow/crashRecovery.test.tsindex 572a42c88..d6be309f7 100644--- a/workflow-recipes/src/workflow/crashRecovery.test.ts+++ b/workflow-recipes/src/workflow/crashRecovery.test.ts@@ -167,5 +167,8 @@ describe('waitForPodDeletion', () => {pollIntervalMs: 5,})).resolves.toBe(false)+ // Liveness witness: `false` also results if the wait never polled. Several GETs+ // that each answered 200 show it kept looking until the timeout.+ expect(readNamespacedPod.mock.calls.length).toBeGreaterThan(1)})})diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 79d94d3cb..671e407e0 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -3247,7 +3247,7 @@ export class WorkflowReconciler {this.log.error('Legacy mcp-servers internet NP delete failed', {recipe: recipeName,...deleteOutcomeFields(outcome),- error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),+ err: outcome.error,})return { retryPending: true }}@@ -3477,7 +3477,7 @@ export class WorkflowReconciler {} catch (error: unknown) {this.log.error('Run-lane NP prune LIST failed; apply outcome is kept', {recipe: recipeName,- error: error instanceof Error ? error.message : String(error),+ err: error,})return { retryPending: true }}@@ -3519,7 +3519,7 @@ export class WorkflowReconciler {recipe: recipeName,policy: name,...deleteOutcomeFields(outcome),- error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),+ err: outcome.error,})retryPending = true}diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex f9087cac6..5a7865788 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -10,6 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import type * as k8s from '@kubernetes/client-node'import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'+import { captureLogger } from '../../../src/reconciler/__tests__/captureLogger'import {resolveStatefulSetHeadlessServiceName,resolveWorkloadMcpServerLabel,@@ -3242,14 +3243,33 @@ describe('WorkflowReconciler — reconcile loop', () => {)}+ /**+ * Liveness witness for the negative prune tests: a catalog member that is not+ * desired sits in the same LIST and must be deleted. If the prune loop never+ * ran, "X was not deleted" would hold anyway; the sibling's delete proves the+ * loop iterated over the listed items.+ */+ function seedPrunableSibling(apiserver: ReturnType<typeof makeApiserverNetworkingApi>): void {+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ }++ function expectPrunableSiblingDeleted(+ apiserver: ReturnType<typeof makeApiserverNetworkingApi>+ ): void {+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)+ }+it('does not delete a listed name that is outside the run-lane catalog', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)+ seedPrunableSibling(apiserver)const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))await applyPolicies(reconciler)expectListWitness(apiserver.api)+ expectPrunableSiblingDeleted(apiserver)expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)})@@ -3356,10 +3376,12 @@ describe('WorkflowReconciler — reconcile loop', () => {it('does not delete a listed policy that is in the desired set', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)+ seedPrunableSibling(apiserver)const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))await applyPolicies(reconciler)+ expectPrunableSiblingDeleted(apiserver)expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)})@@ -3370,11 +3392,13 @@ describe('WorkflowReconciler — reconcile loop', () => {'clerum.io/recipe': RECIPE,'clerum.io/managed-by': 'workflow-recipes',})+ seedPrunableSibling(apiserver)const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))await applyPolicies(reconciler)expectListWitness(apiserver.api)+ expectPrunableSiblingDeleted(apiserver)expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-oauth-broker-egress`)@@ -3460,11 +3484,13 @@ describe('WorkflowReconciler — reconcile loop', () => {'clerum.io/recipe': RECIPE,'clerum.io/managed-by': 'workflow-recipes',})+ seedPrunableSibling(apiserver)const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))await applyPolicies(reconciler, { awaitsTriggeredRun: true })expectListWitness(apiserver.api)+ expectPrunableSiblingDeleted(apiserver)expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)})@@ -3509,6 +3535,76 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)})+ it.each([+ { code: 404, retryPending: false },+ { code: 409, retryPending: true },+ { code: 500, retryPending: true },+ ])(+ 'prune DELETE answering $code sets retryPending=$retryPending',+ async ({ code, retryPending }) => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async ({ name, namespace }: { name: string; namespace: string }) => {+ if (name === GROK_PROXY) throw { code }+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }+ return {}+ }+ )+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )++ const summary = await applyPolicies(reconciler)++ // Liveness witness: the prune reached the DELETE for the leftover.+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)+ expect(summary.retryPending).toBe(retryPending)+ }+ )++ it('logs prune failures as an error object, never as a pre-flattened message string', async () => {+ const errorLog = captureLogger('error')+ try {+ // A client exception whose message embeds the response body and headers: the+ // logger only redacts those when it receives the error itself.+ const failure = Object.assign(new Error('HTTP-Code: 403 Body: "{}" Headers: {}'), {+ code: 403,+ })+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async ({ name, namespace }: { name: string; namespace: string }) => {+ if (name === GROK_PROXY) throw failure+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }+ return {}+ }+ )+ apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce(failure)+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )++ await applyPolicies(reconciler) // LIST fails+ await applyPolicies(reconciler) // LIST ok, DELETE of the leftover fails++ const fields = (message: string) =>+ errorLog.mock.calls.find(([msg]) => msg === message)?.[1] as+ | Record<string, unknown>+ | undefined+ for (const message of [+ 'Run-lane NP prune LIST failed; apply outcome is kept',+ 'Run-lane NP prune delete failed',+ ]) {+ expect(fields(message), message).toBeDefined()+ expect(fields(message)?.err).toBe(failure)+ expect(fields(message)).not.toHaveProperty('error')+ }+ } finally {+ errorLog.mockRestore()+ }+ })+it('marks retryPending when the legacy internet NP DELETE fails', async () => {const apiserver = makeApiserverNetworkingApi()apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(@@ -5260,6 +5356,123 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(legacyDeletes()).toHaveLength(2)expect(second.retryPending).toBe(false)})++ // The retry computes its own verdict. These tests pin it: a proxy the+ // fresh verdict still wants (or cannot decide about) survives, and one+ // it no longer wants goes. The sibling delete is the liveness witness+ // in every case: the prune loop demonstrably iterated the same LIST.+ describe('with a fresh Codex/Grok verdict', () => {+ const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'+ const projection = (overrides: Record<string, unknown>) => ({+ targets: [],+ eligibleTargets: [],+ derivedScopes: [],+ requiresCodexProxyEgress: false,+ requiresGrokProxyEgress: false,+ driftHashInput: '',+ catalogContentHash: null,+ catalogRevision: null,+ connectionRevision: null,+ eligibility: 'ineligible',+ reason: 'static_only',+ ...overrides,+ })++ function seedProxy(apiserver: Converged, name: string) {+ apiserver.live.set(apiserver.key('sandbox-recipes', name), {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ })+ }++ async function retryWith(+ verdict: { codex: Record<string, unknown>; grok: Record<string, unknown> },+ proxies: string[]+ ) {+ const { apiserver, reconciler } = await convergedRunLane()+ apiserver.api.listNamespacedNetworkPolicy.mockClear()+ for (const name of proxies) seedProxy(apiserver, name)+ const verdictSpy = vi.spyOn(reconciler as never, 'codexVerdictFor').mockReturnValue({+ projection: projection(verdict.codex),+ grokProjection: projection(verdict.grok),+ } as never)+ try {+ // An agent recipe with a run id needs mcp-host, the only consumer of+ // the Codex/Grok proxy policies; the snippet spec above never wants them.+ const summary = await reconciler.retryRunLaneNetworkPolicies(+ 'test-wf',+ 'uid-123',+ makeSpec(),+ 'test-wf',+ 'run-1'+ )+ expect(verdictSpy).toHaveBeenCalledTimes(1)+ return { apiserver, summary }+ } finally {+ verdictSpy.mockRestore()+ }+ }++ // The baseline lane is the snippet one, so switching to an agent spec also+ // prunes its snippet policies. Only the proxy policies are under test here.+ const deletedNames = (apiserver: Converged) =>+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls+ .map(([arg]) => arg.name)+ .filter(name => name.endsWith('-proxy'))++ it('keeps a proxy the fresh verdict still requires and prunes the undesired twin', async () => {+ const { apiserver, summary } = await retryWith(+ {+ codex: {+ eligibility: 'eligible',+ requiresCodexProxyEgress: true,+ reason: 'granted',+ },+ grok: {},+ },+ [CODEX_LIVE, LEFTOVER]+ )++ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(deletedNames(apiserver)).toEqual([LEFTOVER])+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)+ expect(summary.retryPending).toBe(false)+ })++ it('keeps both proxies while the verdict is uncertain, and reports no leftover', async () => {+ const { apiserver, summary } = await retryWith(+ {+ codex: { eligibility: 'uncertain', reason: 'forbidden' },+ grok: { eligibility: 'uncertain', reason: 'forbidden' },+ },+ [CODEX_LIVE, LEFTOVER]+ )++ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(deletedNames(apiserver)).toEqual([])+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)+ expect(summary.retryPending).toBe(false)+ })++ it('prunes a proxy once the fresh verdict decides it is no longer wanted', async () => {+ const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, [+ CODEX_LIVE,+ LEFTOVER,+ ])++ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(deletedNames(apiserver).sort()).toEqual([CODEX_LIVE, LEFTOVER].sort())+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(false)+ expect(summary.retryPending).toBe(false)+ })+ })})})})