Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): address PR #855 full-review findings

Alfredo Lopezf07285bf2448
diff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.sh
index fd3f533c5..6599fd5ef 100755
--- a/scripts/minikube-profiles/branch-profile.sh
+++ b/scripts/minikube-profiles/branch-profile.sh
@@ -823,7 +823,11 @@ cmd_prepare_shims() {
rewrite_shim "${SHIMS_DIR}/generate-keys.sh" 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "generate-keys.sh"
rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "seed-test-data.sh REPO_ROOT"
- rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh CONTEXT"
+ # One call per substitution, so an upstream edit that stops matching one of them
+ # fails on its own instead of hiding behind the other. The earlier combined
+ # expression carried two alternatives that matched nothing in the current upstream.
+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=clerum-test\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-clerum-test}}"#g' "seed-test-data.sh CONTEXT default"
+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh ALLOWED_CONTEXTS"
rewrite_shim "${SHIMS_DIR}/build-images.sh" 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "build-images.sh"
# full-setup.sh resolves BRANCH_PROFILE_DEPLOY_DIR natively (ACTIVE_MINIKUBE_*),
# so it needs no path rewrite; rewrite_shim would fail on it as a no-op.
@@ -1098,7 +1102,7 @@ cmd_setup() {
MINIKUBE_PROFILE="${PROFILE}" \
CLERUM_PROJECT_DIR="${REPO_DIR}" \
BRANCH_PROFILE_DEPLOY_DIR="${DEPLOY_SHIM_DIR}" \
- "${SHIMS_DIR}/full-setup.sh" "${setup_args[@]}"
+ "${SHIMS_DIR}/full-setup.sh" ${setup_args[@]+"${setup_args[@]}"}
}
cmd_delete() {
diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.sh
index 9d21919b6..aa7d473b1 100755
--- a/scripts/tests/test-repo-intake-packet.sh
+++ b/scripts/tests/test-repo-intake-packet.sh
@@ -272,16 +272,16 @@ assert_contains "${DETACHED_OUTPUT}" "repo_root:" "detached output includes repo
assert_contains "${DETACHED_OUTPUT}" "${DETACHED_WT}" "detached output uses current worktree root"
assert_contains "${DETACHED_OUTPUT}" "primary_checkout:" "detached output includes primary checkout label"
assert_contains "${DETACHED_OUTPUT}" "${MAIN_REPO}" "detached output resolves primary checkout"
-assert_contains "${DETACHED_OUTPUT}" "detached:" "detached output includes detached label"
-assert_contains "${DETACHED_OUTPUT}" "yes" "detached output reports detached state"
-assert_contains "${DETACHED_OUTPUT}" "profile_helper_local:" "detached output includes local helper label"
-assert_contains "${DETACHED_OUTPUT}" "no" "detached output tolerates missing local scripts helper"
-assert_contains "${DETACHED_OUTPUT}" "profile_helper_primary:" "detached output includes primary helper label"
-assert_contains "${DETACHED_OUTPUT}" "yes" "detached output finds primary scripts helper"
+# A bare "yes"/"no" matches any line of the packet, so each value is asserted on
+# its own label row, formatted exactly as the packet's kv() prints it.
+kv_row() { printf '%-30s %s' "$1:" "$2"; }
+assert_contains "${DETACHED_OUTPUT}" "$(kv_row detached yes)" "detached output reports detached state"
+assert_contains "${DETACHED_OUTPUT}" "$(kv_row profile_helper_local no)" "detached output tolerates missing local scripts helper"
+assert_contains "${DETACHED_OUTPUT}" "$(kv_row profile_helper_primary yes)" "detached output finds primary scripts helper"
assert_not_contains "${DETACHED_OUTPUT}" "unbound variable" "detached output has no shell unbound-variable warnings"
# A run that skipped its assertions must not exit 0.
-if (( PASSED < 44 )); then
- fail "expected at least 44 passing assertions, got ${PASSED}"
+if (( PASSED < 41 )); then
+ fail "expected at least 41 passing assertions, got ${PASSED}"
fi
exit "${FAIL}"
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index f3a166a25..5fb7841e7 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.412",
+ "version": "0.1.413",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.412",
+ "version": "0.1.413",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 341e136e6..171c101d4 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.412",
+ "version": "0.1.413",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 5545401de..271a46756 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -17,8 +17,34 @@ import {
workflowRecipeFromWatchObject,
} from './k8sClient'
import { captureLogger } from './reconciler/__tests__/captureLogger'
+import type { SecretEventType, SecretLike } from './reconciler/secretWatcher'
import type { WorkflowRecipeCRD } from './types'
+// startSecretWatch builds one SecretWatcher per namespace and hands it to a
+// K8sSecretWatchLoop. Capturing that hand-off lets a test drive the REAL
+// watcher instances the client constructed, so the callback wiring is covered
+// end to end instead of being replaced by a hand-injected callback.
+const secretLoops = vi.hoisted(
+ () =>
+ [] as Array<{
+ namespace: string
+ handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }
+ }>
+)
+vi.mock('./k8sSecretWatchLoop', () => ({
+ K8sSecretWatchLoop: class {
+ constructor(
+ _kc: unknown,
+ namespace: string,
+ handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }
+ ) {
+ secretLoops.push({ namespace, handler })
+ }
+ async start(): Promise<void> {}
+ stop(): void {}
+ },
+}))
+
function makeWorkflowRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {
return {
apiVersion: 'clerum.io/v1alpha1',
@@ -3254,7 +3280,7 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issu
describe('handleOAuthBrokerTokenAdded', () => {
type BrokerAddedInternal = {
config: { sandboxNamespace: string }
- reconciler: { invalidateOAuthBrokerDeleteLedger: (recipeName: string) => void }
+ reconciler: { invalidateOAuthBrokerSecretLedger: (recipeName: string) => void }
recipes: Map<string, WorkflowRecipeCRD>
eventQueue: { enqueue: (key: string, task: () => Promise<void>) => unknown }
handleOAuthBrokerTokenAdded: (namespace: string, recipeName: string) => void
@@ -3290,7 +3316,7 @@ describe('handleOAuthBrokerTokenAdded', () => {
const internal = watcher as unknown as BrokerAddedInternal
const invalidate = vi.fn()
const enqueue = vi.fn()
- internal.reconciler = { invalidateOAuthBrokerDeleteLedger: invalidate }
+ internal.reconciler = { invalidateOAuthBrokerSecretLedger: invalidate }
internal.eventQueue = { enqueue }
internal.recipes = new Map([
['no-bg', recipeWith('no-bg', false)],
@@ -3314,6 +3340,36 @@ describe('handleOAuthBrokerTokenAdded', () => {
expect(enqueue).not.toHaveBeenCalled()
})
+ it('startSecretWatch wires each namespace watcher to the handler with its own namespace', async () => {
+ const { internal, invalidate, enqueue } = setup()
+ secretLoops.length = 0
+ await (internal as unknown as { startSecretWatch: () => Promise<void> }).startSecretWatch()
+
+ const brokerToken = {
+ metadata: {
+ name: 'wf-no-bg-oauth-broker-token',
+ labels: { 'clerum.io/component': 'oauth-broker-token', 'clerum.io/recipe': 'no-bg' },
+ },
+ data: { token: 'x' },
+ }
+
+ // Liveness witness: the sandbox-namespace watcher exists and reaches the handler.
+ const sandbox = secretLoops.find(l => l.namespace === internal.config.sandboxNamespace)
+ expect(sandbox).toBeDefined()
+ sandbox?.handler.handleEvent('ADDED', brokerToken)
+ expect(invalidate).toHaveBeenCalledWith('no-bg')
+ expect(enqueue).toHaveBeenCalledTimes(1)
+
+ // Every other namespace's watcher is bound to ITS namespace, so the handler ignores it.
+ const others = secretLoops.filter(l => l.namespace !== internal.config.sandboxNamespace)
+ expect(others.length).toBeGreaterThan(0)
+ for (const other of others) {
+ other.handler.handleEvent('ADDED', brokerToken)
+ }
+ expect(invalidate).toHaveBeenCalledTimes(1)
+ expect(enqueue).toHaveBeenCalledTimes(1)
+ })
+
it('ignores a same-named Secret outside the sandbox namespace', () => {
const { internal, invalidate, enqueue } = setup()
// Liveness witness: the same call in the sandbox namespace does act.
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index 9468489be..2299de08e 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -933,7 +933,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
*/
private handleOAuthBrokerTokenAdded(namespace: string, recipeName: string): void {
if (namespace !== this.config.sandboxNamespace) return
- this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)
+ this.reconciler.invalidateOAuthBrokerSecretLedger(recipeName)
const recipe = this.recipes.get(recipeName)
if (recipe && !recipeHasBackgroundAccessClient(recipe)) {
this.triggerSecretDrivenReconcile(recipeName)
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
new file mode 100644
index 000000000..b3d94a744
--- /dev/null
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
@@ -0,0 +1,68 @@
+import { describe, expect, it } from 'vitest'
+import { OAUTH_BROKER_NP_TTL_MS, OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'
+
+describe('OAuthBrokerDeleteLedger', () => {
+ it('remembers a Secret delete for the recorded generation only', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
+ ledger.recordSecretDelete('r', 4)
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)
+ expect(ledger.shouldDeleteSecret('r', 5)).toBe(true)
+ })
+
+ it('keeps one entry per recipe: a newer generation replaces the older one', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.recordSecretDelete('r', 4)
+ ledger.recordSecretDelete('r', 5)
+ // Generations only increase, so the older key is dead weight. Not remembering it
+ // is what bounds the ledger at one entry per recipe instead of one per spec edit.
+ expect(ledger.shouldDeleteSecret('r', 5)).toBe(false)
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
+
+ ledger.recordPolicyDelete('r', 4, 0)
+ ledger.recordPolicyDelete('r', 5, 0)
+ expect(ledger.shouldDeletePolicy('r', 5, 1)).toBe(false)
+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)
+ })
+
+ it('does not let one recipe suppress another', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.recordSecretDelete('a', 1)
+ expect(ledger.shouldDeleteSecret('a', 1)).toBe(false)
+ expect(ledger.shouldDeleteSecret('a-b', 1)).toBe(true)
+ expect(ledger.shouldDeleteSecret('b', 1)).toBe(true)
+ })
+
+ it('expires the NetworkPolicy entry after the TTL', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.recordPolicyDelete('r', 4, 1_000)
+ expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)
+ expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)
+ })
+
+ it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.recordSecretDelete('r', 4)
+ ledger.recordPolicyDelete('r', 4, 0)
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)
+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)
+
+ ledger.invalidateSecret('r')
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)
+ })
+
+ it('invalidate drops both sides for the named recipe and no other', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.recordSecretDelete('r', 4)
+ ledger.recordPolicyDelete('r', 4, 0)
+ ledger.recordSecretDelete('r2', 4)
+ ledger.recordPolicyDelete('r2', 4, 0)
+
+ ledger.invalidate('r')
+ expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
+ expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)
+ expect(ledger.shouldDeleteSecret('r2', 4)).toBe(false)
+ expect(ledger.shouldDeletePolicy('r2', 4, 1)).toBe(false)
+ })
+})
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
index 76f3a5d4a..0f1667c0b 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
@@ -3,24 +3,33 @@
* Transition is (recipe, metadata.generation). The NetworkPolicy side adds a
* 1h TTL so a later pass can reap a leftover without a watch. Indexing only
* by recipe name is the E.6 vacuity mutation: a generation bump must miss.
+ *
+ * One entry per recipe (the last generation wins): generations only increase,
+ * so older entries can never match again and keeping them would grow the
+ * ledger by one key per spec edit for the life of the process.
*/
export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000
-function ledgerKey(recipeName: string, generation: number | undefined): string {
- return `${recipeName}:${generation ?? 0}`
+interface PolicyEntry {
+ generation: number
+ expiresAt: number
+}
+
+function normalizeGeneration(generation: number | undefined): number {
+ return generation ?? 0
}
export class OAuthBrokerDeleteLedger {
- private readonly secrets = new Set<string>()
- private readonly policies = new Map<string, number>()
+ private readonly secrets = new Map<string, number>()
+ private readonly policies = new Map<string, PolicyEntry>()
shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {
- return !this.secrets.has(ledgerKey(recipeName, generation))
+ return this.secrets.get(recipeName) !== normalizeGeneration(generation)
}
recordSecretDelete(recipeName: string, generation: number | undefined): void {
- this.secrets.add(ledgerKey(recipeName, generation))
+ this.secrets.set(recipeName, normalizeGeneration(generation))
}
shouldDeletePolicy(
@@ -28,21 +37,34 @@ export class OAuthBrokerDeleteLedger {
generation: number | undefined,
nowMs = Date.now()
): boolean {
- const expiresAt = this.policies.get(ledgerKey(recipeName, generation))
- return expiresAt === undefined || nowMs >= expiresAt
+ const entry = this.policies.get(recipeName)
+ return (
+ entry === undefined ||
+ entry.generation !== normalizeGeneration(generation) ||
+ nowMs >= entry.expiresAt
+ )
}
recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {
- this.policies.set(ledgerKey(recipeName, generation), nowMs + OAUTH_BROKER_NP_TTL_MS)
+ this.policies.set(recipeName, {
+ generation: normalizeGeneration(generation),
+ expiresAt: nowMs + OAUTH_BROKER_NP_TTL_MS,
+ })
+ }
+
+ /**
+ * A recreated token Secret re-arms only the Secret side. The NetworkPolicy
+ * TTL is not tied to that Secret, and a watch reconnect replays ADDED for
+ * every existing token, so clearing the policy side here would cost one
+ * extra DELETE per recipe after each reconnect.
+ */
+ invalidateSecret(recipeName: string): void {
+ this.secrets.delete(recipeName)
}
+ /** Recipe deletion drops both sides. */
invalidate(recipeName: string): void {
- const prefix = `${recipeName}:`
- for (const key of [...this.secrets]) {
- if (key.startsWith(prefix)) this.secrets.delete(key)
- }
- for (const key of [...this.policies.keys()]) {
- if (key.startsWith(prefix)) this.policies.delete(key)
- }
+ this.secrets.delete(recipeName)
+ this.policies.delete(recipeName)
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 095f06493..21fe2ca2f 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -7473,6 +7473,35 @@ describe('WorkflowRecipeReconciler', () => {
)
})
+ // The legacy internet NP ledger is keyed by recipeUid and cleared from
+ // cleanupPluginWorkloadSdk only when the uid is passed. The SDK-only delete
+ // path is the one that has to hand it over.
+ it('reconcileDelete hands the recipe uid to the SDK cleanup on the SDK-only path', async () => {
+ const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }
+ }
+ ).workflowReconciler = { cleanupPluginWorkloadSdk }
+ const recipe = makeRecipe({
+ metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },
+ spec: {
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ },
+ status: {
+ phase: 'active',
+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },
+ },
+ } as Partial<WorkflowRecipeCRD>)
+
+ await reconciler.reconcileDelete(recipe)
+
+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)
+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
+ recipeUid: 'uid-sdk-delete',
+ })
+ })
+
// ─── Phase 8: Namespace Splitting ─────────────────────────────────
it('R.8.1 — non-MCP workload deploys to sandbox-recipes namespace', async () => {
@@ -15457,9 +15486,23 @@ describe('WorkflowRecipeReconciler', () => {
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
expect(secretDeletes()).toBe(1)
- reconciler.invalidateOAuthBrokerDeleteLedger('test-recipe')
+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+ })
+
+ it('ADDED invalidation re-arms the Secret delete but keeps the NetworkPolicy TTL', async () => {
+ const recipe = reapRecipe(4)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reapPolicy(recipe)
+ expect(secretDeletes()).toBe(1)
+ expect(policyDeletes()).toBe(1)
+
+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reapPolicy(recipe)
expect(secretDeletes()).toBe(2)
+ expect(policyDeletes()).toBe(1)
})
it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {
@@ -15521,8 +15564,10 @@ describe('WorkflowRecipeReconciler', () => {
expect(policyDeletes()).toBe(2)
})
- it('c3: generation 0 on one recipe does not skip another recipe with undefined generation', async () => {
+ it('c3: a recorded delete for one recipe does not skip another recipe', async () => {
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))
+ // Liveness witness: the first recipe's delete really ran.
+ expect(secretDeletes()).toBe(1)
const other = makeRecipe({
metadata: {
name: 'other-recipe',
@@ -15621,6 +15666,16 @@ describe('WorkflowRecipeReconciler', () => {
labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
},
},
+ {
+ // Liveness witness: an ordinary catalog member that is not desired sits in
+ // the same LIST and must be deleted, so "GFS was not deleted" cannot hold
+ // just because the prune never iterated.
+ metadata: {
+ name: `${RECIPE}-mcp-host-to-grok-proxy`,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
],
})
const inner = installRealInner()
@@ -15666,6 +15721,9 @@ describe('WorkflowRecipeReconciler', () => {
)
expect(result, result.message).toMatchObject({ skipStatusPatch: true })
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })
+ )
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: GFS })
)
@@ -15686,6 +15744,13 @@ describe('WorkflowRecipeReconciler', () => {
labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
},
},
+ {
+ metadata: {
+ name: `${RECIPE}-mcp-host-to-grok-proxy`,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
],
})
const inner = installRealInner()
@@ -15702,6 +15767,9 @@ describe('WorkflowRecipeReconciler', () => {
// absence of a GFS DELETE below is the catalog gate holding, not a retry
// that never reached the prune.
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })
+ )
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: GFS })
)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index a74afaf3c..894bb5165 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -948,8 +948,13 @@ export class WorkflowRecipeReconciler {
WorkflowRecipeReconcilerDeps['verifyWorkflowRunProvenance']
>
- invalidateOAuthBrokerDeleteLedger(recipeName: string): void {
- this.oauthBrokerDeleteLedger.invalidate(recipeName)
+ /**
+ * Re-arms the Secret delete for a recipe whose token Secret was (re)created.
+ * The NetworkPolicy TTL is untouched: a watch reconnect replays ADDED for
+ * every existing token and must not reset it.
+ */
+ invalidateOAuthBrokerSecretLedger(recipeName: string): void {
+ this.oauthBrokerDeleteLedger.invalidateSecret(recipeName)
}
constructor(kc: k8s.KubeConfig, config?: OperatorConfig, deps?: WorkflowRecipeReconcilerDeps) {
diff --git a/workflow-recipes/src/workflow/crashRecovery.test.ts b/workflow-recipes/src/workflow/crashRecovery.test.ts
index 572a42c88..d6be309f7 100644
--- a/workflow-recipes/src/workflow/crashRecovery.test.ts
+++ b/workflow-recipes/src/workflow/crashRecovery.test.ts
@@ -167,5 +167,8 @@ describe('waitForPodDeletion', () => {
pollIntervalMs: 5,
})
).resolves.toBe(false)
+ // Liveness witness: `false` also results if the wait never polled. Several GETs
+ // that each answered 200 show it kept looking until the timeout.
+ expect(readNamespacedPod.mock.calls.length).toBeGreaterThan(1)
})
})
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 79d94d3cb..671e407e0 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -3247,7 +3247,7 @@ export class WorkflowReconciler {
this.log.error('Legacy mcp-servers internet NP delete failed', {
recipe: recipeName,
...deleteOutcomeFields(outcome),
- error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),
+ err: outcome.error,
})
return { retryPending: true }
}
@@ -3477,7 +3477,7 @@ export class WorkflowReconciler {
} catch (error: unknown) {
this.log.error('Run-lane NP prune LIST failed; apply outcome is kept', {
recipe: recipeName,
- error: error instanceof Error ? error.message : String(error),
+ err: error,
})
return { retryPending: true }
}
@@ -3519,7 +3519,7 @@ export class WorkflowReconciler {
recipe: recipeName,
policy: name,
...deleteOutcomeFields(outcome),
- error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),
+ err: outcome.error,
})
retryPending = true
}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index f9087cac6..5a7865788 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -10,6 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'
import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'
+import { captureLogger } from '../../../src/reconciler/__tests__/captureLogger'
import {
resolveStatefulSetHeadlessServiceName,
resolveWorkloadMcpServerLabel,
@@ -3242,14 +3243,33 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
}
+ /**
+ * Liveness witness for the negative prune tests: a catalog member that is not
+ * desired sits in the same LIST and must be deleted. If the prune loop never
+ * ran, "X was not deleted" would hold anyway; the sibling's delete proves the
+ * loop iterated over the listed items.
+ */
+ function seedPrunableSibling(apiserver: ReturnType<typeof makeApiserverNetworkingApi>): void {
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ }
+
+ function expectPrunableSiblingDeleted(
+ apiserver: ReturnType<typeof makeApiserverNetworkingApi>
+ ): void {
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
+ }
+
it('does not delete a listed name that is outside the run-lane catalog', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
+ seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
+ expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
})
@@ -3356,10 +3376,12 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('does not delete a listed policy that is in the desired set', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
+ seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
+ expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
})
@@ -3370,11 +3392,13 @@ describe('WorkflowReconciler — reconcile loop', () => {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'workflow-recipes',
})
+ seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
+ expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(
`${RECIPE}-oauth-broker-egress`
)
@@ -3460,11 +3484,13 @@ describe('WorkflowReconciler — reconcile loop', () => {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'workflow-recipes',
})
+ seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler, { awaitsTriggeredRun: true })
expectListWitness(apiserver.api)
+ expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
})
@@ -3509,6 +3535,76 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
})
+ it.each([
+ { code: 404, retryPending: false },
+ { code: 409, retryPending: true },
+ { code: 500, retryPending: true },
+ ])(
+ 'prune DELETE answering $code sets retryPending=$retryPending',
+ async ({ code, retryPending }) => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, namespace }: { name: string; namespace: string }) => {
+ if (name === GROK_PROXY) throw { code }
+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
+ return {}
+ }
+ )
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+
+ const summary = await applyPolicies(reconciler)
+
+ // Liveness witness: the prune reached the DELETE for the leftover.
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
+ expect(summary.retryPending).toBe(retryPending)
+ }
+ )
+
+ it('logs prune failures as an error object, never as a pre-flattened message string', async () => {
+ const errorLog = captureLogger('error')
+ try {
+ // A client exception whose message embeds the response body and headers: the
+ // logger only redacts those when it receives the error itself.
+ const failure = Object.assign(new Error('HTTP-Code: 403 Body: "{}" Headers: {}'), {
+ code: 403,
+ })
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, namespace }: { name: string; namespace: string }) => {
+ if (name === GROK_PROXY) throw failure
+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
+ return {}
+ }
+ )
+ apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce(failure)
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+
+ await applyPolicies(reconciler) // LIST fails
+ await applyPolicies(reconciler) // LIST ok, DELETE of the leftover fails
+
+ const fields = (message: string) =>
+ errorLog.mock.calls.find(([msg]) => msg === message)?.[1] as
+ | Record<string, unknown>
+ | undefined
+ for (const message of [
+ 'Run-lane NP prune LIST failed; apply outcome is kept',
+ 'Run-lane NP prune delete failed',
+ ]) {
+ expect(fields(message), message).toBeDefined()
+ expect(fields(message)?.err).toBe(failure)
+ expect(fields(message)).not.toHaveProperty('error')
+ }
+ } finally {
+ errorLog.mockRestore()
+ }
+ })
+
it('marks retryPending when the legacy internet NP DELETE fails', async () => {
const apiserver = makeApiserverNetworkingApi()
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
@@ -5260,6 +5356,123 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(legacyDeletes()).toHaveLength(2)
expect(second.retryPending).toBe(false)
})
+
+ // The retry computes its own verdict. These tests pin it: a proxy the
+ // fresh verdict still wants (or cannot decide about) survives, and one
+ // it no longer wants goes. The sibling delete is the liveness witness
+ // in every case: the prune loop demonstrably iterated the same LIST.
+ describe('with a fresh Codex/Grok verdict', () => {
+ const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'
+ const projection = (overrides: Record<string, unknown>) => ({
+ targets: [],
+ eligibleTargets: [],
+ derivedScopes: [],
+ requiresCodexProxyEgress: false,
+ requiresGrokProxyEgress: false,
+ driftHashInput: '',
+ catalogContentHash: null,
+ catalogRevision: null,
+ connectionRevision: null,
+ eligibility: 'ineligible',
+ reason: 'static_only',
+ ...overrides,
+ })
+
+ function seedProxy(apiserver: Converged, name: string) {
+ apiserver.live.set(apiserver.key('sandbox-recipes', name), {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ })
+ }
+
+ async function retryWith(
+ verdict: { codex: Record<string, unknown>; grok: Record<string, unknown> },
+ proxies: string[]
+ ) {
+ const { apiserver, reconciler } = await convergedRunLane()
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+ for (const name of proxies) seedProxy(apiserver, name)
+ const verdictSpy = vi.spyOn(reconciler as never, 'codexVerdictFor').mockReturnValue({
+ projection: projection(verdict.codex),
+ grokProjection: projection(verdict.grok),
+ } as never)
+ try {
+ // An agent recipe with a run id needs mcp-host, the only consumer of
+ // the Codex/Grok proxy policies; the snippet spec above never wants them.
+ const summary = await reconciler.retryRunLaneNetworkPolicies(
+ 'test-wf',
+ 'uid-123',
+ makeSpec(),
+ 'test-wf',
+ 'run-1'
+ )
+ expect(verdictSpy).toHaveBeenCalledTimes(1)
+ return { apiserver, summary }
+ } finally {
+ verdictSpy.mockRestore()
+ }
+ }
+
+ // The baseline lane is the snippet one, so switching to an agent spec also
+ // prunes its snippet policies. Only the proxy policies are under test here.
+ const deletedNames = (apiserver: Converged) =>
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls
+ .map(([arg]) => arg.name)
+ .filter(name => name.endsWith('-proxy'))
+
+ it('keeps a proxy the fresh verdict still requires and prunes the undesired twin', async () => {
+ const { apiserver, summary } = await retryWith(
+ {
+ codex: {
+ eligibility: 'eligible',
+ requiresCodexProxyEgress: true,
+ reason: 'granted',
+ },
+ grok: {},
+ },
+ [CODEX_LIVE, LEFTOVER]
+ )
+
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(deletedNames(apiserver)).toEqual([LEFTOVER])
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
+ expect(summary.retryPending).toBe(false)
+ })
+
+ it('keeps both proxies while the verdict is uncertain, and reports no leftover', async () => {
+ const { apiserver, summary } = await retryWith(
+ {
+ codex: { eligibility: 'uncertain', reason: 'forbidden' },
+ grok: { eligibility: 'uncertain', reason: 'forbidden' },
+ },
+ [CODEX_LIVE, LEFTOVER]
+ )
+
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(deletedNames(apiserver)).toEqual([])
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
+ expect(summary.retryPending).toBe(false)
+ })
+
+ it('prunes a proxy once the fresh verdict decides it is no longer wanted', async () => {
+ const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, [
+ CODEX_LIVE,
+ LEFTOVER,
+ ])
+
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(deletedNames(apiserver).sort()).toEqual([CODEX_LIVE, LEFTOVER].sort())
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(false)
+ expect(summary.retryPending).toBe(false)
+ })
+ })
})
})
})