Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): send no legacy internet policy DELETE from the mid-run retry
Alfredo Lopez46da0bf7e806
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 03812980f..1268ec7db 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.426",+ "version": "0.1.428","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.426",+ "version": "0.1.428","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 9d446dfcb..85237ed10 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.426",+ "version": "0.1.428","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.tsindex f880e1fac..4dab2d0e7 100644--- a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts@@ -2,7 +2,7 @@ import { vi } from 'vitest'import * as logging from '../../observability/logger'/** Observe the service logging boundary without replacing other log levels. */-export function captureLogger(level: 'info' | 'warn' | 'error') {+export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {const original = logging.createLoggerconst calls = vi.fn()const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 78c338e37..4fc76ae9c 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -16156,7 +16156,7 @@ describe('WorkflowRecipeReconciler', () => {).toBe(false)})- it('retryRunLaneNetworkPolicies does not prune reserved GFS', async () => {+ it('retryRunLaneNetworkPolicies prunes nothing: no LIST and no DELETE, reserved GFS included', async () => {mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [{@@ -16201,17 +16201,11 @@ describe('WorkflowRecipeReconciler', () => {)).toBe(true)// The run's pods are live, so the retry revokes no lane: not the- // reserved GFS policy, not a sibling the spec no longer wants, and not- // a Codex/Grok proxy.- for (const name of [- GFS,- `${RECIPE}-snippet-runner-egress`,- `${RECIPE}-mcp-host-to-grok-proxy`,- ]) {- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ name })- )- }+ // reserved GFS policy, not a sibling the spec no longer wants, not a+ // Codex/Grok proxy, and not the legacy mcp-servers internet policy. It+ // does not even LIST what a prune would find.+ expect(mockNetworkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()})})diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 2c6d6dac6..433f31039 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -231,10 +231,13 @@ type RunLaneNetworkPolicyApplyResult =reason: 'terminating' | 'contended'}| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }-/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */+/**+ * Which Codex/Grok proxy policy survives a prune that does not desire it: a+ * proxy whose verdict is uncertain. The skip log names it `reason: 'uncertain'`.+ */interface RunLaneProxyKeepReasons {- codex?: 'uncertain'- grok?: 'uncertain'+ codex: boolean+ grok: boolean}// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most// three writes, the same as the create plus two replaces it replaced.@@ -3067,12 +3070,14 @@ export class WorkflowReconciler {* active short-circuits return before reconcile(), so a policy a previous* pass left pending (terminating or contended) would otherwise stay as it is* until the run ends. Builds the desired set with a verdict computed now and- * applies it, but prunes no run-lane policy: the run's pods are live and may- * still use any lane the spec wanted when they were created, so a revoke here- * cuts their traffic. Only the legacy mcp-servers internet policy, which no- * spec wants, is still deleted. retryPending reflects the applies and that- * delete, so a marker a failed reconcile() prune set clears here and the- * leftover waits for the next reconcile() pass or the finalizer sweep.+ * applies it, but deletes nothing: the run's pods are live and may still use+ * any lane the spec wanted when they were created, so a revoke here cuts+ * their traffic. That includes the legacy mcp-servers internet policy, which+ * only a reconcile() pass deletes. retryPending reflects the applies alone+ * and the summary carries no prune fact, so a pending prune a reconcile()+ * pass published survives this path: a run-lane leftover waits for the next+ * reconcile() pass or the finalizer's label sweep, the legacy policy for the+ * next reconcile() pass.** What each path may delete (verdict = Codex/Grok eligibility; it governs* only the proxy lanes, every other catalog lane follows the spec alone):@@ -3119,10 +3124,7 @@ export class WorkflowReconciler {false,codexVerdict.grokProjection)- const summary = await this.applyNetworkPolicyList(policies)- const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)- if (legacy.deletePending) summary.retryPending = true- return summary+ return this.applyNetworkPolicyList(policies)}async ensureCoordinatorRuntimeCredentials(@@ -3530,8 +3532,8 @@ export class WorkflowReconciler {grokProjection)return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {- codex: codexProjection.eligibility === 'uncertain' ? 'uncertain' : undefined,- grok: grokProjection?.eligibility === 'uncertain' ? 'uncertain' : undefined,+ codex: codexProjection.eligibility === 'uncertain',+ grok: grokProjection?.eligibility === 'uncertain',})}@@ -3560,8 +3562,8 @@ export class WorkflowReconciler {* Two-term selector only (`managed-by=wrc`): a single-term recipe selector* would also return the recipes-lane policies in the same namespace.* Universe is the factory catalog (GFS never in it), not LIST \ desired.- * A Codex/Grok proxy with a keep reason (an uncertain verdict) survives even- * when it is absent from this pass's desired set. A listed policy with an+ * A Codex/Grok proxy whose verdict is uncertain survives even when it is+ * absent from this pass's desired set. A listed policy with an* ownerReference is skipped, matching the apply's ownership veto. Only* reconcile() prunes; the mid-run retry never calls this (see* retryRunLaneNetworkPolicies for the per-path table).@@ -3593,17 +3595,17 @@ export class WorkflowReconciler {for (const policy of list.items ?? []) {const name = policy.metadata?.nameif (!name || desiredNames.has(name)) continue- const keepReason =+ const keep =name === codexProxyPolicyName? keepProxies.codex: name === grokProxyPolicyName? keepProxies.grok- : undefined- if (keepReason) {+ : false+ if (keep) {this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,- reason: keepReason,+ reason: 'uncertain',})continue}diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 3007c0512..f08d8dc5f 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3409,12 +3409,22 @@ describe('WorkflowReconciler — reconcile loop', () => {it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {const uncertain = makeApiserverNetworkingApi()seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)- const uncertainReconciler = new WorkflowReconciler(- makeDeps({ networkingApi: uncertain.api as never })- )- await applyPolicies(uncertainReconciler, {- codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },- })+ const debugLog = captureLogger('debug')+ try {+ const uncertainReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: uncertain.api as never })+ )+ await applyPolicies(uncertainReconciler, {+ codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },+ })+ // The skip names why the proxy survives.+ expect(debugLog).toHaveBeenCalledWith(+ 'Skipping run-lane NP prune',+ expect.objectContaining({ policy: CODEX_PROXY, reason: 'uncertain' })+ )+ } finally {+ debugLog.mockRestore()+ }expectListWitness(uncertain.api)expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)@@ -5468,27 +5478,75 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)})- it('retries a failed legacy internet policy delete', async () => {+ // The legacy mcp-servers internet policy is a prune like any other:+ // only reconcile() deletes it. A failed delete is PrunePending, with+ // no retry flag, and the next reconcile() pass deletes it again.+ const LEGACY = 'test-wf-mcp-servers-egress-internet'+ const legacyDeletes = (apiserver: Converged) =>+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === LEGACY+ )++ it('reports a failed legacy internet policy delete from reconcile() as PrunePending, and deletes it on the next reconcile()', async () => {const apiserver = makeApiserverNetworkingApi()const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))- apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({- code: 403,- message: 'forbidden',- })- const legacyDeletes = () =>- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(- ([arg]) => arg.name === 'test-wf-mcp-servers-egress-internet'+ const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async (arg: { name: string; namespace: string }) => {+ if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {+ throw { code: 403, message: 'forbidden' }+ }+ return liveDelete(arg)+ }+ )++ const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())+ expect(legacyDeletes(apiserver)).toHaveLength(1)+ expect(failed.networkPolicyRetryPending).toBeFalsy()+ const published = failed.networkPolicyOwnershipConditions ?? []+ expect(+ published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')+ ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])++ const pruned = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ spec(),+ {+ conditions: published,+ }+ )+ expect(legacyDeletes(apiserver)).toHaveLength(2)+ expect(pruned.networkPolicyRetryPending).toBeFalsy()+ expect(pruned.networkPolicyOwnershipConditions).toBeDefined()+ expect(+ (pruned.networkPolicyOwnershipConditions ?? []).filter(+ c => c.type === 'WorkflowNetworkPoliciesConverged')+ ).toEqual([])+ })- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(legacyDeletes()).toHaveLength(1)- expect(first.retryPending).toBe(true)+ it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(legacyDeletes()).toHaveLength(2)- expect(second.retryPending).toBe(false)+ const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())++ // Liveness witness: the retry applied the lanes the spec wants.+ expect(+ apiserver.api.createNamespacedNetworkPolicy.mock.calls+ .map(([arg]) => arg.body.metadata?.name)+ .sort()+ ).toEqual(RUN_LANE_POLICY_NAMES.map(key => key.split('/')[1]).sort())+ expect(retried).toEqual({ conflicts: [], retryPending: false })+ expect(legacyDeletes(apiserver)).toEqual([])+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)})// The retry computes its own verdict, and the verdict decides only