Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): send no legacy internet policy DELETE from the mid-run retry

Alfredo Lopez46da0bf7e806
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 03812980f..1268ec7db 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.426",
+ "version": "0.1.428",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.426",
+ "version": "0.1.428",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 9d446dfcb..85237ed10 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.426",
+ "version": "0.1.428",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
index f880e1fac..4dab2d0e7 100644
--- a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
@@ -2,7 +2,7 @@ import { vi } from 'vitest'
import * as logging from '../../observability/logger'
/** Observe the service logging boundary without replacing other log levels. */
-export function captureLogger(level: 'info' | 'warn' | 'error') {
+export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {
const original = logging.createLogger
const calls = vi.fn()
const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 78c338e37..4fc76ae9c 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -16156,7 +16156,7 @@ describe('WorkflowRecipeReconciler', () => {
).toBe(false)
})
- it('retryRunLaneNetworkPolicies does not prune reserved GFS', async () => {
+ it('retryRunLaneNetworkPolicies prunes nothing: no LIST and no DELETE, reserved GFS included', async () => {
mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [
{
@@ -16201,17 +16201,11 @@ describe('WorkflowRecipeReconciler', () => {
)
).toBe(true)
// The run's pods are live, so the retry revokes no lane: not the
- // reserved GFS policy, not a sibling the spec no longer wants, and not
- // a Codex/Grok proxy.
- for (const name of [
- GFS,
- `${RECIPE}-snippet-runner-egress`,
- `${RECIPE}-mcp-host-to-grok-proxy`,
- ]) {
- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ name })
- )
- }
+ // reserved GFS policy, not a sibling the spec no longer wants, not a
+ // Codex/Grok proxy, and not the legacy mcp-servers internet policy. It
+ // does not even LIST what a prune would find.
+ expect(mockNetworkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
})
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 2c6d6dac6..433f31039 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -231,10 +231,13 @@ type RunLaneNetworkPolicyApplyResult =
reason: 'terminating' | 'contended'
}
| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }
-/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */
+/**
+ * Which Codex/Grok proxy policy survives a prune that does not desire it: a
+ * proxy whose verdict is uncertain. The skip log names it `reason: 'uncertain'`.
+ */
interface RunLaneProxyKeepReasons {
- codex?: 'uncertain'
- grok?: 'uncertain'
+ codex: boolean
+ grok: boolean
}
// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most
// three writes, the same as the create plus two replaces it replaced.
@@ -3067,12 +3070,14 @@ export class WorkflowReconciler {
* active short-circuits return before reconcile(), so a policy a previous
* pass left pending (terminating or contended) would otherwise stay as it is
* until the run ends. Builds the desired set with a verdict computed now and
- * applies it, but prunes no run-lane policy: the run's pods are live and may
- * still use any lane the spec wanted when they were created, so a revoke here
- * cuts their traffic. Only the legacy mcp-servers internet policy, which no
- * spec wants, is still deleted. retryPending reflects the applies and that
- * delete, so a marker a failed reconcile() prune set clears here and the
- * leftover waits for the next reconcile() pass or the finalizer sweep.
+ * applies it, but deletes nothing: the run's pods are live and may still use
+ * any lane the spec wanted when they were created, so a revoke here cuts
+ * their traffic. That includes the legacy mcp-servers internet policy, which
+ * only a reconcile() pass deletes. retryPending reflects the applies alone
+ * and the summary carries no prune fact, so a pending prune a reconcile()
+ * pass published survives this path: a run-lane leftover waits for the next
+ * reconcile() pass or the finalizer's label sweep, the legacy policy for the
+ * next reconcile() pass.
*
* What each path may delete (verdict = Codex/Grok eligibility; it governs
* only the proxy lanes, every other catalog lane follows the spec alone):
@@ -3119,10 +3124,7 @@ export class WorkflowReconciler {
false,
codexVerdict.grokProjection
)
- const summary = await this.applyNetworkPolicyList(policies)
- const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
- if (legacy.deletePending) summary.retryPending = true
- return summary
+ return this.applyNetworkPolicyList(policies)
}
async ensureCoordinatorRuntimeCredentials(
@@ -3530,8 +3532,8 @@ export class WorkflowReconciler {
grokProjection
)
return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {
- codex: codexProjection.eligibility === 'uncertain' ? 'uncertain' : undefined,
- grok: grokProjection?.eligibility === 'uncertain' ? 'uncertain' : undefined,
+ codex: codexProjection.eligibility === 'uncertain',
+ grok: grokProjection?.eligibility === 'uncertain',
})
}
@@ -3560,8 +3562,8 @@ export class WorkflowReconciler {
* Two-term selector only (`managed-by=wrc`): a single-term recipe selector
* would also return the recipes-lane policies in the same namespace.
* Universe is the factory catalog (GFS never in it), not LIST \ desired.
- * A Codex/Grok proxy with a keep reason (an uncertain verdict) survives even
- * when it is absent from this pass's desired set. A listed policy with an
+ * A Codex/Grok proxy whose verdict is uncertain survives even when it is
+ * absent from this pass's desired set. A listed policy with an
* ownerReference is skipped, matching the apply's ownership veto. Only
* reconcile() prunes; the mid-run retry never calls this (see
* retryRunLaneNetworkPolicies for the per-path table).
@@ -3593,17 +3595,17 @@ export class WorkflowReconciler {
for (const policy of list.items ?? []) {
const name = policy.metadata?.name
if (!name || desiredNames.has(name)) continue
- const keepReason =
+ const keep =
name === codexProxyPolicyName
? keepProxies.codex
: name === grokProxyPolicyName
? keepProxies.grok
- : undefined
- if (keepReason) {
+ : false
+ if (keep) {
this.log.debug('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
- reason: keepReason,
+ reason: 'uncertain',
})
continue
}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 3007c0512..f08d8dc5f 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3409,12 +3409,22 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {
const uncertain = makeApiserverNetworkingApi()
seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)
- const uncertainReconciler = new WorkflowReconciler(
- makeDeps({ networkingApi: uncertain.api as never })
- )
- await applyPolicies(uncertainReconciler, {
- codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },
- })
+ const debugLog = captureLogger('debug')
+ try {
+ const uncertainReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: uncertain.api as never })
+ )
+ await applyPolicies(uncertainReconciler, {
+ codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },
+ })
+ // The skip names why the proxy survives.
+ expect(debugLog).toHaveBeenCalledWith(
+ 'Skipping run-lane NP prune',
+ expect.objectContaining({ policy: CODEX_PROXY, reason: 'uncertain' })
+ )
+ } finally {
+ debugLog.mockRestore()
+ }
expectListWitness(uncertain.api)
expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)
expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)
@@ -5468,27 +5478,75 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
})
- it('retries a failed legacy internet policy delete', async () => {
+ // The legacy mcp-servers internet policy is a prune like any other:
+ // only reconcile() deletes it. A failed delete is PrunePending, with
+ // no retry flag, and the next reconcile() pass deletes it again.
+ const LEGACY = 'test-wf-mcp-servers-egress-internet'
+ const legacyDeletes = (apiserver: Converged) =>
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === LEGACY
+ )
+
+ it('reports a failed legacy internet policy delete from reconcile() as PrunePending, and deletes it on the next reconcile()', async () => {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
- apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
- code: 403,
- message: 'forbidden',
- })
- const legacyDeletes = () =>
- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
- ([arg]) => arg.name === 'test-wf-mcp-servers-egress-internet'
+ const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!
+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
+ async (arg: { name: string; namespace: string }) => {
+ if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {
+ throw { code: 403, message: 'forbidden' }
+ }
+ return liveDelete(arg)
+ }
+ )
+
+ const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
+ expect(legacyDeletes(apiserver)).toHaveLength(1)
+ expect(failed.networkPolicyRetryPending).toBeFalsy()
+ const published = failed.networkPolicyOwnershipConditions ?? []
+ expect(
+ published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')
+ ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
+
+ const pruned = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ spec(),
+ {
+ conditions: published,
+ }
+ )
+ expect(legacyDeletes(apiserver)).toHaveLength(2)
+ expect(pruned.networkPolicyRetryPending).toBeFalsy()
+ expect(pruned.networkPolicyOwnershipConditions).toBeDefined()
+ expect(
+ (pruned.networkPolicyOwnershipConditions ?? []).filter(
+ c => c.type === 'WorkflowNetworkPoliciesConverged'
)
+ ).toEqual([])
+ })
- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(legacyDeletes()).toHaveLength(1)
- expect(first.retryPending).toBe(true)
+ it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(legacyDeletes()).toHaveLength(2)
- expect(second.retryPending).toBe(false)
+ const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+
+ // Liveness witness: the retry applied the lanes the spec wants.
+ expect(
+ apiserver.api.createNamespacedNetworkPolicy.mock.calls
+ .map(([arg]) => arg.body.metadata?.name)
+ .sort()
+ ).toEqual(RUN_LANE_POLICY_NAMES.map(key => key.split('/')[1]).sort())
+ expect(retried).toEqual({ conflicts: [], retryPending: false })
+ expect(legacyDeletes(apiserver)).toEqual([])
+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
// The retry computes its own verdict, and the verdict decides only