Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): surface a pending NetworkPolicy retry from the egress refresh and the DNS-failure prune

Alfredo Lopez0c158ba0a108
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index acfb7cd02..99dc6227d 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.382",
+ "version": "0.1.383",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.382",
+ "version": "0.1.383",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 0b8cb9cd9..96b61c9dc 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.382",
+ "version": "0.1.383",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 576ee1329..456559168 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -11109,6 +11109,67 @@ describe('WorkflowRecipeReconciler', () => {
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ it('logs an error only when the runtime HTTP egress refresh leaves a NetworkPolicy pending a retry', async () => {
+ const errorLog = captureLogger('error')
+ try {
+ const refreshRuntimeHttpEgressNetworkPolicies = vi
+ .fn()
+ .mockResolvedValueOnce({ conflicts: [], retryPending: true })
+ .mockResolvedValueOnce({
+ conflicts: [
+ { policy: 'run-recipe-snippet-runner-egress', reason: 'owner-reference-mismatch' },
+ ],
+ retryPending: false,
+ })
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ ensureCoordinatorRuntimeCredentials: ReturnType<typeof vi.fn>
+ ensureMcpHostRuntimeCredentials: ReturnType<typeof vi.fn>
+ refreshRuntimeHttpEgressNetworkPolicies: typeof refreshRuntimeHttpEgressNetworkPolicies
+ }
+ }
+ ).workflowReconciler = {
+ ensureCoordinatorRuntimeCredentials: vi.fn().mockResolvedValue(undefined),
+ ensureMcpHostRuntimeCredentials: vi.fn().mockResolvedValue(undefined),
+ refreshRuntimeHttpEgressNetworkPolicies,
+ }
+ const recipe = makeRecipe({
+ metadata: { name: 'run-recipe', namespace: 'sandbox-recipes', uid: 'uid-run' },
+ spec: { steps: [{ id: 'research', instruction: 'run' }] },
+ status: {
+ phase: 'active',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ } as WorkflowRecipeCRD['status'],
+ })
+ const pendingRetryLogs = () =>
+ errorLog.mock.calls.filter(([msg]) =>
+ String(msg).includes('Runtime HTTP egress refresh left a NetworkPolicy pending a retry')
+ )
+
+ await reconciler.refreshInProgressWorkflowRuntimeCredentials(recipe)
+
+ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledTimes(1)
+ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledWith(
+ 'sandbox-recipes',
+ 'run-recipe',
+ 'uid-run',
+ recipe.spec,
+ 'run-recipe'
+ )
+ expect(pendingRetryLogs()).toHaveLength(1)
+
+ // A conflict alone was already logged at warn by the apply; it is not an error.
+ await reconciler.refreshInProgressWorkflowRuntimeCredentials(recipe)
+
+ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledTimes(2)
+ expect(pendingRetryLogs()).toHaveLength(1)
+ } finally {
+ errorLog.mockRestore()
+ }
+ })
+
it('requeues a transient DB provenance failure without downgrading to child identity', async () => {
mockVerifyWorkflowRunProvenance.mockRejectedValue(new Error('ECONNRESET'))
const ensureMcpHostRuntimeCredentials = vi.fn().mockResolvedValue(undefined)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 65af838c4..83235633e 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1430,13 +1430,22 @@ export class WorkflowRecipeReconciler {
)
}
try {
- await this.workflowReconciler.refreshRuntimeHttpEgressNetworkPolicies(
+ const refreshed = await this.workflowReconciler.refreshRuntimeHttpEgressNetworkPolicies(
recipe.metadata.namespace,
recipe.metadata.name,
recipe.metadata.uid ?? recipe.metadata.name,
recipe.spec,
runtimeScopeRecipeName
)
+ // This path publishes no status and schedules no requeue, so a pending
+ // retry would otherwise leave only per-policy warn lines. Conflicts were
+ // already logged at warn by the apply and stay there.
+ if (refreshed.retryPending) {
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Runtime HTTP egress refresh left a NetworkPolicy pending a retry; the next refresh retries it',
+ { name: recipe.metadata.name }
+ )
+ }
} catch (error) {
createLogger('wrc', recipe.metadata.name).error(
'Failed to refresh runtime HTTP egress; keeping the last valid NetworkPolicy',
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 5d3f141d5..3a279fbb9 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -2881,14 +2881,19 @@ export class WorkflowReconciler {
})
}
+ /**
+ * Reapply the runtime HTTP egress policies outside reconcile(). Returns what
+ * the apply could not converge: this path has no status to publish it on, so
+ * the caller logs it.
+ */
async refreshRuntimeHttpEgressNetworkPolicies(
_recipeNamespace: string,
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
_runtimeScopeRecipeName = recipeName
- ): Promise<void> {
- if (!this.needsRuntimeHttpEgressRefresh(spec)) return
+ ): Promise<WorkflowNetworkPolicyApplySummary> {
+ if (!this.needsRuntimeHttpEgressRefresh(spec)) return { conflicts: [], retryPending: false }
let policies: k8s.V1NetworkPolicy[]
try {
policies = await this.buildRuntimeHttpEgressNetworkPoliciesForSpec(
@@ -2900,9 +2905,7 @@ export class WorkflowReconciler {
await this.pruneExpiredRuntimeHttpEgressNetworkPolicyOverlaps(recipeName, recipeUid, spec)
throw error
}
- for (const policy of policies) {
- await this.applyNetworkPolicy(policy)
- }
+ return this.applyNetworkPolicyList(policies)
}
private async buildRuntimeHttpEgressNetworkPoliciesForSpec(
@@ -3025,8 +3028,14 @@ export class WorkflowReconciler {
runtimeHttpEgressPolicyNames,
state
)
- for (const policy of policies) {
- await this.applyNetworkPolicy(policy)
+ const pruned = await this.applyNetworkPolicyList(policies)
+ // The refresh rethrows the DNS error after this prune, so this line is the
+ // only record that a policy still carries the CIDRs the prune meant to drop.
+ if (pruned.retryPending) {
+ this.log.error(
+ 'DNS-failure prune left a NetworkPolicy pending a retry; the next refresh retries it',
+ { recipeName }
+ )
}
}
@@ -3208,17 +3217,7 @@ export class WorkflowReconciler {
eagerSdkMcpHost,
grokProjection
)
- // A conflict or a pending retry leaves that one policy as it is and the
- // loop moves on; the caller surfaces both, so neither fails the pass.
- const summary: WorkflowNetworkPolicyApplySummary = { conflicts: [], retryPending: false }
- for (const policy of policies) {
- const applied = await this.applyNetworkPolicy(policy)
- if (applied.action === 'conflict') {
- summary.conflicts.push({ policy: applied.policy, reason: applied.reason })
- } else if (applied.action === 'retry') {
- summary.retryPending = true
- }
- }
+ const summary = await this.applyNetworkPolicyList(policies)
const policyNames = new Set(policies.map(policy => policy.metadata?.name))
const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`
const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`
@@ -3243,6 +3242,26 @@ export class WorkflowReconciler {
return summary
}
+ /**
+ * Apply each policy and collect what the pass could not converge. A conflict
+ * or a pending retry leaves that one policy as it is and the loop moves on;
+ * each caller decides how to surface both, so neither fails the pass.
+ */
+ private async applyNetworkPolicyList(
+ policies: k8s.V1NetworkPolicy[]
+ ): Promise<WorkflowNetworkPolicyApplySummary> {
+ const summary: WorkflowNetworkPolicyApplySummary = { conflicts: [], retryPending: false }
+ for (const policy of policies) {
+ const applied = await this.applyNetworkPolicy(policy)
+ if (applied.action === 'conflict') {
+ summary.conflicts.push({ policy: applied.policy, reason: applied.reason })
+ } else if (applied.action === 'retry') {
+ summary.retryPending = true
+ }
+ }
+ return summary
+ }
+
private async resolveRuntimeHttpEgressPolicyState(
namespace: string,
policyNames: string[],
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index efa2e1f79..a965df9fe 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -2364,6 +2364,15 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
)
).toBe(true)
+ // The refresh rethrows the DNS error, so the prune's own error line is
+ // the only record that a policy still carries the CIDRs it meant to drop.
+ expect(
+ logs.entries.some(
+ entry =>
+ entry.level === 'error' &&
+ String(entry.msg).includes('DNS-failure prune left a NetworkPolicy pending a retry')
+ )
+ ).toBe(true)
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
} finally {
@@ -3056,7 +3065,12 @@ describe('WorkflowReconciler — reconcile loop', () => {
'uid-123',
spec
)
- ).resolves.toBeUndefined()
+ ).resolves.toEqual({
+ conflicts: [
+ { policy: 'test-wf-snippet-runner-egress', reason: 'owner-reference-mismatch' },
+ ],
+ retryPending: false,
+ })
expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
expect(
@@ -3079,6 +3093,49 @@ describe('WorkflowReconciler — reconcile loop', () => {
}
})
+ it('refresh returns a pending retry for a terminating policy and leaves it unwritten', async () => {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ networkingApi: api as never,
+ resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,
+ })
+ )
+ const spec = runtimeEgressSpec()
+ await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
+ 'sandbox-recipes',
+ 'test-wf',
+ 'uid-123',
+ spec
+ )
+ const terminating = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
+ expect(terminating).toBeDefined()
+ terminating!.metadata = {
+ ...terminating!.metadata,
+ deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
+ }
+
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+
+ await expect(
+ reconciler.refreshRuntimeHttpEgressNetworkPolicies(
+ 'sandbox-recipes',
+ 'test-wf',
+ 'uid-123',
+ spec
+ )
+ ).resolves.toEqual({ conflicts: [], retryPending: true })
+
+ expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
+ expect(
+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-snippet-runner-egress'
+ )
+ ).toHaveLength(0)
+ })
+
describe('runtime HTTP egress resolved-at', () => {
// runtimeEgressSpec has no coordinator, so the snippet runner is the
// only runtime HTTP egress policy.