Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): surface a pending NetworkPolicy retry from the egress refresh and the DNS-failure prune
Alfredo Lopez0c158ba0a108
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex acfb7cd02..99dc6227d 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.382",+ "version": "0.1.383","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.382",+ "version": "0.1.383","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 0b8cb9cd9..96b61c9dc 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.382",+ "version": "0.1.383","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 576ee1329..456559168 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -11109,6 +11109,67 @@ describe('WorkflowRecipeReconciler', () => {expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ it('logs an error only when the runtime HTTP egress refresh leaves a NetworkPolicy pending a retry', async () => {+ const errorLog = captureLogger('error')+ try {+ const refreshRuntimeHttpEgressNetworkPolicies = vi+ .fn()+ .mockResolvedValueOnce({ conflicts: [], retryPending: true })+ .mockResolvedValueOnce({+ conflicts: [+ { policy: 'run-recipe-snippet-runner-egress', reason: 'owner-reference-mismatch' },+ ],+ retryPending: false,+ })+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ ensureCoordinatorRuntimeCredentials: ReturnType<typeof vi.fn>+ ensureMcpHostRuntimeCredentials: ReturnType<typeof vi.fn>+ refreshRuntimeHttpEgressNetworkPolicies: typeof refreshRuntimeHttpEgressNetworkPolicies+ }+ }+ ).workflowReconciler = {+ ensureCoordinatorRuntimeCredentials: vi.fn().mockResolvedValue(undefined),+ ensureMcpHostRuntimeCredentials: vi.fn().mockResolvedValue(undefined),+ refreshRuntimeHttpEgressNetworkPolicies,+ }+ const recipe = makeRecipe({+ metadata: { name: 'run-recipe', namespace: 'sandbox-recipes', uid: 'uid-run' },+ spec: { steps: [{ id: 'research', instruction: 'run' }] },+ status: {+ phase: 'active',+ message: 'Workflow running',+ workflowExecution: { phase: 'running' },+ } as WorkflowRecipeCRD['status'],+ })+ const pendingRetryLogs = () =>+ errorLog.mock.calls.filter(([msg]) =>+ String(msg).includes('Runtime HTTP egress refresh left a NetworkPolicy pending a retry')+ )++ await reconciler.refreshInProgressWorkflowRuntimeCredentials(recipe)++ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledTimes(1)+ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledWith(+ 'sandbox-recipes',+ 'run-recipe',+ 'uid-run',+ recipe.spec,+ 'run-recipe'+ )+ expect(pendingRetryLogs()).toHaveLength(1)++ // A conflict alone was already logged at warn by the apply; it is not an error.+ await reconciler.refreshInProgressWorkflowRuntimeCredentials(recipe)++ expect(refreshRuntimeHttpEgressNetworkPolicies).toHaveBeenCalledTimes(2)+ expect(pendingRetryLogs()).toHaveLength(1)+ } finally {+ errorLog.mockRestore()+ }+ })+it('requeues a transient DB provenance failure without downgrading to child identity', async () => {mockVerifyWorkflowRunProvenance.mockRejectedValue(new Error('ECONNRESET'))const ensureMcpHostRuntimeCredentials = vi.fn().mockResolvedValue(undefined)diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 65af838c4..83235633e 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1430,13 +1430,22 @@ export class WorkflowRecipeReconciler {)}try {- await this.workflowReconciler.refreshRuntimeHttpEgressNetworkPolicies(+ const refreshed = await this.workflowReconciler.refreshRuntimeHttpEgressNetworkPolicies(recipe.metadata.namespace,recipe.metadata.name,recipe.metadata.uid ?? recipe.metadata.name,recipe.spec,runtimeScopeRecipeName)+ // This path publishes no status and schedules no requeue, so a pending+ // retry would otherwise leave only per-policy warn lines. Conflicts were+ // already logged at warn by the apply and stay there.+ if (refreshed.retryPending) {+ createLogger('wrc', recipe.metadata.name).error(+ 'Runtime HTTP egress refresh left a NetworkPolicy pending a retry; the next refresh retries it',+ { name: recipe.metadata.name }+ )+ }} catch (error) {createLogger('wrc', recipe.metadata.name).error('Failed to refresh runtime HTTP egress; keeping the last valid NetworkPolicy',diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 5d3f141d5..3a279fbb9 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -2881,14 +2881,19 @@ export class WorkflowReconciler {})}+ /**+ * Reapply the runtime HTTP egress policies outside reconcile(). Returns what+ * the apply could not converge: this path has no status to publish it on, so+ * the caller logs it.+ */async refreshRuntimeHttpEgressNetworkPolicies(_recipeNamespace: string,recipeName: string,recipeUid: string,spec: WorkflowRecipeSpec,_runtimeScopeRecipeName = recipeName- ): Promise<void> {- if (!this.needsRuntimeHttpEgressRefresh(spec)) return+ ): Promise<WorkflowNetworkPolicyApplySummary> {+ if (!this.needsRuntimeHttpEgressRefresh(spec)) return { conflicts: [], retryPending: false }let policies: k8s.V1NetworkPolicy[]try {policies = await this.buildRuntimeHttpEgressNetworkPoliciesForSpec(@@ -2900,9 +2905,7 @@ export class WorkflowReconciler {await this.pruneExpiredRuntimeHttpEgressNetworkPolicyOverlaps(recipeName, recipeUid, spec)throw error}- for (const policy of policies) {- await this.applyNetworkPolicy(policy)- }+ return this.applyNetworkPolicyList(policies)}private async buildRuntimeHttpEgressNetworkPoliciesForSpec(@@ -3025,8 +3028,14 @@ export class WorkflowReconciler {runtimeHttpEgressPolicyNames,state)- for (const policy of policies) {- await this.applyNetworkPolicy(policy)+ const pruned = await this.applyNetworkPolicyList(policies)+ // The refresh rethrows the DNS error after this prune, so this line is the+ // only record that a policy still carries the CIDRs the prune meant to drop.+ if (pruned.retryPending) {+ this.log.error(+ 'DNS-failure prune left a NetworkPolicy pending a retry; the next refresh retries it',+ { recipeName }+ )}}@@ -3208,17 +3217,7 @@ export class WorkflowReconciler {eagerSdkMcpHost,grokProjection)- // A conflict or a pending retry leaves that one policy as it is and the- // loop moves on; the caller surfaces both, so neither fails the pass.- const summary: WorkflowNetworkPolicyApplySummary = { conflicts: [], retryPending: false }- for (const policy of policies) {- const applied = await this.applyNetworkPolicy(policy)- if (applied.action === 'conflict') {- summary.conflicts.push({ policy: applied.policy, reason: applied.reason })- } else if (applied.action === 'retry') {- summary.retryPending = true- }- }+ const summary = await this.applyNetworkPolicyList(policies)const policyNames = new Set(policies.map(policy => policy.metadata?.name))const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`@@ -3243,6 +3242,26 @@ export class WorkflowReconciler {return summary}+ /**+ * Apply each policy and collect what the pass could not converge. A conflict+ * or a pending retry leaves that one policy as it is and the loop moves on;+ * each caller decides how to surface both, so neither fails the pass.+ */+ private async applyNetworkPolicyList(+ policies: k8s.V1NetworkPolicy[]+ ): Promise<WorkflowNetworkPolicyApplySummary> {+ const summary: WorkflowNetworkPolicyApplySummary = { conflicts: [], retryPending: false }+ for (const policy of policies) {+ const applied = await this.applyNetworkPolicy(policy)+ if (applied.action === 'conflict') {+ summary.conflicts.push({ policy: applied.policy, reason: applied.reason })+ } else if (applied.action === 'retry') {+ summary.retryPending = true+ }+ }+ return summary+ }+private async resolveRuntimeHttpEgressPolicyState(namespace: string,policyNames: string[],diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex efa2e1f79..a965df9fe 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -2364,6 +2364,15 @@ describe('WorkflowReconciler — reconcile loop', () => {))).toBe(true)+ // The refresh rethrows the DNS error, so the prune's own error line is+ // the only record that a policy still carries the CIDRs it meant to drop.+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'error' &&+ String(entry.msg).includes('DNS-failure prune left a NetworkPolicy pending a retry')+ )+ ).toBe(true)expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)} finally {@@ -3056,7 +3065,12 @@ describe('WorkflowReconciler — reconcile loop', () => {'uid-123',spec)- ).resolves.toBeUndefined()+ ).resolves.toEqual({+ conflicts: [+ { policy: 'test-wf-snippet-runner-egress', reason: 'owner-reference-mismatch' },+ ],+ retryPending: false,+ })expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')expect(@@ -3079,6 +3093,49 @@ describe('WorkflowReconciler — reconcile loop', () => {}})+ it('refresh returns a pending retry for a terminating policy and leaves it unwritten', async () => {+ const { api, live, key } = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: api as never,+ resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,+ })+ )+ const spec = runtimeEgressSpec()+ await reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )+ const terminating = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))+ expect(terminating).toBeDefined()+ terminating!.metadata = {+ ...terminating!.metadata,+ deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),+ }++ api.readNamespacedNetworkPolicy.mockClear()+ api.replaceNamespacedNetworkPolicy.mockClear()++ await expect(+ reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )+ ).resolves.toEqual({ conflicts: [], retryPending: true })++ expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')+ expect(+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'test-wf-snippet-runner-egress'+ )+ ).toHaveLength(0)+ })+describe('runtime HTTP egress resolved-at', () => {// runtimeEgressSpec has no coordinator, so the snippet runner is the// only runtime HTTP egress policy.