Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): retry only a pending apply and keep a pending prune at terminal
Alfredo Lopez8db12c8cd130
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex e231ae321..03812980f 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.425",+ "version": "0.1.426","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.425",+ "version": "0.1.426","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 86d586a02..9d446dfcb 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.425",+ "version": "0.1.426","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 0328161b4..78c338e37 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -9981,6 +9981,63 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueAfterMs).toBeUndefined()})+ // A marker naming both facts: the retry still owes an apply, and a+ // reconcile() prune still owes a DELETE.+ const retryAndPruneMarker = {+ ...retryMarker,+ message:+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',+ }++ it('reapplies from the active short-circuit for a marker that names both facts, and keeps the prune fact', async () => {+ const retry = vi.fn().mockResolvedValue({ conflicts: [], retryPending: false })+ installRunLane(retry)+ const recipe = runningRecipe({ conditions: [unrelatedCondition, retryAndPruneMarker] })++ const result = await reconciler.reconcile(recipe)++ expect(retry).toHaveBeenCalledTimes(1)+ const patches = conditionPatches()+ expect(patches).toHaveLength(1)+ expect(patches[0].body.status.conditions).toEqual([+ unrelatedCondition,+ expect.objectContaining({+ type: 'WorkflowNetworkPoliciesConverged',+ status: 'False',+ reason: 'PrunePending',+ lastTransitionTime: retryMarker.lastTransitionTime,+ }),+ ])+ expect(result.requeueAfterMs).toBeUndefined()+ })++ it('rewrites a marker that names both facts to PrunePending from the terminal branch', async () => {+ const retry = vi.fn()+ const stub = installRunLane(retry)+ const recipe = runningRecipe({+ execPhase: 'completed',+ conditions: [unrelatedCondition, retryAndPruneMarker],+ })++ await reconciler.reconcile(recipe)++ // Witness: the terminal branch ran.+ expect(stub.ensureMcpHostRuntimeCredentials).toHaveBeenCalledTimes(1)+ expect(retry).not.toHaveBeenCalled()+ const patches = conditionPatches()+ expect(patches).toHaveLength(1)+ expect(Object.keys(patches[0].body.status)).toEqual(['conditions'])+ expect(patches[0].body.status.conditions).toEqual([+ unrelatedCondition,+ expect.objectContaining({+ type: 'WorkflowNetworkPoliciesConverged',+ status: 'False',+ reason: 'PrunePending',+ lastTransitionTime: retryMarker.lastTransitionTime,+ }),+ ])+ })+it('opens no patch from the terminal branch without the marker', async () => {const retry = vi.fn()const stub = installRunLane(retry)diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 554bd7755..8995960cf 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -68,8 +68,10 @@ import {WORKFLOW_OUTPUT_CONDITION_TYPES,WorkflowReconciler,WorkflowReconcilerDeps,+ buildNetworkPolicyConvergedCondition,hasNetworkPolicyRetryPendingMarker,networkPolicyConditionsChanged,+ networkPolicyMarkerFacts,translateNetworkPolicyApplySummary,} from '../workflow/workflowReconciler'import { evaluateComputedValues } from './computedValuesEvaluator'@@ -1472,14 +1474,16 @@ export class WorkflowRecipeReconciler {/*** Applies the run-lane NetworkPolicies again from a short-circuit when the- * published status carries the retry marker. Those short-circuits never- * reach `WorkflowReconciler.reconcile()`, so without this a policy left- * pending a retry would stay unapplied until the run ends. The retry only- * applies the desired policies, with a verdict computed for this pass; it- * prunes no run-lane policy, because the run's pods may still use any lane.- * A marker set by a failed reconcile() prune clears here once the applies- * converge, and the leftover waits for the next reconcile() pass or the- * finalizer sweep (see `WorkflowReconciler.retryRunLaneNetworkPolicies`).+ * published marker says an apply is pending a retry. Those short-circuits+ * never reach `WorkflowReconciler.reconcile()`, so without this a policy+ * left pending a retry would stay unapplied until the run ends. The retry+ * only applies the desired policies, with a verdict computed for this pass;+ * it prunes no run-lane policy, because the run's pods may still use any+ * lane. A marker that carries only a pending prune (`PrunePending`) does not+ * enter: the retry could not clear it. A pending prune carried next to the+ * apply survives the retry, and the leftover waits for the next reconcile()+ * pass or the finalizer sweep (see+ * `WorkflowReconciler.retryRunLaneNetworkPolicies`).** Returns the requeue delay the short-circuit must add: the progress base* while a policy is still pending, the transient base when the apply threw@@ -1534,17 +1538,27 @@ export class WorkflowRecipeReconciler {}/**- * Drops the retry marker once the run is terminal: the run no longer needs- * its policies, so there is nothing left to retry. The ownership condition,- * if any, stays published.+ * Settles the marker once the run is terminal. The run no longer needs its+ * policies, so a pending apply is dropped: there is nothing left to retry.+ * A pending prune is kept: the terminal teardown deletes the run's pods, not+ * its run-lane policies, so the leftover is still live and only a later+ * reconcile() prune or the finalizer removes it. The ownership condition, if+ * any, stays published. Opens a patch only when the marker changes.*/- private async clearNetworkPolicyRetryMarker(recipe: WorkflowRecipeCRD): Promise<void> {+ private async settleNetworkPolicyMarkerForTerminalRun(recipe: WorkflowRecipeCRD): Promise<void> {const existing = recipe.status?.conditions- if (!hasNetworkPolicyRetryPendingMarker(existing)) return- await this.publishNetworkPolicyConditions(- recipe,- (existing ?? []).filter(c => c.type !== NETWORK_POLICIES_CONVERGED_CONDITION_TYPE)+ const { prunePending } = networkPolicyMarkerFacts(existing)+ const marker = buildNetworkPolicyConvergedCondition(+ { applyPending: false, prunePending },+ new Date().toISOString(),+ existing)+ const settled = [+ ...(existing ?? []).filter(c => c.type !== NETWORK_POLICIES_CONVERGED_CONDITION_TYPE),+ ...(marker ? [marker] : []),+ ]+ if (!networkPolicyConditionsChanged(existing, settled)) return+ await this.publishNetworkPolicyConditions(recipe, settled)}/**@@ -2051,7 +2065,7 @@ export class WorkflowRecipeReconciler {// torn down above; this teardown is idempotent/404-tolerant).const terminalOwnership = await this.revokeOrRequeueSteadyWorkflow(recipe, derivedPhase)if (terminalOwnership) return terminalOwnership- await this.clearNetworkPolicyRetryMarker(recipe)+ await this.settleNetworkPolicyMarkerForTerminalRun(recipe)// The recipe phase may remain active across running -> completed. Still// patch once if the top-level message is stale so UIs do not show a// completed execution as still running.@@ -2461,9 +2475,12 @@ export class WorkflowRecipeReconciler {// WorkflowNetworkPoliciesConverged=False/RetryPending marker. The// requeued pass can stop at a short-circuit above, before// WorkflowReconciler.reconcile(): the in-progress and active- // short-circuits reapply the run-lane policies while the marker is- // present, and the terminal branch removes it. The awaiting-trigger- // short-circuit does not reapply; the triggered run applies them.+ // short-circuits reapply the run-lane policies while the marker names+ // a pending apply, and the terminal branch drops that fact. A pending+ // prune (PrunePending) adds no requeue and survives both; only a+ // later reconcile() prune or the finalizer clears it. The+ // awaiting-trigger short-circuit does not reapply; the triggered run+ // applies them.requeueAfterMs: result.skipStatusPatch? TRANSIENT_REQUEUE_BASE_MS: result.phase === 'deploying' ||diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 516012754..2c6d6dac6 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -332,13 +332,15 @@ export function networkPolicyConditionsChanged(return signal(existing ?? []) !== signal(fresh)}-/** True when the published status still carries the retry marker. */+/**+ * True when the published marker says an apply is pending a retry. A marker+ * that carries only a pending prune does not count: the retry prunes nothing,+ * so reapplying for it would do no work.+ */export function hasNetworkPolicyRetryPendingMarker(conditions: StatusCondition[] | undefined): boolean {- return (conditions ?? []).some(- c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'- )+ return networkPolicyMarkerFacts(conditions).applyPending}/**