Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): retry only a pending apply and keep a pending prune at terminal

Alfredo Lopez8db12c8cd130
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index e231ae321..03812980f 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.425",
+ "version": "0.1.426",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.425",
+ "version": "0.1.426",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 86d586a02..9d446dfcb 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.425",
+ "version": "0.1.426",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 0328161b4..78c338e37 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -9981,6 +9981,63 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueAfterMs).toBeUndefined()
})
+ // A marker naming both facts: the retry still owes an apply, and a
+ // reconcile() prune still owes a DELETE.
+ const retryAndPruneMarker = {
+ ...retryMarker,
+ message:
+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
+ }
+
+ it('reapplies from the active short-circuit for a marker that names both facts, and keeps the prune fact', async () => {
+ const retry = vi.fn().mockResolvedValue({ conflicts: [], retryPending: false })
+ installRunLane(retry)
+ const recipe = runningRecipe({ conditions: [unrelatedCondition, retryAndPruneMarker] })
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(retry).toHaveBeenCalledTimes(1)
+ const patches = conditionPatches()
+ expect(patches).toHaveLength(1)
+ expect(patches[0].body.status.conditions).toEqual([
+ unrelatedCondition,
+ expect.objectContaining({
+ type: 'WorkflowNetworkPoliciesConverged',
+ status: 'False',
+ reason: 'PrunePending',
+ lastTransitionTime: retryMarker.lastTransitionTime,
+ }),
+ ])
+ expect(result.requeueAfterMs).toBeUndefined()
+ })
+
+ it('rewrites a marker that names both facts to PrunePending from the terminal branch', async () => {
+ const retry = vi.fn()
+ const stub = installRunLane(retry)
+ const recipe = runningRecipe({
+ execPhase: 'completed',
+ conditions: [unrelatedCondition, retryAndPruneMarker],
+ })
+
+ await reconciler.reconcile(recipe)
+
+ // Witness: the terminal branch ran.
+ expect(stub.ensureMcpHostRuntimeCredentials).toHaveBeenCalledTimes(1)
+ expect(retry).not.toHaveBeenCalled()
+ const patches = conditionPatches()
+ expect(patches).toHaveLength(1)
+ expect(Object.keys(patches[0].body.status)).toEqual(['conditions'])
+ expect(patches[0].body.status.conditions).toEqual([
+ unrelatedCondition,
+ expect.objectContaining({
+ type: 'WorkflowNetworkPoliciesConverged',
+ status: 'False',
+ reason: 'PrunePending',
+ lastTransitionTime: retryMarker.lastTransitionTime,
+ }),
+ ])
+ })
+
it('opens no patch from the terminal branch without the marker', async () => {
const retry = vi.fn()
const stub = installRunLane(retry)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 554bd7755..8995960cf 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -68,8 +68,10 @@ import {
WORKFLOW_OUTPUT_CONDITION_TYPES,
WorkflowReconciler,
WorkflowReconcilerDeps,
+ buildNetworkPolicyConvergedCondition,
hasNetworkPolicyRetryPendingMarker,
networkPolicyConditionsChanged,
+ networkPolicyMarkerFacts,
translateNetworkPolicyApplySummary,
} from '../workflow/workflowReconciler'
import { evaluateComputedValues } from './computedValuesEvaluator'
@@ -1472,14 +1474,16 @@ export class WorkflowRecipeReconciler {
/**
* Applies the run-lane NetworkPolicies again from a short-circuit when the
- * published status carries the retry marker. Those short-circuits never
- * reach `WorkflowReconciler.reconcile()`, so without this a policy left
- * pending a retry would stay unapplied until the run ends. The retry only
- * applies the desired policies, with a verdict computed for this pass; it
- * prunes no run-lane policy, because the run's pods may still use any lane.
- * A marker set by a failed reconcile() prune clears here once the applies
- * converge, and the leftover waits for the next reconcile() pass or the
- * finalizer sweep (see `WorkflowReconciler.retryRunLaneNetworkPolicies`).
+ * published marker says an apply is pending a retry. Those short-circuits
+ * never reach `WorkflowReconciler.reconcile()`, so without this a policy
+ * left pending a retry would stay unapplied until the run ends. The retry
+ * only applies the desired policies, with a verdict computed for this pass;
+ * it prunes no run-lane policy, because the run's pods may still use any
+ * lane. A marker that carries only a pending prune (`PrunePending`) does not
+ * enter: the retry could not clear it. A pending prune carried next to the
+ * apply survives the retry, and the leftover waits for the next reconcile()
+ * pass or the finalizer sweep (see
+ * `WorkflowReconciler.retryRunLaneNetworkPolicies`).
*
* Returns the requeue delay the short-circuit must add: the progress base
* while a policy is still pending, the transient base when the apply threw
@@ -1534,17 +1538,27 @@ export class WorkflowRecipeReconciler {
}
/**
- * Drops the retry marker once the run is terminal: the run no longer needs
- * its policies, so there is nothing left to retry. The ownership condition,
- * if any, stays published.
+ * Settles the marker once the run is terminal. The run no longer needs its
+ * policies, so a pending apply is dropped: there is nothing left to retry.
+ * A pending prune is kept: the terminal teardown deletes the run's pods, not
+ * its run-lane policies, so the leftover is still live and only a later
+ * reconcile() prune or the finalizer removes it. The ownership condition, if
+ * any, stays published. Opens a patch only when the marker changes.
*/
- private async clearNetworkPolicyRetryMarker(recipe: WorkflowRecipeCRD): Promise<void> {
+ private async settleNetworkPolicyMarkerForTerminalRun(recipe: WorkflowRecipeCRD): Promise<void> {
const existing = recipe.status?.conditions
- if (!hasNetworkPolicyRetryPendingMarker(existing)) return
- await this.publishNetworkPolicyConditions(
- recipe,
- (existing ?? []).filter(c => c.type !== NETWORK_POLICIES_CONVERGED_CONDITION_TYPE)
+ const { prunePending } = networkPolicyMarkerFacts(existing)
+ const marker = buildNetworkPolicyConvergedCondition(
+ { applyPending: false, prunePending },
+ new Date().toISOString(),
+ existing
)
+ const settled = [
+ ...(existing ?? []).filter(c => c.type !== NETWORK_POLICIES_CONVERGED_CONDITION_TYPE),
+ ...(marker ? [marker] : []),
+ ]
+ if (!networkPolicyConditionsChanged(existing, settled)) return
+ await this.publishNetworkPolicyConditions(recipe, settled)
}
/**
@@ -2051,7 +2065,7 @@ export class WorkflowRecipeReconciler {
// torn down above; this teardown is idempotent/404-tolerant).
const terminalOwnership = await this.revokeOrRequeueSteadyWorkflow(recipe, derivedPhase)
if (terminalOwnership) return terminalOwnership
- await this.clearNetworkPolicyRetryMarker(recipe)
+ await this.settleNetworkPolicyMarkerForTerminalRun(recipe)
// The recipe phase may remain active across running -> completed. Still
// patch once if the top-level message is stale so UIs do not show a
// completed execution as still running.
@@ -2461,9 +2475,12 @@ export class WorkflowRecipeReconciler {
// WorkflowNetworkPoliciesConverged=False/RetryPending marker. The
// requeued pass can stop at a short-circuit above, before
// WorkflowReconciler.reconcile(): the in-progress and active
- // short-circuits reapply the run-lane policies while the marker is
- // present, and the terminal branch removes it. The awaiting-trigger
- // short-circuit does not reapply; the triggered run applies them.
+ // short-circuits reapply the run-lane policies while the marker names
+ // a pending apply, and the terminal branch drops that fact. A pending
+ // prune (PrunePending) adds no requeue and survives both; only a
+ // later reconcile() prune or the finalizer clears it. The
+ // awaiting-trigger short-circuit does not reapply; the triggered run
+ // applies them.
requeueAfterMs: result.skipStatusPatch
? TRANSIENT_REQUEUE_BASE_MS
: result.phase === 'deploying' ||
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 516012754..2c6d6dac6 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -332,13 +332,15 @@ export function networkPolicyConditionsChanged(
return signal(existing ?? []) !== signal(fresh)
}
-/** True when the published status still carries the retry marker. */
+/**
+ * True when the published marker says an apply is pending a retry. A marker
+ * that carries only a pending prune does not count: the retry prunes nothing,
+ * so reapplying for it would do no work.
+ */
export function hasNetworkPolicyRetryPendingMarker(
conditions: StatusCondition[] | undefined
): boolean {
- return (conditions ?? []).some(
- c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'
- )
+ return networkPolicyMarkerFacts(conditions).applyPending
}
/**