Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): publish a pending prune as PrunePending, apart from the retry

Alfredo Lopez1fa15a26a6d4
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 6904a6a11..dab435349 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.423",
+ "version": "0.1.424",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.423",
+ "version": "0.1.424",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index e3ece4603..22b458bec 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.423",
+ "version": "0.1.424",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
index e574c969c..73b59af20 100644
--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
@@ -54,10 +54,16 @@ export type EagerSdkMcpHostStatus =
* names a live policy another controller owns; WRC leaves it untouched. A
* pending retry means a policy was left unwritten for a later pass: it was
* terminating, or still contended after the bounded apply rounds.
+ *
+ * `prune` is set only by a pass that pruned: `pending` when a DELETE the
+ * pass owed did not land (a non-404 prune or legacy DELETE, or a failed
+ * LIST), `converged` otherwise. A pass that did not prune leaves it
+ * undefined, and the published prune fact is carried over unchanged.
*/
export type WorkflowNetworkPolicyApplySummary = {
conflicts: { policy: string; reason: NetworkPolicyConflictReason }[]
retryPending: boolean
+ prune?: 'converged' | 'pending'
}
export type EagerSdkMcpHostResult = {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index c6e1f818d..d0fa0b1f9 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -1727,9 +1727,11 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {
// The conflict did not stop the rest of the pass.
expect(createdNames()).toContain('sdk-only-workload-to-mcp-host-sdk-egress')
expect(result.phase).toBe('active')
+ // The eager apply prunes too, so its summary carries the prune fact.
expect(result.networkPolicies).toEqual({
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: false,
+ prune: 'converged',
})
expect(writesTo(foreignName)).toHaveLength(0)
})
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 74995487d..516012754 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -270,15 +270,38 @@ export interface PluginWorkloadSdkCleanupOptions {
export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'
/**
- * Published only as `False/RetryPending` while a run-lane NetworkPolicy is
- * left pending a retry (terminating or contended), and removed once the
- * policies converge. The WRC short-circuits of a running workflow never reach
- * `reconcile()`, so this marker is what tells them to apply the policies again.
+ * Published only as `False` while the run-lane policies have not converged,
+ * and removed once they have. It carries two independent facts (see
+ * `networkPolicyMarkerFacts`):
+ *
+ * - apply-pending: a desired policy was left unwritten (terminating or
+ * contended). Reason `RetryPending`; the pass requeues, and the WRC
+ * short-circuits of a running workflow, which never reach `reconcile()`,
+ * apply the policies again while it is set.
+ * - prune-pending: a DELETE the prune owed did not land (a non-404 prune or
+ * legacy DELETE, or a failed LIST). Reason `PrunePending` when it is the
+ * only fact; no requeue, since nothing but `reconcile()` prunes. Only a
+ * later `reconcile()` prune that lands, or the finalizer, clears it; every
+ * other path carries it over.
+ *
+ * Both facts together are one `RetryPending` marker with its own message.
*/
export const NETWORK_POLICIES_CONVERGED_CONDITION_TYPE = 'WorkflowNetworkPoliciesConverged'
+const NETWORK_POLICY_RETRY_PENDING_REASON = 'RetryPending'
+const NETWORK_POLICY_PRUNE_PENDING_REASON = 'PrunePending'
const NETWORK_POLICY_RETRY_PENDING_MESSAGE =
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)'
+const NETWORK_POLICY_PRUNE_PENDING_MESSAGE =
+ 'One or more run-lane NetworkPolicies the spec no longer wants are pending a delete; the next reconcile() pass or the finalizer removes them'
+const NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE =
+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete'
+
+/** The two facts the WorkflowNetworkPoliciesConverged marker carries. */
+export interface NetworkPolicyMarkerFacts {
+ applyPending: boolean
+ prunePending: boolean
+}
/**
* Merged by its own group in `patchStatus`, apart from the workflow-output
@@ -318,18 +341,55 @@ export function hasNetworkPolicyRetryPendingMarker(
)
}
-function buildNetworkPolicyRetryPendingCondition(
+/**
+ * Reads the facts a published marker carries. This and
+ * `buildNetworkPolicyConvergedCondition` are the only places that encode what
+ * the marker means. A `False` marker with any other reason was not written by
+ * this controller and carries neither fact.
+ */
+export function networkPolicyMarkerFacts(
+ conditions: StatusCondition[] | undefined
+): NetworkPolicyMarkerFacts {
+ const marker = (conditions ?? []).find(
+ c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'
+ )
+ if (marker?.reason === NETWORK_POLICY_PRUNE_PENDING_REASON) {
+ return { applyPending: false, prunePending: true }
+ }
+ if (marker?.reason === NETWORK_POLICY_RETRY_PENDING_REASON) {
+ return {
+ applyPending: true,
+ prunePending: marker.message === NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE,
+ }
+ }
+ return { applyPending: false, prunePending: false }
+}
+
+/**
+ * The marker for `facts`, or `undefined` when both are clear (the marker is
+ * then removed). `lastTransitionTime` follows `status`: a marker that stays
+ * `False` keeps the time it was first published, whichever fact it now names.
+ */
+export function buildNetworkPolicyConvergedCondition(
+ facts: NetworkPolicyMarkerFacts,
now: string,
existingConditions?: StatusCondition[]
-): StatusCondition {
+): StatusCondition | undefined {
+ if (!facts.applyPending && !facts.prunePending) return undefined
const existing = existingConditions?.find(
c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'
)
return {
type: NETWORK_POLICIES_CONVERGED_CONDITION_TYPE,
status: 'False',
- reason: 'RetryPending',
- message: NETWORK_POLICY_RETRY_PENDING_MESSAGE,
+ reason: facts.applyPending
+ ? NETWORK_POLICY_RETRY_PENDING_REASON
+ : NETWORK_POLICY_PRUNE_PENDING_REASON,
+ message: facts.applyPending
+ ? facts.prunePending
+ ? NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE
+ : NETWORK_POLICY_RETRY_PENDING_MESSAGE
+ : NETWORK_POLICY_PRUNE_PENDING_MESSAGE,
lastTransitionTime: existing?.lastTransitionTime ?? now,
}
}
@@ -380,8 +440,11 @@ export function buildNetworkPolicyOwnershipConditions(
* and the SDK-only lane so both translate a summary the same way. No summary
* (the pass returned before the apply) yields neither field, which keeps the
* published conditions. A summary always yields the group's conditions, `[]`
- * included: the ownership condition when a policy conflicts, and the retry
- * marker plus the retry flag when a policy was left pending a retry.
+ * included: the ownership condition when a policy conflicts, and the
+ * converged marker while a policy is pending a retry or a delete. Only a
+ * pending retry sets the retry flag (and with it a requeue). A summary without
+ * a prune fact (a path that did not prune) carries the published prune fact
+ * over, so only a pass that pruned can clear it.
*/
export function translateNetworkPolicyApplySummary(
summary: WorkflowNetworkPolicyApplySummary | undefined,
@@ -389,12 +452,19 @@ export function translateNetworkPolicyApplySummary(
now: string
): Pick<WorkflowReconcileResult, 'networkPolicyOwnershipConditions' | 'networkPolicyRetryPending'> {
if (summary === undefined) return {}
+ const prunePending =
+ summary.prune === undefined
+ ? networkPolicyMarkerFacts(existingConditions).prunePending
+ : summary.prune === 'pending'
+ const marker = buildNetworkPolicyConvergedCondition(
+ { applyPending: summary.retryPending, prunePending },
+ now,
+ existingConditions
+ )
return {
networkPolicyOwnershipConditions: [
...buildNetworkPolicyOwnershipConditions(summary, now, existingConditions),
- ...(summary.retryPending
- ? [buildNetworkPolicyRetryPendingCondition(now, existingConditions)]
- : []),
+ ...(marker ? [marker] : []),
],
...(summary.retryPending ? { networkPolicyRetryPending: true } : {}),
}
@@ -3049,7 +3119,7 @@ export class WorkflowReconciler {
)
const summary = await this.applyNetworkPolicyList(policies)
const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
- if (legacy.retryPending) summary.retryPending = true
+ if (legacy.deletePending) summary.retryPending = true
return summary
}
@@ -3251,13 +3321,13 @@ export class WorkflowReconciler {
private async pruneLegacyMcpServersInternetEgressPolicy(
recipeName: string,
recipeUid?: string
- ): Promise<{ retryPending: boolean }> {
+ ): Promise<{ deletePending: boolean }> {
const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)
if (this.prunedLegacyMcpServersInternetEgress.has(key)) {
this.log.debug('Skipping legacy mcp-servers internet NP delete; already observed gone', {
recipe: recipeName,
})
- return { retryPending: false }
+ return { deletePending: false }
}
const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
@@ -3271,7 +3341,7 @@ export class WorkflowReconciler {
...deleteOutcomeFields(outcome),
err: outcome.error,
})
- return { retryPending: true }
+ return { deletePending: true }
}
this.log.info('Legacy mcp-servers internet NP delete', {
recipe: recipeName,
@@ -3280,7 +3350,7 @@ export class WorkflowReconciler {
if (shouldRecordDelete(outcome)) {
this.prunedLegacyMcpServersInternetEgress.add(key)
}
- return { retryPending: false }
+ return { deletePending: false }
}
private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {
@@ -3478,9 +3548,8 @@ export class WorkflowReconciler {
catalog,
keepProxies
)
- if (prune.retryPending) summary.retryPending = true
const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
- if (legacy.retryPending) summary.retryPending = true
+ summary.prune = prune.deletePending || legacy.deletePending ? 'pending' : 'converged'
return summary
}
@@ -3500,7 +3569,7 @@ export class WorkflowReconciler {
desiredNames: Set<string | undefined>,
catalog: Set<string>,
keepProxies: RunLaneProxyKeepReasons
- ): Promise<{ retryPending: boolean }> {
+ ): Promise<{ deletePending: boolean }> {
const namespace = this.deps.config.sandboxNamespace
const labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
let list: { items?: k8s.V1NetworkPolicy[] }
@@ -3514,11 +3583,11 @@ export class WorkflowReconciler {
recipe: recipeName,
err: error,
})
- return { retryPending: true }
+ return { deletePending: true }
}
const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`
const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`
- let retryPending = false
+ let deletePending = false
for (const policy of list.items ?? []) {
const name = policy.metadata?.name
if (!name || desiredNames.has(name)) continue
@@ -3566,10 +3635,10 @@ export class WorkflowReconciler {
...deleteOutcomeFields(outcome),
err: outcome.error,
})
- retryPending = true
+ deletePending = true
}
}
- return { retryPending }
+ return { deletePending }
}
/**
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 2c6b03464..3007c0512 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3559,7 +3559,9 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
})
- it('marks retryPending when a leftover prune DELETE fails', async () => {
+ // A prune DELETE that did not land is prune-pending, not a pending retry:
+ // it publishes PrunePending and does not requeue (R3-L3).
+ it('marks the prune pending, not a retry, when a leftover prune DELETE fails', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
@@ -3572,37 +3574,34 @@ describe('WorkflowReconciler — reconcile loop', () => {
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
- expect(summary.retryPending).toBe(true)
+ expect(summary.prune).toBe('pending')
+ expect(summary.retryPending).toBe(false)
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
})
it.each([
- { code: 404, retryPending: false },
- { code: 409, retryPending: true },
- { code: 500, retryPending: true },
- ])(
- 'prune DELETE answering $code sets retryPending=$retryPending',
- async ({ code, retryPending }) => {
- const apiserver = makeApiserverNetworkingApi()
- seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
- apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
- async ({ name, namespace }: { name: string; namespace: string }) => {
- if (name === GROK_PROXY) throw { code }
- if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
- return {}
- }
- )
- const reconciler = new WorkflowReconciler(
- makeDeps({ networkingApi: apiserver.api as never })
- )
+ { code: 404, prune: 'converged' },
+ { code: 409, prune: 'pending' },
+ { code: 500, prune: 'pending' },
+ ])('prune DELETE answering $code leaves the prune $prune', async ({ code, prune }) => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, namespace }: { name: string; namespace: string }) => {
+ if (name === GROK_PROXY) throw { code }
+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
+ return {}
+ }
+ )
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
- const summary = await applyPolicies(reconciler)
+ const summary = await applyPolicies(reconciler)
- // Liveness witness: the prune reached the DELETE for the leftover.
- expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
- expect(summary.retryPending).toBe(retryPending)
- }
- )
+ // Liveness witness: the prune reached the DELETE for the leftover.
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
+ expect(summary.prune).toBe(prune)
+ expect(summary.retryPending).toBe(false)
+ })
it('logs prune failures as an error object, never as a pre-flattened message string', async () => {
const errorLog = captureLogger('error')
@@ -3646,7 +3645,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
}
})
- it('marks retryPending when the legacy internet NP DELETE fails', async () => {
+ it('marks the prune pending, not a retry, when the legacy internet NP DELETE fails', async () => {
const apiserver = makeApiserverNetworkingApi()
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name, namespace }: { name: string; namespace: string }) => {
@@ -3660,7 +3659,14 @@ describe('WorkflowReconciler — reconcile loop', () => {
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
- expect(summary.retryPending).toBe(true)
+ // Liveness witness: the legacy delete was attempted.
+ expect(
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
+ )
+ ).toHaveLength(1)
+ expect(summary.prune).toBe('pending')
+ expect(summary.retryPending).toBe(false)
})
it('keeps the apply outcome when the prune LIST fails', async () => {
@@ -3670,7 +3676,8 @@ describe('WorkflowReconciler — reconcile loop', () => {
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
- expect(summary.retryPending).toBe(true)
+ expect(summary.prune).toBe('pending')
+ expect(summary.retryPending).toBe(false)
expect(sandboxDeletes(apiserver.api)).toEqual([])
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
// The apply outcome survives: the legacy internet policy prune that runs
@@ -5375,9 +5382,9 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
- // A reconcile() prune that failed sets the retry marker. The retry
- // does not prune, so it clears the marker once its applies converge,
- // and the leftover waits for the next reconcile() pass.
+ // A reconcile() prune that failed publishes PrunePending with no retry
+ // flag. The retry does not prune and carries no prune fact, so the
+ // marker stays until the next reconcile() pass prunes the leftover.
describe('when a prune left a leftover behind', () => {
// A catalog member the snippet lane does not want and that no
// eligibility verdict governs.
@@ -5399,7 +5406,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
}
- it('keeps it through the retry, which clears the marker, and prunes it on the next reconcile()', async () => {
+ it('keeps it and its PrunePending marker through the retry, and prunes it on the next reconcile()', async () => {
const { apiserver, reconciler } = await convergedRunLane()
seedLeftover(apiserver)
apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
@@ -5408,7 +5415,11 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
- expect(failed.networkPolicyRetryPending).toBe(true)
+ expect(failed.networkPolicyRetryPending).toBeFalsy()
+ const published = failed.networkPolicyOwnershipConditions ?? []
+ expect(
+ published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')
+ ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
apiserver.api.listNamespacedNetworkPolicy.mockClear()
@@ -5429,8 +5440,24 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
- const pruned = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
+ const pruned = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ spec(),
+ {
+ conditions: published,
+ }
+ )
expect(pruned.networkPolicyRetryPending).toBeFalsy()
+ // The prune that landed clears the marker: the pass publishes its
+ // conditions and none of them is of that type any more.
+ expect(pruned.networkPolicyOwnershipConditions).toBeDefined()
+ expect(
+ (pruned.networkPolicyOwnershipConditions ?? []).filter(
+ c => c.type === 'WorkflowNetworkPoliciesConverged'
+ )
+ ).toEqual([])
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'sandbox-recipes',
labelSelector: TWO_TERM_SELECTOR,