Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): publish a pending prune as PrunePending, apart from the retry
Alfredo Lopez1fa15a26a6d4
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 6904a6a11..dab435349 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.423",+ "version": "0.1.424","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.423",+ "version": "0.1.424","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex e3ece4603..22b458bec 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.423",+ "version": "0.1.424","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tsindex e574c969c..73b59af20 100644--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts@@ -54,10 +54,16 @@ export type EagerSdkMcpHostStatus =* names a live policy another controller owns; WRC leaves it untouched. A* pending retry means a policy was left unwritten for a later pass: it was* terminating, or still contended after the bounded apply rounds.+ *+ * `prune` is set only by a pass that pruned: `pending` when a DELETE the+ * pass owed did not land (a non-404 prune or legacy DELETE, or a failed+ * LIST), `converged` otherwise. A pass that did not prune leaves it+ * undefined, and the published prune fact is carried over unchanged.*/export type WorkflowNetworkPolicyApplySummary = {conflicts: { policy: string; reason: NetworkPolicyConflictReason }[]retryPending: boolean+ prune?: 'converged' | 'pending'}export type EagerSdkMcpHostResult = {diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex c6e1f818d..d0fa0b1f9 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -1727,9 +1727,11 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {// The conflict did not stop the rest of the pass.expect(createdNames()).toContain('sdk-only-workload-to-mcp-host-sdk-egress')expect(result.phase).toBe('active')+ // The eager apply prunes too, so its summary carries the prune fact.expect(result.networkPolicies).toEqual({conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],retryPending: false,+ prune: 'converged',})expect(writesTo(foreignName)).toHaveLength(0)})diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 74995487d..516012754 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -270,15 +270,38 @@ export interface PluginWorkloadSdkCleanupOptions {export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'/**- * Published only as `False/RetryPending` while a run-lane NetworkPolicy is- * left pending a retry (terminating or contended), and removed once the- * policies converge. The WRC short-circuits of a running workflow never reach- * `reconcile()`, so this marker is what tells them to apply the policies again.+ * Published only as `False` while the run-lane policies have not converged,+ * and removed once they have. It carries two independent facts (see+ * `networkPolicyMarkerFacts`):+ *+ * - apply-pending: a desired policy was left unwritten (terminating or+ * contended). Reason `RetryPending`; the pass requeues, and the WRC+ * short-circuits of a running workflow, which never reach `reconcile()`,+ * apply the policies again while it is set.+ * - prune-pending: a DELETE the prune owed did not land (a non-404 prune or+ * legacy DELETE, or a failed LIST). Reason `PrunePending` when it is the+ * only fact; no requeue, since nothing but `reconcile()` prunes. Only a+ * later `reconcile()` prune that lands, or the finalizer, clears it; every+ * other path carries it over.+ *+ * Both facts together are one `RetryPending` marker with its own message.*/export const NETWORK_POLICIES_CONVERGED_CONDITION_TYPE = 'WorkflowNetworkPoliciesConverged'+const NETWORK_POLICY_RETRY_PENDING_REASON = 'RetryPending'+const NETWORK_POLICY_PRUNE_PENDING_REASON = 'PrunePending'const NETWORK_POLICY_RETRY_PENDING_MESSAGE ='One or more run-lane NetworkPolicies are pending a retry (terminating or contended)'+const NETWORK_POLICY_PRUNE_PENDING_MESSAGE =+ 'One or more run-lane NetworkPolicies the spec no longer wants are pending a delete; the next reconcile() pass or the finalizer removes them'+const NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE =+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete'++/** The two facts the WorkflowNetworkPoliciesConverged marker carries. */+export interface NetworkPolicyMarkerFacts {+ applyPending: boolean+ prunePending: boolean+}/*** Merged by its own group in `patchStatus`, apart from the workflow-output@@ -318,18 +341,55 @@ export function hasNetworkPolicyRetryPendingMarker()}-function buildNetworkPolicyRetryPendingCondition(+/**+ * Reads the facts a published marker carries. This and+ * `buildNetworkPolicyConvergedCondition` are the only places that encode what+ * the marker means. A `False` marker with any other reason was not written by+ * this controller and carries neither fact.+ */+export function networkPolicyMarkerFacts(+ conditions: StatusCondition[] | undefined+): NetworkPolicyMarkerFacts {+ const marker = (conditions ?? []).find(+ c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'+ )+ if (marker?.reason === NETWORK_POLICY_PRUNE_PENDING_REASON) {+ return { applyPending: false, prunePending: true }+ }+ if (marker?.reason === NETWORK_POLICY_RETRY_PENDING_REASON) {+ return {+ applyPending: true,+ prunePending: marker.message === NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE,+ }+ }+ return { applyPending: false, prunePending: false }+}++/**+ * The marker for `facts`, or `undefined` when both are clear (the marker is+ * then removed). `lastTransitionTime` follows `status`: a marker that stays+ * `False` keeps the time it was first published, whichever fact it now names.+ */+export function buildNetworkPolicyConvergedCondition(+ facts: NetworkPolicyMarkerFacts,now: string,existingConditions?: StatusCondition[]-): StatusCondition {+): StatusCondition | undefined {+ if (!facts.applyPending && !facts.prunePending) return undefinedconst existing = existingConditions?.find(c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False')return {type: NETWORK_POLICIES_CONVERGED_CONDITION_TYPE,status: 'False',- reason: 'RetryPending',- message: NETWORK_POLICY_RETRY_PENDING_MESSAGE,+ reason: facts.applyPending+ ? NETWORK_POLICY_RETRY_PENDING_REASON+ : NETWORK_POLICY_PRUNE_PENDING_REASON,+ message: facts.applyPending+ ? facts.prunePending+ ? NETWORK_POLICY_RETRY_AND_PRUNE_PENDING_MESSAGE+ : NETWORK_POLICY_RETRY_PENDING_MESSAGE+ : NETWORK_POLICY_PRUNE_PENDING_MESSAGE,lastTransitionTime: existing?.lastTransitionTime ?? now,}}@@ -380,8 +440,11 @@ export function buildNetworkPolicyOwnershipConditions(* and the SDK-only lane so both translate a summary the same way. No summary* (the pass returned before the apply) yields neither field, which keeps the* published conditions. A summary always yields the group's conditions, `[]`- * included: the ownership condition when a policy conflicts, and the retry- * marker plus the retry flag when a policy was left pending a retry.+ * included: the ownership condition when a policy conflicts, and the+ * converged marker while a policy is pending a retry or a delete. Only a+ * pending retry sets the retry flag (and with it a requeue). A summary without+ * a prune fact (a path that did not prune) carries the published prune fact+ * over, so only a pass that pruned can clear it.*/export function translateNetworkPolicyApplySummary(summary: WorkflowNetworkPolicyApplySummary | undefined,@@ -389,12 +452,19 @@ export function translateNetworkPolicyApplySummary(now: string): Pick<WorkflowReconcileResult, 'networkPolicyOwnershipConditions' | 'networkPolicyRetryPending'> {if (summary === undefined) return {}+ const prunePending =+ summary.prune === undefined+ ? networkPolicyMarkerFacts(existingConditions).prunePending+ : summary.prune === 'pending'+ const marker = buildNetworkPolicyConvergedCondition(+ { applyPending: summary.retryPending, prunePending },+ now,+ existingConditions+ )return {networkPolicyOwnershipConditions: [...buildNetworkPolicyOwnershipConditions(summary, now, existingConditions),- ...(summary.retryPending- ? [buildNetworkPolicyRetryPendingCondition(now, existingConditions)]- : []),+ ...(marker ? [marker] : []),],...(summary.retryPending ? { networkPolicyRetryPending: true } : {}),}@@ -3049,7 +3119,7 @@ export class WorkflowReconciler {)const summary = await this.applyNetworkPolicyList(policies)const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)- if (legacy.retryPending) summary.retryPending = true+ if (legacy.deletePending) summary.retryPending = truereturn summary}@@ -3251,13 +3321,13 @@ export class WorkflowReconciler {private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string,recipeUid?: string- ): Promise<{ retryPending: boolean }> {+ ): Promise<{ deletePending: boolean }> {const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)if (this.prunedLegacyMcpServersInternetEgress.has(key)) {this.log.debug('Skipping legacy mcp-servers internet NP delete; already observed gone', {recipe: recipeName,})- return { retryPending: false }+ return { deletePending: false }}const outcome = await observeNamespacedDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({@@ -3271,7 +3341,7 @@ export class WorkflowReconciler {...deleteOutcomeFields(outcome),err: outcome.error,})- return { retryPending: true }+ return { deletePending: true }}this.log.info('Legacy mcp-servers internet NP delete', {recipe: recipeName,@@ -3280,7 +3350,7 @@ export class WorkflowReconciler {if (shouldRecordDelete(outcome)) {this.prunedLegacyMcpServersInternetEgress.add(key)}- return { retryPending: false }+ return { deletePending: false }}private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {@@ -3478,9 +3548,8 @@ export class WorkflowReconciler {catalog,keepProxies)- if (prune.retryPending) summary.retryPending = trueconst legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)- if (legacy.retryPending) summary.retryPending = true+ summary.prune = prune.deletePending || legacy.deletePending ? 'pending' : 'converged'return summary}@@ -3500,7 +3569,7 @@ export class WorkflowReconciler {desiredNames: Set<string | undefined>,catalog: Set<string>,keepProxies: RunLaneProxyKeepReasons- ): Promise<{ retryPending: boolean }> {+ ): Promise<{ deletePending: boolean }> {const namespace = this.deps.config.sandboxNamespaceconst labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`let list: { items?: k8s.V1NetworkPolicy[] }@@ -3514,11 +3583,11 @@ export class WorkflowReconciler {recipe: recipeName,err: error,})- return { retryPending: true }+ return { deletePending: true }}const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`- let retryPending = false+ let deletePending = falsefor (const policy of list.items ?? []) {const name = policy.metadata?.nameif (!name || desiredNames.has(name)) continue@@ -3566,10 +3635,10 @@ export class WorkflowReconciler {...deleteOutcomeFields(outcome),err: outcome.error,})- retryPending = true+ deletePending = true}}- return { retryPending }+ return { deletePending }}/**diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 2c6b03464..3007c0512 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3559,7 +3559,9 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)})- it('marks retryPending when a leftover prune DELETE fails', async () => {+ // A prune DELETE that did not land is prune-pending, not a pending retry:+ // it publishes PrunePending and does not requeue (R3-L3).+ it('marks the prune pending, not a retry, when a leftover prune DELETE fails', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(@@ -3572,37 +3574,34 @@ describe('WorkflowReconciler — reconcile loop', () => {const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))const summary = await applyPolicies(reconciler)- expect(summary.retryPending).toBe(true)+ expect(summary.prune).toBe('pending')+ expect(summary.retryPending).toBe(false)expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)})it.each([- { code: 404, retryPending: false },- { code: 409, retryPending: true },- { code: 500, retryPending: true },- ])(- 'prune DELETE answering $code sets retryPending=$retryPending',- async ({ code, retryPending }) => {- const apiserver = makeApiserverNetworkingApi()- seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)- apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(- async ({ name, namespace }: { name: string; namespace: string }) => {- if (name === GROK_PROXY) throw { code }- if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }- return {}- }- )- const reconciler = new WorkflowReconciler(- makeDeps({ networkingApi: apiserver.api as never })- )+ { code: 404, prune: 'converged' },+ { code: 409, prune: 'pending' },+ { code: 500, prune: 'pending' },+ ])('prune DELETE answering $code leaves the prune $prune', async ({ code, prune }) => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async ({ name, namespace }: { name: string; namespace: string }) => {+ if (name === GROK_PROXY) throw { code }+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }+ return {}+ }+ )+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))- const summary = await applyPolicies(reconciler)+ const summary = await applyPolicies(reconciler)- // Liveness witness: the prune reached the DELETE for the leftover.- expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)- expect(summary.retryPending).toBe(retryPending)- }- )+ // Liveness witness: the prune reached the DELETE for the leftover.+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)+ expect(summary.prune).toBe(prune)+ expect(summary.retryPending).toBe(false)+ })it('logs prune failures as an error object, never as a pre-flattened message string', async () => {const errorLog = captureLogger('error')@@ -3646,7 +3645,7 @@ describe('WorkflowReconciler — reconcile loop', () => {}})- it('marks retryPending when the legacy internet NP DELETE fails', async () => {+ it('marks the prune pending, not a retry, when the legacy internet NP DELETE fails', async () => {const apiserver = makeApiserverNetworkingApi()apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => {@@ -3660,7 +3659,14 @@ describe('WorkflowReconciler — reconcile loop', () => {const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))const summary = await applyPolicies(reconciler)- expect(summary.retryPending).toBe(true)+ // Liveness witness: the legacy delete was attempted.+ expect(+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`+ )+ ).toHaveLength(1)+ expect(summary.prune).toBe('pending')+ expect(summary.retryPending).toBe(false)})it('keeps the apply outcome when the prune LIST fails', async () => {@@ -3670,7 +3676,8 @@ describe('WorkflowReconciler — reconcile loop', () => {const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))const summary = await applyPolicies(reconciler)- expect(summary.retryPending).toBe(true)+ expect(summary.prune).toBe('pending')+ expect(summary.retryPending).toBe(false)expect(sandboxDeletes(apiserver.api)).toEqual([])expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)// The apply outcome survives: the legacy internet policy prune that runs@@ -5375,9 +5382,9 @@ describe('WorkflowReconciler — reconcile loop', () => {})})- // A reconcile() prune that failed sets the retry marker. The retry- // does not prune, so it clears the marker once its applies converge,- // and the leftover waits for the next reconcile() pass.+ // A reconcile() prune that failed publishes PrunePending with no retry+ // flag. The retry does not prune and carries no prune fact, so the+ // marker stays until the next reconcile() pass prunes the leftover.describe('when a prune left a leftover behind', () => {// A catalog member the snippet lane does not want and that no// eligibility verdict governs.@@ -5399,7 +5406,7 @@ describe('WorkflowReconciler — reconcile loop', () => {})}- it('keeps it through the retry, which clears the marker, and prunes it on the next reconcile()', async () => {+ it('keeps it and its PrunePending marker through the retry, and prunes it on the next reconcile()', async () => {const { apiserver, reconciler } = await convergedRunLane()seedLeftover(apiserver)apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({@@ -5408,7 +5415,11 @@ describe('WorkflowReconciler — reconcile loop', () => {})const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())- expect(failed.networkPolicyRetryPending).toBe(true)+ expect(failed.networkPolicyRetryPending).toBeFalsy()+ const published = failed.networkPolicyOwnershipConditions ?? []+ expect(+ published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')+ ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)apiserver.api.listNamespacedNetworkPolicy.mockClear()@@ -5429,8 +5440,24 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)- const pruned = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())+ const pruned = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ spec(),+ {+ conditions: published,+ }+ )expect(pruned.networkPolicyRetryPending).toBeFalsy()+ // The prune that landed clears the marker: the pass publishes its+ // conditions and none of them is of that type any more.+ expect(pruned.networkPolicyOwnershipConditions).toBeDefined()+ expect(+ (pruned.networkPolicyOwnershipConditions ?? []).filter(+ c => c.type === 'WorkflowNetworkPoliciesConverged'+ )+ ).toEqual([])expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({namespace: 'sandbox-recipes',labelSelector: TWO_TERM_SELECTOR,