Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): read run-lane NetworkPolicies before writing them

Alfredo Lopez9731c615f63c
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 7617292a8..fa59ea564 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.372",
+ "version": "0.1.373",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.372",
+ "version": "0.1.373",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index bc2495ef8..e989368d5 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.372",
+ "version": "0.1.373",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
index 02b9ab08f..5cf95ac09 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
@@ -171,7 +171,9 @@ function createHarness() {
}
const mockNetworkingApi = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ // Absent by default: applyNetworkPolicy reads first, so create is the
+ // witness that a policy was applied.
+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
index c59dc27c9..2998acf69 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
@@ -110,7 +110,9 @@ function createHarness(grokSubscriptionEnabled: boolean, pluginWorkloadSdkEnable
}
const mockNetworkingApi = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ // Absent by default: applyNetworkPolicy reads first, so create is the
+ // witness that a policy was applied.
+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 56f8eda77..9501c81d4 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -31,6 +31,11 @@ import {
getErrorCode,
isRetryableInfraError,
} from '../reconciler/k8sErrors'
+import {
+ type NetworkPolicyConvergenceDecision,
+ buildNetworkPolicyReplacement,
+ decideNetworkPolicyConvergence,
+} from '../reconciler/networkPolicyConvergence'
import {
resolveStatefulSetHeadlessServiceName,
resolveWorkloadMcpServerLabel,
@@ -194,6 +199,24 @@ const RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION =
const RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION =
'clerum.io/runtime-http-egress-previous-cidr-expiries'
const RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION = 'clerum.io/runtime-http-egress-resolved-at'
+// The run lane authors these keys even when desired omits them, so a stale
+// live value is drift and the replacement drops it.
+const RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS: ReadonlySet<string> = new Set([
+ RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION,
+ RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION,
+ RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION,
+ RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION,
+ RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION,
+])
+
+type RunLaneNetworkPolicyApplyResult =
+ | { policy: string; action: 'created' | 'replaced' | 'unchanged' }
+ | { policy: string; action: 'retry'; reason: 'terminating' | 'absent-after-write-conflict' }
+ | {
+ policy: string
+ action: 'conflict'
+ reason: Extract<NetworkPolicyConvergenceDecision, { action: 'conflict' }>['reason']
+ }
const MCP_HOST_READINESS_WAIT_TIMEOUT_MS = 4 * 60_000
const DNS_SUBDOMAIN_RE =
/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/
@@ -4589,41 +4612,96 @@ export class WorkflowReconciler {
}
}
- private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {
+ private async readNetworkPolicyOrNull(
+ name: string,
+ namespace: string
+ ): Promise<k8s.V1NetworkPolicy | null> {
+ try {
+ return await this.deps.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) return null
+ throw error
+ }
+ }
+
+ /**
+ * Read first, then write only what the live object lacks. Ownership conflicts
+ * and terminating objects are returned, not thrown: the workflow reconcile
+ * catch turns every non-transport error into a terminal `failed`.
+ */
+ private async applyNetworkPolicy(
+ policy: k8s.V1NetworkPolicy
+ ): Promise<RunLaneNetworkPolicyApplyResult> {
const name = policy.metadata?.name
const namespace = policy.metadata?.namespace
if (!name || !namespace)
throw new Error('NetworkPolicy metadata.name and namespace are required')
- try {
- await this.deps.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
- this.log.info(`Created NetworkPolicy "${name}"`)
- return
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) throw error
+ let existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) {
+ try {
+ await this.deps.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
+ this.log.info(`Created NetworkPolicy "${name}"`)
+ return { policy: name, action: 'created' }
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 409) throw error
+ }
+ existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)
}
for (let attempt = 0; attempt < 2; attempt += 1) {
- const existing = await this.deps.networkingApi.readNamespacedNetworkPolicy({
- name,
- namespace,
- })
- policy.metadata = {
- ...policy.metadata,
- resourceVersion: existing.metadata?.resourceVersion,
+ const decision: NetworkPolicyConvergenceDecision = decideNetworkPolicyConvergence(
+ 'workload-egress',
+ policy,
+ existing,
+ RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS
+ )
+ if (decision.action === 'unchanged') {
+ this.log.info(`Unchanged NetworkPolicy "${name}"; skipping update`, { namespace })
+ return { policy: name, action: 'unchanged' }
+ }
+ if (decision.action === 'conflict') {
+ this.log.warn(`NetworkPolicy "${name}" is not owned by this workflow; leaving it`, {
+ namespace,
+ reason: decision.reason,
+ })
+ return { policy: name, action: 'conflict', reason: decision.reason }
+ }
+ if (decision.action === 'retry') {
+ this.log.warn(`NetworkPolicy "${name}" is terminating; retrying later`, { namespace })
+ return { policy: name, action: 'retry', reason: decision.reason }
}
try {
await this.deps.networkingApi.replaceNamespacedNetworkPolicy({
name,
namespace,
- body: policy,
+ body: buildNetworkPolicyReplacement(
+ policy,
+ existing,
+ RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS
+ ),
})
- this.log.info(`Updated NetworkPolicy "${name}"`)
- return
+ this.log.info(`Updated NetworkPolicy "${name}"`, { namespace, reason: decision.reason })
+ return { policy: name, action: 'replaced' }
} catch (error: unknown) {
+ // A stale resourceVersion gets one re-read and re-decision.
if (getErrorCode(error) !== 409 || attempt === 1) throw error
}
+ existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)
}
+ throw new Error(`NetworkPolicy "${name}" apply loop exited without a result`)
+ }
+
+ private networkPolicyAbsentAfterConflict(
+ name: string,
+ namespace: string
+ ): RunLaneNetworkPolicyApplyResult {
+ this.log.warn(`NetworkPolicy "${name}" vanished after a write conflict; retrying later`, {
+ namespace,
+ })
+ return { policy: name, action: 'retry', reason: 'absent-after-write-conflict' }
}
private async safeDelete(deleteFn: () => Promise<unknown>): Promise<void> {
diff --git a/workflow-recipes/tests/unit/workflow/asApiserverNetworkPolicy.ts b/workflow-recipes/tests/unit/workflow/asApiserverNetworkPolicy.ts
new file mode 100644
index 000000000..d6cbe2277
--- /dev/null
+++ b/workflow-recipes/tests/unit/workflow/asApiserverNetworkPolicy.ts
@@ -0,0 +1,118 @@
+import type * as k8s from '@kubernetes/client-node'
+
+/**
+ * Copied from host-context-controller/src/__tests__/asApiserverNetworkPolicy.ts.
+ * The packages are separate npm projects without a workspace, so the fixture
+ * cannot be imported across them.
+ *
+ * Recorded kube-apiserver GET of a NetworkPolicy after client-node decode
+ * (`from` → `_from`), not a field list copied from the comparator. Nested port
+ * defaults are merged from this blob (and `RECORDED_DEFAULT_PORT`) onto desired
+ * ports so a newer apiserver default-fill goes red until the comparator learns
+ * it. Do not reconstruct ports from desired alone.
+ */
+
+/** Recorded apiserver default-fill on a port that omits protocol. */
+const RECORDED_DEFAULT_PORT: k8s.V1NetworkPolicyPort = { protocol: 'TCP' }
+
+/** Recorded apiserver policyTypes when the live object has egress rules. */
+const RECORDED_EGRESS_POLICY_TYPES: string[] = ['Ingress', 'Egress']
+
+export const RECORDED_NETWORKPOLICY: k8s.V1NetworkPolicy = {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ annotations: {
+ 'kubectl.kubernetes.io/last-applied-configuration': '{}',
+ },
+ creationTimestamp: new Date('2026-04-01T00:00:00.000Z'),
+ generation: 1,
+ managedFields: [
+ {
+ apiVersion: 'networking.k8s.io/v1',
+ fieldsType: 'FieldsV1',
+ fieldsV1: { 'f:spec': { 'f:podSelector': {} } },
+ manager: 'kube-apiserver',
+ operation: 'Update',
+ time: new Date('2026-04-01T00:00:00.000Z'),
+ },
+ ],
+ name: 'recorded-np',
+ namespace: 'recorded-ns',
+ resourceVersion: '1783417',
+ uid: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
+ selfLink: '/apis/networking.k8s.io/v1/namespaces/recorded-ns/networkpolicies/recorded-np',
+ },
+ spec: {
+ podSelector: { matchLabels: { app: 'recorded' } },
+ policyTypes: ['Ingress'],
+ ingress: [
+ {
+ _from: [{ podSelector: { matchLabels: { app: 'peer' } } }],
+ ports: [{ port: 8080, protocol: 'TCP' }],
+ },
+ ],
+ },
+}
+
+function stampPorts(
+ rules: k8s.V1NetworkPolicyIngressRule[] | k8s.V1NetworkPolicyEgressRule[] | undefined,
+ recordedRules: typeof rules
+): typeof rules {
+ if (!rules) return rules
+ // Unlike the HCC original, recorded peers are not merged into the rule: a
+ // desired rule without `_from`/`to` (allow-all) must not gain a recorded peer
+ // the apiserver never adds. Only port defaults come from the recording.
+ return rules.map((rule, ruleIndex) => {
+ const recordedRule = recordedRules?.[ruleIndex]
+ if (!rule.ports) return { ...rule, ports: undefined }
+ return {
+ ...rule,
+ ports: rule.ports.map((port, portIndex) => {
+ const recordedPort = recordedRule?.ports?.[portIndex]
+ return {
+ ...RECORDED_DEFAULT_PORT,
+ ...recordedPort,
+ ...port,
+ protocol: port.protocol ?? recordedPort?.protocol ?? RECORDED_DEFAULT_PORT.protocol,
+ }
+ }),
+ }
+ })
+}
+
+/**
+ * The object a GET returns after the apiserver stored `desired`: recorded
+ * server metadata, desired identity labels/annotations, and default-filled
+ * spec fields. The caller sets the resourceVersion it wants to model.
+ */
+export function asApiserverNetworkPolicy(desired: k8s.V1NetworkPolicy): k8s.V1NetworkPolicy {
+ const recorded = structuredClone(RECORDED_NETWORKPOLICY)
+ const hasEgress = (desired.spec?.egress?.length ?? 0) > 0
+ const explicitTypes = desired.spec?.policyTypes
+ const policyTypes =
+ explicitTypes && explicitTypes.length > 0
+ ? explicitTypes
+ : hasEgress
+ ? [...RECORDED_EGRESS_POLICY_TYPES]
+ : [...(recorded.spec?.policyTypes ?? ['Ingress'])]
+ return {
+ ...recorded,
+ metadata: {
+ ...recorded.metadata,
+ name: desired.metadata?.name,
+ namespace: desired.metadata?.namespace,
+ labels: desired.metadata?.labels,
+ annotations: desired.metadata?.annotations,
+ ownerReferences: desired.metadata?.ownerReferences,
+ selfLink: `/apis/networking.k8s.io/v1/namespaces/${desired.metadata?.namespace}/networkpolicies/${desired.metadata?.name}`,
+ },
+ spec: {
+ ...recorded.spec,
+ podSelector: desired.spec?.podSelector ?? {},
+ policyTypes,
+ ingress: stampPorts(desired.spec?.ingress, recorded.spec?.ingress),
+ egress: stampPorts(desired.spec?.egress, recorded.spec?.egress),
+ },
+ }
+}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index c6a16fe5e..89717173e 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -7,6 +7,7 @@
* Deps: { coreApi, customApi, networkingApi, batchApi?, config, tokenFactory }
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type * as k8s from '@kubernetes/client-node'
import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'
import {
resolveStatefulSetHeadlessServiceName,
@@ -34,6 +35,7 @@ import {
WorkflowReconciler,
type WorkflowReconcilerDeps,
} from '../../../src/workflow/workflowReconciler'
+import { asApiserverNetworkPolicy } from './asApiserverNetworkPolicy'
vi.mock('../../../src/workflow/mcpHostRuntimeTokenIssuerClient', () => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
@@ -364,6 +366,102 @@ function makeNetworkingApi() {
}
}
+/**
+ * Stateful NetworkPolicy double. Stored objects are what the apiserver would
+ * return (`asApiserverNetworkPolicy`), never the body that was sent, so a
+ * second pass only converges if the comparison tolerates apiserver defaults.
+ * create rejects 409 when present; replace rejects 404 when absent and 409 on a
+ * stale resourceVersion.
+ */
+function makeApiserverNetworkingApi() {
+ const live = new Map<string, k8s.V1NetworkPolicy>()
+ let revision = 100
+ const key = (namespace: string, name: string) => `${namespace}/${name}`
+ const store = (namespace: string, body: k8s.V1NetworkPolicy): k8s.V1NetworkPolicy => {
+ revision += 1
+ const stored = asApiserverNetworkPolicy({
+ ...body,
+ metadata: { ...body.metadata, namespace },
+ })
+ stored.metadata = { ...stored.metadata, resourceVersion: String(revision) }
+ live.set(key(namespace, stored.metadata.name!), stored)
+ return structuredClone(stored)
+ }
+ const api = {
+ createNamespacedNetworkPolicy: vi.fn(
+ async ({ namespace, body }: { namespace: string; body: k8s.V1NetworkPolicy }) => {
+ if (live.has(key(namespace, body.metadata!.name!))) throw { code: 409 }
+ return store(namespace, body)
+ }
+ ),
+ readNamespacedNetworkPolicy: vi.fn(
+ async ({ name, namespace }: { name: string; namespace: string }) => {
+ const found = live.get(key(namespace, name))
+ if (!found) throw { code: 404 }
+ return structuredClone(found)
+ }
+ ),
+ replaceNamespacedNetworkPolicy: vi.fn(
+ async ({
+ name,
+ namespace,
+ body,
+ }: {
+ name: string
+ namespace: string
+ body: k8s.V1NetworkPolicy
+ }) => {
+ const found = live.get(key(namespace, name))
+ if (!found) throw { code: 404 }
+ if (body.metadata?.resourceVersion !== found.metadata?.resourceVersion) {
+ throw { code: 409 }
+ }
+ return store(namespace, body)
+ }
+ ),
+ deleteNamespacedNetworkPolicy: vi.fn(
+ async ({ name, namespace }: { name: string; namespace: string }) => {
+ if (!live.delete(key(namespace, name))) throw { code: 404 }
+ return {}
+ }
+ ),
+ listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
+ }
+ return { api, live, key }
+}
+
+function createdPolicyNames(networkingApi: ReturnType<typeof makeNetworkingApi>): string[] {
+ return networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => (arg as { body?: k8s.V1NetworkPolicy }).body?.metadata?.name ?? ''
+ )
+}
+
+function captureRunLaneNetworkPolicyLogs(): {
+ entries: Array<Record<string, unknown>>
+ restore: () => void
+} {
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const entries: Array<Record<string, unknown>> = []
+ const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
+ for (const line of text.split('\n')) {
+ if (!line.startsWith('{')) continue
+ const parsed = JSON.parse(line) as Record<string, unknown>
+ if (String(parsed.msg).includes('NetworkPolicy')) entries.push(parsed)
+ }
+ return true
+ }) as unknown as typeof process.stdout.write)
+ return {
+ entries,
+ restore: () => {
+ spy.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ },
+ }
+}
+
function hasPublicHttpEgressRule(policy: {
spec?: {
egress?: Array<{
@@ -1907,7 +2005,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('refreshes runtime HTTP egress policies with DNS overlap for active snippets', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
@@ -1954,6 +2051,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
expect(replaced!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
@@ -1967,7 +2065,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('refreshes coordinator and snippet runtime HTTP egress policies with one overlap window', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
const laterPreviousExpiresAt = new Date(Date.now() + 10 * 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
if (name === 'test-wf-coord-to-wrc') {
@@ -2042,6 +2139,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
for (const name of ['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress']) {
const policy = replacedPolicies[name]
expect(policy).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain(name)
expect(publicHttpEgressCidrs(policy!)).toEqual([
'93.184.216.33/32',
'93.184.216.34/32',
@@ -2069,7 +2167,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('does not extend older runtime HTTP egress CIDR expiries during a new DNS rollover', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
const olderPreviousExpiresAt = new Date(Date.now() + 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
@@ -2120,6 +2217,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual([
'93.184.216.34/32',
'93.184.216.35/32',
@@ -2136,7 +2234,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('prunes expired runtime HTTP egress overlap CIDRs even when DNS refresh fails', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
const activePreviousExpiresAt = new Date(Date.now() + 5 * 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
@@ -2190,6 +2287,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32', '93.184.216.36/32'])
expect(replaced!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
@@ -2252,7 +2350,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('drops expired runtime HTTP egress overlap CIDRs on refresh', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
@@ -2299,6 +2396,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32'])
expect(
replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']
@@ -2310,7 +2408,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('drops invalid, private, or untrusted-future runtime HTTP egress overlap CIDRs from annotations', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
@@ -2359,6 +2456,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
expect(replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']).toBe(
'93.184.216.34/32'
@@ -2629,7 +2727,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('replaces existing NetworkPolicies so removed HTTP egress cannot persist', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: { name: 'test-wf-coord-to-wrc', resourceVersion: 'rv-1' },
})
@@ -2661,6 +2758,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)?.[0].body
expect(coordPolicy).toBeDefined()
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
expect(coordPolicy!.metadata!.resourceVersion).toBe('rv-1')
expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(false)
})
@@ -2694,7 +2792,6 @@ describe('WorkflowReconciler — reconcile loop', () => {
it('retries NetworkPolicy replace once when resourceVersion is stale', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
- networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
const readCounts = new Map<string, number>()
const coordResourceVersions: Array<string | undefined> = []
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
@@ -2740,6 +2837,172 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
expect(coordCalls).toHaveLength(2)
expect(coordResourceVersions).toEqual(['rv-1', 'rv-2'])
+ expect(readCounts.get('test-wf-coord-to-wrc')).toBe(2)
+ expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
+ })
+
+ describe('run-lane NetworkPolicy convergence against apiserver-shaped live objects', () => {
+ const RUN_LANE_POLICY_NAMES = [
+ 'sandbox-recipes/test-wf-coord-to-snippet-runner',
+ 'sandbox-recipes/test-wf-coord-to-snippet-runner-ingress',
+ 'sandbox-recipes/test-wf-coord-to-wrc',
+ 'sandbox-recipes/test-wf-snippet-runner-egress',
+ 'sandbox-recipes/test-wf-wrc-to-artifact-reader',
+ ]
+
+ function readPolicyNames(api: ReturnType<typeof makeApiserverNetworkingApi>['api']): string[] {
+ return api.readNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name).sort()
+ }
+
+ function runtimeEgressSpec(): WorkflowRecipeSpec {
+ return makeSpec({
+ agent: undefined,
+ runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
+ steps: [
+ {
+ id: 'snippet',
+ run: {
+ type: 'snippet',
+ language: 'typescript',
+ code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
+ capabilities: { http: { allowedHosts: ['api.example.com'] } },
+ },
+ },
+ ],
+ })
+ }
+
+ it('writes nothing on a second pass once every policy is live', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ const { api, live } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ expect(first.workflowPhase).not.toBe('failed')
+ expect([...live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)
+
+ api.createNamespacedNetworkPolicy.mockClear()
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+ logs.entries.length = 0
+
+ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(second.workflowPhase).not.toBe('failed')
+ const expectedNames = RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
+ expect(readPolicyNames(api)).toEqual(expectedNames)
+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ for (const name of expectedNames) {
+ expect(
+ logs.entries.some(entry =>
+ String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
+ )
+ ).toBe(true)
+ }
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('replaces exactly the drifted policy with its live resourceVersion', async () => {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ const driftedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
+ const drifted = live.get(driftedKey)
+ expect(drifted).toBeDefined()
+ drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { app: 'edited-by-hand' } } }
+ const liveResourceVersion = drifted!.metadata!.resourceVersion
+
+ api.createNamespacedNetworkPolicy.mockClear()
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+
+ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
+ expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const [replaceArg] = api.replaceNamespacedNetworkPolicy.mock.calls[0]!
+ expect(replaceArg.name).toBe('test-wf-coord-to-wrc')
+ expect(replaceArg.body.metadata?.resourceVersion).toBe(liveResourceVersion)
+ expect(live.get(driftedKey)!.spec!.podSelector).not.toEqual({
+ matchLabels: { app: 'edited-by-hand' },
+ })
+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+
+ it('refresh leaves a policy owned by another controller untouched and warns', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ networkingApi: api as never,
+ resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,
+ })
+ )
+ const spec = runtimeEgressSpec()
+ await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
+ 'sandbox-recipes',
+ 'test-wf',
+ 'uid-123',
+ spec
+ )
+ const foreign = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
+ expect(foreign).toBeDefined()
+ foreign!.metadata = {
+ ...foreign!.metadata,
+ ownerReferences: [
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ name: 'another-controller',
+ uid: 'foreign-owner-uid',
+ controller: true,
+ },
+ ],
+ }
+
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+ logs.entries.length = 0
+
+ await expect(
+ reconciler.refreshRuntimeHttpEgressNetworkPolicies(
+ 'sandbox-recipes',
+ 'test-wf',
+ 'uid-123',
+ spec
+ )
+ ).resolves.toBeUndefined()
+
+ expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
+ expect(
+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-snippet-runner-egress'
+ )
+ ).toHaveLength(0)
+ // Only the ownership branch emits this reason; a vanished-after-conflict
+ // warning also names the policy and must not satisfy this witness.
+ expect(
+ logs.entries.some(
+ entry =>
+ entry.level === 'warn' &&
+ entry.reason === 'owner-reference-mismatch' &&
+ String(entry.msg).includes('test-wf-snippet-runner-egress')
+ )
+ ).toBe(true)
+ } finally {
+ logs.restore()
+ }
+ })
})
it('resolves snippet MCP transport workloads into the mounted runner config', async () => {