Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): propagate non-404 gate reads and close re-review gaps

Alfredo Lopezdc2fa7c67f59
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 3a7c19686..9dfd0c21a 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.367",
+ "version": "0.1.368",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.367",
+ "version": "0.1.368",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 0fd1f05c6..774903918 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.367",
+ "version": "0.1.368",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
index 4d9819a22..b656cb992 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
@@ -2307,7 +2307,7 @@ describe('delegateTransportWorkloads', () => {
// any other failure on the way to it.
expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {
workloadId: 'redis-mcp',
- error: expect.objectContaining({
+ err: expect.objectContaining({
message: expect.stringContaining('already exists for recipe "other-recipe"'),
}),
})
@@ -2463,7 +2463,7 @@ describe('delegateTransportWorkloads', () => {
expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)
expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {
workloadId: 'redis-mcp',
- error: expect.objectContaining({
+ err: expect.objectContaining({
message: expect.stringContaining('failed to update after conflict retries'),
}),
})
@@ -3419,7 +3419,7 @@ describe('transport Service read-first apply', () => {
expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })
expect(errorLog).toHaveBeenCalledWith(
'Failed to delegate workload',
- expect.objectContaining({ error: { code: 503 } })
+ expect.objectContaining({ err: { code: 503 } })
)
expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)
expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
@@ -3452,7 +3452,7 @@ describe('transport Service read-first apply', () => {
expect(errorLog).toHaveBeenCalledWith(
'Failed to delegate workload',
expect.objectContaining({
- error: expect.objectContaining({
+ err: expect.objectContaining({
name: 'RetryableReconcileError',
message: expect.stringContaining(
'disappeared after create conflict; a fresh reconciliation is required'
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.ts
index b898560f7..b613bd3ab 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts
@@ -593,6 +593,12 @@ export function buildTransportService(
* clusterIP (spec.clusterIP is immutable once assigned). Any read failure other
* than a 404 propagates: the replace needs the live resourceVersion and
* clusterIP, which a failed read cannot provide.
+ *
+ * The steady-state skip does not hold for a stdio transport. HCC also writes
+ * that Service (host-context-controller/src/reconciler.ts) with its own
+ * controller ownerReference and keeps WRC's spec-hash annotation, so the owner
+ * uid never matches and WRC replaces the Service on every pass, as it did before
+ * the read-first gate. Which controller owns it is left to a follow-up.
*/
async function ensureTransportService(
deps: DelegationDeps,
@@ -1170,7 +1176,7 @@ export async function preDeployMcpServers(
preDeployed.push(r.value)
} else if (r.status === 'rejected') {
const workloadId = transportWorkloads[i].id
- log.warn('Pre-deploy failed for workload', { workloadId, error: r.reason })
+ log.warn('Pre-deploy failed for workload', { workloadId, err: r.reason })
errors.push({ workloadId, error: r.reason })
}
}
@@ -1481,7 +1487,7 @@ export async function delegateTransportWorkloads(
delegated.push(r.value)
} else {
const workloadId = transportWorkloads[i].id
- log.error('Failed to delegate workload', { workloadId, error: r.reason })
+ log.error('Failed to delegate workload', { workloadId, err: r.reason })
errors.push({ workloadId, error: r.reason })
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index fbb200aa6..6e2a71115 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -223,6 +223,12 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R
* 404; one it created or replaced reads back as written, carrying `status` —
* the same "ready" status the observation path used to get from a static read.
* `admit` models fields the apiserver assigns on create (a Service's clusterIP).
+ *
+ * `unseenReplace` says what a PUT to an object the store never held does.
+ * `'reject'` answers 404, as the apiserver does. `'accept'` exists for kinds
+ * whose tests override `read` to supply a live object with a specific status
+ * (the readiness tests): the object exists in that test's model, so its PUT
+ * succeeds.
*/
function installLiveStore(
api: {
@@ -231,7 +237,8 @@ function installLiveStore(
replace: ReturnType<typeof vi.fn>
},
status: Record<string, unknown>,
- admit?: (stored: LiveManifest) => void
+ admit?: (stored: LiveManifest) => void,
+ unseenReplace: 'accept' | 'reject' = 'accept'
): void {
const live = new Map<string, LiveManifest>()
const key = (namespace: string, name: string) => `${namespace}/${name}`
@@ -267,8 +274,7 @@ function installLiveStore(
}) => {
const k = key(namespace, name)
const current = live.get(k)
- // A test that overrides `read` supplies the live object itself, so an
- // object the store never saw is accepted here rather than 404'd.
+ if (!current && unseenReplace === 'reject') throw { code: 404 }
if (current && body.metadata.resourceVersion !== current.metadata.resourceVersion) {
throw { code: 409 }
}
@@ -296,7 +302,8 @@ function installLiveServiceAndConfigMapStores(): void {
{},
stored => {
stored.spec = { ...(stored.spec as Record<string, unknown>), clusterIP: LIVE_CLUSTER_IP }
- }
+ },
+ 'reject'
)
installLiveStore(
{
@@ -304,7 +311,9 @@ function installLiveServiceAndConfigMapStores(): void {
create: mockCoreApi.createNamespacedConfigMap,
replace: mockCoreApi.replaceNamespacedConfigMap,
},
- {}
+ {},
+ undefined,
+ 'reject'
)
}
@@ -375,6 +384,15 @@ describe('WorkflowRecipeReconciler', () => {
{ succeeded: 1 }
)
installLiveServiceAndConfigMapStores()
+ // clearAllMocks keeps implementations and `Once` queues, so a Secret store
+ // or a create override installed by one test would reach the next. Every
+ // test starts with no live Secret; tests that need one install it.
+ mockCoreApi.readNamespacedSecret.mockReset()
+ mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
+ mockCoreApi.createNamespacedSecret.mockReset()
+ mockCoreApi.createNamespacedSecret.mockResolvedValue({})
+ mockCoreApi.replaceNamespacedSecret.mockReset()
+ mockCoreApi.replaceNamespacedSecret.mockResolvedValue({})
mockCoreApi.readNamespacedPersistentVolumeClaim.mockReset()
mockCoreApi.readNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 404 })
mockCustomApi.createNamespacedCustomObject.mockReset()
@@ -2715,19 +2733,6 @@ describe('WorkflowRecipeReconciler', () => {
expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)
})
- it('creates a Deployment whose gate read resolves to nothing', async () => {
- const recipe = makeRecipe()
- const store = deploymentStore()
- mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce(null)
-
- await ensureDeployment(recipe.spec.workloads![0], recipe)
-
- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(store.live()).not.toBeNull()
- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)
- })
-
// The POST is mocked to 409 only so that the pre-read-first code reaches
// the gate at all; with the read first, it must never be sent.
it('propagates an unclassifiable gate-read error and writes nothing', async () => {
@@ -2774,21 +2779,42 @@ describe('WorkflowRecipeReconciler', () => {
'a transport reset',
Object.assign(new Error('connect ECONNRESET 10.0.0.1:443'), { code: 'ECONNRESET' }),
],
- ])('falls through to the replace when the gate read fails with %s', async (_, failure) => {
+ ])('propagates a gate read that fails with %s without writing', async (_, failure) => {
const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
const recipe = makeRecipe({ spec: { workloads: [workload] } })
- // The first pass creates the headless Service, so the replace has a
- // live object to read back.
+ // The first pass creates the headless Service, so a replace would have
+ // a live object to write: only the gate's decision can keep the PUT out.
await ensureStatefulSet(workload, recipe)
+ mockCoreApi.readNamespacedService.mockClear()
mockCoreApi.createNamespacedService.mockClear()
mockCoreApi.replaceNamespacedService.mockClear()
mockCoreApi.readNamespacedService.mockRejectedValueOnce(failure)
- await ensureStatefulSet(workload, recipe)
+ await expect(ensureStatefulSet(workload, recipe)).rejects.toBe(failure)
- // Witness: the PUT went out, so the fall-through ran.
- expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)
+ // Witness: the gate read ran and failed; nothing followed it.
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)
expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('propagates a failed re-read after a create conflict without writing', async () => {
+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
+ const recipe = makeRecipe({ spec: { workloads: [workload] } })
+ const unavailable = { code: 503 }
+ // The gate read misses, the POST loses a race, and the conflict re-read
+ // fails: nothing says whether the object exists, so nothing is written.
+ mockCoreApi.readNamespacedService
+ .mockRejectedValueOnce({ code: 404 })
+ .mockRejectedValueOnce(unavailable)
+ mockCoreApi.createNamespacedService.mockRejectedValueOnce({ code: 409 })
+
+ await expect(ensureStatefulSet(workload, recipe)).rejects.toBe(unavailable)
+
+ // Witness: the missed read, the POST and the failed re-read all ran.
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(2)
+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(1)
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
})
@@ -2852,6 +2878,26 @@ describe('WorkflowRecipeReconciler', () => {
expect(replaced.body.metadata.resourceVersion).toBe('1')
})
+ // Before the read-first change the POST went first, and a 503 on it kept
+ // the phase and requeued. The same 503 on the gate read must do the same,
+ // not reach a replace whose re-read 404s into a terminal `failed`.
+ it('keeps the phase and requeues when the gate read of an absent Service fails with 503', async () => {
+ const recipe = makeRecipe({
+ status: { phase: 'active', message: 'All workloads deployed' },
+ })
+ mockCoreApi.readNamespacedService.mockRejectedValueOnce({ code: 503 })
+
+ const result = await reconciler.reconcile(recipe)
+
+ // Witness: the gate read ran exactly once and nothing was written.
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)
+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ expect(result.phase).toBe('active')
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ })
+
it('sends neither POST nor PUT for an unchanged workflow-lane Service', async () => {
const recipe = makeRecipe({
spec: {
@@ -3252,6 +3298,97 @@ describe('WorkflowRecipeReconciler', () => {
expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)
})
+
+ // Each case leaves data equal to the desired Secret and changes one other
+ // field secretMatchesDesired compares, so only that comparison can force
+ // the PUT.
+ it.each([
+ ['its type', (live: SecretManifest) => ({ ...live, type: 'kubernetes.io/basic-auth' })],
+ [
+ 'a label that is not the ownership label',
+ (live: SecretManifest) => {
+ const labels = { ...(live.metadata.labels as Record<string, string>) }
+ const key = Object.keys(labels).find(k => k !== 'clerum.io/recipe')
+ if (!key) throw new Error('the built Secret carries no non-ownership label')
+ delete labels[key]
+ return { ...live, metadata: { ...live.metadata, labels } }
+ },
+ ],
+ [
+ 'the uid of its controller owner (recipe recreated under the same name)',
+ (live: SecretManifest) => {
+ const refs = live.metadata.ownerReferences as Array<{
+ controller?: boolean
+ uid?: string
+ }>
+ if (!refs?.some(ref => ref.controller && ref.uid)) {
+ throw new Error('the built Secret carries no controller owner uid')
+ }
+ return {
+ ...live,
+ metadata: {
+ ...live.metadata,
+ ownerReferences: refs.map(ref =>
+ ref.controller ? { ...ref, uid: 'uid-old' } : ref
+ ),
+ },
+ }
+ },
+ ],
+ ])('replaces a user-data Secret whose live copy differs in %s', async (_field, drift) => {
+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ const live = await readLive(name, created.namespace)
+ await mockCoreApi.replaceNamespacedSecret({
+ name,
+ namespace: created.namespace,
+ body: drift(live),
+ })
+ clearSecretCalls()
+
+ await ensureSecret(res, recipe)
+
+ // Witness: the gate read the drifted Secret before deciding to write.
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(1)
+ const replaced = mockCoreApi.replaceNamespacedSecret.mock.calls[0][0] as {
+ body: SecretManifest
+ }
+ expect(replaced.body).toMatchObject({
+ type: created.body.type,
+ metadata: {
+ labels: created.body.metadata.labels,
+ ownerReferences: created.body.metadata.ownerReferences,
+ },
+ })
+ })
+
+ it.each([
+ ['403', { code: 403 }],
+ ['503', { code: 503 }],
+ ])(
+ 'propagates a Secret read that fails with %s without writing',
+ async (_status, failure) => {
+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ clearSecretCalls()
+ mockCoreApi.readNamespacedSecret.mockRejectedValueOnce(failure)
+
+ await expect(ensureSecret(res, recipe)).rejects.toBe(failure)
+
+ // Witness: the failing read is the only call; nothing was written blind.
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(0)
+ expect(mockCoreApi.replaceNamespacedSecret).toHaveBeenCalledTimes(0)
+ }
+ )
})
it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {
@@ -3763,32 +3900,34 @@ describe('WorkflowRecipeReconciler', () => {
expect(mockAppsApi.replaceNamespacedDaemonSet).toHaveBeenCalledTimes(0)
})
- it('falls open and replaces when a transport failure hides the existing object (never silently skips)', async () => {
+ it('propagates a transport failure on the gate read instead of writing blind', async () => {
const recipe = makeRecipe()
const workload = recipe.spec.workloads![0]
+ const timeout = Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), {
+ code: 'ETIMEDOUT',
+ })
- // First read (the gate) times out at the socket; the replace path's own read then succeeds.
+ // The gate read times out at the socket; a later read would succeed, so
+ // only the gate's decision can keep the replace path from running.
mockAppsApi.readNamespacedDeployment
.mockReset()
- .mockRejectedValueOnce(
- Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), { code: 'ETIMEDOUT' })
- )
+ .mockRejectedValueOnce(timeout)
.mockResolvedValue({ metadata: { resourceVersion: '9' } })
mockAppsApi.createNamespacedDeployment.mockClear()
mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})
- await (
- reconciler as unknown as {
- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
- }
- ).ensureDeployment(workload, recipe, 'minimal', {})
+ await expect(
+ (
+ reconciler as unknown as {
+ ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ }
+ ).ensureDeployment(workload, recipe, 'minimal', {})
+ ).rejects.toBe(timeout)
- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
+ // Witness: the gate read ran once; neither write followed it.
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(
- mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion
- ).toBe('9')
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)
})
})
@@ -4572,18 +4711,19 @@ describe('WorkflowRecipeReconciler', () => {
it('creates resources before workloads (3.11a resources)', async () => {
const callOrder: string[] = []
- mockCoreApi.createNamespacedSecret.mockImplementation(async () => {
- callOrder.push('secret')
- return {}
- })
- mockCoreApi.createNamespacedConfigMap.mockImplementation(async () => {
- callOrder.push('configmap')
- return {}
- })
- mockAppsApi.createNamespacedDeployment.mockImplementation(async () => {
- callOrder.push('deployment')
- return {}
- })
+ // Record the order, then hand the call to the live store installed by
+ // beforeEach, so the created objects exist for the readiness reads.
+ const recordThenStore = (fn: ReturnType<typeof vi.fn>, kind: string) => {
+ const store = fn.getMockImplementation()
+ if (!store) throw new Error(`beforeEach installed no implementation for the ${kind} create`)
+ fn.mockImplementation(async (args: unknown) => {
+ callOrder.push(kind)
+ return store(args)
+ })
+ }
+ recordThenStore(mockCoreApi.createNamespacedSecret, 'secret')
+ recordThenStore(mockCoreApi.createNamespacedConfigMap, 'configmap')
+ recordThenStore(mockAppsApi.createNamespacedDeployment, 'deployment')
const recipe = makeRecipe({
spec: {
@@ -4594,7 +4734,12 @@ describe('WorkflowRecipeReconciler', () => {
],
},
})
- await reconciler.reconcile(recipe)
+ const result = await reconciler.reconcile(recipe)
+ expect(result.phase).toBe('active')
+ // Witness: each object was created, so an index of -1 cannot pass the order check.
+ expect(callOrder.filter(c => c === 'secret')).toHaveLength(1)
+ expect(callOrder.filter(c => c === 'configmap')).toHaveLength(1)
+ expect(callOrder).toContain('deployment')
expect(callOrder.indexOf('secret')).toBeLessThan(callOrder.indexOf('deployment'))
expect(callOrder.indexOf('configmap')).toBeLessThan(callOrder.indexOf('deployment'))
})
@@ -7027,8 +7172,10 @@ describe('WorkflowRecipeReconciler', () => {
resources: [{ id: 'creds', type: 'secret', data: { key: 'val' } }],
},
})
- await reconciler.reconcile(recipe)
+ const result = await reconciler.reconcile(recipe)
+ expect(result.phase).toBe('active')
// Secret not mounted by any workload → stays with the recipe runtime.
+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)
const secretCall = mockCoreApi.createNamespacedSecret.mock.calls[0][0]
expect(secretCall.namespace).toBe('sandbox-recipes')
})
@@ -12222,6 +12369,29 @@ describe('WorkflowRecipeReconciler', () => {
expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
})
+
+ it('propagates a policy read that fails with 503 instead of creating blind', async () => {
+ stubSteadyWorkflowRuntime()
+ installGfsPolicyStore()
+ const failure = { code: 503 }
+ const storeRead = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ async (args: { name: string; namespace: string }) => {
+ if (args.name === gfsPolicyName) throw failure
+ return storeRead(args)
+ }
+ )
+ clearPolicyCalls()
+
+ await expect(reconciler.reconcile(activeWorkflow(publishTargets))).rejects.toBe(failure)
+
+ // Witness: the failing read is the only call on the policy.
+ expect(
+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')
+ ).toBe(1)
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ })
})
it('refuses to delete a homonymous policy not owned by the exact recipe', async () => {
diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
index a0160891b..4886756cd 100644
--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts
+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
@@ -926,6 +926,21 @@ describe('buildSecret', () => {
expect(secret.metadata?.ownerReferences).toHaveLength(1)
expect(secret.metadata?.ownerReferences![0].kind).toBe('WorkflowRecipe')
})
+
+ // The reconciler's secretMatchesDesired skips the PUT by comparing exactly
+ // these fields. A field added here (annotations, immutable, stringData) must
+ // be added to that comparison too, or a change to it is never written.
+ it('emits only the fields secretMatchesDesired compares', () => {
+ const res: SecretResourceDef = { id: 'creds', type: 'secret', data: { password: 's3cret' } }
+ const secret = buildSecret(res, makeRecipe())
+ expect(Object.keys(secret).sort()).toEqual(['apiVersion', 'data', 'kind', 'metadata', 'type'])
+ expect(Object.keys(secret.metadata!).sort()).toEqual([
+ 'labels',
+ 'name',
+ 'namespace',
+ 'ownerReferences',
+ ])
+ })
})
// ─── ConfigMap Builder ──────────────────────────────────────────────
diff --git a/workflow-recipes/src/reconciler/specHash.property.test.ts b/workflow-recipes/src/reconciler/specHash.property.test.ts
new file mode 100644
index 000000000..61151afec
--- /dev/null
+++ b/workflow-recipes/src/reconciler/specHash.property.test.ts
@@ -0,0 +1,57 @@
+import { describe, expect, it } from 'vitest'
+import fc from 'fast-check'
+import { SPEC_HASH_ANNOTATION, computeSpecHash, stampSpecHash } from './specHash'
+
+// Rebuild every object with its keys in reverse insertion order; arrays keep
+// their order. Object.fromEntries defines own properties, so a generated
+// "__proto__" key stays a key instead of replacing the prototype.
+function reverseKeys(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(reverseKeys)
+ if (value === null || typeof value !== 'object') return value
+ return Object.fromEntries(
+ Object.entries(value as Record<string, unknown>)
+ .reverse()
+ .map(([key, child]) => [key, reverseKeys(child)])
+ )
+}
+
+const manifest = fc.record({
+ metadata: fc.record({
+ name: fc.string({ minLength: 1 }),
+ labels: fc.dictionary(fc.string({ minLength: 1 }), fc.string()),
+ }),
+ spec: fc.jsonValue({ maxDepth: 4 }),
+})
+
+describe('computeSpecHash properties', () => {
+ it('does not depend on the key order of nested objects', () => {
+ fc.assert(
+ fc.property(manifest, m => {
+ expect(computeSpecHash(reverseKeys(m) as object)).toBe(computeSpecHash(m))
+ })
+ )
+ })
+
+ it('depends on the element order of arrays', () => {
+ fc.assert(
+ fc.property(fc.uniqueArray(fc.string(), { minLength: 2, maxLength: 6 }), containers => {
+ const reordered = [...containers].reverse()
+ expect(computeSpecHash({ spec: { containers: reordered } })).not.toBe(
+ computeSpecHash({ spec: { containers } })
+ )
+ })
+ )
+ })
+
+ it('gives a stamped manifest the hash it had before stamping', () => {
+ fc.assert(
+ fc.property(manifest, m => {
+ const before = computeSpecHash(m)
+ const stamped = JSON.parse(JSON.stringify(m)) as Parameters<typeof stampSpecHash>[0]
+ expect(stampSpecHash(stamped)).toBe(before)
+ expect(stamped.metadata?.annotations?.[SPEC_HASH_ANNOTATION]).toBe(before)
+ expect(computeSpecHash(stamped)).toBe(before)
+ })
+ )
+ })
+})
diff --git a/workflow-recipes/src/reconciler/specHash.ts b/workflow-recipes/src/reconciler/specHash.ts
index bc8943d2f..2bd5e5542 100644
--- a/workflow-recipes/src/reconciler/specHash.ts
+++ b/workflow-recipes/src/reconciler/specHash.ts
@@ -46,9 +46,15 @@ export const HASH_EXCLUDED_ANNOTATIONS = [PRE_DEPLOY_ANNOTATION] as const
// this without needing an index signature.
type AnnotatableManifest = { metadata?: { annotations?: { [key: string]: string } } }
-// Fields the API server owns/defaults; they must never influence the hash or the
-// computed hash would differ between what we send and what we read back, defeating
-// the gate. `annotations[SPEC_HASH_ANNOTATION]` is stripped to avoid self-reference.
+// Metadata that is not recipe-derived. The hash is only ever computed from a
+// manifest WRC built, never from an object read back: the gate compares the live
+// object's annotation string with the desired hash. The builders never set the
+// server-assigned fields, so excluding them keeps the hash a function of the
+// recipe even if a caller hashes a manifest that already carries, say, a
+// resourceVersion. ownerReferences carry the apiserver-assigned recipe uid and are
+// dropped for a cross-namespace object; the gate compares the controller owner uid
+// on its own (controllerOwnerUidMatches).
+// `annotations[SPEC_HASH_ANNOTATION]` is stripped to avoid self-reference.
const VOLATILE_METADATA_FIELDS = [
'resourceVersion',
'uid',
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 8acb63a65..45c2d013f 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -4062,8 +4062,9 @@ export class WorkflowRecipeReconciler {
* Create-or-replace a managed object. When `idempotency` is supplied, the
* desired manifest is stamped with a spec-hash annotation and the object is
* READ FIRST: absent → POST, present with the desired hash → no write at all,
- * present with another hash → PUT. Without `idempotency` the object is
- * created first and replaced on 409.
+ * present with another hash → PUT; a read failure other than 404 propagates
+ * before any write. Without `idempotency` the object is created first and
+ * replaced on 409.
*
* This is what stops the generation churn: WRC reconciles every workload on a
* periodic resync, and an unconditional full replace re-defaults server-managed
@@ -4079,7 +4080,7 @@ export class WorkflowRecipeReconciler {
label: string,
idempotency?: {
manifest: GatedManifest
- readExisting: () => Promise<GatedManifest | null>
+ readExisting: () => Promise<GatedManifest>
/** Throws when the live object belongs to someone else; runs before the hash compare. */
assertOwned?: (existing: GatedManifest) => void
}
@@ -4142,34 +4143,32 @@ export class WorkflowRecipeReconciler {
}
/**
- * Read the live object and decide the write: `absent` (404) → create,
+ * Read the live object and decide the write: `absent` (404; the client never
+ * resolves a read to null, it returns the object or throws) → create,
* `unchanged` (it carries the desired spec-hash and the same controller owner
* uid) → no write, `changed` → replace. When the caller passes `assertOwned`,
* it runs on the live object before the hash compare, so a matching hash never
* stands in for ownership.
*
- * A read failure that carries an apiserver status or a transport signature
- * returns `changed` — never skip an update we cannot prove is unnecessary.
- * The caller then goes straight to the replace, not to a create first: after
- * a 403 on the read, the PUT usually fails with 403 as well and that error
- * propagates like any other failed write. Any other failure is not a read
- * outcome but a domain or programming error, and it propagates instead of
- * turning into a silent overwrite.
+ * Any read failure other than a 404 propagates. The gate then cannot say
+ * whether the object exists, and every replace path re-reads before it
+ * writes, so a write would meet the same failure one call later. The
+ * reconcile catch classifies the error exactly as it classified a failed
+ * create before the read-first change: 429, 5xx and socket errors keep the
+ * phase and requeue; other statuses are terminal.
*/
private async applyGate(idempotency: {
manifest: GatedManifest
- readExisting: () => Promise<GatedManifest | null>
+ readExisting: () => Promise<GatedManifest>
assertOwned?: (existing: GatedManifest) => void
}): Promise<'absent' | 'unchanged' | 'changed'> {
- let existing: GatedManifest | null
+ let existing: GatedManifest
try {
existing = await idempotency.readExisting()
} catch (error) {
- if (getErrorCode(error) === 404) return 'absent'
- if (getErrorCode(error) !== undefined || isRetryableInfraError(error)) return 'changed'
- throw error
+ if (getErrorCode(error) !== 404) throw error
+ return 'absent'
}
- if (existing == null) return 'absent'
idempotency.assertOwned?.(existing)
return specHashUnchanged(idempotency.manifest, existing) &&
controllerOwnerUidMatches(idempotency.manifest, existing)
@@ -7242,6 +7241,11 @@ export class WorkflowRecipeReconciler {
* A live Secret needs no write when its data and type equal the desired ones,
* it carries every desired label, and its controller ownerReference points at
* the same owner uid (none on either side for a cross-namespace Secret).
+ *
+ * These are all the fields `buildSecret` emits; annotations, `immutable` and
+ * `stringData` are not compared because it never sets them. A field added to
+ * the builder must be added here too, or a change to it is never written
+ * (pinned by the `buildSecret` field-set test in resourceBuilder.test.ts).
*/
private secretMatchesDesired(desired: k8s.V1Secret, existing: k8s.V1Secret): boolean {
if (