Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): propagate non-404 gate reads and close re-review gaps
Alfredo Lopezdc2fa7c67f59
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 3a7c19686..9dfd0c21a 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.367",+ "version": "0.1.368","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.367",+ "version": "0.1.368","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 0fd1f05c6..774903918 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.367",+ "version": "0.1.368","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex 4d9819a22..b656cb992 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -2307,7 +2307,7 @@ describe('delegateTransportWorkloads', () => {// any other failure on the way to it.expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {workloadId: 'redis-mcp',- error: expect.objectContaining({+ err: expect.objectContaining({message: expect.stringContaining('already exists for recipe "other-recipe"'),}),})@@ -2463,7 +2463,7 @@ describe('delegateTransportWorkloads', () => {expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {workloadId: 'redis-mcp',- error: expect.objectContaining({+ err: expect.objectContaining({message: expect.stringContaining('failed to update after conflict retries'),}),})@@ -3419,7 +3419,7 @@ describe('transport Service read-first apply', () => {expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload',- expect.objectContaining({ error: { code: 503 } })+ expect.objectContaining({ err: { code: 503 } }))expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)@@ -3452,7 +3452,7 @@ describe('transport Service read-first apply', () => {expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload',expect.objectContaining({- error: expect.objectContaining({+ err: expect.objectContaining({name: 'RetryableReconcileError',message: expect.stringContaining('disappeared after create conflict; a fresh reconciliation is required'diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex b898560f7..b613bd3ab 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -593,6 +593,12 @@ export function buildTransportService(* clusterIP (spec.clusterIP is immutable once assigned). Any read failure other* than a 404 propagates: the replace needs the live resourceVersion and* clusterIP, which a failed read cannot provide.+ *+ * The steady-state skip does not hold for a stdio transport. HCC also writes+ * that Service (host-context-controller/src/reconciler.ts) with its own+ * controller ownerReference and keeps WRC's spec-hash annotation, so the owner+ * uid never matches and WRC replaces the Service on every pass, as it did before+ * the read-first gate. Which controller owns it is left to a follow-up.*/async function ensureTransportService(deps: DelegationDeps,@@ -1170,7 +1176,7 @@ export async function preDeployMcpServers(preDeployed.push(r.value)} else if (r.status === 'rejected') {const workloadId = transportWorkloads[i].id- log.warn('Pre-deploy failed for workload', { workloadId, error: r.reason })+ log.warn('Pre-deploy failed for workload', { workloadId, err: r.reason })errors.push({ workloadId, error: r.reason })}}@@ -1481,7 +1487,7 @@ export async function delegateTransportWorkloads(delegated.push(r.value)} else {const workloadId = transportWorkloads[i].id- log.error('Failed to delegate workload', { workloadId, error: r.reason })+ log.error('Failed to delegate workload', { workloadId, err: r.reason })errors.push({ workloadId, error: r.reason })}}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex fbb200aa6..6e2a71115 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -223,6 +223,12 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R* 404; one it created or replaced reads back as written, carrying `status` —* the same "ready" status the observation path used to get from a static read.* `admit` models fields the apiserver assigns on create (a Service's clusterIP).+ *+ * `unseenReplace` says what a PUT to an object the store never held does.+ * `'reject'` answers 404, as the apiserver does. `'accept'` exists for kinds+ * whose tests override `read` to supply a live object with a specific status+ * (the readiness tests): the object exists in that test's model, so its PUT+ * succeeds.*/function installLiveStore(api: {@@ -231,7 +237,8 @@ function installLiveStore(replace: ReturnType<typeof vi.fn>},status: Record<string, unknown>,- admit?: (stored: LiveManifest) => void+ admit?: (stored: LiveManifest) => void,+ unseenReplace: 'accept' | 'reject' = 'accept'): void {const live = new Map<string, LiveManifest>()const key = (namespace: string, name: string) => `${namespace}/${name}`@@ -267,8 +274,7 @@ function installLiveStore(}) => {const k = key(namespace, name)const current = live.get(k)- // A test that overrides `read` supplies the live object itself, so an- // object the store never saw is accepted here rather than 404'd.+ if (!current && unseenReplace === 'reject') throw { code: 404 }if (current && body.metadata.resourceVersion !== current.metadata.resourceVersion) {throw { code: 409 }}@@ -296,7 +302,8 @@ function installLiveServiceAndConfigMapStores(): void {{},stored => {stored.spec = { ...(stored.spec as Record<string, unknown>), clusterIP: LIVE_CLUSTER_IP }- }+ },+ 'reject')installLiveStore({@@ -304,7 +311,9 @@ function installLiveServiceAndConfigMapStores(): void {create: mockCoreApi.createNamespacedConfigMap,replace: mockCoreApi.replaceNamespacedConfigMap,},- {}+ {},+ undefined,+ 'reject')}@@ -375,6 +384,15 @@ describe('WorkflowRecipeReconciler', () => {{ succeeded: 1 })installLiveServiceAndConfigMapStores()+ // clearAllMocks keeps implementations and `Once` queues, so a Secret store+ // or a create override installed by one test would reach the next. Every+ // test starts with no live Secret; tests that need one install it.+ mockCoreApi.readNamespacedSecret.mockReset()+ mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })+ mockCoreApi.createNamespacedSecret.mockReset()+ mockCoreApi.createNamespacedSecret.mockResolvedValue({})+ mockCoreApi.replaceNamespacedSecret.mockReset()+ mockCoreApi.replaceNamespacedSecret.mockResolvedValue({})mockCoreApi.readNamespacedPersistentVolumeClaim.mockReset()mockCoreApi.readNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 404 })mockCustomApi.createNamespacedCustomObject.mockReset()@@ -2715,19 +2733,6 @@ describe('WorkflowRecipeReconciler', () => {expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)})- it('creates a Deployment whose gate read resolves to nothing', async () => {- const recipe = makeRecipe()- const store = deploymentStore()- mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce(null)-- await ensureDeployment(recipe.spec.workloads![0], recipe)-- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(store.live()).not.toBeNull()- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)- })-// The POST is mocked to 409 only so that the pre-read-first code reaches// the gate at all; with the read first, it must never be sent.it('propagates an unclassifiable gate-read error and writes nothing', async () => {@@ -2774,21 +2779,42 @@ describe('WorkflowRecipeReconciler', () => {'a transport reset',Object.assign(new Error('connect ECONNRESET 10.0.0.1:443'), { code: 'ECONNRESET' }),],- ])('falls through to the replace when the gate read fails with %s', async (_, failure) => {+ ])('propagates a gate read that fails with %s without writing', async (_, failure) => {const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }const recipe = makeRecipe({ spec: { workloads: [workload] } })- // The first pass creates the headless Service, so the replace has a- // live object to read back.+ // The first pass creates the headless Service, so a replace would have+ // a live object to write: only the gate's decision can keep the PUT out.await ensureStatefulSet(workload, recipe)+ mockCoreApi.readNamespacedService.mockClear()mockCoreApi.createNamespacedService.mockClear()mockCoreApi.replaceNamespacedService.mockClear()mockCoreApi.readNamespacedService.mockRejectedValueOnce(failure)- await ensureStatefulSet(workload, recipe)+ await expect(ensureStatefulSet(workload, recipe)).rejects.toBe(failure)- // Witness: the PUT went out, so the fall-through ran.- expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)+ // Witness: the gate read ran and failed; nothing followed it.+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('propagates a failed re-read after a create conflict without writing', async () => {+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }+ const recipe = makeRecipe({ spec: { workloads: [workload] } })+ const unavailable = { code: 503 }+ // The gate read misses, the POST loses a race, and the conflict re-read+ // fails: nothing says whether the object exists, so nothing is written.+ mockCoreApi.readNamespacedService+ .mockRejectedValueOnce({ code: 404 })+ .mockRejectedValueOnce(unavailable)+ mockCoreApi.createNamespacedService.mockRejectedValueOnce({ code: 409 })++ await expect(ensureStatefulSet(workload, recipe)).rejects.toBe(unavailable)++ // Witness: the missed read, the POST and the failed re-read all ran.+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(2)+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(1)+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)})})@@ -2852,6 +2878,26 @@ describe('WorkflowRecipeReconciler', () => {expect(replaced.body.metadata.resourceVersion).toBe('1')})+ // Before the read-first change the POST went first, and a 503 on it kept+ // the phase and requeued. The same 503 on the gate read must do the same,+ // not reach a replace whose re-read 404s into a terminal `failed`.+ it('keeps the phase and requeues when the gate read of an absent Service fails with 503', async () => {+ const recipe = makeRecipe({+ status: { phase: 'active', message: 'All workloads deployed' },+ })+ mockCoreApi.readNamespacedService.mockRejectedValueOnce({ code: 503 })++ const result = await reconciler.reconcile(recipe)++ // Witness: the gate read ran exactly once and nothing was written.+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ expect(result.phase).toBe('active')+ expect(result.skipStatusPatch).toBe(true)+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ })+it('sends neither POST nor PUT for an unchanged workflow-lane Service', async () => {const recipe = makeRecipe({spec: {@@ -3252,6 +3298,97 @@ describe('WorkflowRecipeReconciler', () => {expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)})++ // Each case leaves data equal to the desired Secret and changes one other+ // field secretMatchesDesired compares, so only that comparison can force+ // the PUT.+ it.each([+ ['its type', (live: SecretManifest) => ({ ...live, type: 'kubernetes.io/basic-auth' })],+ [+ 'a label that is not the ownership label',+ (live: SecretManifest) => {+ const labels = { ...(live.metadata.labels as Record<string, string>) }+ const key = Object.keys(labels).find(k => k !== 'clerum.io/recipe')+ if (!key) throw new Error('the built Secret carries no non-ownership label')+ delete labels[key]+ return { ...live, metadata: { ...live.metadata, labels } }+ },+ ],+ [+ 'the uid of its controller owner (recipe recreated under the same name)',+ (live: SecretManifest) => {+ const refs = live.metadata.ownerReferences as Array<{+ controller?: boolean+ uid?: string+ }>+ if (!refs?.some(ref => ref.controller && ref.uid)) {+ throw new Error('the built Secret carries no controller owner uid')+ }+ return {+ ...live,+ metadata: {+ ...live.metadata,+ ownerReferences: refs.map(ref =>+ ref.controller ? { ...ref, uid: 'uid-old' } : ref+ ),+ },+ }+ },+ ],+ ])('replaces a user-data Secret whose live copy differs in %s', async (_field, drift) => {+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ const live = await readLive(name, created.namespace)+ await mockCoreApi.replaceNamespacedSecret({+ name,+ namespace: created.namespace,+ body: drift(live),+ })+ clearSecretCalls()++ await ensureSecret(res, recipe)++ // Witness: the gate read the drifted Secret before deciding to write.+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(1)+ const replaced = mockCoreApi.replaceNamespacedSecret.mock.calls[0][0] as {+ body: SecretManifest+ }+ expect(replaced.body).toMatchObject({+ type: created.body.type,+ metadata: {+ labels: created.body.metadata.labels,+ ownerReferences: created.body.metadata.ownerReferences,+ },+ })+ })++ it.each([+ ['403', { code: 403 }],+ ['503', { code: 503 }],+ ])(+ 'propagates a Secret read that fails with %s without writing',+ async (_status, failure) => {+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ clearSecretCalls()+ mockCoreApi.readNamespacedSecret.mockRejectedValueOnce(failure)++ await expect(ensureSecret(res, recipe)).rejects.toBe(failure)++ // Witness: the failing read is the only call; nothing was written blind.+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(0)+ expect(mockCoreApi.replaceNamespacedSecret).toHaveBeenCalledTimes(0)+ }+ )})it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {@@ -3763,32 +3900,34 @@ describe('WorkflowRecipeReconciler', () => {expect(mockAppsApi.replaceNamespacedDaemonSet).toHaveBeenCalledTimes(0)})- it('falls open and replaces when a transport failure hides the existing object (never silently skips)', async () => {+ it('propagates a transport failure on the gate read instead of writing blind', async () => {const recipe = makeRecipe()const workload = recipe.spec.workloads![0]+ const timeout = Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), {+ code: 'ETIMEDOUT',+ })- // First read (the gate) times out at the socket; the replace path's own read then succeeds.+ // The gate read times out at the socket; a later read would succeed, so+ // only the gate's decision can keep the replace path from running.mockAppsApi.readNamespacedDeployment.mockReset()- .mockRejectedValueOnce(- Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), { code: 'ETIMEDOUT' })- )+ .mockRejectedValueOnce(timeout).mockResolvedValue({ metadata: { resourceVersion: '9' } })mockAppsApi.createNamespacedDeployment.mockClear()mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})- await (- reconciler as unknown as {- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>- }- ).ensureDeployment(workload, recipe, 'minimal', {})+ await expect(+ (+ reconciler as unknown as {+ ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ }+ ).ensureDeployment(workload, recipe, 'minimal', {})+ ).rejects.toBe(timeout)- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)+ // Witness: the gate read ran once; neither write followed it.+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(- mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion- ).toBe('9')+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)})})@@ -4572,18 +4711,19 @@ describe('WorkflowRecipeReconciler', () => {it('creates resources before workloads (3.11a resources)', async () => {const callOrder: string[] = []- mockCoreApi.createNamespacedSecret.mockImplementation(async () => {- callOrder.push('secret')- return {}- })- mockCoreApi.createNamespacedConfigMap.mockImplementation(async () => {- callOrder.push('configmap')- return {}- })- mockAppsApi.createNamespacedDeployment.mockImplementation(async () => {- callOrder.push('deployment')- return {}- })+ // Record the order, then hand the call to the live store installed by+ // beforeEach, so the created objects exist for the readiness reads.+ const recordThenStore = (fn: ReturnType<typeof vi.fn>, kind: string) => {+ const store = fn.getMockImplementation()+ if (!store) throw new Error(`beforeEach installed no implementation for the ${kind} create`)+ fn.mockImplementation(async (args: unknown) => {+ callOrder.push(kind)+ return store(args)+ })+ }+ recordThenStore(mockCoreApi.createNamespacedSecret, 'secret')+ recordThenStore(mockCoreApi.createNamespacedConfigMap, 'configmap')+ recordThenStore(mockAppsApi.createNamespacedDeployment, 'deployment')const recipe = makeRecipe({spec: {@@ -4594,7 +4734,12 @@ describe('WorkflowRecipeReconciler', () => {],},})- await reconciler.reconcile(recipe)+ const result = await reconciler.reconcile(recipe)+ expect(result.phase).toBe('active')+ // Witness: each object was created, so an index of -1 cannot pass the order check.+ expect(callOrder.filter(c => c === 'secret')).toHaveLength(1)+ expect(callOrder.filter(c => c === 'configmap')).toHaveLength(1)+ expect(callOrder).toContain('deployment')expect(callOrder.indexOf('secret')).toBeLessThan(callOrder.indexOf('deployment'))expect(callOrder.indexOf('configmap')).toBeLessThan(callOrder.indexOf('deployment'))})@@ -7027,8 +7172,10 @@ describe('WorkflowRecipeReconciler', () => {resources: [{ id: 'creds', type: 'secret', data: { key: 'val' } }],},})- await reconciler.reconcile(recipe)+ const result = await reconciler.reconcile(recipe)+ expect(result.phase).toBe('active')// Secret not mounted by any workload → stays with the recipe runtime.+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)const secretCall = mockCoreApi.createNamespacedSecret.mock.calls[0][0]expect(secretCall.namespace).toBe('sandbox-recipes')})@@ -12222,6 +12369,29 @@ describe('WorkflowRecipeReconciler', () => {expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)})++ it('propagates a policy read that fails with 503 instead of creating blind', async () => {+ stubSteadyWorkflowRuntime()+ installGfsPolicyStore()+ const failure = { code: 503 }+ const storeRead = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ if (args.name === gfsPolicyName) throw failure+ return storeRead(args)+ }+ )+ clearPolicyCalls()++ await expect(reconciler.reconcile(activeWorkflow(publishTargets))).rejects.toBe(failure)++ // Witness: the failing read is the only call on the policy.+ expect(+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')+ ).toBe(1)+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ })})it('refuses to delete a homonymous policy not owned by the exact recipe', async () => {diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.tsindex a0160891b..4886756cd 100644--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts@@ -926,6 +926,21 @@ describe('buildSecret', () => {expect(secret.metadata?.ownerReferences).toHaveLength(1)expect(secret.metadata?.ownerReferences![0].kind).toBe('WorkflowRecipe')})++ // The reconciler's secretMatchesDesired skips the PUT by comparing exactly+ // these fields. A field added here (annotations, immutable, stringData) must+ // be added to that comparison too, or a change to it is never written.+ it('emits only the fields secretMatchesDesired compares', () => {+ const res: SecretResourceDef = { id: 'creds', type: 'secret', data: { password: 's3cret' } }+ const secret = buildSecret(res, makeRecipe())+ expect(Object.keys(secret).sort()).toEqual(['apiVersion', 'data', 'kind', 'metadata', 'type'])+ expect(Object.keys(secret.metadata!).sort()).toEqual([+ 'labels',+ 'name',+ 'namespace',+ 'ownerReferences',+ ])+ })})// ─── ConfigMap Builder ──────────────────────────────────────────────diff --git a/workflow-recipes/src/reconciler/specHash.property.test.ts b/workflow-recipes/src/reconciler/specHash.property.test.tsnew file mode 100644index 000000000..61151afec--- /dev/null+++ b/workflow-recipes/src/reconciler/specHash.property.test.ts@@ -0,0 +1,57 @@+import { describe, expect, it } from 'vitest'+import fc from 'fast-check'+import { SPEC_HASH_ANNOTATION, computeSpecHash, stampSpecHash } from './specHash'++// Rebuild every object with its keys in reverse insertion order; arrays keep+// their order. Object.fromEntries defines own properties, so a generated+// "__proto__" key stays a key instead of replacing the prototype.+function reverseKeys(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(reverseKeys)+ if (value === null || typeof value !== 'object') return value+ return Object.fromEntries(+ Object.entries(value as Record<string, unknown>)+ .reverse()+ .map(([key, child]) => [key, reverseKeys(child)])+ )+}++const manifest = fc.record({+ metadata: fc.record({+ name: fc.string({ minLength: 1 }),+ labels: fc.dictionary(fc.string({ minLength: 1 }), fc.string()),+ }),+ spec: fc.jsonValue({ maxDepth: 4 }),+})++describe('computeSpecHash properties', () => {+ it('does not depend on the key order of nested objects', () => {+ fc.assert(+ fc.property(manifest, m => {+ expect(computeSpecHash(reverseKeys(m) as object)).toBe(computeSpecHash(m))+ })+ )+ })++ it('depends on the element order of arrays', () => {+ fc.assert(+ fc.property(fc.uniqueArray(fc.string(), { minLength: 2, maxLength: 6 }), containers => {+ const reordered = [...containers].reverse()+ expect(computeSpecHash({ spec: { containers: reordered } })).not.toBe(+ computeSpecHash({ spec: { containers } })+ )+ })+ )+ })++ it('gives a stamped manifest the hash it had before stamping', () => {+ fc.assert(+ fc.property(manifest, m => {+ const before = computeSpecHash(m)+ const stamped = JSON.parse(JSON.stringify(m)) as Parameters<typeof stampSpecHash>[0]+ expect(stampSpecHash(stamped)).toBe(before)+ expect(stamped.metadata?.annotations?.[SPEC_HASH_ANNOTATION]).toBe(before)+ expect(computeSpecHash(stamped)).toBe(before)+ })+ )+ })+})diff --git a/workflow-recipes/src/reconciler/specHash.ts b/workflow-recipes/src/reconciler/specHash.tsindex bc8943d2f..2bd5e5542 100644--- a/workflow-recipes/src/reconciler/specHash.ts+++ b/workflow-recipes/src/reconciler/specHash.ts@@ -46,9 +46,15 @@ export const HASH_EXCLUDED_ANNOTATIONS = [PRE_DEPLOY_ANNOTATION] as const// this without needing an index signature.type AnnotatableManifest = { metadata?: { annotations?: { [key: string]: string } } }-// Fields the API server owns/defaults; they must never influence the hash or the-// computed hash would differ between what we send and what we read back, defeating-// the gate. `annotations[SPEC_HASH_ANNOTATION]` is stripped to avoid self-reference.+// Metadata that is not recipe-derived. The hash is only ever computed from a+// manifest WRC built, never from an object read back: the gate compares the live+// object's annotation string with the desired hash. The builders never set the+// server-assigned fields, so excluding them keeps the hash a function of the+// recipe even if a caller hashes a manifest that already carries, say, a+// resourceVersion. ownerReferences carry the apiserver-assigned recipe uid and are+// dropped for a cross-namespace object; the gate compares the controller owner uid+// on its own (controllerOwnerUidMatches).+// `annotations[SPEC_HASH_ANNOTATION]` is stripped to avoid self-reference.const VOLATILE_METADATA_FIELDS = ['resourceVersion','uid',diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 8acb63a65..45c2d013f 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -4062,8 +4062,9 @@ export class WorkflowRecipeReconciler {* Create-or-replace a managed object. When `idempotency` is supplied, the* desired manifest is stamped with a spec-hash annotation and the object is* READ FIRST: absent → POST, present with the desired hash → no write at all,- * present with another hash → PUT. Without `idempotency` the object is- * created first and replaced on 409.+ * present with another hash → PUT; a read failure other than 404 propagates+ * before any write. Without `idempotency` the object is created first and+ * replaced on 409.** This is what stops the generation churn: WRC reconciles every workload on a* periodic resync, and an unconditional full replace re-defaults server-managed@@ -4079,7 +4080,7 @@ export class WorkflowRecipeReconciler {label: string,idempotency?: {manifest: GatedManifest- readExisting: () => Promise<GatedManifest | null>+ readExisting: () => Promise<GatedManifest>/** Throws when the live object belongs to someone else; runs before the hash compare. */assertOwned?: (existing: GatedManifest) => void}@@ -4142,34 +4143,32 @@ export class WorkflowRecipeReconciler {}/**- * Read the live object and decide the write: `absent` (404) → create,+ * Read the live object and decide the write: `absent` (404; the client never+ * resolves a read to null, it returns the object or throws) → create,* `unchanged` (it carries the desired spec-hash and the same controller owner* uid) → no write, `changed` → replace. When the caller passes `assertOwned`,* it runs on the live object before the hash compare, so a matching hash never* stands in for ownership.*- * A read failure that carries an apiserver status or a transport signature- * returns `changed` — never skip an update we cannot prove is unnecessary.- * The caller then goes straight to the replace, not to a create first: after- * a 403 on the read, the PUT usually fails with 403 as well and that error- * propagates like any other failed write. Any other failure is not a read- * outcome but a domain or programming error, and it propagates instead of- * turning into a silent overwrite.+ * Any read failure other than a 404 propagates. The gate then cannot say+ * whether the object exists, and every replace path re-reads before it+ * writes, so a write would meet the same failure one call later. The+ * reconcile catch classifies the error exactly as it classified a failed+ * create before the read-first change: 429, 5xx and socket errors keep the+ * phase and requeue; other statuses are terminal.*/private async applyGate(idempotency: {manifest: GatedManifest- readExisting: () => Promise<GatedManifest | null>+ readExisting: () => Promise<GatedManifest>assertOwned?: (existing: GatedManifest) => void}): Promise<'absent' | 'unchanged' | 'changed'> {- let existing: GatedManifest | null+ let existing: GatedManifesttry {existing = await idempotency.readExisting()} catch (error) {- if (getErrorCode(error) === 404) return 'absent'- if (getErrorCode(error) !== undefined || isRetryableInfraError(error)) return 'changed'- throw error+ if (getErrorCode(error) !== 404) throw error+ return 'absent'}- if (existing == null) return 'absent'idempotency.assertOwned?.(existing)return specHashUnchanged(idempotency.manifest, existing) &&controllerOwnerUidMatches(idempotency.manifest, existing)@@ -7242,6 +7241,11 @@ export class WorkflowRecipeReconciler {* A live Secret needs no write when its data and type equal the desired ones,* it carries every desired label, and its controller ownerReference points at* the same owner uid (none on either side for a cross-namespace Secret).+ *+ * These are all the fields `buildSecret` emits; annotations, `immutable` and+ * `stringData` are not compared because it never sets them. A field added to+ * the builder must be added here too, or a change to it is never written+ * (pinned by the `buildSecret` field-set test in resourceBuilder.test.ts).*/private secretMatchesDesired(desired: k8s.V1Secret, existing: k8s.V1Secret): boolean {if (