Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): read the StatefulSet before creating it (#760)
Alfredo Lopez71fa3b2fc434
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 70f2708e1..e555ad593 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.461",+ "version": "0.1.462","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.461",+ "version": "0.1.462","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 51dbd71f3..c052cf6f7 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.461",+ "version": "0.1.462","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex dfa7f6765..029dc0388 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -367,6 +367,31 @@ function installLiveStatefulSetStore(): void {)}+/**+ * Observe the StatefulSet WRC created with a test-chosen generation, spec and+ * status. Reads still go through the live store, so an object WRC has not+ * created reads as a 404 and the read before create sees what a real+ * apiserver would.+ */+function observeLiveStatefulSetAs(observed: {+ generation: number+ spec: Record<string, unknown>+ status: Record<string, unknown>+}): void {+ const storeRead = mockAppsApi.readNamespacedStatefulSet.getMockImplementation()!+ mockAppsApi.readNamespacedStatefulSet.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ const live = await storeRead(args)+ return {+ ...live,+ metadata: { ...live.metadata, generation: observed.generation },+ spec: { ...live.spec, ...observed.spec },+ status: observed.status,+ }+ }+ )+}+function installLiveServiceAndConfigMapStores(): void {installLiveStore({@@ -2323,8 +2348,8 @@ describe('WorkflowRecipeReconciler', () => {})it('treats a fully-ready StatefulSet as ready when observedGeneration transiently lags', async () => {- mockAppsApi.readNamespacedStatefulSet.mockRejectedValueOnce({ code: 404 }).mockResolvedValue({- metadata: { resourceVersion: '265', generation: 265 },+ observeLiveStatefulSetAs({+ generation: 265,spec: { replicas: 1 },status: { observedGeneration: 264, readyReplicas: 1 },})@@ -2410,8 +2435,8 @@ describe('WorkflowRecipeReconciler', () => {})it('degrades when a child StatefulSet was externally scaled below recipe replicas', async () => {- mockAppsApi.readNamespacedStatefulSet.mockRejectedValueOnce({ code: 404 }).mockResolvedValue({- metadata: { resourceVersion: '2', generation: 2 },+ observeLiveStatefulSetAs({+ generation: 2,spec: { replicas: 0 },status: { observedGeneration: 2, readyReplicas: 0 },})@@ -4220,19 +4245,38 @@ describe('WorkflowRecipeReconciler', () => {expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },spec: createdBody!.spec,})mockAppsApi.replaceNamespacedStatefulSet.mockClear()+ mockAppsApi.readNamespacedStatefulSet.mockClear()+ const name = (createdBody! as unknown as { metadata: { name: string } }).metadata.name- await (- reconciler as unknown as {- ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>- }- ).ensureStatefulSet(workload, recipe, 'minimal', {})+ // Liveness witnesses for the negative assertions below: the writer read+ // this object and took the unchanged-hash branch.+ const logs = captureLoggerLevels(['info', 'debug'] as const)+ try {+ await (+ reconciler as unknown as {+ ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ }+ ).ensureStatefulSet(workload, recipe, 'minimal', {})+ expect(+ logs.calls.info.mock.calls.filter(+ ([msg, ctx]) =>+ msg === 'StatefulSet spec hash unchanged; skipping update' &&+ (ctx as { name?: string })?.name === name+ )+ ).toHaveLength(1)+ } finally {+ logs.restore()+ }+ expect(readsOf(mockAppsApi.readNamespacedStatefulSet, name, 'sandbox-recipes')).toBe(1)expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).not.toHaveBeenCalled()})@@ -4257,6 +4301,7 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(createdBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-or-missing' }mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4268,6 +4313,7 @@ describe('WorkflowRecipeReconciler', () => {).ensureStatefulSet(workload, recipe, 'minimal', {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).toHaveBeenCalledTimes(1)expect(mockAppsApi.patchNamespacedStatefulSet.mock.calls[0][0].body).toEqual({metadata: { annotations: { [SPEC_HASH]: createdBody!.metadata.annotations[SPEC_HASH] } },@@ -4297,6 +4343,7 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(existingBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-old-template' }mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4308,6 +4355,7 @@ describe('WorkflowRecipeReconciler', () => {).ensureStatefulSet(newWorkload, newRecipe, 'minimal', {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).toHaveBeenCalledTimes(1)const patchBody = mockAppsApi.patchNamespacedStatefulSet.mock.calls[0][0].bodyexpect(patchBody.spec.template.spec.containers[0].image).toBe('postgres:16')@@ -4363,6 +4411,7 @@ describe('WorkflowRecipeReconciler', () => {status: { phase: 'Pending' },}))mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4374,6 +4423,7 @@ describe('WorkflowRecipeReconciler', () => {).ensureStatefulSet(newWorkload, newRecipe, 'minimal', {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).toHaveBeenCalledTimes(1)const patchBody = mockAppsApi.patchNamespacedStatefulSet.mock.calls[0][0].bodyexpect(patchBody.spec.template.spec.containers[0].image).toBe('postgres:16')@@ -4458,6 +4508,7 @@ describe('WorkflowRecipeReconciler', () => {status: { phase: 'Pending' },}))mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4469,6 +4520,7 @@ describe('WorkflowRecipeReconciler', () => {).ensureStatefulSet(newWorkload, newRecipe, 'minimal', {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).toHaveBeenCalledTimes(1)const patchBody = mockAppsApi.patchNamespacedStatefulSet.mock.calls[0][0].bodyexpect(patchBody.spec.template.spec.containers[0].env).toContainEqual({@@ -4526,6 +4578,7 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(existingBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-storage' }mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4545,6 +4598,7 @@ describe('WorkflowRecipeReconciler', () => {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).not.toHaveBeenCalled()})@@ -4570,6 +4624,7 @@ describe('WorkflowRecipeReconciler', () => {existing.metadata.annotations = { [SPEC_HASH]: 'stale-immutable-drift' }existing.spec.serviceName = 'manually-mutated-headless-service'mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })+ mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()@@ -4589,6 +4644,7 @@ describe('WorkflowRecipeReconciler', () => {})expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.patchNamespacedStatefulSet).not.toHaveBeenCalled()})@@ -7392,14 +7448,8 @@ describe('WorkflowRecipeReconciler', () => {},})- mockAppsApi.readNamespacedStatefulSet.mockImplementation(({ name }) => {- if (name === 'db') return Promise.reject({ code: 404 })- return Promise.resolve({- metadata: { resourceVersion: '1' },- spec: { replicas: 1 },- status: { readyReplicas: 1 },- })- })+ // StatefulSet reads use the live store: the raw "db" was never created, so it+ // reads as a 404, and the scoped one WRC creates reads back ready.mockCoreApi.readNamespacedService.mockImplementation(({ name }) => {if (name === 'db-headless') {return Promise.resolve({diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 1a716ffab..1b1fd421d 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -4812,22 +4812,53 @@ export class WorkflowRecipeReconciler {name: string): Promise<void> {const desiredHash = stampSpecHash(statefulSet)+ let existing = await this.readStatefulSetIfPresent(name, namespace)+ if (!existing) {+ try {+ await this.appsApi.createNamespacedStatefulSet({ namespace, body: statefulSet })+ createLogger('wrc', 'workflow-recipes').info('Created StatefulSet', { name, namespace })+ return+ } catch (error: unknown) {+ if (getErrorCode(error) !== 409) {+ createLogger('wrc', 'workflow-recipes').error('Failed to create StatefulSet', {+ name,+ namespace,+ err: error,+ })+ throw error+ }+ // Another writer created it between our read and this POST.+ existing = await this.readStatefulSetIfPresent(name, namespace)+ if (!existing) {+ throw new ResourceVanishedAfterConflictError(`StatefulSet "${name}" in ${namespace}`, {+ cause: error,+ })+ }+ }+ }+ await this.reconcileExistingStatefulSet(statefulSet, existing, desiredHash, namespace, name)+ }++ /** The live StatefulSet, or null on a 404; every other read error is rethrown. */+ private async readStatefulSetIfPresent(+ name: string,+ namespace: string+ ): Promise<k8s.V1StatefulSet | null> {try {- await this.appsApi.createNamespacedStatefulSet({ namespace, body: statefulSet })- createLogger('wrc', 'workflow-recipes').info('Created StatefulSet', { name, namespace })- return+ return await this.appsApi.readNamespacedStatefulSet({ name, namespace })} catch (error: unknown) {- if (getErrorCode(error) !== 409) {- createLogger('wrc', 'workflow-recipes').error('Failed to create StatefulSet', {- name,- namespace,- err: error,- })- throw error- }+ if (getErrorCode(error) === 404) return null+ throw error}+ }- const existing = await this.appsApi.readNamespacedStatefulSet({ name, namespace })+ private async reconcileExistingStatefulSet(+ statefulSet: k8s.V1StatefulSet,+ existing: k8s.V1StatefulSet,+ desiredHash: string,+ namespace: string,+ name: string+ ): Promise<void> {if (existing.metadata?.annotations?.[SPEC_HASH_ANNOTATION] === desiredHash) {createLogger('wrc', 'workflow-recipes').info('StatefulSet spec hash unchanged; skipping update',diff --git a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.tsindex b9b1c7b0f..355f4cc9a 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts@@ -12,7 +12,8 @@ const mockAppsApi = {replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),deleteNamespacedDeployment: vi.fn().mockResolvedValue({}),createNamespacedStatefulSet: vi.fn().mockResolvedValue({}),- readNamespacedStatefulSet: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedStatefulSet: vi.fn().mockRejectedValue({ code: 404 }),+ patchNamespacedStatefulSet: vi.fn().mockResolvedValue({}),replaceNamespacedStatefulSet: vi.fn().mockResolvedValue({}),deleteNamespacedStatefulSet: vi.fn().mockResolvedValue({}),createNamespacedDaemonSet: vi.fn().mockResolvedValue({}),@@ -117,11 +118,9 @@ describe('Workflow Reconciler Bifurcation', () => {spec: { replicas: 1 },status: { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 },})- mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({- metadata: { resourceVersion: '1' },- spec: { replicas: 1 },- status: { readyReplicas: 1 },- })+ // StatefulSets are applied read-first: by default none exists, so the read+ // before the create is a 404, as for a freshly approved recipe.+ mockAppsApi.readNamespacedStatefulSet.mockReset().mockRejectedValue({ code: 404 })mockAppsApi.readNamespacedDaemonSet.mockResolvedValue({metadata: { resourceVersion: '1' },status: { desiredNumberScheduled: 1, numberReady: 1 },