Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(workflow-recipes): an existing StatefulSet is read before any create (#760)
Alfredo Lopez240f0ec9715b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 90496da1a..70f2708e1 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.460",+ "version": "0.1.461","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.460",+ "version": "0.1.461","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex ac7b81a9a..51dbd71f3 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.460",+ "version": "0.1.461","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex d6f00f38e..dfa7f6765 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -27,7 +27,7 @@ import {} from '../workflow/workflowReconciler'import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'import { defaultFqdnLookup } from './fqdnResolver'-import { isRetryableInfraError } from './k8sErrors'+import { ResourceVanishedAfterConflictError, isRetryableInfraError } from './k8sErrors'import type { NetworkPolicyFamily } from './networkPolicyConvergence'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'import {@@ -246,12 +246,19 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R* whose tests override `read` to supply a live object with a specific status* (the readiness tests): the object exists in that test's model, so its PUT* succeeds.+ *+ * `patch`, when given, merges `metadata.annotations` and the top-level `spec`+ * keys of the body into the stored object and bumps its resourceVersion. It is+ * not a strategic merge: assertions about what a patch sent read the request+ * body, not the stored object. A patch to an object the store never held+ * follows `unseenReplace`.*/function installLiveStore(api: {read: ReturnType<typeof vi.fn>create: ReturnType<typeof vi.fn>replace: ReturnType<typeof vi.fn>+ patch?: ReturnType<typeof vi.fn>},status: Record<string, unknown>,admit?: (stored: LiveManifest) => void,@@ -305,10 +312,61 @@ function installLiveStore(return {}})+ api.patch?.mockReset().mockImplementation(+ async ({+ name,+ namespace,+ body,+ }: {+ name: string+ namespace: string+ body: {+ metadata?: { annotations?: Record<string, string> }+ spec?: Record<string, unknown>+ }+ }) => {+ const k = key(namespace, name)+ const current = live.get(k)+ if (!current) {+ if (unseenReplace === 'reject') throw { code: 404 }+ return {}+ }+ const metadata = current.metadata as LiveManifest['metadata'] & {+ annotations?: Record<string, string>+ resourceVersion?: string+ }+ live.set(k, {+ ...current,+ metadata: {+ ...metadata,+ annotations: { ...metadata.annotations, ...body.metadata?.annotations },+ resourceVersion: String(Number(metadata.resourceVersion ?? '1') + 1),+ },+ spec: { ...(current.spec as Record<string, unknown>), ...body.spec },+ })+ return {}+ }+ )}const LIVE_CLUSTER_IP = '10.0.0.7'+// StatefulSet double used by every test unless a test overrides it: reads are+// 404 until something creates the object, then return it with a ready status.+const LIVE_STATEFULSET_STATUS = { observedGeneration: 1, readyReplicas: 1 }++function installLiveStatefulSetStore(): void {+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedStatefulSet,+ create: mockAppsApi.createNamespacedStatefulSet,+ replace: mockAppsApi.replaceNamespacedStatefulSet,+ patch: mockAppsApi.patchNamespacedStatefulSet,+ },+ LIVE_STATEFULSET_STATUS+ )+}+function installLiveServiceAndConfigMapStores(): void {installLiveStore({@@ -368,14 +426,7 @@ describe('WorkflowRecipeReconciler', () => {},{ observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 })- mockAppsApi.readNamespacedStatefulSet.mockReset()- mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({- metadata: { resourceVersion: '1' },- spec: { replicas: 1 },- status: { readyReplicas: 1 },- })- mockAppsApi.patchNamespacedStatefulSet.mockReset()- mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})+ installLiveStatefulSetStore()installLiveStore({read: mockAppsApi.readNamespacedDaemonSet,@@ -3180,12 +3231,7 @@ describe('WorkflowRecipeReconciler', () => {.mockReset().mockResolvedValue({ metadata: { resourceVersion: '1' } })mockAppsApi.replaceNamespacedDeployment.mockReset().mockResolvedValue({})- mockAppsApi.createNamespacedStatefulSet.mockReset().mockResolvedValue({})- mockAppsApi.readNamespacedStatefulSet- .mockReset()- .mockResolvedValue({ metadata: { resourceVersion: '1' } })- mockAppsApi.patchNamespacedStatefulSet.mockReset().mockResolvedValue({})- mockAppsApi.replaceNamespacedStatefulSet.mockReset().mockResolvedValue({})+ installLiveStatefulSetStore()mockAppsApi.createNamespacedDaemonSet.mockReset().mockResolvedValue({})mockAppsApi.readNamespacedDaemonSet.mockReset()@@ -3993,6 +4039,169 @@ describe('WorkflowRecipeReconciler', () => {)})+ // #760: the StatefulSet writer reads before it creates, so a pass over an+ // object that already exists writes nothing. Every count below is filtered+ // by name: the same read serves the readiness probe and the legacy checks.+ describe('read-first StatefulSet apply (#760)', () => {+ type StsBody = {+ metadata: { name: string; annotations: Record<string, string> }+ spec: Record<string, unknown>+ }+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }+ const recipe = makeRecipe({ spec: { workloads: [workload] } })+ const namespace = 'sandbox-recipes'+ const ensure = () =>+ (+ reconciler as unknown as {+ ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ }+ ).ensureStatefulSet(workload, recipe, 'minimal', {})+ const create = mockAppsApi.createNamespacedStatefulSet+ const read = mockAppsApi.readNamespacedStatefulSet+ const patch = mockAppsApi.patchNamespacedStatefulSet+ const createdBody = (call = 0) => create.mock.calls[call][0].body as StsBody+ const loggedFor = (level: ReturnType<typeof vi.fn>, message: string, name: string) =>+ level.mock.calls.filter(+ ([msg, ctx]) => msg === message && (ctx as { name?: string })?.name === name+ ).length++ it('S1: an existing StatefulSet with the same hash is read and nothing is written', async () => {+ await ensure()+ const name = createdBody().metadata.name+ create.mockClear()+ read.mockClear()+ patch.mockClear()++ const logs = captureLoggerLevels(['info', 'debug'] as const)+ try {+ await ensure()+ expect(+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)+ ).toBe(1)+ } finally {+ logs.restore()+ }+ expect(readsOf(read, name, namespace)).toBe(1)+ expect(postsOf(create, name)).toBe(0)+ expect(patch).toHaveBeenCalledTimes(0)+ })++ it('S2: an absent StatefulSet is read, then created', async () => {+ const logs = captureLoggerLevels(['info', 'debug'] as const)+ try {+ await ensure()+ const name = createdBody().metadata.name+ expect(loggedFor(logs.calls.info, 'Created StatefulSet', name)).toBe(1)+ expect(postsOf(create, name)).toBe(1)+ expect(readsOf(read, name, namespace)).toBe(1)+ const readOrder =+ read.mock.invocationCallOrder[+ read.mock.calls.findIndex(([a]) => (a as { name: string }).name === name)+ ]+ expect(readOrder).toBeLessThan(create.mock.invocationCallOrder[0])+ } finally {+ logs.restore()+ }+ })++ it('S3: a create conflict with an identical object re-reads it and skips the update', async () => {+ const storeCreate = create.getMockImplementation()!+ create.mockImplementationOnce(async (args: { namespace: string; body: StsBody }) => {+ await storeCreate(args)+ throw { code: 409 }+ })++ const logs = captureLoggerLevels(['info', 'debug'] as const)+ try {+ await ensure()+ const name = createdBody().metadata.name+ expect(+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)+ ).toBe(1)+ expect(readsOf(read, name, namespace)).toBe(2)+ } finally {+ logs.restore()+ }+ expect(patch).toHaveBeenCalledTimes(0)+ })++ it('S3b: a create conflict with a drifted object re-reads it and patches it', async () => {+ const storeCreate = create.getMockImplementation()!+ create.mockImplementationOnce(async (args: { namespace: string; body: StsBody }) => {+ const other = structuredClone(args.body)+ other.metadata.annotations = {+ ...other.metadata.annotations,+ [SPEC_HASH]: 'other-writer',+ }+ other.spec = { ...other.spec, replicas: 3 }+ await storeCreate({ namespace: args.namespace, body: other })+ throw { code: 409 }+ })++ await ensure()++ const desired = createdBody()+ expect(readsOf(read, desired.metadata.name, namespace)).toBe(2)+ expect(patch).toHaveBeenCalledTimes(1)+ const sent = patch.mock.calls[0][0] as {+ name: string+ body: {+ metadata: { annotations: Record<string, string> }+ spec?: Record<string, unknown>+ }+ }+ expect(sent.name).toBe(desired.metadata.name)+ expect(sent.body.metadata.annotations[SPEC_HASH]).toBe(+ desired.metadata.annotations[SPEC_HASH]+ )+ expect(sent.body.spec?.replicas).toBe(desired.spec.replicas)+ })++ it('S4: a read error other than 404 is rethrown and nothing is created', async () => {+ const failure = { code: 500 }+ read.mockImplementation(async () => {+ throw failure+ })++ await expect(ensure()).rejects.toBe(failure)++ // Witness: the failing read happened; the create (which would succeed) did not.+ expect(read).toHaveBeenCalledTimes(1)+ expect(create).toHaveBeenCalledTimes(0)+ expect(patch).toHaveBeenCalledTimes(0)+ })++ it('S5: an object that vanishes after the create conflict asks for a fresh reconcile', async () => {+ create.mockRejectedValueOnce({ code: 409 })++ const outcome = ensure()++ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)+ const name = createdBody().metadata.name+ expect(readsOf(read, name, namespace)).toBe(2)+ expect(patch).toHaveBeenCalledTimes(0)+ })++ it('S6: the second reconcile of a recipe whose StatefulSet exists writes no StatefulSet', async () => {+ await reconciler.reconcile(recipe)+ const name = createdBody().metadata.name+ // Witness for the first pass: it did create the object.+ expect(postsOf(create, name)).toBe(1)+ create.mockClear()++ const logs = captureLoggerLevels(['info', 'debug'] as const)+ try {+ await reconciler.reconcile(recipe)+ expect(+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)+ ).toBe(1)+ } finally {+ logs.restore()+ }+ expect(postsOf(create, name)).toBe(0)+ })+ })+it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }const recipe = makeRecipe({ spec: { workloads: [workload] } })@@ -6839,14 +7048,21 @@ describe('WorkflowRecipeReconciler', () => {const expectedDb = resolveScopedStatefulSetResourceName(recipe, 'db')// Raw "db" exists and is owned by this recipe; the scoped workload is not ready- // yet → cleanup must defer. Reads carry the clerum.io/recipe ownership label so- // the owned-probe recognizes them (issue #571 S2).- mockAppsApi.readNamespacedStatefulSet.mockImplementation(({ name }) =>- Promise.resolve({- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },- spec: { replicas: 1 },- status: { readyReplicas: name === 'db' ? 1 : 0 },- })+ // yet → cleanup must defer. The raw read carries the clerum.io/recipe ownership+ // label so the owned-probe recognizes it (issue #571 S2). The scoped name goes+ // to the live store, so the StatefulSet WRC writes is the one it reads back.+ const storeRead = mockAppsApi.readNamespacedStatefulSet.getMockImplementation()!+ const ownedRawDb = () => ({+ metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },+ spec: { replicas: 1 },+ status: { readyReplicas: 1 },+ })+ mockAppsApi.readNamespacedStatefulSet.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ if (args.name === 'db') return ownedRawDb()+ const scoped = await storeRead(args)+ return { ...scoped, status: { ...scoped.status, readyReplicas: 0 } }+ })// The legacy raw Services are live and owned by this recipe.for (const legacy of ['db', 'db-headless']) {@@ -6879,12 +7095,11 @@ describe('WorkflowRecipeReconciler', () => {vi.clearAllMocks()// Second reconcile: scoped workload now ready, raw "db" still present and owned →- // teardown fires. Re-establish owned reads after clearAllMocks.- mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },- spec: { replicas: 1 },- status: { readyReplicas: 1 },- })+ // teardown fires. The scoped read returns the stored object with its ready status.+ mockAppsApi.readNamespacedStatefulSet.mockImplementation(+ async (args: { name: string; namespace: string }) =>+ args.name === 'db' ? ownedRawDb() : storeRead(args)+ )await reconciler.reconcile(recipe)