Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(workflow-recipes): an existing StatefulSet is read before any create (#760)

Alfredo Lopez240f0ec9715b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 90496da1a..70f2708e1 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.460",
+ "version": "0.1.461",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.460",
+ "version": "0.1.461",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index ac7b81a9a..51dbd71f3 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.460",
+ "version": "0.1.461",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index d6f00f38e..dfa7f6765 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -27,7 +27,7 @@ import {
} from '../workflow/workflowReconciler'
import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'
import { defaultFqdnLookup } from './fqdnResolver'
-import { isRetryableInfraError } from './k8sErrors'
+import { ResourceVanishedAfterConflictError, isRetryableInfraError } from './k8sErrors'
import type { NetworkPolicyFamily } from './networkPolicyConvergence'
import * as brokerIssuer from './oauthBrokerTokenIssuerClient'
import {
@@ -246,12 +246,19 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R
* whose tests override `read` to supply a live object with a specific status
* (the readiness tests): the object exists in that test's model, so its PUT
* succeeds.
+ *
+ * `patch`, when given, merges `metadata.annotations` and the top-level `spec`
+ * keys of the body into the stored object and bumps its resourceVersion. It is
+ * not a strategic merge: assertions about what a patch sent read the request
+ * body, not the stored object. A patch to an object the store never held
+ * follows `unseenReplace`.
*/
function installLiveStore(
api: {
read: ReturnType<typeof vi.fn>
create: ReturnType<typeof vi.fn>
replace: ReturnType<typeof vi.fn>
+ patch?: ReturnType<typeof vi.fn>
},
status: Record<string, unknown>,
admit?: (stored: LiveManifest) => void,
@@ -305,10 +312,61 @@ function installLiveStore(
return {}
}
)
+ api.patch?.mockReset().mockImplementation(
+ async ({
+ name,
+ namespace,
+ body,
+ }: {
+ name: string
+ namespace: string
+ body: {
+ metadata?: { annotations?: Record<string, string> }
+ spec?: Record<string, unknown>
+ }
+ }) => {
+ const k = key(namespace, name)
+ const current = live.get(k)
+ if (!current) {
+ if (unseenReplace === 'reject') throw { code: 404 }
+ return {}
+ }
+ const metadata = current.metadata as LiveManifest['metadata'] & {
+ annotations?: Record<string, string>
+ resourceVersion?: string
+ }
+ live.set(k, {
+ ...current,
+ metadata: {
+ ...metadata,
+ annotations: { ...metadata.annotations, ...body.metadata?.annotations },
+ resourceVersion: String(Number(metadata.resourceVersion ?? '1') + 1),
+ },
+ spec: { ...(current.spec as Record<string, unknown>), ...body.spec },
+ })
+ return {}
+ }
+ )
}
const LIVE_CLUSTER_IP = '10.0.0.7'
+// StatefulSet double used by every test unless a test overrides it: reads are
+// 404 until something creates the object, then return it with a ready status.
+const LIVE_STATEFULSET_STATUS = { observedGeneration: 1, readyReplicas: 1 }
+
+function installLiveStatefulSetStore(): void {
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedStatefulSet,
+ create: mockAppsApi.createNamespacedStatefulSet,
+ replace: mockAppsApi.replaceNamespacedStatefulSet,
+ patch: mockAppsApi.patchNamespacedStatefulSet,
+ },
+ LIVE_STATEFULSET_STATUS
+ )
+}
+
function installLiveServiceAndConfigMapStores(): void {
installLiveStore(
{
@@ -368,14 +426,7 @@ describe('WorkflowRecipeReconciler', () => {
},
{ observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 }
)
- mockAppsApi.readNamespacedStatefulSet.mockReset()
- mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- spec: { replicas: 1 },
- status: { readyReplicas: 1 },
- })
- mockAppsApi.patchNamespacedStatefulSet.mockReset()
- mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
+ installLiveStatefulSetStore()
installLiveStore(
{
read: mockAppsApi.readNamespacedDaemonSet,
@@ -3180,12 +3231,7 @@ describe('WorkflowRecipeReconciler', () => {
.mockReset()
.mockResolvedValue({ metadata: { resourceVersion: '1' } })
mockAppsApi.replaceNamespacedDeployment.mockReset().mockResolvedValue({})
- mockAppsApi.createNamespacedStatefulSet.mockReset().mockResolvedValue({})
- mockAppsApi.readNamespacedStatefulSet
- .mockReset()
- .mockResolvedValue({ metadata: { resourceVersion: '1' } })
- mockAppsApi.patchNamespacedStatefulSet.mockReset().mockResolvedValue({})
- mockAppsApi.replaceNamespacedStatefulSet.mockReset().mockResolvedValue({})
+ installLiveStatefulSetStore()
mockAppsApi.createNamespacedDaemonSet.mockReset().mockResolvedValue({})
mockAppsApi.readNamespacedDaemonSet
.mockReset()
@@ -3993,6 +4039,169 @@ describe('WorkflowRecipeReconciler', () => {
)
})
+ // #760: the StatefulSet writer reads before it creates, so a pass over an
+ // object that already exists writes nothing. Every count below is filtered
+ // by name: the same read serves the readiness probe and the legacy checks.
+ describe('read-first StatefulSet apply (#760)', () => {
+ type StsBody = {
+ metadata: { name: string; annotations: Record<string, string> }
+ spec: Record<string, unknown>
+ }
+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
+ const recipe = makeRecipe({ spec: { workloads: [workload] } })
+ const namespace = 'sandbox-recipes'
+ const ensure = () =>
+ (
+ reconciler as unknown as {
+ ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ }
+ ).ensureStatefulSet(workload, recipe, 'minimal', {})
+ const create = mockAppsApi.createNamespacedStatefulSet
+ const read = mockAppsApi.readNamespacedStatefulSet
+ const patch = mockAppsApi.patchNamespacedStatefulSet
+ const createdBody = (call = 0) => create.mock.calls[call][0].body as StsBody
+ const loggedFor = (level: ReturnType<typeof vi.fn>, message: string, name: string) =>
+ level.mock.calls.filter(
+ ([msg, ctx]) => msg === message && (ctx as { name?: string })?.name === name
+ ).length
+
+ it('S1: an existing StatefulSet with the same hash is read and nothing is written', async () => {
+ await ensure()
+ const name = createdBody().metadata.name
+ create.mockClear()
+ read.mockClear()
+ patch.mockClear()
+
+ const logs = captureLoggerLevels(['info', 'debug'] as const)
+ try {
+ await ensure()
+ expect(
+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)
+ ).toBe(1)
+ } finally {
+ logs.restore()
+ }
+ expect(readsOf(read, name, namespace)).toBe(1)
+ expect(postsOf(create, name)).toBe(0)
+ expect(patch).toHaveBeenCalledTimes(0)
+ })
+
+ it('S2: an absent StatefulSet is read, then created', async () => {
+ const logs = captureLoggerLevels(['info', 'debug'] as const)
+ try {
+ await ensure()
+ const name = createdBody().metadata.name
+ expect(loggedFor(logs.calls.info, 'Created StatefulSet', name)).toBe(1)
+ expect(postsOf(create, name)).toBe(1)
+ expect(readsOf(read, name, namespace)).toBe(1)
+ const readOrder =
+ read.mock.invocationCallOrder[
+ read.mock.calls.findIndex(([a]) => (a as { name: string }).name === name)
+ ]
+ expect(readOrder).toBeLessThan(create.mock.invocationCallOrder[0])
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('S3: a create conflict with an identical object re-reads it and skips the update', async () => {
+ const storeCreate = create.getMockImplementation()!
+ create.mockImplementationOnce(async (args: { namespace: string; body: StsBody }) => {
+ await storeCreate(args)
+ throw { code: 409 }
+ })
+
+ const logs = captureLoggerLevels(['info', 'debug'] as const)
+ try {
+ await ensure()
+ const name = createdBody().metadata.name
+ expect(
+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)
+ ).toBe(1)
+ expect(readsOf(read, name, namespace)).toBe(2)
+ } finally {
+ logs.restore()
+ }
+ expect(patch).toHaveBeenCalledTimes(0)
+ })
+
+ it('S3b: a create conflict with a drifted object re-reads it and patches it', async () => {
+ const storeCreate = create.getMockImplementation()!
+ create.mockImplementationOnce(async (args: { namespace: string; body: StsBody }) => {
+ const other = structuredClone(args.body)
+ other.metadata.annotations = {
+ ...other.metadata.annotations,
+ [SPEC_HASH]: 'other-writer',
+ }
+ other.spec = { ...other.spec, replicas: 3 }
+ await storeCreate({ namespace: args.namespace, body: other })
+ throw { code: 409 }
+ })
+
+ await ensure()
+
+ const desired = createdBody()
+ expect(readsOf(read, desired.metadata.name, namespace)).toBe(2)
+ expect(patch).toHaveBeenCalledTimes(1)
+ const sent = patch.mock.calls[0][0] as {
+ name: string
+ body: {
+ metadata: { annotations: Record<string, string> }
+ spec?: Record<string, unknown>
+ }
+ }
+ expect(sent.name).toBe(desired.metadata.name)
+ expect(sent.body.metadata.annotations[SPEC_HASH]).toBe(
+ desired.metadata.annotations[SPEC_HASH]
+ )
+ expect(sent.body.spec?.replicas).toBe(desired.spec.replicas)
+ })
+
+ it('S4: a read error other than 404 is rethrown and nothing is created', async () => {
+ const failure = { code: 500 }
+ read.mockImplementation(async () => {
+ throw failure
+ })
+
+ await expect(ensure()).rejects.toBe(failure)
+
+ // Witness: the failing read happened; the create (which would succeed) did not.
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(create).toHaveBeenCalledTimes(0)
+ expect(patch).toHaveBeenCalledTimes(0)
+ })
+
+ it('S5: an object that vanishes after the create conflict asks for a fresh reconcile', async () => {
+ create.mockRejectedValueOnce({ code: 409 })
+
+ const outcome = ensure()
+
+ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)
+ const name = createdBody().metadata.name
+ expect(readsOf(read, name, namespace)).toBe(2)
+ expect(patch).toHaveBeenCalledTimes(0)
+ })
+
+ it('S6: the second reconcile of a recipe whose StatefulSet exists writes no StatefulSet', async () => {
+ await reconciler.reconcile(recipe)
+ const name = createdBody().metadata.name
+ // Witness for the first pass: it did create the object.
+ expect(postsOf(create, name)).toBe(1)
+ create.mockClear()
+
+ const logs = captureLoggerLevels(['info', 'debug'] as const)
+ try {
+ await reconciler.reconcile(recipe)
+ expect(
+ loggedFor(logs.calls.info, 'StatefulSet spec hash unchanged; skipping update', name)
+ ).toBe(1)
+ } finally {
+ logs.restore()
+ }
+ expect(postsOf(create, name)).toBe(0)
+ })
+ })
+
it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {
const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
const recipe = makeRecipe({ spec: { workloads: [workload] } })
@@ -6839,14 +7048,21 @@ describe('WorkflowRecipeReconciler', () => {
const expectedDb = resolveScopedStatefulSetResourceName(recipe, 'db')
// Raw "db" exists and is owned by this recipe; the scoped workload is not ready
- // yet → cleanup must defer. Reads carry the clerum.io/recipe ownership label so
- // the owned-probe recognizes them (issue #571 S2).
- mockAppsApi.readNamespacedStatefulSet.mockImplementation(({ name }) =>
- Promise.resolve({
- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },
- spec: { replicas: 1 },
- status: { readyReplicas: name === 'db' ? 1 : 0 },
- })
+ // yet → cleanup must defer. The raw read carries the clerum.io/recipe ownership
+ // label so the owned-probe recognizes it (issue #571 S2). The scoped name goes
+ // to the live store, so the StatefulSet WRC writes is the one it reads back.
+ const storeRead = mockAppsApi.readNamespacedStatefulSet.getMockImplementation()!
+ const ownedRawDb = () => ({
+ metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },
+ spec: { replicas: 1 },
+ status: { readyReplicas: 1 },
+ })
+ mockAppsApi.readNamespacedStatefulSet.mockImplementation(
+ async (args: { name: string; namespace: string }) => {
+ if (args.name === 'db') return ownedRawDb()
+ const scoped = await storeRead(args)
+ return { ...scoped, status: { ...scoped.status, readyReplicas: 0 } }
+ }
)
// The legacy raw Services are live and owned by this recipe.
for (const legacy of ['db', 'db-headless']) {
@@ -6879,12 +7095,11 @@ describe('WorkflowRecipeReconciler', () => {
vi.clearAllMocks()
// Second reconcile: scoped workload now ready, raw "db" still present and owned →
- // teardown fires. Re-establish owned reads after clearAllMocks.
- mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({
- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },
- spec: { replicas: 1 },
- status: { readyReplicas: 1 },
- })
+ // teardown fires. The scoped read returns the stored object with its ready status.
+ mockAppsApi.readNamespacedStatefulSet.mockImplementation(
+ async (args: { name: string; namespace: string }) =>
+ args.name === 'db' ? ownedRawDb() : storeRead(args)
+ )
await reconciler.reconcile(recipe)