Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
127 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import {
  WorkflowRecipeWatcher,
  controllerErrorTelemetryProjection,
  externalEgressRefreshIntervalMs,
  recipeHasExternalExactHostEgress,
  recipeReferencesNamedSecrets,
  reconcileOutcomeTelemetryProjection,
  rotateBrokerTokensOnce,
  runtimeCredentialRefreshIntervalMs,
  shouldPatchRecipeStatus,
  workflowNeedsInfrastructureReconcile,
  workflowNeedsOwnershipBackstop,
  workflowNeedsRuntimeCredentialRefresh,
  workflowNeedsTerminalRunTeardown,
  workflowNeedsWorkloadStatusRefresh,
  workflowRecipeFromWatchObject,
} from './k8sClient'
import { captureLogger } from './reconciler/__tests__/captureLogger'
import { buildOAuthBrokerTokenSecret } from './reconciler/resourceBuilder'
import type { SecretEventType, SecretLike } from './reconciler/secretWatcher'
import type { WorkflowRecipeCRD } from './types'

// startSecretWatch builds one SecretWatcher per namespace and hands it to a
// K8sSecretWatchLoop. Capturing that hand-off lets a test drive the REAL
// watcher instances the client constructed, so the callback wiring is covered
// end to end instead of being replaced by a hand-injected callback.
const secretLoops = vi.hoisted(
  () =>
    [] as Array<{
      namespace: string
      handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }
    }>
)
vi.mock('./k8sSecretWatchLoop', () => ({
  K8sSecretWatchLoop: class {
    constructor(
      _kc: unknown,
      namespace: string,
      handler: { handleEvent: (type: SecretEventType, secret: SecretLike) => void }
    ) {
      secretLoops.push({ namespace, handler })
    }
    async start(): Promise<void> {}
    stop(): void {}
  },
}))

function makeWorkflowRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {
  return {
    apiVersion: 'clerum.io/v1alpha1',
    kind: 'WorkflowRecipe',
    metadata: {
      name: 'recipe',
      namespace: 'sandbox-recipes',
      ...(overrides.metadata ?? {}),
    },
    spec: {
      steps: [
        { id: 'step-1', run: { type: 'snippet', language: 'typescript', code: 'return {}' } },
      ],
      ...(overrides.spec ?? {}),
    },
    status: overrides.status,
  }
}

describe('workflowRecipeFromWatchObject', () => {
  it('preserves ownerReferences from Kubernetes watch objects', () => {
    const recipe = workflowRecipeFromWatchObject(
      {
        metadata: {
          name: 'child-run',
          uid: 'uid-child',
          labels: { 'clerum.io/parent-recipe': 'parent-recipe' },
          ownerReferences: [
            {
              apiVersion: 'clerum.io/v1alpha1',
              kind: 'WorkflowRecipe',
              name: 'parent-recipe',
              uid: 'uid-parent-recipe',
              controller: true,
              blockOwnerDeletion: true,
            },
          ],
        },
        spec: { steps: [{ id: 'step-1', instruction: 'run' }] },
      },
      'sandbox-recipes'
    )

    expect(recipe.metadata.namespace).toBe('sandbox-recipes')
    expect(recipe.metadata.ownerReferences).toEqual([
      {
        apiVersion: 'clerum.io/v1alpha1',
        kind: 'WorkflowRecipe',
        name: 'parent-recipe',
        uid: 'uid-parent-recipe',
        controller: true,
        blockOwnerDeletion: true,
      },
    ])
  })
})

describe('persisted transport external-egress admission', () => {
  const pending = {
    type: 'TransportExternalEgressReady',
    status: 'False' as const,
    reason: 'ExternalEgressPending',
    message: 'Awaiting HCC admission',
    lastTransitionTime: '2026-09-05T00:00:00.000Z',
  }

  it('publishes explicit admission clearance without treating observation as clearance', () => {
    const recipe = makeWorkflowRecipe({
      spec: { steps: [] },
      status: { phase: 'active', message: 'Ready', conditions: [pending] },
    })
    const observed = { phase: 'active' as const, message: 'Ready', workloadStatuses: [] }
    expect(shouldPatchRecipeStatus(recipe, observed)).toBe(false)
    expect(shouldPatchRecipeStatus(recipe, { ...observed, transportNetworkConditions: [] })).toBe(
      true
    )
    expect(
      shouldPatchRecipeStatus(recipe, {
        ...observed,
        transportNetworkConditions: [
          { ...pending, lastTransitionTime: '2026-09-06T00:00:00.000Z' },
        ],
      })
    ).toBe(false)
  })

  it('restores retries only for live, pending admission rather than Pod-only degradation or terminal runs', () => {
    const recipe = makeWorkflowRecipe({
      spec: { steps: [] },
      status: { phase: 'degraded', conditions: [pending] },
    })
    expect(workflowNeedsInfrastructureReconcile(recipe)).toBe(true)
    expect(workflowNeedsInfrastructureReconcile({ ...recipe, status: { phase: 'degraded' } })).toBe(
      false
    )
    expect(
      workflowNeedsInfrastructureReconcile({
        ...recipe,
        status: { phase: 'failed', conditions: [pending] },
      })
    ).toBe(false)
    expect(
      workflowNeedsInfrastructureReconcile({
        ...recipe,
        metadata: { ...recipe.metadata, deletionTimestamp: '2026-09-05T00:00:00.000Z' },
      })
    ).toBe(false)
    const workflow = {
      ...recipe,
      spec: { ...recipe.spec, steps: [{ id: 's', instruction: 'run' }] },
    }
    expect(workflowNeedsInfrastructureReconcile(workflow)).toBe(true)
    expect(
      workflowNeedsInfrastructureReconcile({
        ...workflow,
        status: {
          phase: 'active',
          conditions: [pending],
          workflowExecution: { phase: 'completed' },
        },
      })
    ).toBe(false)
  })
})

describe('rotateBrokerTokensOnce', () => {
  function makeRecipe(name: string, withBackgroundAccess: boolean): WorkflowRecipeCRD {
    return {
      apiVersion: 'clerum.io/v1alpha1',
      kind: 'WorkflowRecipe',
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [{ id: 's1', instruction: 'noop' }],
        oauthClients: withBackgroundAccess
          ? [
              {
                id: 'gmail',
                provider: 'google',
                clientIdRef: { name: 'creds', key: 'client-id' },
                clientSecretRef: { name: 'creds', key: 'client-secret' },
                scopes: ['https://www.googleapis.com/auth/gmail.readonly'],
                backgroundAccess: true,
              },
            ]
          : [],
      } as WorkflowRecipeCRD['spec'],
    }
  }

  it('refreshes only recipes with a backgroundAccess client', async () => {
    const refresh = vi.fn().mockResolvedValue(undefined)
    const recipes = [
      makeRecipe('with-bg', true),
      makeRecipe('without-bg', false),
      makeRecipe('also-with-bg', true),
    ]
    await rotateBrokerTokensOnce(recipes, refresh)
    expect(refresh).toHaveBeenCalledTimes(2)
    const names = refresh.mock.calls.map(c => c[0].metadata.name).sort()
    expect(names).toEqual(['also-with-bg', 'with-bg'])
  })

  it('continues iterating after a refresh throws and reports it via onError', async () => {
    const boom = new Error('apiserver 503')
    const refresh = vi
      .fn<(r: WorkflowRecipeCRD) => Promise<void>>()
      .mockResolvedValueOnce(undefined)
      .mockRejectedValueOnce(boom)
      .mockResolvedValueOnce(undefined)
    const onError = vi.fn()
    const recipes = [makeRecipe('first', true), makeRecipe('boom', true), makeRecipe('third', true)]
    await rotateBrokerTokensOnce(recipes, refresh, onError)
    expect(refresh).toHaveBeenCalledTimes(3)
    expect(onError).toHaveBeenCalledTimes(1)
    expect(onError.mock.calls[0][0].metadata.name).toBe('boom')
    expect(onError.mock.calls[0][1]).toBe(boom)
  })
})

describe('shouldPatchRecipeStatus', () => {
  describe('WorkflowNetworkPolicyOwnership condition', () => {
    const conflict = {
      type: 'WorkflowNetworkPolicyOwnership',
      status: 'False' as const,
      reason: 'OwnershipConflict',
      message: 'NetworkPolicy ownership conflict: idle-wf-coord-to-wrc (owner-reference-mismatch)',
      lastTransitionTime: '2026-09-23T10:00:00.000Z',
    }
    const steadyRecipe = (conditions?: (typeof conflict)[]): WorkflowRecipeCRD => ({
      apiVersion: 'clerum.io/v1alpha1',
      kind: 'WorkflowRecipe',
      metadata: { name: 'idle-wf', namespace: 'sandbox-recipes' },
      spec: { steps: [{ id: 'run', instruction: 'run' }] },
      status: {
        phase: 'active',
        message: 'Workflow idle',
        ...(conditions ? { conditions } : {}),
      },
    })
    const steadyResult = (networkPolicyOwnershipConditions?: (typeof conflict)[]) => ({
      phase: 'active' as const,
      message: 'Workflow idle',
      workloadStatuses: [],
      ...(networkPolicyOwnershipConditions !== undefined
        ? { networkPolicyOwnershipConditions }
        : {}),
    })

    it('patches when the condition appears', () => {
      expect(shouldPatchRecipeStatus(steadyRecipe(), steadyResult([conflict]))).toBe(true)
    })

    it('patches when the condition disappears', () => {
      expect(shouldPatchRecipeStatus(steadyRecipe([conflict]), steadyResult([]))).toBe(true)
    })

    it('does not patch when only lastTransitionTime differs or the result carries no ownership field', () => {
      // Witness: the same steady pair does patch once the condition set changes.
      expect(shouldPatchRecipeStatus(steadyRecipe(), steadyResult([conflict]))).toBe(true)
      expect(
        shouldPatchRecipeStatus(
          steadyRecipe([conflict]),
          steadyResult([{ ...conflict, lastTransitionTime: '2026-09-23T11:00:00.000Z' }])
        )
      ).toBe(false)
      expect(shouldPatchRecipeStatus(steadyRecipe([conflict]), steadyResult())).toBe(false)
    })

    const retryMarker = {
      type: 'WorkflowNetworkPoliciesConverged',
      status: 'False' as const,
      reason: 'RetryPending',
      message:
        'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
      lastTransitionTime: '2026-09-23T10:00:00.000Z',
    }

    it('patches when the retry marker appears or disappears, the ownership condition unchanged', () => {
      expect(shouldPatchRecipeStatus(steadyRecipe(), steadyResult([retryMarker]))).toBe(true)
      expect(
        shouldPatchRecipeStatus(steadyRecipe([conflict, retryMarker]), steadyResult([conflict]))
      ).toBe(true)
      // Witness: the same pair with the marker on both sides opens no patch.
      expect(
        shouldPatchRecipeStatus(
          steadyRecipe([conflict, retryMarker]),
          steadyResult([conflict, { ...retryMarker, lastTransitionTime: '2026-09-24T00:00:00Z' }])
        )
      ).toBe(false)
    })
  })

  it('patches when a terminal workflow keeps the same phase but updates the message', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'e2e-scheduled-recipe', namespace: 'sandbox-recipes' },
          spec: { steps: [{ id: 'run', instruction: 'run' }] },
          status: {
            phase: 'active',
            message: 'Workflow running',
            lastTransitionTime: '2026-05-09T00:00:00Z',
          },
        },
        {
          phase: 'active',
          message: 'Workflow completed',
          workloadStatuses: [],
          skipStatusPatch: false,
        }
      )
    ).toBe(true)
  })

  it('patches active parent workflow recipes when stale workflowExecution must be cleared', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'e2e-scheduled-recipe', namespace: 'sandbox-recipes' },
          spec: { steps: [{ id: 'run', instruction: 'run' }] },
          status: {
            phase: 'active',
            workflowExecution: { phase: 'pending', message: 'stale parent run state' },
          },
        },
        {
          phase: 'active',
          message: 'Workflow trigger infrastructure registered (workflow-agentic)',
          workloadStatuses: [],
          clearWorkflowExecution: true,
        }
      )
    ).toBe(true)
  })

  it('patches when internal dependency readiness is missing on an otherwise steady active recipe', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'helpdesk', namespace: 'sandbox-recipes' },
          spec: { workloads: [{ id: 'api', type: 'deployment', image: 'api:test' }] },
          status: {
            phase: 'active',
            message: 'All workloads deployed',
            conditions: [
              { type: 'WebhookGatewayNotReady', status: 'False', lastTransitionTime: 'old' },
            ],
          },
        },
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          internalDependencyConditions: [
            {
              type: 'InternalDependenciesReady',
              status: 'True',
              reason: 'Reconciled',
              message: 'Reconciled 6 WRC internal dependency rule(s)',
              lastTransitionTime: 'now',
            },
          ],
        }
      )
    ).toBe(true)
  })

  it('patches when stale internal dependency readiness must be cleared', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'policy-blocked', namespace: 'sandbox-recipes' },
          spec: { workloads: [{ id: 'api', type: 'deployment', image: 'api:test' }] },
          status: {
            phase: 'failed',
            message: 'Policy violation: blocked',
            conditions: [
              {
                type: 'InternalDependenciesReady',
                status: 'False',
                reason: 'InvalidInternalDependency',
                message: 'stale',
                lastTransitionTime: 'old',
              },
            ],
          },
        },
        {
          phase: 'failed',
          message: 'Policy violation: blocked',
          workloadStatuses: [],
        }
      )
    ).toBe(true)
  })

  it('does not patch steady state when phase and message already match', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'e2e-scheduled-recipe', namespace: 'sandbox-recipes' },
          spec: { steps: [{ id: 'run', instruction: 'run' }] },
          status: {
            phase: 'active',
            message: 'Workflow completed',
            lastTransitionTime: '2026-05-09T00:00:00Z',
          },
        },
        {
          phase: 'active',
          message: 'Workflow completed',
          workloadStatuses: [],
          skipStatusPatch: false,
        }
      )
    ).toBe(false)
  })

  it('patches a contradictory persisted SDK capability message', () => {
    expect(
      shouldPatchRecipeStatus(
        makeWorkflowRecipe({
          spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
          status: {
            phase: 'active',
            message: 'All workloads deployed',
            pluginWorkloadSdk: {
              state: 'validated',
              promptBridge: true,
              clientNotifications: false,
              message: 'promptBridge bootstrap policy proof is not ready',
            },
            conditions: [
              {
                type: 'PluginWorkloadSdkCapability',
                status: 'True',
                reason: 'Validated',
                message: 'Capability validated (promptBridge)',
                lastTransitionTime: 'now',
              },
            ],
          },
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
        }
      )
    ).toBe(true)
  })

  it('patches when only the persisted workload status projection changes', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'api-recipe', namespace: 'sandbox-recipes' },
          spec: { workloads: [{ id: 'api', type: 'deployment', image: 'api:test' }] },
          status: {
            phase: 'degraded',
            message: 'Some workloads not ready',
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                phase: 'degraded',
                ready: false,
                message: 'Deployment "api" updated/ready/available/desired 1/0/0/1',
              },
            ],
          },
        },
        {
          phase: 'degraded',
          message: 'Some workloads not ready',
          workloadStatuses: [
            {
              id: 'api',
              phase: 'degraded',
              ready: false,
              message: 'Deployment "api" updated/ready/available/desired 1/1/0/1',
            },
          ],
        }
      )
    ).toBe(true)
  })

  it('does not patch steady active workflow recipes without observable status changes', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'e2e-scheduled-recipe', namespace: 'sandbox-recipes' },
          spec: { steps: [{ id: 'run', instruction: 'run' }] },
          status: {
            phase: 'active',
            message: 'Workflow trigger infrastructure registered (workflow-agentic)',
          },
        },
        {
          phase: 'active',
          message: 'Workflow trigger infrastructure registered (workflow-agentic)',
          workloadStatuses: [],
        }
      )
    ).toBe(false)
  })

  it('honors explicit skipStatusPatch', () => {
    expect(
      shouldPatchRecipeStatus(
        {
          apiVersion: 'clerum.io/v1alpha1',
          kind: 'WorkflowRecipe',
          metadata: { name: 'e2e-scheduled-recipe', namespace: 'sandbox-recipes' },
          spec: { steps: [{ id: 'run', instruction: 'run' }] },
          status: {
            phase: 'active',
            message: 'Workflow running',
            lastTransitionTime: '2026-05-09T00:00:00Z',
          },
        },
        {
          phase: 'active',
          message: 'Workflow completed',
          workloadStatuses: [],
          skipStatusPatch: true,
        }
      )
    ).toBe(false)
  })

  // ── issue #375: publish computed Plugin Workload SDK state transitions ──
  // The dominant failure mode: the reconciler computes awaiting_policy→validated
  // but no OTHER observable diff (phase, top-level message, workloads, owned
  // conditions) changes, so the old shouldPatchRecipeStatus returned false and
  // the transition was never published — status stayed awaiting_policy for ~40
  // min. shouldPatchRecipeStatus must now compare the SDK capability projection
  // carried on result.pluginWorkloadSdkProjection.
  function validatedProjection() {
    return {
      conditions: [],
      capability: {
        state: 'validated' as const,
        promptBridge: true,
        clientNotifications: false,
        message: 'Capability validated (promptBridge)',
        bootstrapPodUid: 'pod-abc',
        policyRevision: 3,
        defaultTargetRef: 'primary-openai',
        validatedAt: new Date().toISOString(),
        verifiedAt: new Date().toISOString(),
      },
    }
  }

  function sdkRecipe(persistedSdk: Record<string, unknown>): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
      status: {
        phase: 'active',
        message: 'All workloads deployed',
        // No PluginWorkloadSdkCapability condition, so the pre-existing
        // message-vs-condition guard stays inert and this asserts ONLY the new
        // projection comparison.
        pluginWorkloadSdk: persistedSdk as never,
      },
    })
  }

  it('patches a computed awaiting_policy → validated transition (issue #375 incident replica)', () => {
    // The exact incident shape: phase and top-level message are unchanged, the
    // ONLY change is the SDK capability state. Before the fix this returned
    // false and the validated state was never published.
    // M2 (jozer review): the persisted verifiedAt is FRESH, so the >5-min
    // throttle branch stays CLOSED and this can only pass through the semantic
    // field comparisons — a hardcoded stale timestamp made it pass via the
    // throttle, pinning nothing about the incident's mechanism.
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'awaiting_policy',
          promptBridge: true,
          clientNotifications: false,
          message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',
          verifiedAt: new Date(Date.now() - 10_000).toISOString(),
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: validatedProjection(),
        }
      )
    ).toBe(true)
  })

  it('refreshes a stale verifiedAt on an otherwise-identical validated projection (throttle open)', () => {
    const staleVerifiedAt = new Date(Date.now() - 6 * 60_000).toISOString()
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'validated',
          promptBridge: true,
          clientNotifications: false,
          message: 'Capability validated (promptBridge)',
          bootstrapPodUid: 'pod-abc',
          policyRevision: 3,
          defaultTargetRef: 'primary-openai',
          verifiedAt: staleVerifiedAt,
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: validatedProjection(),
        }
      )
    ).toBe(true)
  })

  it('does not patch a fresh verifiedAt on an identical validated projection (throttle closed / anti-loop)', () => {
    const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'validated',
          promptBridge: true,
          clientNotifications: false,
          message: 'Capability validated (promptBridge)',
          bootstrapPodUid: 'pod-abc',
          policyRevision: 3,
          defaultTargetRef: 'primary-openai',
          verifiedAt: freshVerifiedAt,
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: validatedProjection(),
        }
      )
    ).toBe(false)
  })

  it('patches a computed validated → awaiting_policy transition (revocation symmetry)', () => {
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'validated',
          promptBridge: true,
          clientNotifications: false,
          message: 'Capability validated (promptBridge)',
          bootstrapPodUid: 'pod-abc',
          policyRevision: 3,
          defaultTargetRef: 'primary-openai',
          verifiedAt: new Date(Date.now() - 10_000).toISOString(),
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: {
            conditions: [],
            capability: {
              state: 'awaiting_policy' as const,
              promptBridge: true,
              clientNotifications: false,
              message: 'Plugin Workload SDK promptBridge is awaiting an operator grant',
              bootstrapPodUid: 'pod-abc',
              verifiedAt: new Date().toISOString(),
            },
          },
        }
      )
    ).toBe(true)
  })

  it('does not patch when the SDK projection is semantically unchanged and verifiedAt is fresh', () => {
    // Anti-loop guard: two back-to-back reconciles with the same computed state
    // and a still-fresh verifiedAt must NOT keep emitting patches.
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'validated',
          promptBridge: true,
          clientNotifications: false,
          message: 'Capability validated (promptBridge)',
          bootstrapPodUid: 'pod-abc',
          policyRevision: 3,
          defaultTargetRef: 'primary-openai',
          verifiedAt: new Date(Date.now() - 5_000).toISOString(),
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: validatedProjection(),
        }
      )
    ).toBe(false)
  })

  it('patches an in-place re-bootstrap that only changes bootstrapModel/provider/contractVersion (issue #375 B1)', () => {
    // A validated promptBridge recipe re-bootstraps in place (SAME pod, same
    // policyRevision) against a DIFFERENT model/provider — e.g. the operator
    // corrects the target model. state stays 'validated', message is
    // families-only, pod/policyRevision unchanged, verifiedAt is fresh (throttle
    // closed). Only bootstrapModel/provider/contractVersion move — persisted
    // semantic fields read by control-api at runtime. They MUST force a patch.
    const freshVerifiedAt = new Date(Date.now() - 10_000).toISOString()
    expect(
      shouldPatchRecipeStatus(
        sdkRecipe({
          state: 'validated',
          promptBridge: true,
          clientNotifications: false,
          message: 'Capability validated (promptBridge)',
          bootstrapPodUid: 'pod-abc',
          bootstrapContractVersion: 2,
          bootstrapProvider: 'openai',
          bootstrapModel: 'gpt-5.4-mini',
          policyRevision: 3,
          defaultTargetRef: 'primary-openai',
          verifiedAt: freshVerifiedAt,
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: {
            conditions: [],
            capability: {
              state: 'validated' as const,
              promptBridge: true,
              clientNotifications: false,
              message: 'Capability validated (promptBridge)',
              bootstrapPodUid: 'pod-abc',
              bootstrapContractVersion: 2,
              bootstrapProvider: 'anthropic',
              bootstrapModel: 'claude-sonnet-4-6',
              policyRevision: 3,
              defaultTargetRef: 'primary-openai',
              verifiedAt: freshVerifiedAt,
            },
          },
        }
      )
    ).toBe(true)
  })

  // Symmetric to #375 (computed-but-not-published): when a recipe DROPS
  // spec.pluginWorkloadSdk while a capability is still persisted in status, the
  // projection computes `capability === null` (clear the field). The
  // `projected === null` branch is the ONLY thing that publishes that clear — a
  // regression to `return false` there would leave a stale validated/
  // awaiting_policy record proclaimed forever.
  it('patches to clear status.pluginWorkloadSdk when the capability is removed from spec', () => {
    expect(
      shouldPatchRecipeStatus(
        makeWorkflowRecipe({
          spec: { steps: [] },
          status: {
            phase: 'active',
            message: 'All workloads deployed',
            pluginWorkloadSdk: {
              state: 'validated',
              promptBridge: true,
              clientNotifications: false,
            } as never,
          },
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: { conditions: [], capability: null },
        }
      )
    ).toBe(true)
  })

  it('does not patch when the capability is absent both in the projection and in status', () => {
    expect(
      shouldPatchRecipeStatus(
        makeWorkflowRecipe({
          spec: { steps: [] },
          status: { phase: 'active', message: 'All workloads deployed' },
        }),
        {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: { conditions: [], capability: null },
        }
      )
    ).toBe(false)
  })

  // ── issue #375 M2 (jozer review): every projection field comparison is pinned
  // INDIVIDUALLY. Each case varies exactly ONE field between the persisted
  // capability and the projection, with a FRESH persisted verifiedAt so the
  // >5-min throttle branch stays closed — removing that single field's
  // comparison in pluginWorkloadSdkProjectionChanged turns that case RED.
  describe('per-field projection comparisons (issue #375 M2)', () => {
    function persistedBase(): Record<string, unknown> {
      return {
        state: 'validated',
        promptBridge: true,
        clientNotifications: false,
        message: 'Capability validated (promptBridge)',
        policyRevision: 3,
        policyHash: 'sha256:policy-a',
        defaultTargetRef: 'primary-openai',
        bootstrapPodUid: 'pod-abc',
        bootstrapContractVersion: 2,
        bootstrapProvider: 'openai',
        bootstrapModel: 'gpt-5.4-mini',
        // FRESH: keeps the verifiedAt throttle CLOSED so a `true` can only come
        // from the varied field's own comparison.
        verifiedAt: new Date(Date.now() - 10_000).toISOString(),
      }
    }

    function projectionFrom(
      persisted: Record<string, unknown>,
      override: Record<string, unknown>
    ): { conditions: never[]; capability: never } {
      const { verifiedAt: _persistedVerifiedAt, ...semantic } = persisted
      return {
        conditions: [],
        capability: {
          ...semantic,
          verifiedAt: new Date().toISOString(),
          validatedAt: new Date().toISOString(),
          ...override,
        } as never,
      }
    }

    const cases: Array<[field: string, override: Record<string, unknown>]> = [
      ['state', { state: 'awaiting_policy' }],
      ['message', { message: 'Capability validated (promptBridge, clientNotifications)' }],
      ['policyRevision', { policyRevision: 4 }],
      ['policyHash', { policyHash: 'sha256:policy-b' }],
      ['defaultTargetRef', { defaultTargetRef: 'fallback-anthropic' }],
      ['bootstrapPodUid', { bootstrapPodUid: 'pod-def' }],
      ['bootstrapContractVersion', { bootstrapContractVersion: null }],
      ['bootstrapProvider', { bootstrapProvider: 'anthropic' }],
      ['bootstrapModel', { bootstrapModel: 'claude-sonnet-4-6' }],
      ['promptBridge', { promptBridge: false }],
      ['clientNotifications', { clientNotifications: true }],
    ]

    for (const [field, override] of cases) {
      it(`a lone ${field} change forces a patch (throttle closed)`, () => {
        const persisted = persistedBase()
        expect(
          shouldPatchRecipeStatus(sdkRecipe(persisted), {
            phase: 'active',
            message: 'All workloads deployed',
            workloadStatuses: [],
            pluginWorkloadSdkProjection: projectionFrom(persisted, override),
          })
        ).toBe(true)
      })
    }

    it('control: with NO field varied and a fresh persisted verifiedAt, no patch is emitted', () => {
      const persisted = persistedBase()
      expect(
        shouldPatchRecipeStatus(sdkRecipe(persisted), {
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
          pluginWorkloadSdkProjection: projectionFrom(persisted, {}),
        })
      ).toBe(false)
    })

    it('first-ever publish: a computed capability with NOTHING persisted forces a patch', () => {
      // Pins the `if (!current) return true` branch — the first publish of
      // status.pluginWorkloadSdk after a recipe declares the capability.
      expect(
        shouldPatchRecipeStatus(
          makeWorkflowRecipe({
            spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
            status: { phase: 'active', message: 'All workloads deployed' },
          }),
          {
            phase: 'active',
            message: 'All workloads deployed',
            workloadStatuses: [],
            pluginWorkloadSdkProjection: projectionFrom(persistedBase(), {}),
          }
        )
      ).toBe(true)
    })
  })
})

describe('infrastructure telemetry projections', () => {
  it('builds stable reconcile_outcome source ids bound to the canonical workflow run label', () => {
    const recipe = makeWorkflowRecipe({
      metadata: {
        name: 'run-child',
        namespace: 'sandbox-recipes',
        generation: 3,
        resourceVersion: 'rv-9',
        labels: { 'clerum.io/workflow-run-id': 'run-123' },
      },
    })

    expect(
      reconcileOutcomeTelemetryProjection(
        recipe,
        { phase: 'active', message: 'ok', workloadStatuses: [] },
        '2026-07-11T10:00:00.000Z'
      )
    ).toEqual({
      sourceEventId: 'workflow-reconcile:sandbox-recipes:run-child:generation:3:active',
      occurredAt: '2026-07-11T10:00:00.000Z',
      telemetryType: 'reconcile_outcome',
      runId: 'run-123',
      payload: { phase: 'active', status: 'patched' },
    })
  })

  it('skips infrastructure telemetry for recipes without a canonical workflow run label', () => {
    const recipe = makeWorkflowRecipe({
      metadata: { name: 'parent', namespace: 'sandbox-recipes' },
    })

    expect(
      reconcileOutcomeTelemetryProjection(recipe, {
        phase: 'active',
        message: 'ok',
        workloadStatuses: [],
      })
    ).toBeUndefined()
    expect(controllerErrorTelemetryProjection(recipe, new Error('boom'))).toBeUndefined()
  })
})

describe('runtime credential refresh loop helpers', () => {
  it('uses a quarter of the refresh window bounded between 5s and 30s', () => {
    expect(runtimeCredentialRefreshIntervalMs({ runtimeTokenRefreshBeforeSeconds: 45 })).toBe(
      11_250
    )
    expect(runtimeCredentialRefreshIntervalMs({ runtimeTokenRefreshBeforeSeconds: 1 })).toBe(5_000)
    expect(runtimeCredentialRefreshIntervalMs({ runtimeTokenRefreshBeforeSeconds: 300 })).toBe(
      30_000
    )
  })

  it('selects only non-deleting WorkflowRecipes with in-progress workflow execution', () => {
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({ status: { phase: 'active', workflowExecution: { phase: 'running' } } })
      )
    ).toBe(true)
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({
          status: { phase: 'deploying', workflowExecution: { phase: 'initializing' } },
        })
      )
    ).toBe(true)
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({
          status: { phase: 'deploying', workflowExecution: { phase: 'recovering' } },
        })
      )
    ).toBe(true)
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({
          status: { phase: 'active', workflowExecution: { phase: 'completed' } },
        })
      )
    ).toBe(false)
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({
          metadata: { name: 'deleting', namespace: 'sandbox-recipes', deletionTimestamp: 'now' },
          status: { phase: 'active', workflowExecution: { phase: 'running' } },
        })
      )
    ).toBe(false)
    expect(
      workflowNeedsRuntimeCredentialRefresh(
        makeWorkflowRecipe({
          spec: {
            steps: [],
            workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
          },
          status: { phase: 'active', workflowExecution: { phase: 'running' } },
        })
      )
    ).toBe(false)
  })

  it('selects initializing and recovering workflows for infrastructure reconcile retries', () => {
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          status: { phase: 'deploying', workflowExecution: { phase: 'initializing' } },
        })
      )
    ).toBe(true)
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          status: { phase: 'deploying', workflowExecution: { phase: 'recovering' } },
        })
      )
    ).toBe(true)
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          status: { phase: 'active', workflowExecution: { phase: 'running' } },
        })
      )
    ).toBe(false)
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          metadata: { name: 'deleting', namespace: 'sandbox-recipes', deletionTimestamp: 'now' },
          status: { phase: 'deploying', workflowExecution: { phase: 'initializing' } },
        })
      )
    ).toBe(false)
  })

  it('forces timer-driven infrastructure retries past the status-only generation guard', async () => {
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    const sdkOnly = makeWorkflowRecipe({
      metadata: { name: 'sdk-only', namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
        pluginWorkloadSdk: { promptBridge: {} },
      },
      status: { phase: 'active' },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      eventQueue: { enqueue: typeof enqueue }
      handleRecipeEvent: typeof handleRecipeEvent
      refreshRuntimeCredentialsForInProgressRecipes: () => Promise<void>
      stopped: boolean
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[sdkOnly.metadata.name, sdkOnly]])
    internal.eventQueue = { enqueue }
    internal.handleRecipeEvent = handleRecipeEvent
    internal.stopped = false

    await internal.refreshRuntimeCredentialsForInProgressRecipes()

    expect(enqueue).toHaveBeenCalledWith(sdkOnly.metadata.name, expect.any(Function))
    expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', sdkOnly, {
      forceReconcile: true,
    })
  })

  it('keeps reconciling a promptBridge SDK recipe with no run so the eager mcp-host stays configured', () => {
    // phase=active, workflowExecution cleared (eager SDK path) — must still requeue.
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          spec: {
            steps: [{ id: 's1', instruction: 'ack' }],
            agent: { provider: 'zai', model: 'glm-4.7' },
            pluginWorkloadSdk: { promptBridge: {} },
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(true)
  })

  it('keeps reconciling a clientNotifications-only SDK recipe with no run so the eager mcp-host self-heals', () => {
    // Both families run the always-on eager SDK mcp-host. A clientNotifications-only
    // recipe still needs the level-triggered requeue so a dead eager pod
    // (restartPolicy: Never) is recreated without an unrelated spec edit.
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          spec: {
            steps: [{ id: 's1', instruction: 'ack' }],
            pluginWorkloadSdk: { clientNotifications: { allowedEventTypes: ['e2e.test'] } },
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(true)
  })

  it('keeps reconciling a stepless SDK-only recipe through the full capability lane', () => {
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          spec: {
            steps: [],
            workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
            pluginWorkloadSdk: { promptBridge: {} },
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(true)
  })

  it('does NOT keep reconciling a recipe with no Plugin Workload SDK and no run', () => {
    // No pluginWorkloadSdk → no eager mcp-host → no eager-reconcile requeue needed.
    expect(
      workflowNeedsInfrastructureReconcile(
        makeWorkflowRecipe({
          spec: {
            steps: [{ id: 's1', instruction: 'ack' }],
            agent: { provider: 'zai', model: 'glm-4.7' },
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(false)
  })

  it('refreshes cached in-progress workflow credentials without patching status', async () => {
    const refreshInProgressWorkflowRuntimeCredentials = vi.fn().mockResolvedValue(undefined)
    const running = makeWorkflowRecipe({
      metadata: { name: 'running-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const completed = makeWorkflowRecipe({
      metadata: { name: 'completed-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'active', workflowExecution: { phase: 'completed' } },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      reconciler: {
        refreshInProgressWorkflowRuntimeCredentials: typeof refreshInProgressWorkflowRuntimeCredentials
      }
      refreshRuntimeCredentialsForInProgressRecipes: () => Promise<void>
      stopped: boolean
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([
      [running.metadata.name, running],
      [completed.metadata.name, completed],
    ])
    internal.reconciler = { refreshInProgressWorkflowRuntimeCredentials }
    internal.stopped = false

    await internal.refreshRuntimeCredentialsForInProgressRecipes()

    expect(refreshInProgressWorkflowRuntimeCredentials).toHaveBeenCalledTimes(1)
    expect(refreshInProgressWorkflowRuntimeCredentials).toHaveBeenCalledWith(running)
  })

  it('re-enqueues initializing workflows so pod readiness can progress without CRD events', async () => {
    const refreshInProgressWorkflowRuntimeCredentials = vi.fn().mockResolvedValue(undefined)
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    const initializing = makeWorkflowRecipe({
      metadata: { name: 'initializing-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'deploying', workflowExecution: { phase: 'initializing' } },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      reconciler: {
        refreshInProgressWorkflowRuntimeCredentials: typeof refreshInProgressWorkflowRuntimeCredentials
      }
      eventQueue: { enqueue: typeof enqueue }
      handleRecipeEvent: typeof handleRecipeEvent
      refreshRuntimeCredentialsForInProgressRecipes: () => Promise<void>
      stopped: boolean
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[initializing.metadata.name, initializing]])
    internal.reconciler = { refreshInProgressWorkflowRuntimeCredentials }
    internal.eventQueue = { enqueue }
    internal.handleRecipeEvent = handleRecipeEvent
    internal.stopped = false

    await internal.refreshRuntimeCredentialsForInProgressRecipes()

    expect(refreshInProgressWorkflowRuntimeCredentials).toHaveBeenCalledWith(initializing)
    expect(enqueue).toHaveBeenCalledWith(initializing.metadata.name, expect.any(Function))
    expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', initializing, {
      forceReconcile: true,
    })
  })
})

describe('workload status refresh loop helpers', () => {
  function makeWorkloadRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      ...overrides,
      spec: {
        steps: [],
        workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
        ...(overrides.spec ?? {}),
      },
    })
  }

  it('selects active and degraded workload recipes for observed status refresh', () => {
    expect(
      workflowNeedsWorkloadStatusRefresh(makeWorkloadRecipe({ status: { phase: 'active' } }))
    ).toBe(true)
    expect(
      workflowNeedsWorkloadStatusRefresh(makeWorkloadRecipe({ status: { phase: 'degraded' } }))
    ).toBe(true)
    expect(
      workflowNeedsWorkloadStatusRefresh(makeWorkloadRecipe({ status: { phase: 'failed' } }))
    ).toBe(false)
    expect(
      workflowNeedsWorkloadStatusRefresh(
        makeWorkloadRecipe({
          metadata: { name: 'deleting', namespace: 'sandbox-recipes', deletionTimestamp: 'now' },
          status: { phase: 'active' },
        })
      )
    ).toBe(false)
    expect(
      workflowNeedsWorkloadStatusRefresh(
        makeWorkloadRecipe({
          spec: {
            steps: [],
            pluginWorkloadSdk: { promptBridge: {} },
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(false)
    expect(
      workflowNeedsWorkloadStatusRefresh(makeWorkflowRecipe({ status: { phase: 'active' } }))
    ).toBe(false)
    expect(
      workflowNeedsWorkloadStatusRefresh(
        makeWorkloadRecipe({
          spec: {
            steps: [{ id: 'step-1', instruction: 'run' }],
            workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
          },
          status: { phase: 'active' },
        })
      )
    ).toBe(false)
  })

  it('recipeReferencesNamedSecrets detects envSecret and imagePullSecrets (Issue #637 backstop)', () => {
    expect(recipeReferencesNamedSecrets(makeWorkloadRecipe())).toBe(false)
    expect(
      recipeReferencesNamedSecrets(
        makeWorkloadRecipe({
          spec: {
            steps: [],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                envSecret: { name: 'creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
              },
            ],
          },
        })
      )
    ).toBe(true)
    expect(
      recipeReferencesNamedSecrets(
        makeWorkloadRecipe({
          spec: {
            steps: [],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                imagePullSecrets: ['reg'],
              },
            ],
          },
        })
      )
    ).toBe(true)
  })

  it('workflowNeedsOwnershipBackstop selects ONLY workflow recipes that reference named Secrets (Issue #637)', () => {
    // Non-workflow recipe (steps==0): already covered by workflowNeedsWorkloadStatusRefresh.
    expect(
      workflowNeedsOwnershipBackstop(
        makeWorkloadRecipe({
          spec: {
            steps: [],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                envSecret: { name: 'creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
              },
            ],
          },
        })
      )
    ).toBe(false)
    // Workflow recipe (steps>0) without a named Secret: no backstop needed.
    expect(
      workflowNeedsOwnershipBackstop(
        makeWorkloadRecipe({
          spec: {
            steps: [{ id: 'step-1', instruction: 'run' }],
            workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
          },
        })
      )
    ).toBe(false)
    // Workflow recipe (steps>0) WITH an envSecret: backstop fires.
    expect(
      workflowNeedsOwnershipBackstop(
        makeWorkloadRecipe({
          spec: {
            steps: [{ id: 'step-1', instruction: 'run' }],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                envSecret: { name: 'creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
              },
            ],
          },
        })
      )
    ).toBe(true)
    // Workflow recipe with imagePullSecrets: backstop fires.
    expect(
      workflowNeedsOwnershipBackstop(
        makeWorkloadRecipe({
          spec: {
            steps: [{ id: 'step-1', instruction: 'run' }],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                imagePullSecrets: ['reg'],
              },
            ],
          },
        })
      )
    ).toBe(true)
    // Deleting recipe: never.
    expect(
      workflowNeedsOwnershipBackstop(
        makeWorkloadRecipe({
          metadata: { name: 'deleting', namespace: 'sandbox-recipes', deletionTimestamp: 'now' },
          spec: {
            steps: [{ id: 'step-1', instruction: 'run' }],
            workloads: [
              {
                id: 'api',
                type: 'deployment',
                image: 'clerum/api:test',
                envSecret: { name: 'creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
              },
            ],
          },
        })
      )
    ).toBe(false)
  })

  it('re-enqueues steady workload recipes so readiness drift is observed without CRD events', async () => {
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const refreshWorkloadStatusOnly = vi.fn().mockResolvedValue(undefined)
    const active = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'active' },
    })
    const degraded = makeWorkloadRecipe({
      metadata: { name: 'degraded-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'degraded' },
    })
    const failed = makeWorkloadRecipe({
      metadata: { name: 'failed-recipe', namespace: 'sandbox-recipes' },
      status: { phase: 'failed' },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      eventQueue: { enqueue: typeof enqueue }
      refreshWorkloadStatusOnly: typeof refreshWorkloadStatusOnly
      refreshWorkloadStatusesForSteadyRecipes: () => Promise<void>
      stopped: boolean
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([
      [active.metadata.name, active],
      [degraded.metadata.name, degraded],
      [failed.metadata.name, failed],
    ])
    internal.eventQueue = { enqueue }
    internal.refreshWorkloadStatusOnly = refreshWorkloadStatusOnly
    internal.stopped = false

    await internal.refreshWorkloadStatusesForSteadyRecipes()

    expect(enqueue).toHaveBeenCalledTimes(2)
    expect(enqueue).toHaveBeenCalledWith(active.metadata.name, expect.any(Function))
    expect(enqueue).toHaveBeenCalledWith(degraded.metadata.name, expect.any(Function))
    // Non-workflow recipes go through the full status-refresh path (ownershipOnly: false).
    expect(refreshWorkloadStatusOnly).toHaveBeenCalledWith(active, { ownershipOnly: false })
    expect(refreshWorkloadStatusOnly).toHaveBeenCalledWith(degraded, { ownershipOnly: false })
  })

  it('re-enqueues steady WORKFLOW recipes that project a named Secret for an ownership-only backstop (Issue #637)', async () => {
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const refreshWorkloadStatusOnly = vi.fn().mockResolvedValue(undefined)
    // Workflow recipe (steps>0) WITH an envSecret: excluded from the status refresh,
    // but must still get the periodic ownership re-check (its only other revocation
    // path is the event-driven SecretWatcher).
    const workflowWithSecret = makeWorkloadRecipe({
      metadata: { name: 'wf-secret', namespace: 'sandbox-recipes' },
      spec: {
        steps: [{ id: 'run', instruction: 'go' }],
        workloads: [
          {
            id: 'api',
            type: 'deployment',
            image: 'clerum/api:test',
            envSecret: { name: 'creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
          },
        ],
      },
      status: { phase: 'active' },
    })
    // Workflow recipe (steps>0) WITHOUT a named Secret: no backstop, not enqueued.
    const workflowNoSecret = makeWorkloadRecipe({
      metadata: { name: 'wf-plain', namespace: 'sandbox-recipes' },
      spec: {
        steps: [{ id: 'run', instruction: 'go' }],
        workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
      },
      status: { phase: 'active' },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      eventQueue: { enqueue: typeof enqueue }
      refreshWorkloadStatusOnly: typeof refreshWorkloadStatusOnly
      refreshWorkloadStatusesForSteadyRecipes: () => Promise<void>
      stopped: boolean
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([
      [workflowWithSecret.metadata.name, workflowWithSecret],
      [workflowNoSecret.metadata.name, workflowNoSecret],
    ])
    internal.eventQueue = { enqueue }
    internal.refreshWorkloadStatusOnly = refreshWorkloadStatusOnly
    internal.stopped = false

    await internal.refreshWorkloadStatusesForSteadyRecipes()

    // Only the workflow-with-secret is enqueued, in ownership-only mode (no status patch).
    expect(enqueue).toHaveBeenCalledTimes(1)
    expect(enqueue).toHaveBeenCalledWith(workflowWithSecret.metadata.name, expect.any(Function))
    expect(refreshWorkloadStatusOnly).toHaveBeenCalledWith(workflowWithSecret, {
      ownershipOnly: true,
    })
  })

  it('triggerSecretDrivenReconcile forces a full reconcile so a Secret re-label reaches the ownership gate (Issue #637)', async () => {
    // A Secret re-label does NOT bump the recipe's generation, so a plain 'MODIFIED'
    // event is status-only and short-circuits before reconcile() (and the ownership
    // gate). The Secret-driven trigger MUST pass forceReconcile, or the event-driven
    // revocation never re-evaluates ownership and falls back to the slow 30s backstop.
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const recipe = makeWorkloadRecipe({
      metadata: { name: 'secret-driven', namespace: 'sandbox-recipes' },
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      eventQueue: { enqueue: typeof enqueue }
      handleRecipeEvent: typeof handleRecipeEvent
      triggerSecretDrivenReconcile: (name: string) => void
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[recipe.metadata.name, recipe]])
    internal.eventQueue = { enqueue }
    internal.handleRecipeEvent = handleRecipeEvent

    internal.triggerSecretDrivenReconcile(recipe.metadata.name)

    expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', recipe, { forceReconcile: true })
  })

  it('updates the cache but skips full reconcile for status-only watch events', async () => {
    const cached = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const statusOnly = makeWorkloadRecipe({
      metadata: {
        name: 'active-recipe',
        namespace: 'sandbox-recipes',
        generation: 7,
        resourceVersion: 'next',
      },
      status: { phase: 'degraded', message: 'Some workloads not ready' },
    })
    const reconcile = vi.fn().mockResolvedValue({
      phase: 'active',
      message: 'All workloads deployed',
      workloadStatuses: [],
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      reconciler: { reconcile: typeof reconcile }
      transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
      handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[cached.metadata.name, cached]])
    internal.transientRetries = new Map()
    internal.reconciler = { reconcile }

    await internal.handleRecipeEvent('MODIFIED', statusOnly)

    expect(internal.recipes.get(statusOnly.metadata.name)).toBe(statusOnly)
    expect(reconcile).not.toHaveBeenCalled()
  })

  it('retries a transient degraded result even when the first pass patches status', async () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkloadRecipe({
        metadata: { name: 'policy-race', namespace: 'sandbox-recipes', generation: 1 },
        status: { phase: 'candidate' },
      })
      const reconcile = vi
        .fn()
        .mockResolvedValueOnce({
          phase: 'degraded',
          message: 'NetworkPolicy changed during replace',
          workloadStatuses: [],
          requeueAfterMs: 5_000,
        })
        .mockResolvedValueOnce({
          phase: 'active',
          message: 'All workloads deployed',
          workloadStatuses: [],
        })
      const patchStatus = vi.fn().mockResolvedValue(undefined)
      const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())

      type InternalWatcher = {
        recipes: Map<string, WorkflowRecipeCRD>
        transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
        eventQueue: { enqueue: typeof enqueue }
        stopped: boolean
        traceReporter: null
        dbRunProcessor: null
        reconciler: {
          reconcile: typeof reconcile
          isRecipeStillActive: ReturnType<typeof vi.fn>
          ensureFinalizer: ReturnType<typeof vi.fn>
          patchStatus: typeof patchStatus
        }
        handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
      }

      const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
      internal.recipes = new Map([[recipe.metadata.name, recipe]])
      internal.transientRetries = new Map()
      internal.eventQueue = { enqueue }
      internal.stopped = false
      internal.traceReporter = null
      internal.dbRunProcessor = null
      internal.reconciler = {
        reconcile,
        isRecipeStillActive: vi.fn().mockResolvedValue(true),
        ensureFinalizer: vi.fn().mockResolvedValue(undefined),
        patchStatus,
      }

      await internal.handleRecipeEvent('ADDED', recipe)

      expect(patchStatus).toHaveBeenCalledTimes(1)
      expect(internal.transientRetries.has(recipe.metadata.name)).toBe(true)

      await vi.advanceTimersByTimeAsync(5_000)
      await Promise.resolve()

      expect(enqueue).toHaveBeenCalledWith(recipe.metadata.name, expect.any(Function))
      expect(reconcile).toHaveBeenCalledTimes(2)
      expect(internal.transientRetries.has(recipe.metadata.name)).toBe(false)
    } finally {
      vi.useRealTimers()
    }
  })

  it.each([
    { from: 'failed', to: 'candidate', runs: 1 },
    { from: 'active', to: 'active', runs: 0 },
    { from: 'active', to: 'degraded', runs: 0 },
    { from: 'degraded', to: 'active', runs: 0 },
    { from: 'failed', to: 'failed', runs: 0 },
    { from: 'candidate', to: 'candidate', runs: 0 },
  ] as const)(
    'handles the supported retry transition $from -> $to without a generation bump',
    async ({ from, to, runs }) => {
      const cached = makeWorkloadRecipe({
        metadata: { name: 'retry-recipe', namespace: 'sandbox-recipes', generation: 7 },
        status: { phase: from },
      })
      const requested = {
        ...cached,
        status: { phase: to, message: 'Manual retry requested by operator' },
      }
      const reconcile = vi.fn().mockResolvedValue({
        phase: 'active',
        message: 'All workloads deployed',
        workloadStatuses: [],
      })
      const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {
        recipes: Map<string, WorkflowRecipeCRD>
        handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
      }
      Object.assign(watcher, {
        recipes: new Map([[cached.metadata.name, cached]]),
        transientRetries: new Map(),
        stopped: false,
        traceReporter: null,
        dbRunProcessor: null,
        reconciler: {
          reconcile,
          isRecipeStillActive: vi.fn().mockResolvedValue(true),
          ensureFinalizer: vi.fn().mockResolvedValue(undefined),
          patchStatus: vi.fn().mockResolvedValue(undefined),
        },
      })
      await watcher.handleRecipeEvent('MODIFIED', requested)
      expect(reconcile).toHaveBeenCalledTimes(runs)
      expect(watcher.recipes.get(cached.metadata.name)).toEqual(requested)
      // Duplicate status events must not repeatedly execute the retry transition.
      await watcher.handleRecipeEvent('MODIFIED', requested)
      expect(reconcile).toHaveBeenCalledTimes(runs)
    }
  )

  it('reconciles when the canonical subscription annotation changes without a generation bump', async () => {
    const cached = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const granted = makeWorkloadRecipe({
      metadata: {
        name: 'active-recipe',
        namespace: 'sandbox-recipes',
        generation: 7,
        resourceVersion: 'next',
        annotations: { 'clerum.io/subscription-connection-ref': 'personal-pro' },
      },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const reconcile = vi.fn().mockResolvedValue({
      phase: 'active',
      message: 'All workloads deployed',
      workloadStatuses: [],
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
      stopped: boolean
      reconciler: {
        reconcile: typeof reconcile
        isRecipeStillActive: ReturnType<typeof vi.fn>
        ensureFinalizer: ReturnType<typeof vi.fn>
        patchStatus: ReturnType<typeof vi.fn>
      }
      handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
      scheduleTransientRetry: ReturnType<typeof vi.fn>
      clearTransientRetry: ReturnType<typeof vi.fn>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[cached.metadata.name, cached]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.reconciler = {
      reconcile,
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    internal.handleRecipeEvent = WorkflowRecipeWatcher.prototype.handleRecipeEvent
    internal.scheduleTransientRetry = vi.fn()
    internal.clearTransientRetry = vi.fn()
    await internal.handleRecipeEvent('MODIFIED', granted)
    expect(reconcile).toHaveBeenCalledTimes(1)
  })

  it('reconciles when Codex connection-ref changes without a generation bump', async () => {
    const cached = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const granted = makeWorkloadRecipe({
      metadata: {
        name: 'active-recipe',
        namespace: 'sandbox-recipes',
        generation: 7,
        resourceVersion: 'next',
        annotations: { 'clerum.io/codex-connection-ref': 'codex-ba832480dffd9095' },
      },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const reconcile = vi.fn().mockResolvedValue({
      phase: 'active',
      message: 'All workloads deployed',
      workloadStatuses: [],
    })

    type InternalWatcher = {
      recipes: Map<string, WorkflowRecipeCRD>
      transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
      stopped: boolean
      reconciler: {
        reconcile: typeof reconcile
        isRecipeStillActive: ReturnType<typeof vi.fn>
        ensureFinalizer: ReturnType<typeof vi.fn>
        patchStatus: ReturnType<typeof vi.fn>
      }
      handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
      scheduleTransientRetry: ReturnType<typeof vi.fn>
      clearTransientRetry: ReturnType<typeof vi.fn>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[cached.metadata.name, cached]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.reconciler = {
      reconcile,
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    internal.scheduleTransientRetry = vi.fn()
    internal.clearTransientRetry = vi.fn()

    await internal.handleRecipeEvent('MODIFIED', granted)

    expect(internal.recipes.get(granted.metadata.name)).toBe(granted)
    expect(reconcile).toHaveBeenCalledWith(granted)
  })

  it('forces a full reconcile when status-only refresh observes workload drift', async () => {
    const recipe = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const result = {
      phase: 'degraded' as const,
      message: 'Some workloads not ready',
      workloadStatuses: [{ id: 'api', phase: 'degraded', ready: false }],
    }
    const observeCurrentWorkloadStatus = vi.fn().mockResolvedValue(result)
    const patchStatus = vi.fn().mockResolvedValue(undefined)
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)

    type InternalWatcher = {
      reconciler: {
        observeCurrentWorkloadStatus: typeof observeCurrentWorkloadStatus
        patchStatus: typeof patchStatus
      }
      handleRecipeEvent: typeof handleRecipeEvent
      refreshWorkloadStatusOnly: (recipe: WorkflowRecipeCRD) => Promise<void>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.reconciler = { observeCurrentWorkloadStatus, patchStatus }
    internal.handleRecipeEvent = handleRecipeEvent

    await internal.refreshWorkloadStatusOnly(recipe)

    expect(patchStatus).toHaveBeenCalledWith(recipe, result)
    expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', recipe, { forceReconcile: true })
  })

  it('forces a full reconcile for an ALL-READY recipe that references a named Secret (Issue #637 ownership backstop)', async () => {
    const recipe = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
      spec: {
        workloads: [
          {
            id: 'api',
            type: 'deployment',
            image: 'clerum/api:test',
            envSecret: { name: 'api-creds', keys: [{ secretKey: 'k', envVar: 'K' }] },
          },
        ],
      },
    })
    // NO drift: every workload is ready → the `ready === false` term is false, so the
    // forced reconcile can ONLY come from the ownershipRecheck term. This isolates
    // `ownershipRecheck`: reverting the `ownershipRecheck ||` guard makes it red (the
    // deterministic backstop that re-runs the Secret-ownership gate would be lost).
    const result = {
      phase: 'active' as const,
      message: 'All workloads deployed',
      workloadStatuses: [{ id: 'api', phase: 'running', ready: true }],
    }
    const observeCurrentWorkloadStatus = vi.fn().mockResolvedValue(result)
    const patchStatus = vi.fn().mockResolvedValue(undefined)
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)

    type InternalWatcher = {
      reconciler: {
        observeCurrentWorkloadStatus: typeof observeCurrentWorkloadStatus
        patchStatus: typeof patchStatus
      }
      handleRecipeEvent: typeof handleRecipeEvent
      refreshWorkloadStatusOnly: (recipe: WorkflowRecipeCRD) => Promise<void>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.reconciler = { observeCurrentWorkloadStatus, patchStatus }
    internal.handleRecipeEvent = handleRecipeEvent

    await internal.refreshWorkloadStatusOnly(recipe)

    expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', recipe, { forceReconcile: true })
  })

  it('does NOT force a reconcile for an all-ready recipe with NO named Secret refs', async () => {
    // Proves the ownershipRecheck term is load-bearing (not always-true): a steady
    // recipe with no envSecret/imagePullSecrets and no drift must stay cheap.
    const recipe = makeWorkloadRecipe({
      metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
      status: { phase: 'active', message: 'All workloads deployed' },
    })
    const result = {
      phase: 'active' as const,
      message: 'All workloads deployed',
      workloadStatuses: [{ id: 'api', phase: 'running', ready: true }],
    }
    const observeCurrentWorkloadStatus = vi.fn().mockResolvedValue(result)
    const patchStatus = vi.fn().mockResolvedValue(undefined)
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)

    type InternalWatcher = {
      reconciler: {
        observeCurrentWorkloadStatus: typeof observeCurrentWorkloadStatus
        patchStatus: typeof patchStatus
      }
      handleRecipeEvent: typeof handleRecipeEvent
      refreshWorkloadStatusOnly: (recipe: WorkflowRecipeCRD) => Promise<void>
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.reconciler = { observeCurrentWorkloadStatus, patchStatus }
    internal.handleRecipeEvent = handleRecipeEvent

    await internal.refreshWorkloadStatusOnly(recipe)

    expect(handleRecipeEvent).not.toHaveBeenCalled()
  })
})

describe('workflowNeedsTerminalRunTeardown', () => {
  it.each(['completed', 'failed', 'cancelled'] as const)(
    'selects run-scoped workflow recipes when the terminal phase is %s',
    phase => {
      expect(
        workflowNeedsTerminalRunTeardown(
          makeWorkflowRecipe({
            metadata: {
              name: `terminal-${phase}`,
              namespace: 'sandbox-recipes',
              labels: { 'clerum.io/workflow-run-id': `run-${phase}` },
            },
            status: { phase: 'active', workflowExecution: { phase } },
          })
        )
      ).toBe(true)
    }
  )

  it('does not select non-terminal, non-run-scoped, non-workflow, or deleting recipes', () => {
    expect(
      workflowNeedsTerminalRunTeardown(
        makeWorkflowRecipe({
          metadata: {
            name: 'running',
            namespace: 'sandbox-recipes',
            labels: { 'clerum.io/workflow-run-id': 'run-running' },
          },
          status: { phase: 'active', workflowExecution: { phase: 'running' } },
        })
      )
    ).toBe(false)

    expect(
      workflowNeedsTerminalRunTeardown(
        makeWorkflowRecipe({
          metadata: { name: 'not-run-scoped', namespace: 'sandbox-recipes', labels: {} },
          status: { phase: 'active', workflowExecution: { phase: 'completed' } },
        })
      )
    ).toBe(false)

    expect(
      workflowNeedsTerminalRunTeardown(
        makeWorkflowRecipe({
          metadata: {
            name: 'blank-run-label',
            namespace: 'sandbox-recipes',
            labels: { 'clerum.io/workflow-run-id': '   ' },
          },
          status: { phase: 'active', workflowExecution: { phase: 'completed' } },
        })
      )
    ).toBe(false)

    expect(
      workflowNeedsTerminalRunTeardown(
        makeWorkflowRecipe({
          metadata: {
            name: 'workload-only',
            namespace: 'sandbox-recipes',
            labels: { 'clerum.io/workflow-run-id': 'run-workload-only' },
          },
          spec: {
            steps: [],
            workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
          },
          status: { phase: 'active', workflowExecution: { phase: 'completed' } },
        })
      )
    ).toBe(false)

    expect(
      workflowNeedsTerminalRunTeardown(
        makeWorkflowRecipe({
          metadata: {
            name: 'deleting',
            namespace: 'sandbox-recipes',
            deletionTimestamp: '2026-06-16T00:00:00Z',
            labels: { 'clerum.io/workflow-run-id': 'run-deleting' },
          },
          status: { phase: 'active', workflowExecution: { phase: 'completed' } },
        })
      )
    ).toBe(false)
  })
})

describe('status-only events drive DB sync (terminal + heartbeat)', () => {
  // Bug: a workflow RUN completes (status.workflowExecution.phase = completed)
  // but its workflow_runs DB row stays Running forever — the UI shows "running"
  // and dbRunProcessor keeps logging "orphaned running run reclaimed". Cause:
  // a completion is a .status change, so .metadata.generation is unchanged ⇒
  // isStatusOnlyModifiedEvent → true ⇒ the old early return skipped
  // syncFromRecipeExecution (the ONLY CRD->DB terminal sync caller). The fix
  // runs syncFromRecipeExecution for status-only events too, while still
  // skipping the expensive full reconcile.

  function makeRunChild(
    overrides: Omit<Partial<WorkflowRecipeCRD>, 'metadata'> & {
      metadata?: Partial<WorkflowRecipeCRD['metadata']>
    } = {},
    runId = 'run-abc'
  ): WorkflowRecipeCRD {
    const { metadata: metaOverride, ...rest } = overrides
    return makeWorkflowRecipe({
      ...rest,
      metadata: {
        name: 'run-child',
        namespace: 'sandbox-recipes',
        generation: 7,
        labels: { 'clerum.io/workflow-run-id': runId },
        ...(metaOverride ?? {}),
      },
    })
  }

  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
    stopped: boolean
    reconciler: {
      reconcile: ReturnType<typeof vi.fn>
      isRecipeStillActive: ReturnType<typeof vi.fn>
      ensureFinalizer: ReturnType<typeof vi.fn>
      patchStatus: ReturnType<typeof vi.fn>
    }
    dbRunProcessor: { syncFromRecipeExecution: ReturnType<typeof vi.fn> }
    handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
  }

  function makeInternal(
    cached: WorkflowRecipeCRD,
    syncImpl: () => Promise<void> = () => Promise.resolve()
  ): {
    internal: InternalWatcher
    reconcile: ReturnType<typeof vi.fn>
    sync: ReturnType<typeof vi.fn>
  } {
    const reconcile = vi.fn().mockResolvedValue({
      phase: 'completed',
      message: 'done',
      workloadStatuses: [],
    })
    const sync = vi.fn(syncImpl)
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[cached.metadata.name, cached]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.reconciler = {
      reconcile,
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    internal.dbRunProcessor = { syncFromRecipeExecution: sync }
    return { internal, reconcile, sync }
  }

  it('does not reconcile a run child until the workflow_runs sync barrier completes', async () => {
    let markSyncStarted!: () => void
    let releaseSync!: () => void
    const syncStarted = new Promise<void>(resolve => {
      markSyncStarted = resolve
    })
    const syncBarrier = new Promise<void>(resolve => {
      releaseSync = resolve
    })
    const child = makeRunChild()
    const { internal, reconcile, sync } = makeInternal(child, async () => {
      markSyncStarted()
      await syncBarrier
    })

    const event = internal.handleRecipeEvent('ADDED', child)
    await syncStarted

    expect(sync).toHaveBeenCalledTimes(1)
    expect(reconcile).not.toHaveBeenCalled()

    releaseSync()
    await event

    expect(reconcile).toHaveBeenCalledTimes(1)
    expect(sync.mock.invocationCallOrder[0]).toBeLessThan(reconcile.mock.invocationCallOrder[0])
  })

  it.each(['completed', 'failed', 'cancelled'] as const)(
    'runs syncFromRecipeExecution AND reconciles a %s run-scoped status-only event',
    async phase => {
      // Bug repro: completed run was left Running in DB because the status-only
      // completion event skipped the CRD->DB sync. PR #568 also requires the
      // terminal event to reach reconcile(), where run-scoped compute teardown
      // lives.
      const cached = makeRunChild({
        status: { phase: 'active', workflowExecution: { phase: 'running' } },
      })
      // Same generation (7) ⇒ status-only event; phase flips to completed.
      const completed = makeRunChild({
        metadata: { resourceVersion: 'next' },
        status: { phase: 'active', workflowExecution: { phase } },
      })
      const { internal, reconcile, sync } = makeInternal(cached)

      await internal.handleRecipeEvent('MODIFIED', completed)

      // CRD->DB terminal sync fires so the DB row transitions Running→Succeeded.
      expect(sync).toHaveBeenCalledTimes(1)
      expect(sync).toHaveBeenCalledWith(completed)
      expect(reconcile).toHaveBeenCalledWith(completed)
      // Cache is updated.
      expect(internal.recipes.get(completed.metadata.name)).toBe(completed)
    }
  )

  it('still returns early (no full reconcile/patch path) for non-terminal status-only events', async () => {
    // Guards the reason the status-only fast-path exists: it must NOT fall
    // through to reconcile/patchStatus even though it now runs the DB sync.
    const cached = makeRunChild({
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const statusOnly = makeRunChild({
      metadata: { resourceVersion: 'next' },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const { internal, reconcile } = makeInternal(cached)

    await internal.handleRecipeEvent('MODIFIED', statusOnly)

    expect(reconcile).not.toHaveBeenCalled()
  })

  it('still returns early for terminal status-only events that are not run-scoped', async () => {
    const cached = makeRunChild({
      metadata: { labels: {} },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const completed = makeRunChild({
      metadata: { labels: {}, resourceVersion: 'next' },
      status: { phase: 'active', workflowExecution: { phase: 'completed' } },
    })
    const { internal, reconcile, sync } = makeInternal(cached)

    await internal.handleRecipeEvent('MODIFIED', completed)

    expect(sync).toHaveBeenCalledTimes(1)
    expect(reconcile).not.toHaveBeenCalled()
  })

  it('bumps the heartbeat: an in-progress status-only event still calls syncFromRecipeExecution', async () => {
    // The same call that fixes the terminal-stuck bug also bumps
    // last_reconciled_at (UPDATE_RUN_PROGRESS) for non-terminal runs, so a
    // running run is not falsely reclaimed as orphaned.
    const cached = makeRunChild({
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const stillRunning = makeRunChild({
      metadata: { resourceVersion: 'next' },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const { internal, sync, reconcile } = makeInternal(cached)

    await internal.handleRecipeEvent('MODIFIED', stillRunning)

    expect(sync).toHaveBeenCalledTimes(1)
    expect(sync).toHaveBeenCalledWith(stillRunning)
    expect(reconcile).not.toHaveBeenCalled()
  })

  it('does not crash the watch loop if syncFromRecipeExecution throws on a status-only event', async () => {
    // A DB sync failure must be logged, not propagated — the watch loop has to
    // keep processing events even if one completion sync fails transiently.
    const cached = makeRunChild({
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const completed = makeRunChild({
      metadata: { resourceVersion: 'next' },
      status: { phase: 'active', workflowExecution: { phase: 'completed' } },
    })
    const { internal, reconcile, sync } = makeInternal(cached, () =>
      Promise.reject(new Error('db down'))
    )
    const errSpy = captureLogger('error').mockImplementation(() => undefined)

    await expect(internal.handleRecipeEvent('MODIFIED', completed)).resolves.toBeUndefined()

    expect(sync).toHaveBeenCalledTimes(1)
    expect(reconcile).toHaveBeenCalledWith(completed)
    expect(errSpy).toHaveBeenCalled()
    errSpy.mockRestore()
  })

  it('regression: a non-status-only event (generation changed) runs syncFromRecipeExecution AND proceeds to reconcile', async () => {
    // Existing behavior must be preserved: when .spec changes (generation bump),
    // the full reconcile path still runs (with its own syncFromRecipeExecution
    // call at the start of the reconcile branch).
    const cached = makeRunChild({
      metadata: { generation: 7 },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const specChanged = makeRunChild({
      metadata: { generation: 8, resourceVersion: 'next' },
      status: { phase: 'active', workflowExecution: { phase: 'running' } },
    })
    const { internal, reconcile, sync } = makeInternal(cached)

    await internal.handleRecipeEvent('MODIFIED', specChanged)

    // Generation changed ⇒ NOT status-only ⇒ full reconcile branch runs, which
    // also calls syncFromRecipeExecution at its start.
    expect(sync).toHaveBeenCalledWith(specChanged)
    expect(reconcile).toHaveBeenCalledWith(specChanged)
  })
})

describe('infrastructure telemetry enqueue isolation', () => {
  function makeRunChild(): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: {
        name: 'run-child',
        namespace: 'sandbox-recipes',
        generation: 8,
        resourceVersion: 'rv-1',
        labels: { 'clerum.io/workflow-run-id': 'run-telemetry' },
      },
      status: { phase: 'deploying', workflowExecution: { phase: 'initializing' } },
    })
  }

  function makeInternal(
    recipe: WorkflowRecipeCRD,
    traceReporter: { enqueueInfrastructureTelemetry: ReturnType<typeof vi.fn> }
  ) {
    const result = {
      phase: 'active' as const,
      message: 'Workflow infrastructure ready',
      workloadStatuses: [],
    }
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as {
      recipes: Map<string, WorkflowRecipeCRD>
      transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
      stopped: boolean
      traceReporter: typeof traceReporter
      dbRunProcessor: null
      reconciler: {
        reconcile: ReturnType<typeof vi.fn>
        isRecipeStillActive: ReturnType<typeof vi.fn>
        ensureFinalizer: ReturnType<typeof vi.fn>
        patchStatus: ReturnType<typeof vi.fn>
      }
      handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
    }
    internal.recipes = new Map([[recipe.metadata.name, recipe]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.traceReporter = traceReporter
    internal.dbRunProcessor = null
    internal.reconciler = {
      reconcile: vi.fn().mockResolvedValue(result),
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    return { internal, result }
  }

  it('enqueues reconcile_outcome only after the status patch commits', async () => {
    const recipe = makeRunChild()
    const enqueueInfrastructureTelemetry = vi.fn()
    const { internal, result } = makeInternal(recipe, { enqueueInfrastructureTelemetry })
    internal.reconciler.patchStatus.mockImplementation(async () => {
      expect(enqueueInfrastructureTelemetry).not.toHaveBeenCalled()
    })

    await internal.handleRecipeEvent('ADDED', recipe)

    expect(enqueueInfrastructureTelemetry).toHaveBeenCalledWith(
      expect.objectContaining({
        telemetryType: 'reconcile_outcome',
        runId: 'run-telemetry',
        payload: { phase: result.phase, status: 'patched' },
      })
    )
  })

  it('does not enqueue reconcile_outcome when the status patch fails', async () => {
    const recipe = makeRunChild()
    const enqueueInfrastructureTelemetry = vi.fn()
    const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })
    internal.reconciler.patchStatus.mockRejectedValue(new Error('patch failed'))
    const errSpy = captureLogger('error').mockImplementation(() => undefined)

    await internal.handleRecipeEvent('ADDED', recipe)

    expect(enqueueInfrastructureTelemetry).toHaveBeenCalledTimes(1)
    expect(enqueueInfrastructureTelemetry).toHaveBeenCalledWith(
      expect.objectContaining({
        telemetryType: 'controller_error',
        runId: 'run-telemetry',
      })
    )
    errSpy.mockRestore()
  })

  it('isolates enqueue failures after a committed status patch', async () => {
    const recipe = makeRunChild()
    const enqueueInfrastructureTelemetry = vi.fn(() => {
      throw new Error('trace queue down')
    })
    const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })
    const errSpy = captureLogger('error').mockImplementation(() => undefined)

    await expect(internal.handleRecipeEvent('ADDED', recipe)).resolves.toBeUndefined()

    expect(internal.reconciler.patchStatus).toHaveBeenCalled()
    expect(enqueueInfrastructureTelemetry).toHaveBeenCalled()
    expect(errSpy).toHaveBeenCalledWith(
      '[WR-K8s] Infrastructure telemetry enqueue failed:',
      expect.objectContaining({ err: expect.any(Error) })
    )
    errSpy.mockRestore()
  })
})

describe('transient-result requeue (scheduleTransientRetry)', () => {
  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
    handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
    transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
    stopped: boolean
    scheduleTransientRetry: (name: string, baseMs: number, fixedInterval?: boolean) => void
    clearTransientRetry: (name: string) => void
  }

  function makeInternal(recipe: WorkflowRecipeCRD) {
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[recipe.metadata.name, recipe]])
    internal.eventQueue = { enqueue }
    internal.handleRecipeEvent = handleRecipeEvent
    internal.transientRetries = new Map()
    internal.stopped = false
    return { internal, enqueue, handleRecipeEvent }
  }

  it('re-enqueues a transient result with NO status patch after the backoff delay', () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkflowRecipe({
        metadata: { name: 'flaky', namespace: 'sandbox-recipes' },
      })
      const { internal, enqueue, handleRecipeEvent } = makeInternal(recipe)

      internal.scheduleTransientRetry('flaky', 5_000)
      // Nothing fires synchronously — the retry is purely timer-driven (no patch,
      // no MODIFIED event needed).
      expect(handleRecipeEvent).not.toHaveBeenCalled()

      vi.advanceTimersByTime(5_000)
      expect(enqueue).toHaveBeenCalledWith('flaky', expect.any(Function))
      expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', recipe, {
        forceReconcile: true,
      })
    } finally {
      vi.useRealTimers()
    }
  })

  it('applies bounded exponential backoff across consecutive transient results', () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkflowRecipe({
        metadata: { name: 'flaky', namespace: 'sandbox-recipes' },
      })
      const { internal, handleRecipeEvent } = makeInternal(recipe)

      internal.scheduleTransientRetry('flaky', 5_000) // attempt 0 → 5s
      vi.advanceTimersByTime(5_000)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)

      handleRecipeEvent.mockClear()
      internal.scheduleTransientRetry('flaky', 5_000) // attempt 1 → 10s
      vi.advanceTimersByTime(5_000)
      expect(handleRecipeEvent).not.toHaveBeenCalled() // 5s < 10s backoff
      vi.advanceTimersByTime(5_000)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
    } finally {
      vi.useRealTimers()
    }
  })

  it('clearTransientRetry cancels a pending retry and resets backoff', () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkflowRecipe({
        metadata: { name: 'flaky', namespace: 'sandbox-recipes' },
      })
      const { internal, handleRecipeEvent } = makeInternal(recipe)

      internal.scheduleTransientRetry('flaky', 5_000)
      internal.clearTransientRetry('flaky')
      vi.advanceTimersByTime(60_000)
      expect(handleRecipeEvent).not.toHaveBeenCalled()
      expect(internal.transientRetries.has('flaky')).toBe(false)
    } finally {
      vi.useRealTimers()
    }
  })

  // §5 (the regression this PR exists for): a `deploying` PROGRESS requeue uses a
  // FIXED interval and must NOT inherit the transient-ERROR exponential backoff.
  // If it did, repeated deploying sub-phase passes would grow the delay toward
  // the 60s cap and the 240s mcp-host readiness deadline could STILL be exceeded.
  // Here we drive N consecutive fixed-interval requeues and assert each one fires
  // at exactly the base 5s (no growth), in direct contrast to the exponential
  // test above.
  it('fixed-interval (progress) requeues keep firing at the base delay — no backoff growth', () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkflowRecipe({
        metadata: { name: 'progressing', namespace: 'sandbox-recipes' },
      })
      const { internal, handleRecipeEvent } = makeInternal(recipe)

      // Simulate 4 consecutive deploying sub-phase passes. Each re-schedules a
      // fixed-interval requeue (as the watcher does when result.phase ===
      // 'deploying' && !skipStatusPatch). Every one must fire after exactly 5s.
      for (let pass = 1; pass <= 4; pass++) {
        handleRecipeEvent.mockClear()
        internal.scheduleTransientRetry('progressing', 5_000, true)
        // 1ms before the base delay: must NOT have fired yet.
        vi.advanceTimersByTime(4_999)
        expect(handleRecipeEvent).not.toHaveBeenCalled()
        // At the base delay: fires. If this had inherited exponential backoff,
        // pass 4 would need ~40s (5s * 2**3) and this assertion would fail.
        vi.advanceTimersByTime(1)
        expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
        // attempts pinned at 0 so a later genuine transient error backs off fresh.
        expect(internal.transientRetries.get('progressing')?.attempts).toBe(0)
      }
    } finally {
      vi.useRealTimers()
    }
  })

  // §5 boundary: switching from an exponential transient backoff into a
  // fixed-interval progress requeue RESETS attempts to 0, so the next progress
  // poll is base (5s), not a continuation of the inflated backoff curve.
  it('a fixed-interval requeue resets backoff accumulated by prior transient retries', () => {
    vi.useFakeTimers()
    try {
      const recipe = makeWorkflowRecipe({
        metadata: { name: 'mixed', namespace: 'sandbox-recipes' },
      })
      const { internal, handleRecipeEvent } = makeInternal(recipe)

      // Two transient (error) passes inflate attempts to 2 → next would be 20s.
      internal.scheduleTransientRetry('mixed', 5_000) // attempt 0 → 5s
      vi.advanceTimersByTime(5_000)
      internal.scheduleTransientRetry('mixed', 5_000) // attempt 1 → 10s
      vi.advanceTimersByTime(10_000)
      expect(internal.transientRetries.get('mixed')?.attempts).toBe(2)

      // Now the run starts progressing → fixed-interval requeue. attempts resets
      // to 0 and the timer fires at the base 5s, NOT at 5s*2**2 = 20s.
      handleRecipeEvent.mockClear()
      internal.scheduleTransientRetry('mixed', 5_000, true)
      expect(internal.transientRetries.get('mixed')?.attempts).toBe(0)
      vi.advanceTimersByTime(5_000)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
    } finally {
      vi.useRealTimers()
    }
  })
})

describe('reconcile failure re-schedules the transient retry chain (issue #375 P2)', () => {
  // A transient (non-404) throw inside handleRecipeEvent used to log the failure
  // and return WITHOUT re-arming the per-recipe retry chain, so the 5s self-heal
  // died on the first error and only the 30s watchdog could recover it. The fix
  // keeps the loud error log AND re-schedules the bounded backoff retry; the 404
  // "already gone" branch must still clear and return without a retry.
  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
    stopped: boolean
    traceReporter: null
    dbRunProcessor: null
    eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
    reconciler: {
      reconcile: ReturnType<typeof vi.fn>
      isRecipeStillActive: ReturnType<typeof vi.fn>
      ensureFinalizer: ReturnType<typeof vi.fn>
      patchStatus: ReturnType<typeof vi.fn>
    }
    handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
  }

  function makeInternal(recipe: WorkflowRecipeCRD, reconcileError: unknown): InternalWatcher {
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[recipe.metadata.name, recipe]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.traceReporter = null
    internal.dbRunProcessor = null
    internal.eventQueue = { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) }
    internal.reconciler = {
      reconcile: vi.fn().mockRejectedValue(reconcileError),
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    return internal
  }

  function clearRetries(internal: InternalWatcher): void {
    for (const { timer } of internal.transientRetries.values()) clearTimeout(timer)
  }

  it('re-arms the transient retry after a non-404 reconcile failure (keeps the loud log)', async () => {
    const errSpy = captureLogger('error').mockImplementation(() => undefined)
    const recipe = makeWorkflowRecipe({ metadata: { name: 'flaky', namespace: 'sandbox-recipes' } })
    const internal = makeInternal(recipe, new Error('control-api unreachable'))
    try {
      await internal.handleRecipeEvent('ADDED', recipe)

      // fail-loud preserved: the reconciliation-failed error is still logged.
      expect(errSpy).toHaveBeenCalledWith(
        expect.stringContaining('Reconciliation failed'),
        expect.objectContaining({ name: 'flaky', err: expect.any(Error) })
      )
      // NEW: the 5s self-heal chain is re-armed instead of dying on first error.
      expect(internal.transientRetries.has('flaky')).toBe(true)
    } finally {
      clearRetries(internal)
      errSpy.mockRestore()
    }
  })

  it('does not re-arm a retry when the recipe is already gone (404)', async () => {
    const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
    const recipe = makeWorkflowRecipe({ metadata: { name: 'gone', namespace: 'sandbox-recipes' } })
    const internal = makeInternal(recipe, { code: 404 })
    try {
      await internal.handleRecipeEvent('ADDED', recipe)

      expect(internal.transientRetries.has('gone')).toBe(false)
      expect(internal.recipes.has('gone')).toBe(false)
    } finally {
      clearRetries(internal)
      warnSpy.mockRestore()
    }
  })
})

describe('Plugin Workload SDK computed-vs-published state divergence log (issue #375 P4)', () => {
  // Forensic observability: when the reconciler COMPUTES an SDK capability state
  // that differs from what is currently published, log old→new and whether a
  // patch was emitted. This one line would have discriminated H1 (transition
  // computed but not published) from H2 in minutes during the incident.
  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
    stopped: boolean
    traceReporter: null
    dbRunProcessor: null
    reconciler: {
      reconcile: ReturnType<typeof vi.fn>
      isRecipeStillActive: ReturnType<typeof vi.fn>
      ensureFinalizer: ReturnType<typeof vi.fn>
      patchStatus: ReturnType<typeof vi.fn>
    }
    handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
  }

  function makeInternal(recipe: WorkflowRecipeCRD, result: unknown): InternalWatcher {
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map([[recipe.metadata.name, recipe]])
    internal.transientRetries = new Map()
    internal.stopped = false
    internal.traceReporter = null
    internal.dbRunProcessor = null
    internal.reconciler = {
      reconcile: vi.fn().mockResolvedValue(result),
      isRecipeStillActive: vi.fn().mockResolvedValue(true),
      ensureFinalizer: vi.fn().mockResolvedValue(undefined),
      patchStatus: vi.fn().mockResolvedValue(undefined),
    }
    return internal
  }

  function sdkRecipe(persistedState: 'awaiting_policy' | 'validated'): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
      spec: { steps: [], pluginWorkloadSdk: { promptBridge: {} } },
      status: {
        phase: 'active',
        message: 'All workloads deployed',
        pluginWorkloadSdk: {
          state: persistedState,
          promptBridge: true,
          clientNotifications: false,
        },
      },
    })
  }

  it('logs the old→new SDK state and patchEmitted when the computed state diverges', async () => {
    const logSpy = captureLogger('info').mockImplementation(() => undefined)
    const recipe = sdkRecipe('awaiting_policy')
    const internal = makeInternal(recipe, {
      phase: 'active',
      message: 'All workloads deployed',
      workloadStatuses: [],
      pluginWorkloadSdkProjection: {
        conditions: [],
        capability: {
          state: 'validated' as const,
          promptBridge: true,
          clientNotifications: false,
          verifiedAt: new Date().toISOString(),
        },
      },
    })
    try {
      await internal.handleRecipeEvent('ADDED', recipe)
      expect(logSpy).toHaveBeenCalledWith(
        expect.stringContaining('Plugin Workload SDK state'),
        expect.objectContaining({
          name: 'sdk-recipe',
          value2: 'awaiting_policy',
          projectedSdkState: 'validated',
          willPatch: true,
        })
      )
    } finally {
      logSpy.mockRestore()
    }
  })

  it('does not log a transition when the computed SDK state is unchanged', async () => {
    const logSpy = captureLogger('info').mockImplementation(() => undefined)
    const recipe = sdkRecipe('validated')
    const internal = makeInternal(recipe, {
      phase: 'active',
      message: 'All workloads deployed',
      workloadStatuses: [],
      pluginWorkloadSdkProjection: {
        conditions: [],
        capability: {
          state: 'validated' as const,
          promptBridge: true,
          clientNotifications: false,
          verifiedAt: new Date().toISOString(),
        },
      },
    })
    try {
      await internal.handleRecipeEvent('ADDED', recipe)
      const transitionLogs = logSpy.mock.calls.filter(([msg]) =>
        String(msg).includes('Plugin Workload SDK state')
      )
      expect(transitionLogs).toHaveLength(0)
    } finally {
      logSpy.mockRestore()
    }
  })
})

describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {
  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    eventQueue: { enqueue: ReturnType<typeof vi.fn> }
    handleRecipeEvent: ReturnType<typeof vi.fn>
    handleGrantUpdateNotification: (recipeNamespace: string, recipeName: string) => void
  }

  function makeInternal(recipes: WorkflowRecipeCRD[]): {
    internal: InternalWatcher
    tasks: Array<() => Promise<void>>
  } {
    const tasks: Array<() => Promise<void>> = []
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map(recipes.map(r => [r.metadata.name, r]))
    internal.eventQueue = {
      enqueue: vi.fn((_key: string, task: () => Promise<void>) => {
        tasks.push(task)
        return Promise.resolve()
      }),
    }
    internal.handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    return { internal, tasks }
  }

  it('enqueues a forceReconcile for a matching recipe and reconciles the CURRENT cached recipe (H5)', async () => {
    const recipe = makeWorkflowRecipe({
      metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 1 },
    })
    const { internal, tasks } = makeInternal([recipe])

    internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')

    expect(internal.eventQueue.enqueue).toHaveBeenCalledWith('sdk-recipe', expect.any(Function))
    expect(tasks).toHaveLength(1)

    // H5: a newer MODIFIED lands before the queued job runs. The job must
    // reconcile the FRESH cached recipe, not the snapshot captured at NOTIFY time.
    const fresher = makeWorkflowRecipe({
      metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes', generation: 2 },
    })
    internal.recipes.set('sdk-recipe', fresher)

    await tasks[0]()

    expect(internal.handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', fresher, {
      forceReconcile: true,
    })
    expect(internal.handleRecipeEvent).not.toHaveBeenCalledWith('MODIFIED', recipe, {
      forceReconcile: true,
    })
  })

  it('discards and warns on a namespace mismatch — no enqueue (H4)', () => {
    const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
    const recipe = makeWorkflowRecipe({
      metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
    })
    const { internal } = makeInternal([recipe])
    try {
      internal.handleGrantUpdateNotification('other-namespace', 'sdk-recipe')
      expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
      expect(warnSpy).toHaveBeenCalledWith(
        expect.stringContaining('Discarding grant-update NOTIFY'),
        expect.objectContaining({ recipeNamespace: 'other-namespace', recipeName: 'sdk-recipe' })
      )
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('discards and warns on an unknown recipe — no enqueue (H4)', () => {
    const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
    const { internal } = makeInternal([])
    try {
      internal.handleGrantUpdateNotification('sandbox-recipes', 'nope')
      expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
      expect(warnSpy).toHaveBeenCalledWith(
        expect.stringContaining('Discarding grant-update NOTIFY'),
        expect.objectContaining({ value3: 'unknown recipe' })
      )
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('the queued job re-checks the guard and does nothing if the recipe vanished while queued (H5)', async () => {
    const recipe = makeWorkflowRecipe({
      metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
    })
    const { internal, tasks } = makeInternal([recipe])

    internal.handleGrantUpdateNotification('sandbox-recipes', 'sdk-recipe')
    // Recipe deleted between NOTIFY and job execution.
    internal.recipes.delete('sdk-recipe')

    await tasks[0]()

    expect(internal.handleRecipeEvent).not.toHaveBeenCalled()
  })
})

// issue #299 §3.2/§5 — the periodic external-egress requeue timer. GitHub rotates
// its api.github.com A record (~15s TTL over 140.82.112.0/20) WITHOUT bumping the
// WorkflowRecipe CRD generation, so a generation-only reconcile trigger never
// re-runs the accumulator and the sliding-window set never converges on a fresh
// IP. This timer re-enqueues cached recipes that declare an exact-host EXTERNAL
// egress on a fixed cadence so the accumulator folds each new snapshot.
describe('external egress periodic refresh (issue #299 §3.2/§5)', () => {
  function makeUiEgressRecipe(name: string): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [{ id: 'ui', type: 'deployment', image: 'clerum/ui:test' }],
        ui: {
          workloadRef: 'ui',
          port: 8080,
          egress: { external: [{ fqdn: 'api.github.com', port: 443 }] },
        },
      },
    })
  }

  function makeWorkloadEgressRecipe(name: string): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [
          {
            id: 'api',
            type: 'deployment',
            image: 'clerum/api:test',
            egressBindings: [{ dns: 'api.github.com', port: 443 }],
          },
        ],
      },
    })
  }

  function makeClusterLocalOnlyRecipe(name: string): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [
          { id: 'db', type: 'deployment', image: 'clerum/db:test' },
          {
            id: 'api',
            type: 'deployment',
            image: 'clerum/api:test',
            egressBindings: [{ dns: 'db.sandbox-recipes.svc.cluster.local', port: 5432 }],
          },
        ],
      },
    })
  }

  function makeTransportOnlyRecipe(name: string): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [
          {
            id: 'mcp',
            type: 'deployment',
            image: 'clerum/mcp:test',
            transport: { type: 'streamableHttp' },
            egressBindings: [{ egressClass: 'public-web' }],
          },
        ],
      },
    })
  }

  function makePlainRecipe(name: string): WorkflowRecipeCRD {
    return makeWorkflowRecipe({
      metadata: { name, namespace: 'sandbox-recipes' },
      spec: {
        steps: [],
        workloads: [{ id: 'api', type: 'deployment', image: 'clerum/api:test' }],
      },
    })
  }

  describe('recipeHasExternalExactHostEgress', () => {
    it('is true for a UI external egress FQDN', () => {
      expect(recipeHasExternalExactHostEgress(makeUiEgressRecipe('ui'))).toBe(true)
    })

    it('is true for a non-transport workload egressBinding with an external FQDN', () => {
      expect(recipeHasExternalExactHostEgress(makeWorkloadEgressRecipe('wl'))).toBe(true)
    })

    it('is false when the only binding is a cluster-local sibling (not external)', () => {
      expect(recipeHasExternalExactHostEgress(makeClusterLocalOnlyRecipe('cl'))).toBe(false)
    })

    it('is false for a transport-only (HCC-delegated) egress workload', () => {
      // Transport egress is delegated to HCC/McpServer, NOT folded by the WRC
      // accumulator, so the timer must not select it.
      expect(recipeHasExternalExactHostEgress(makeTransportOnlyRecipe('tr'))).toBe(false)
    })

    it('is false for a recipe with no egress at all', () => {
      expect(recipeHasExternalExactHostEgress(makePlainRecipe('plain'))).toBe(false)
    })

    it('is false while the recipe is being deleted', () => {
      const recipe = makeUiEgressRecipe('deleting')
      recipe.metadata.deletionTimestamp = '2026-08-08T00:00:00Z'
      expect(recipeHasExternalExactHostEgress(recipe)).toBe(false)
    })

    it('excludes the UI workload id from the workload-binding scan (it is covered by ui.egress)', () => {
      // A recipe whose ONLY external binding lives on the ui workload itself must
      // still resolve true via ui.egress.external, never via the workload scan.
      const recipe = makeUiEgressRecipe('ui-wl')
      recipe.spec.workloads = [
        {
          id: 'ui',
          type: 'deployment',
          image: 'clerum/ui:test',
          egressBindings: [{ dns: 'db.sandbox-recipes.svc.cluster.local', port: 5432 }],
        },
      ]
      // ui.egress.external drives the true result; the ui workload's own binding
      // (cluster-local here) is skipped.
      expect(recipeHasExternalExactHostEgress(recipe)).toBe(true)
    })
  })

  describe('externalEgressRefreshIntervalMs', () => {
    it('returns the configured interval (seconds → ms) when above the floor', () => {
      expect(
        externalEgressRefreshIntervalMs({
          externalEgressRefreshIntervalSeconds: 60,
          externalEgressRefreshFloorSeconds: 5,
        })
      ).toBe(60_000)
    })

    it('clamps up to the floor when the interval is below it', () => {
      expect(
        externalEgressRefreshIntervalMs({
          externalEgressRefreshIntervalSeconds: 3,
          externalEgressRefreshFloorSeconds: 5,
        })
      ).toBe(5_000)
    })

    // H2 (issue #299): advance the refresh to <= observed TTL / 2 so a rotating
    // low-TTL host (GitHub api, TTL ~15s) is sampled every rotation, not every 60s.
    it('advances to TTL/2 when the observed TTL makes it faster than the configured interval', () => {
      expect(
        externalEgressRefreshIntervalMs(
          { externalEgressRefreshIntervalSeconds: 60, externalEgressRefreshFloorSeconds: 5 },
          15_000 // observed min TTL 15s → 7.5s refresh
        )
      ).toBe(7_500)
    })

    it('never drops below the floor even for a tiny TTL', () => {
      expect(
        externalEgressRefreshIntervalMs(
          { externalEgressRefreshIntervalSeconds: 60, externalEgressRefreshFloorSeconds: 5 },
          6_000 // TTL/2 = 3s < floor 5s → floor
        )
      ).toBe(5_000)
    })

    it('keeps the configured interval when the observed TTL is large', () => {
      expect(
        externalEgressRefreshIntervalMs(
          { externalEgressRefreshIntervalSeconds: 60, externalEgressRefreshFloorSeconds: 5 },
          600_000 // TTL/2 = 300s > interval 60s → interval wins
        )
      ).toBe(60_000)
    })

    it('ignores a non-finite/absent observed TTL (falls back to the configured interval)', () => {
      expect(
        externalEgressRefreshIntervalMs(
          { externalEgressRefreshIntervalSeconds: 60, externalEgressRefreshFloorSeconds: 5 },
          Infinity
        )
      ).toBe(60_000)
    })
  })

  type InternalWatcher = {
    recipes: Map<string, WorkflowRecipeCRD>
    eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
    handleRecipeEvent: (
      type: string,
      recipe: WorkflowRecipeCRD,
      options?: { forceReconcile?: boolean }
    ) => Promise<void>
    stopped: boolean
    config: {
      externalEgressRefreshIntervalSeconds: number
      externalEgressRefreshFloorSeconds: number
    }
    externalEgressRefreshTimer: ReturnType<typeof setInterval> | null
    externalEgressRefreshRunning: boolean
    // H2 (issue #299): the refresh loop reads the reconciler's observed min TTL.
    reconciler: { externalEgressRefreshMinTtlMs: number }
    refreshExternalEgressForActiveRecipes: () => Promise<void>
    startExternalEgressRefreshLoop: () => void
    stop: () => Promise<void>
  }

  function makeInternal(recipes: WorkflowRecipeCRD[]) {
    const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
    const handleRecipeEvent = vi.fn().mockResolvedValue(undefined)
    const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
    internal.recipes = new Map(recipes.map(r => [r.metadata.name, r]))
    internal.eventQueue = { enqueue }
    internal.handleRecipeEvent = handleRecipeEvent
    internal.stopped = false
    internal.config = {
      externalEgressRefreshIntervalSeconds: 60,
      externalEgressRefreshFloorSeconds: 5,
    }
    internal.externalEgressRefreshTimer = null
    internal.externalEgressRefreshRunning = false
    internal.reconciler = { externalEgressRefreshMinTtlMs: Infinity }
    return { internal, enqueue, handleRecipeEvent }
  }

  it('refreshExternalEgressForActiveRecipes re-enqueues ONLY recipes with external exact-host egress', async () => {
    const { internal, enqueue, handleRecipeEvent } = makeInternal([
      makeUiEgressRecipe('with-ui-egress'),
      makeWorkloadEgressRecipe('with-wl-egress'),
      makePlainRecipe('plain'),
      makeClusterLocalOnlyRecipe('cluster-local'),
      makeTransportOnlyRecipe('transport'),
    ])

    await internal.refreshExternalEgressForActiveRecipes()

    expect(enqueue).toHaveBeenCalledWith('with-ui-egress', expect.any(Function))
    expect(enqueue).toHaveBeenCalledWith('with-wl-egress', expect.any(Function))
    expect(enqueue).not.toHaveBeenCalledWith('plain', expect.any(Function))
    expect(enqueue).not.toHaveBeenCalledWith('cluster-local', expect.any(Function))
    expect(enqueue).not.toHaveBeenCalledWith('transport', expect.any(Function))
    expect(enqueue).toHaveBeenCalledTimes(2)

    // A full reconcile is forced (generation is unchanged, so a plain MODIFIED
    // would short-circuit before the accumulator runs).
    for (const name of ['with-ui-egress', 'with-wl-egress']) {
      expect(handleRecipeEvent).toHaveBeenCalledWith('MODIFIED', internal.recipes.get(name), {
        forceReconcile: true,
      })
    }
  })

  it('refreshExternalEgressForActiveRecipes is a no-op while stopped', async () => {
    const { internal, enqueue } = makeInternal([makeUiEgressRecipe('with-ui-egress')])
    internal.stopped = true
    await internal.refreshExternalEgressForActiveRecipes()
    expect(enqueue).not.toHaveBeenCalled()
  })

  it('the timer re-enqueues external-egress recipes on each tick at the clamped interval', async () => {
    vi.useFakeTimers()
    try {
      const { internal, handleRecipeEvent } = makeInternal([
        makeUiEgressRecipe('with-ui-egress'),
        makePlainRecipe('plain'),
      ])

      internal.startExternalEgressRefreshLoop()
      expect(internal.externalEgressRefreshTimer).not.toBeNull()
      expect(handleRecipeEvent).not.toHaveBeenCalled()

      // 1ms before the 60s interval: nothing has fired.
      await vi.advanceTimersByTimeAsync(59_999)
      expect(handleRecipeEvent).not.toHaveBeenCalled()

      await vi.advanceTimersByTimeAsync(1)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
      expect(handleRecipeEvent).toHaveBeenCalledWith(
        'MODIFIED',
        internal.recipes.get('with-ui-egress'),
        { forceReconcile: true }
      )

      // Second tick fires again — the requeue is periodic, not one-shot.
      await vi.advanceTimersByTimeAsync(60_000)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(2)
    } finally {
      vi.useRealTimers()
    }
  })

  it('advances the refresh to TTL/2 when the reconciler has observed a low TTL (H2 wiring)', async () => {
    vi.useFakeTimers()
    try {
      const { internal, handleRecipeEvent } = makeInternal([makeUiEgressRecipe('with-ui-egress')])
      // The reconciler observed GitHub's ~15s TTL → the loop must fire at 7.5s,
      // not the 60s configured interval.
      internal.reconciler = { externalEgressRefreshMinTtlMs: 15_000 }

      internal.startExternalEgressRefreshLoop()
      await vi.advanceTimersByTimeAsync(7_499)
      expect(handleRecipeEvent).not.toHaveBeenCalled()
      await vi.advanceTimersByTimeAsync(1)
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
    } finally {
      vi.useRealTimers()
    }
  })

  it('startExternalEgressRefreshLoop is idempotent (does not create a second timer)', () => {
    vi.useFakeTimers()
    try {
      const { internal } = makeInternal([makeUiEgressRecipe('with-ui-egress')])
      internal.startExternalEgressRefreshLoop()
      const first = internal.externalEgressRefreshTimer
      internal.startExternalEgressRefreshLoop()
      expect(internal.externalEgressRefreshTimer).toBe(first)
    } finally {
      vi.useRealTimers()
    }
  })

  it('does not start a second refresh pass while one is still running (no concurrent duplicates)', async () => {
    vi.useFakeTimers()
    try {
      // handleRecipeEvent returns a promise that stays pending until we release it,
      // so the first pass is still in-flight when the second tick fires.
      let release: (() => void) | undefined
      const gate = new Promise<void>(resolve => {
        release = resolve
      })
      const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
      const handleRecipeEvent = vi.fn().mockReturnValue(gate)
      const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
      internal.recipes = new Map([['with-ui-egress', makeUiEgressRecipe('with-ui-egress')]])
      internal.eventQueue = { enqueue }
      internal.handleRecipeEvent = handleRecipeEvent
      internal.stopped = false
      internal.config = {
        externalEgressRefreshIntervalSeconds: 60,
        externalEgressRefreshFloorSeconds: 5,
      }
      internal.externalEgressRefreshTimer = null
      internal.externalEgressRefreshRunning = false
      internal.reconciler = { externalEgressRefreshMinTtlMs: Infinity }

      internal.startExternalEgressRefreshLoop()
      await vi.advanceTimersByTimeAsync(60_000) // first tick — pass starts, stays in-flight
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)
      expect(internal.externalEgressRefreshRunning).toBe(true)

      await vi.advanceTimersByTimeAsync(60_000) // second tick — must be skipped
      expect(handleRecipeEvent).toHaveBeenCalledTimes(1)

      release?.()
      await gate
    } finally {
      vi.useRealTimers()
    }
  })

  it('stop() clears the refresh timer so no further ticks fire', async () => {
    vi.useFakeTimers()
    try {
      const { internal, handleRecipeEvent } = makeInternal([makeUiEgressRecipe('with-ui-egress')])
      // Minimal shape for stop() — every other subsystem is absent in this fixture.
      const stoppableInternal = internal as unknown as {
        transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
        runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null
        workloadStatusRefreshTimer: ReturnType<typeof setInterval> | null
        watchRequest: { abort: () => void } | null
        secretWatchLoops: { stop: () => void }[]
        brokerRotationTimer: ReturnType<typeof setInterval> | null
        dbRunProcessor: { stop: () => Promise<void> } | null
        traceReporter: null
      }
      stoppableInternal.transientRetries = new Map()
      stoppableInternal.runtimeCredentialRefreshTimer = null
      stoppableInternal.workloadStatusRefreshTimer = null
      stoppableInternal.watchRequest = null
      stoppableInternal.secretWatchLoops = []
      stoppableInternal.brokerRotationTimer = null
      stoppableInternal.dbRunProcessor = null
      stoppableInternal.traceReporter = null

      internal.startExternalEgressRefreshLoop()
      expect(internal.externalEgressRefreshTimer).not.toBeNull()

      await internal.stop()
      expect(internal.externalEgressRefreshTimer).toBeNull()
      expect(internal.stopped).toBe(true)

      await vi.advanceTimersByTimeAsync(120_000)
      expect(handleRecipeEvent).not.toHaveBeenCalled()
    } finally {
      vi.useRealTimers()
    }
  })
})

// ── issue #375 M1/M4 (jozer review): the REAL constructor + start()/stop()
// wiring of the grant-update listener. Every prior watcher test hand-installed
// collaborators onto Object.create(prototype), so removing the constructor
// wiring or the start()/stop() calls left the whole suite green. These tests
// construct the real watcher and go RED if:
//   - the constructor stops creating the listener,
//   - the listener is handed the SHARED pool instead of its dedicated one (M4),
//   - `await this.grantUpdateListener.start()` is removed from start(),
//   - `await this.grantUpdateListener.stop()` is removed from stop().
describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issue #375 M1/M4)', () => {
  function makeFakeKubeConfig() {
    return {
      makeApiClient: vi.fn(() => ({})),
    } as unknown as import('@kubernetes/client-node').KubeConfig
  }

  type ListenerStub = { start: () => Promise<void>; stop: () => Promise<void> }
  type InternalWatcher = {
    grantUpdateListener: ListenerStub | null
    grantUpdateListenerPool: { connect?: unknown; end: () => Promise<void> } | null
    dbRunProcessor: { start: () => Promise<void>; stop: () => Promise<void> } | null
    traceReporter: null
    tryInitializeWorkflow: () => Promise<void>
    startWatch: () => Promise<void>
    startSecretWatch: () => Promise<void>
    startBrokerTokenRotationLoop: () => void
    startRuntimeCredentialRefreshLoop: () => void
    startWorkloadStatusRefreshLoop: () => void
    startExternalEgressRefreshLoop: () => void
  }

  it('constructor wires a listener backed by a DEDICATED pool — LISTEN never checks out of the shared pool (M4)', async () => {
    const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }
    const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)
    const internal = watcher as unknown as InternalWatcher

    expect(internal.grantUpdateListener).not.toBeNull()
    expect(internal.grantUpdateListenerPool).not.toBeNull()
    // M4: a lifetime-pinned LISTEN session must not burn one of the shared
    // pool's 4 slots — the listener owns a dedicated single-connection pool.
    expect(internal.grantUpdateListenerPool).not.toBe(sharedPool)

    // Behavioral pin: starting the listener checks out of the DEDICATED pool
    // and never touches the shared one.
    const fakeClient = {
      on: vi.fn(),
      query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
      release: vi.fn(),
      removeAllListeners: vi.fn(),
    }
    const dedicatedConnect = vi.fn().mockResolvedValue(fakeClient)
    ;(internal.grantUpdateListenerPool as { connect?: unknown }).connect = dedicatedConnect
    await (internal.grantUpdateListener as ListenerStub).start()
    expect(dedicatedConnect).toHaveBeenCalledTimes(1)
    expect(sharedPool.connect).not.toHaveBeenCalled()
    await (internal.grantUpdateListener as ListenerStub).stop()
  })

  it('does not create the listener (or its pool) without a DB pool', () => {
    const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), null)
    const internal = watcher as unknown as InternalWatcher
    expect(internal.grantUpdateListener).toBeNull()
    expect(internal.grantUpdateListenerPool).toBeNull()
  })

  it('start() starts and stop() stops the grant-update listener and ends its dedicated pool (M1)', async () => {
    const sharedPool = { connect: vi.fn(), query: vi.fn(), on: vi.fn() }
    const watcher = new WorkflowRecipeWatcher(makeFakeKubeConfig(), sharedPool as never)
    const internal = watcher as unknown as InternalWatcher

    // Stub the heavy collaborators. The REAL start()/stop() method bodies stay
    // in play, so neutering the listener wiring inside them goes RED here.
    internal.tryInitializeWorkflow = vi.fn().mockResolvedValue(undefined)
    internal.startWatch = vi.fn().mockResolvedValue(undefined)
    internal.startSecretWatch = vi.fn().mockResolvedValue(undefined)
    internal.startBrokerTokenRotationLoop = vi.fn()
    internal.startRuntimeCredentialRefreshLoop = vi.fn()
    internal.startWorkloadStatusRefreshLoop = vi.fn()
    internal.startExternalEgressRefreshLoop = vi.fn()
    internal.dbRunProcessor = {
      start: vi.fn().mockResolvedValue(undefined),
      stop: vi.fn().mockResolvedValue(undefined),
    }
    internal.traceReporter = null
    const listenerStart = vi.fn().mockResolvedValue(undefined)
    const listenerStop = vi.fn().mockResolvedValue(undefined)
    internal.grantUpdateListener = { start: listenerStart, stop: listenerStop }
    const poolEnd = vi.fn().mockResolvedValue(undefined)
    internal.grantUpdateListenerPool = { end: poolEnd }

    await watcher.start()
    expect(listenerStart).toHaveBeenCalledTimes(1)

    await watcher.stop()
    expect(listenerStop).toHaveBeenCalledTimes(1)
    // M4: the watcher owns the dedicated pool and must close its connection.
    expect(poolEnd).toHaveBeenCalledTimes(1)
  })
})

describe('handleOAuthBrokerTokenAdded', () => {
  type BrokerAddedInternal = {
    config: { sandboxNamespace: string }
    reconciler: { invalidateOAuthBrokerSecretLedger: (recipeName: string) => void }
    recipes: Map<string, WorkflowRecipeCRD>
    eventQueue: { enqueue: (key: string, task: () => Promise<void>) => unknown }
    handleOAuthBrokerTokenAdded: (namespace: string, recipeName: string) => void
  }

  function recipeWith(name: string, backgroundAccess: boolean): WorkflowRecipeCRD {
    const recipe = makeWorkflowRecipe({ metadata: { name, namespace: 'sandbox-recipes' } })
    recipe.spec = {
      ...recipe.spec,
      oauthClients: backgroundAccess
        ? [
            {
              id: 'gmail',
              provider: 'google',
              clientIdRef: { name: 'creds', key: 'client-id' },
              clientSecretRef: { name: 'creds', key: 'client-secret' },
              scopes: ['https://www.googleapis.com/auth/gmail.readonly'],
              backgroundAccess: true,
            },
          ]
        : [],
    } as WorkflowRecipeCRD['spec']
    return recipe
  }

  function setup() {
    const watcher = new WorkflowRecipeWatcher(
      {
        makeApiClient: vi.fn(() => ({})),
      } as unknown as import('@kubernetes/client-node').KubeConfig,
      null
    )
    const internal = watcher as unknown as BrokerAddedInternal
    const invalidate = vi.fn()
    const enqueue = vi.fn()
    internal.reconciler = { invalidateOAuthBrokerSecretLedger: invalidate }
    internal.eventQueue = { enqueue }
    internal.recipes = new Map([
      ['no-bg', recipeWith('no-bg', false)],
      ['with-bg', recipeWith('with-bg', true)],
    ])
    return { internal, invalidate, enqueue }
  }

  it('re-arms the ledger and re-reconciles a recipe that must not have a broker token', () => {
    const { internal, invalidate, enqueue } = setup()
    internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')
    expect(invalidate).toHaveBeenCalledWith('no-bg')
    expect(enqueue).toHaveBeenCalledTimes(1)
    expect(enqueue.mock.calls[0]?.[0]).toBe('no-bg')
  })

  it('only invalidates for an opted-in recipe: a watch reconnect replays ADDED for every token', () => {
    const { internal, invalidate, enqueue } = setup()
    internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'with-bg')
    expect(invalidate).toHaveBeenCalledWith('with-bg')
    expect(enqueue).not.toHaveBeenCalled()
  })

  it('re-arms the ledger and enqueues nothing for a recipe not yet cached', () => {
    // The recipe's own ADDED event is queued already; that its pass then
    // reaps the re-armed token is pinned in reconciler.test.ts by the B3
    // ledger test "deletes the Secret again after ADDED invalidation".
    const { internal, invalidate, enqueue } = setup()
    internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'not-cached')
    // Liveness witness: the handler ran and armed the ledger for that recipe.
    expect(invalidate).toHaveBeenCalledWith('not-cached')
    expect(enqueue).not.toHaveBeenCalled()
  })

  it('startSecretWatch wires each namespace watcher to the handler with its own namespace', async () => {
    const { internal, invalidate, enqueue } = setup()
    secretLoops.length = 0
    await (internal as unknown as { startSecretWatch: () => Promise<void> }).startSecretWatch()

    // The Secret WRC itself writes, not a hand-written copy of its labels.
    const brokerToken = buildOAuthBrokerTokenSecret(
      'no-bg',
      'jwt',
      internal.config.sandboxNamespace
    )

    // Liveness witness: the sandbox-namespace watcher exists and reaches the handler.
    const sandbox = secretLoops.find(l => l.namespace === internal.config.sandboxNamespace)
    expect(sandbox).toBeDefined()
    sandbox?.handler.handleEvent('ADDED', brokerToken)
    expect(invalidate).toHaveBeenCalledWith('no-bg')
    expect(enqueue).toHaveBeenCalledTimes(1)

    // Every other namespace's watcher is bound to ITS namespace, so the handler ignores it.
    const others = secretLoops.filter(l => l.namespace !== internal.config.sandboxNamespace)
    expect(others.length).toBeGreaterThan(0)
    for (const other of others) {
      other.handler.handleEvent('ADDED', brokerToken)
    }
    expect(invalidate).toHaveBeenCalledTimes(1)
    expect(enqueue).toHaveBeenCalledTimes(1)
  })

  it('ignores a same-named Secret outside the sandbox namespace', () => {
    const { internal, invalidate, enqueue } = setup()
    // Liveness witness: the same call in the sandbox namespace does act.
    internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')
    expect(invalidate).toHaveBeenCalledTimes(1)
    expect(enqueue).toHaveBeenCalledTimes(1)
    internal.handleOAuthBrokerTokenAdded('mcp-server', 'no-bg')
    expect(invalidate).toHaveBeenCalledTimes(1)
    expect(enqueue).toHaveBeenCalledTimes(1)
  })
})