Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
43.5 KB
Raw Blame
#!/usr/bin/env node
'use strict'

// Product/controller E2E, not a browser journey and not a T2 attestation.
// All mutation goes through the explicit-context client and inherited lease.
const fs = require('node:fs')
const path = require('node:path')
const dns = require('node:dns/promises')
const { BlockList, isIPv4 } = require('node:net')
const { randomUUID } = require('node:crypto')
const { execFileSync } = require('node:child_process')
const { assertPolicy } = require('./wrc-egress-proof.cjs')
const {
  Journal,
  journalPath,
  readJournal,
  assertJournalBinding,
  createOwned,
  deleteOwned,
  waitFor,
  createCommandRunner,
  createKubeClient,
  invariant,
  RUN_LABEL,
  resourceIdentity,
} = require('./wrc-egress-lifecycle.cjs')

const STATE = 'clerum.io/egress-fqdn-state'
const LOOPBACK = [127, 0, 0, 1].join('.')
const endpointHost = 'example.com' // Existing mock-MCP fixture public endpoint contract.
const deploymentReady = object =>
  Boolean(
    object &&
    object.spec?.replicas > 0 &&
    object.status?.observedGeneration >= object.metadata.generation &&
    object.status.readyReplicas === object.spec.replicas &&
    object.status.availableReplicas === object.spec.replicas
  )
const same = (left, right) => JSON.stringify(left) === JSON.stringify(right)
const id = (kind, namespace, name) => ({ kind, namespace, name })

// Called after the complete policy oracle succeeds. Fresh DNS traffic alone
// cannot satisfy recovery; every tuple needs a newer persisted observation.
function policyObservation(policy, newerThan) {
  const entries = JSON.parse(policy.metadata.annotations[STATE])
  invariant(
    entries.length > 0 &&
      entries.every(
        entry => Number.isFinite(entry.lastObservedAt) && Number.isFinite(entry.expiresAt)
      ),
    'MISSING_ACCEPTED_DNS_OBSERVATION'
  )
  if (newerThan !== undefined && !entries.every(entry => entry.lastObservedAt > newerThan))
    return null
  return {
    lastAcceptedAt: Math.max(...entries.map(entry => entry.lastObservedAt)),
    latestExpiry: Math.max(...entries.map(entry => entry.expiresAt)),
  }
}

function acceptPolicyIntentChange(lanes, before, after, recipeUid, targets) {
  invariant(
    after.metadata?.uid === recipeUid &&
      before.metadata?.uid === recipeUid &&
      after.metadata.generation > before.metadata.generation,
    'HOST_MIGRATION_NOT_APPLIED'
  )
  for (const lane of lanes) {
    const target = targets.find(target => target.lane === (lane.lane === 'ui' ? 'worker' : 'ui'))
    invariant(target, 'MISSING_MIGRATION_TARGET')
    lane.target = target
    lane.uid = undefined
  }
}

// A test setup restriction, not an alternate production address classifier.
const blocked = new BlockList()
for (const cidr of [
  '0.0.0.0/8',
  '10.0.0.0/8',
  '100.64.0.0/10',
  '127.0.0.0/8',
  '169.254.0.0/16',
  '172.16.0.0/12',
  '192.0.0.0/24',
  '192.0.2.0/24',
  '192.31.196.0/24',
  '192.52.193.0/24',
  '192.88.99.0/24',
  '192.168.0.0/16',
  '192.175.48.0/24',
  '198.18.0.0/15',
  '198.51.100.0/24',
  '203.0.113.0/24',
  '224.0.0.0/4',
  '240.0.0.0/4',
]) {
  const [address, prefix] = cidr.split('/')
  blocked.addSubnet(address, Number(prefix), 'ipv4')
}

const trafficCode = `
const net=require('node:net'),https=require('node:https');
const [ip,portText,host,mode]=process.argv.slice(1); const port=Number(portText);
if(mode==='https') {
  const request=https.get({hostname:ip,port,servername:host,path:'/',agent:false,
    headers:{host,connection:'close'}},response=>{
    let body=''; response.on('data',chunk=>{body+=chunk;if(body.length>65536)request.destroy(new Error('body limit'))});
    response.once('aborted',()=>{process.exitCode=43;});
    response.once('error',()=>{process.exitCode=43;});
    response.once('end',()=>{if(response.statusCode===200&&body.includes('Example Domain')){
      process.stdout.write(JSON.stringify({allowed:true,contentVerified:true}));
    }else process.exitCode=44;});
  });
  const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);
  request.once('close',()=>clearTimeout(timer)); request.once('error',()=>{process.exitCode=43;});
} else {
  const socket=net.connect({host:ip,port});
  const timer=setTimeout(()=>{socket.destroy();process.exitCode=42;},2500);
  socket.once('connect',()=>{clearTimeout(timer);socket.destroy();process.stdout.write('connected');});
  socket.once('error',()=>{clearTimeout(timer);process.exitCode=43;});
}`

const businessCode = `
const {Client}=require('@modelcontextprotocol/sdk/client/index.js');
const {StreamableHTTPClientTransport}=require('@modelcontextprotocol/sdk/client/streamableHttp.js');
const [url,key,value,operation]=process.argv.slice(1);
const client=new Client({name:'wrc-egress-business-witness',version:'1.0.0'},{capabilities:{}});
const timer=setTimeout(()=>process.exit(45),10000);
(async()=>{try{
  await client.connect(new StreamableHTTPClientTransport(new URL(url)));
  if(operation==='record'){
    const stored=await client.callTool({name:'record',arguments:{key,value}});
    if(stored.isError)throw new Error('record failed');
  }
  const response=await client.callTool({name:'recall',arguments:{key}});
  const payload=JSON.parse(response.content.find(item=>item.type==='text').text);
  if(response.isError||payload.key!==key||payload.value!==value)throw new Error('business state lost');
  process.stdout.write(JSON.stringify({businessVerified:true}));
}finally{await client.close();clearTimeout(timer);}})().catch(()=>{process.exitCode=46;});`

const dnsControlCode = `
const http=require('node:http'); const [host,route,method]=process.argv.slice(1);
const request=http.request({host,port:8090,path:route,method},response=>{
  let body='';response.on('data',chunk=>{body+=chunk;if(body.length>262144)request.destroy()});
  response.once('end',()=>{if(response.statusCode!==200)process.exitCode=47;else process.stdout.write(body);});
});
const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);
request.once('close',()=>clearTimeout(timer));request.once('error',()=>{process.exitCode=48;});request.end();`

function fixtureObjects(config, runId, wrcImage, answerIp, upstream) {
  const suffix = runId.slice(0, 12)
  const names = {
    proxy: `e2e-wrc-dns-${suffix}`,
    script: `e2e-wrc-dns-script-${suffix}`,
    policy: `e2e-wrc-dns-policy-${suffix}`,
    fromWrc: `e2e-wrc-from-${suffix}`,
    recipe: `e2e-wrc-${suffix}`,
    canary: `e2e-wrc-canary-${suffix}`,
  }
  const labels = { 'e2e.clerum.io/suite': 'wrc-egress-degradation', [RUN_LABEL]: runId }
  const metadata = (name, namespace = config.wrcNamespace) => ({ name, namespace, labels })
  const targets = [
    { lane: 'ui', fqdn: `ui-${suffix}.example.com`, ip: answerIp },
    { lane: 'worker', fqdn: `worker-${suffix}.example.com`, ip: answerIp },
    { lane: 'canary', fqdn: `canary-${suffix}.example.com`, ip: answerIp },
  ]
  const securityContext = {
    allowPrivilegeEscalation: false,
    capabilities: { drop: ['ALL'] },
    readOnlyRootFilesystem: true,
    runAsNonRoot: true,
    runAsUser: 1000,
    runAsGroup: 1000,
    seccompProfile: { type: 'RuntimeDefault' },
  }
  const ports = [
    { protocol: 'UDP', port: 53 },
    { protocol: 'TCP', port: 53 },
    { protocol: 'UDP', port: 8053 },
    { protocol: 'TCP', port: 8053 },
  ]
  const proxyPeer = { podSelector: { matchLabels: { app: names.proxy } } }
  const objects = [
    {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: metadata(names.script),
      data: {
        'server.cjs': fs.readFileSync(
          path.join(config.repository, 'tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs'),
          'utf8'
        ),
      },
    },
    {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: metadata(names.policy),
      spec: {
        podSelector: proxyPeer.podSelector,
        policyTypes: ['Ingress', 'Egress'],
        ingress: [
          { from: [{ podSelector: { matchLabels: { app: config.wrcDeployment } } }], ports },
        ],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' },
                },
                podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } },
              },
            ],
            ports: ports.slice(0, 2),
          },
        ],
      },
    },
    {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: metadata(names.fromWrc),
      spec: {
        podSelector: { matchLabels: { app: config.wrcDeployment } },
        policyTypes: ['Egress'],
        egress: [{ to: [proxyPeer], ports }],
      },
    },
    {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: metadata(names.proxy),
      spec: {
        selector: { app: names.proxy },
        ports: [
          { name: 'dns-udp', protocol: 'UDP', port: 53, targetPort: 8053 },
          { name: 'dns-tcp', protocol: 'TCP', port: 53, targetPort: 8053 },
        ],
      },
    },
    {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: metadata(names.proxy),
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: names.proxy } },
        template: {
          metadata: { labels: { ...labels, app: names.proxy } },
          spec: {
            automountServiceAccountToken: false,
            terminationGracePeriodSeconds: 1,
            containers: [
              {
                name: 'dns-proxy',
                image: wrcImage,
                imagePullPolicy: 'IfNotPresent',
                command: ['node', '/fixture/server.cjs'],
                env: [
                  { name: 'DNS_TARGETS_JSON', value: JSON.stringify(targets) },
                  { name: 'DNS_TTL_SECONDS', value: '5' },
                  { name: 'UPSTREAM_DNS', value: upstream },
                ],
                ports: [
                  { name: 'dns-udp', containerPort: 8053, protocol: 'UDP' },
                  { name: 'dns-tcp', containerPort: 8053, protocol: 'TCP' },
                ],
                readinessProbe: {
                  tcpSocket: { port: 8053 },
                  periodSeconds: 1,
                  failureThreshold: 10,
                },
                volumeMounts: [{ name: 'fixture', mountPath: '/fixture', readOnly: true }],
                resources: {
                  requests: { cpu: '5m', memory: '20Mi' },
                  limits: { cpu: '100m', memory: '96Mi' },
                },
                securityContext,
              },
            ],
            volumes: [{ name: 'fixture', configMap: { name: names.script } }],
          },
        },
      },
    },
  ]
  const workload = name => ({
    id: name,
    type: 'deployment',
    image: 'clerum/mock-mcp-server:test',
    port: 3000,
    healthCheck: { type: 'http', path: '/', port: 3001 },
  })
  const bindings = fqdn => [80, 443].map(port => ({ dns: fqdn, port, protocol: 'TCP' }))
  const recipe = {
    apiVersion: 'clerum.io/v1alpha1',
    kind: 'WorkflowRecipe',
    metadata: metadata(names.recipe, config.recipeNamespace),
    spec: {
      contextRef: 'context1',
      description: 'WRC authorization continuity fixture',
      workloads: [
        workload('frontend'),
        { ...workload('worker'), egressBindings: bindings(targets[1].fqdn) },
      ],
      ui: {
        workloadRef: 'frontend',
        port: 3000,
        title: 'WRC continuity',
        defaultPath: '/mcp',
        egress: {
          internal: [
            { workloadRef: 'worker', port: 3000 },
            { workloadRef: 'worker', port: 3001 },
          ],
          external: [80, 443].map(port => ({ fqdn: targets[0].fqdn, port })),
        },
      },
      security: { isolationLevel: 'minimal' },
    },
  }
  const canary = {
    apiVersion: 'clerum.io/v1alpha1',
    kind: 'WorkflowRecipe',
    metadata: metadata(names.canary, config.recipeNamespace),
    spec: {
      contextRef: 'context1',
      description: 'Independent WRC business witness',
      workloads: [{ ...workload('witness'), egressBindings: bindings(targets[2].fqdn) }],
      security: { isolationLevel: 'minimal' },
    },
  }
  return { names, targets, objects, recipe, canary }
}

async function readyPod(kube, namespace, recipe, workload, timeoutMs = 120_000) {
  let pod
  await waitFor('ready-fixture-pod', timeoutMs, async (remaining, signal) => {
    const pods = await kube.list(
      'pods',
      namespace,
      `clerum.io/recipe=${recipe},clerum.io/workload=${workload}`,
      remaining,
      signal
    )
    const ready = pods.filter(
      item =>
        !item.metadata.deletionTimestamp &&
        item.status?.conditions?.some(
          condition => condition.type === 'Ready' && condition.status === 'True'
        )
    )
    invariant(ready.length <= 1, 'AMBIGUOUS_FIXTURE_POD')
    pod = ready[0]
    return Boolean(pod?.metadata.uid)
  })
  return pod
}

const identityKey = identity => `${identity.kind}/${identity.namespace}/${identity.name}`
const checkedIdentity = identity => resourceIdentity({ kind: identity.kind, metadata: identity })

function uiChildIdentities(ui) {
  const identities = []
  if (ui.physicalName !== null) {
    for (const kind of ['Deployment', 'Service']) {
      identities.push(checkedIdentity({ kind, namespace: ui.namespace, name: ui.physicalName }))
    }
  }
  identities.push(
    checkedIdentity({
      kind: 'NetworkPolicy',
      namespace: ui.namespace,
      name: `ui-egress-${ui.parent.name}`,
    })
  )
  return identities
}

function uiCleanupRecord(journal) {
  const ui = journal.state.uiCleanup
  // Journals from before UI capture cannot silently certify cross-namespace
  // teardown once a recipe was attempted. They require explicit inspection.
  invariant(
    ui !== undefined || !journal.state.resources.some(entry => entry.kind === 'WorkflowRecipe'),
    'UI_CLEANUP_RECORD_MISSING'
  )
  if (ui === null || ui === undefined) return null
  invariant(
    ui.runId === journal.state.runId &&
      ui.parent?.kind === 'WorkflowRecipe' &&
      typeof ui.workload === 'string' &&
      /^[a-z0-9][a-z0-9.-]*$/.test(ui.workload) &&
      Array.isArray(ui.children) &&
      ui.children.length <= 3,
    'INVALID_UI_CLEANUP_RECORD'
  )
  checkedIdentity(ui.parent)
  const allowed = new Set(uiChildIdentities(ui).map(identityKey))
  const seen = new Set()
  for (const child of ui.children) {
    const key = identityKey(checkedIdentity(child))
    invariant(
      allowed.has(key) &&
        !seen.has(key) &&
        (child.uid === null || (typeof child.uid === 'string' && child.uid.length > 0)),
      'INVALID_UI_CHILD_RECORD'
    )
    seen.add(key)
  }
  return ui
}

function assertUiChild(object, identity, ui) {
  invariant(
    identityKey(resourceIdentity(object)) === identityKey(identity) &&
      typeof object.metadata.uid === 'string' &&
      object.metadata.uid.length > 0 &&
      typeof object.metadata.resourceVersion === 'string' &&
      object.metadata.resourceVersion.length > 0,
    'UI_CHILD_IDENTITY_UNPROVEN'
  )
  const labels = object.metadata.labels ?? {}
  invariant(
    labels['clerum.io/managed-by'] === 'workflow-recipes' &&
      labels['clerum.io/recipe'] === ui.parent.name,
    'UI_CHILD_OWNER_UNPROVEN'
  )
  invariant(
    (labels['clerum.io/recipe-name'] === undefined ||
      labels['clerum.io/recipe-name'] === ui.parent.name) &&
      (labels['clerum.io/recipe-namespace'] === undefined ||
        labels['clerum.io/recipe-namespace'] === ui.parent.namespace) &&
      (labels['clerum.io/sandbox-ui'] === undefined || labels['clerum.io/sandbox-ui'] === 'true'),
    'UI_CHILD_SCOPE_CONFLICT'
  )
  if (identity.kind !== 'NetworkPolicy') {
    invariant(labels['clerum.io/workload'] === ui.workload, 'UI_CHILD_WORKLOAD_UNPROVEN')
  }
  // Deployment metadata may omit sandbox-ui labels; its physical status name
  // plus ordinary WRC recipe/workload labels supply the ownership proof.
  if (identity.kind !== 'Deployment') {
    invariant(
      labels['clerum.io/recipe-name'] === ui.parent.name &&
        labels['clerum.io/recipe-namespace'] === ui.parent.namespace,
      'UI_CHILD_SCOPE_UNPROVEN'
    )
  }
  if (identity.kind === 'Service') {
    invariant(labels['clerum.io/sandbox-ui'] === 'true', 'UI_SERVICE_SCOPE_UNPROVEN')
  }
  for (const owner of object.metadata.ownerReferences ?? []) {
    invariant(
      owner.kind === 'WorkflowRecipe' &&
        owner.name === ui.parent.name &&
        owner.uid === ui.parentUid,
      'UI_CHILD_HAS_FOREIGN_OWNER'
    )
  }
}

async function captureUiChildren(kube, journal) {
  const ui = uiCleanupRecord(journal)
  if (!ui) return null
  const parentIdentity = checkedIdentity(ui.parent)
  const parentRecord = journal.state.resources.find(
    entry => identityKey(entry) === identityKey(parentIdentity)
  )
  if (ui.parentUid !== null) {
    invariant(parentRecord?.uid === ui.parentUid, 'UI_PARENT_RECORD_MISMATCH')
  }
  const parent = await kube.get(parentIdentity)
  // On recovery, absence cannot authorize adopting a same-name child. Only
  // identities durably captured while the exact parent existed remain usable.
  if (!parent) return ui
  invariant(
    parentRecord?.uid &&
      identityKey(resourceIdentity(parent)) === identityKey(parentIdentity) &&
      parent.metadata.uid === parentRecord.uid &&
      parent.metadata.labels?.[RUN_LABEL] === ui.runId &&
      parent.spec?.ui?.workloadRef === ui.workload &&
      parent.spec.workloads?.some(
        workload => workload.id === ui.workload && workload.type === 'deployment'
      ),
    'UI_PARENT_OWNERSHIP_UNPROVEN'
  )
  ui.parentUid = parentRecord.uid
  const physicalName = parent.status?.workloadInstances?.[ui.workload]
  if (physicalName !== undefined) {
    // Status is persisted before WRC creates workloads. Validate it before any
    // CLI argument; never substitute the raw logical workload ID when absent.
    checkedIdentity({ kind: 'Deployment', namespace: ui.namespace, name: physicalName })
    invariant(
      ui.physicalName === null || ui.physicalName === physicalName,
      'UI_PHYSICAL_NAME_CHANGED'
    )
    ui.physicalName = physicalName
  }
  for (const identity of uiChildIdentities(ui)) {
    const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))
    const object = await kube.get(identity)
    if (object) {
      assertUiChild(object, identity, ui)
      invariant(!recorded || recorded.uid === object.metadata.uid, 'UI_CHILD_IDENTITY_CHANGED')
    }
    if (!recorded) ui.children.push({ ...identity, uid: object?.metadata.uid ?? null })
    journal.save()
  }
  return ui
}

async function cleanupUiChildren(kube, ui) {
  invariant(!(await kube.get(checkedIdentity(ui.parent))), 'UI_PARENT_STILL_PRESENT')
  for (const child of ui.children) {
    const identity = checkedIdentity(child)
    const live = await kube.get(identity)
    if (!live) continue
    invariant(child.uid && live.metadata.uid === child.uid, 'UI_CHILD_IDENTITY_CHANGED')
    assertUiChild(live, identity, ui)
    try {
      await kube.delete(identity, {
        uid: child.uid,
        resourceVersion: live.metadata.resourceVersion,
      })
    } catch (error) {
      if (await kube.get(identity)) throw error
    }
    await waitFor('UI-child-absence', 90_000, async (remaining, signal) => {
      const current = await kube.get(identity, remaining, signal)
      if (!current) return true
      invariant(current.metadata.uid === child.uid, 'UI_CHILD_RECREATED_DURING_CLEANUP')
      return false
    })
  }
  // WRC's best-effort finalizer can delete a Deployment in Background. Its
  // ReplicaSets/Pods must also disappear before a restored verdict is valid.
  // Discovery is read-only: an uncaptured managed child is never adopted here.
  await waitFor('UI-resource-absence', 90_000, async (remaining, signal) => {
    const deadline = Date.now() + remaining
    let empty = true
    for (const kind of ['deployments', 'services', 'networkpolicies', 'replicasets', 'pods']) {
      const objects = await kube.list(
        kind,
        ui.namespace,
        `clerum.io/recipe=${ui.parent.name}`,
        deadline - Date.now(),
        signal
      )
      for (const object of objects) {
        if (!['replicasets', 'pods'].includes(kind)) {
          const identity = resourceIdentity(object)
          const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))
          invariant(
            recorded?.uid && recorded.uid === object.metadata.uid,
            'UI_CHILD_IDENTITY_NOT_CAPTURED'
          )
        }
        empty = false
      }
    }
    return empty
  })
}

async function restore(kube, journal) {
  await kube.setCleanup(true)
  journal.state.phase = 'restoring'
  journal.save()
  const failures = []
  let ui
  const wrc = journal.state.wrc
  if (wrc?.attempted) {
    try {
      const live = await kube.get(wrc.identity)
      invariant(
        live?.metadata.uid === wrc.uid && live.spec.replicas === wrc.replicas,
        'WRC_RESTORE_IDENTITY_CHANGED'
      )
      const current = live.spec.template.spec
      const alreadyRestored =
        (current.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&
        (current.dnsConfig ?? null) === null
      if (!alreadyRestored) {
        invariant(
          current.dnsPolicy === 'None' && same(current.dnsConfig, wrc.injectedConfig),
          'WRC_DNS_CHANGED_BY_ANOTHER_OWNER'
        )
        await kube.patch(wrc.identity, [
          { op: 'test', path: '/metadata/uid', value: wrc.uid },
          { op: 'test', path: '/metadata/resourceVersion', value: live.metadata.resourceVersion },
          { op: 'replace', path: '/spec/template/spec/dnsPolicy', value: wrc.originalPolicy },
          { op: 'remove', path: '/spec/template/spec/dnsConfig' },
        ])
      }
      await kube.rollout(wrc.identity.namespace, wrc.identity.name)
      const restored = await kube.get(wrc.identity)
      invariant(
        restored?.metadata.uid === wrc.uid &&
          restored.spec.replicas === wrc.replicas &&
          deploymentReady(restored) &&
          (restored.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&
          (restored.spec.template.spec.dnsConfig ?? null) === null,
        'WRC_RESTORE_NOT_PROVEN'
      )
      journal.state.wrc.attempted = false
      journal.save()
    } catch (error) {
      failures.push(error)
    }
  }
  // Keep the DNS fixture if restoration is uncertain; deleting it would remove
  // the nameserver from a WRC Deployment that may still depend on it.
  if (failures.length === 0) {
    try {
      ui = await captureUiChildren(kube, journal)
    } catch (error) {
      failures.push(error)
    }
  }
  if (failures.length === 0) {
    for (const entry of [...journal.state.resources].reverse()) {
      try {
        await deleteOwned(kube, journal, entry)
      } catch (error) {
        failures.push(error)
      }
    }
  }
  if (failures.length === 0 && ui) {
    try {
      await cleanupUiChildren(kube, ui)
    } catch (error) {
      failures.push(error)
    }
  }
  if (failures.length) {
    journal.state.phase = 'recovery-required'
    journal.save()
    throw new AggregateError(failures, 'WRC_EGRESS_RECOVERY_REQUIRED')
  }
  journal.state.phase = 'restored'
  journal.save()
  journal.remove()
}

async function protectedRun(kube, journal, exercise, cancellation) {
  let original
  try {
    await exercise()
  } catch (error) {
    original = error
  }
  try {
    await restore(kube, journal)
  } catch (cleanupError) {
    throw new AggregateError(
      [...(original ? [original] : []), cleanupError],
      'WRC_EGRESS_RECOVERY_REQUIRED'
    )
  }
  if (cancellation.signal.aborted) throw cancellation.signal.reason
  if (original) throw original
}

async function exercise(config, kube, journal) {
  const wrcIdentity = id('Deployment', config.wrcNamespace, config.wrcDeployment)
  const wrc = await kube.get(wrcIdentity)
  invariant(deploymentReady(wrc) && wrc.spec.replicas === 1, 'WRC_NOT_READY_OR_NOT_SINGLE_WRITER')
  invariant(
    (wrc.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&
      (wrc.spec.template.spec.dnsConfig ?? null) === null,
    'WRC_DNS_NOT_BASELINE'
  )
  const image = wrc.spec.template.spec.containers.find(
    container => container.name === 'workflow-recipes'
  )?.image
  invariant(image, 'MISSING_WRC_IMAGE')
  const upstream = (await kube.get(id('Service', 'kube-system', 'kube-dns')))?.spec.clusterIP
  invariant(upstream, 'MISSING_CLUSTER_DNS')
  const resolver = new dns.Resolver({ timeout: 5000, tries: 1 })
  const answers = await resolver.resolve4(endpointHost)
  const answerIp = answers.find(address => isIPv4(address) && !blocked.check(address, 'ipv4'))
  invariant(answerIp, 'PUBLIC_ENDPOINT_UNAVAILABLE')
  const fixture = fixtureObjects(config, journal.state.runId, image, answerIp, upstream)
  const { names, targets } = fixture
  for (const object of fixture.objects) await createOwned(kube, journal, object)
  await kube.rollout(config.wrcNamespace, names.proxy, 90_000)
  const proxy = (await kube.list('pods', config.wrcNamespace, `app=${names.proxy}`)).find(
    pod =>
      !pod.metadata.deletionTimestamp &&
      pod.status?.conditions?.some(c => c.type === 'Ready' && c.status === 'True')
  )
  invariant(proxy?.metadata.uid, 'DNS_PROXY_NOT_READY')
  const proxyIp = (await kube.get(id('Service', config.wrcNamespace, names.proxy)))?.spec.clusterIP
  invariant(proxyIp, 'DNS_PROXY_ADDRESS_MISSING')
  const dnsControl = async (route = '/state', method = 'GET', remaining = 15_000, signal) => {
    const state = JSON.parse(
      await kube.exec(
        config.wrcNamespace,
        proxy.metadata.name,
        dnsControlCode,
        [LOOPBACK, route, method],
        Math.min(15_000, remaining),
        signal
      )
    )
    invariant(state.fault === null && Array.isArray(state.lanes), 'DNS_FIXTURE_FAULT')
    return state
  }
  const setAffectedDns = async mode => {
    await dnsControl(`/mode/ui/${mode}`, 'POST')
    await dnsControl(`/mode/worker/${mode}`, 'POST')
  }
  await dnsControl()
  const injectedConfig = {
    nameservers: [proxyIp],
    options: [
      { name: 'ndots', value: '1' },
      { name: 'timeout', value: '2' },
      { name: 'attempts', value: '1' },
    ],
  }
  journal.state.wrc = {
    identity: wrcIdentity,
    uid: wrc.metadata.uid,
    replicas: 1,
    originalPolicy: 'ClusterFirst',
    injectedConfig,
    attempted: true,
  }
  journal.state.phase = 'DNS-injected'
  journal.save()
  const beforeInjection = await kube.get(wrcIdentity)
  invariant(
    beforeInjection?.metadata.uid === wrc.metadata.uid &&
      deploymentReady(beforeInjection) &&
      (beforeInjection.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&
      (beforeInjection.spec.template.spec.dnsConfig ?? null) === null,
    'WRC_CHANGED_BEFORE_INJECTION'
  )
  await kube.patch(wrcIdentity, [
    { op: 'test', path: '/metadata/uid', value: wrc.metadata.uid },
    {
      op: 'test',
      path: '/metadata/resourceVersion',
      value: beforeInjection.metadata.resourceVersion,
    },
    { op: 'add', path: '/spec/template/spec/dnsPolicy', value: 'None' },
    { op: 'add', path: '/spec/template/spec/dnsConfig', value: injectedConfig },
  ])
  await kube.rollout(config.wrcNamespace, config.wrcDeployment)
  journal.state.uiCleanup = {
    parent: resourceIdentity(fixture.recipe),
    parentUid: null,
    runId: journal.state.runId,
    namespace: config.uiNamespace,
    workload: fixture.recipe.spec.ui.workloadRef,
    physicalName: null,
    children: [],
  }
  journal.save()
  const recipe = await createOwned(kube, journal, fixture.recipe)
  await createOwned(kube, journal, fixture.canary)
  const recipeIdentity = id('WorkflowRecipe', config.recipeNamespace, names.recipe)
  const active = async (name, remaining, signal) => {
    const current = await kube.get(
      id('WorkflowRecipe', config.recipeNamespace, name),
      remaining,
      signal
    )
    return current?.status?.phase === 'active'
  }
  await waitFor('active-recipes', 180_000, async (remaining, signal) => {
    const deadline = Date.now() + remaining
    return (
      (await active(names.recipe, remaining, signal)) &&
      (await active(names.canary, deadline - Date.now(), signal))
    )
  })
  const frontend = await readyPod(kube, config.uiNamespace, names.recipe, 'frontend')
  const worker = await readyPod(kube, config.recipeNamespace, names.recipe, 'worker')
  const canary = await readyPod(kube, config.recipeNamespace, names.canary, 'witness')
  const lanes = [
    {
      lane: 'ui',
      workload: 'frontend',
      namespace: config.uiNamespace,
      pod: frontend,
      policy: `ui-egress-${names.recipe}`,
      target: targets[0],
    },
    {
      lane: 'workload',
      workload: 'worker',
      namespace: config.recipeNamespace,
      pod: worker,
      policy: `wl-egress-${names.recipe}-worker`,
      target: targets[1],
    },
  ]
  const policiesMatch = async (ports, internalPorts, remaining = 30_000, signal) => {
    const deadline = Date.now() + remaining
    for (const lane of lanes) {
      const policy = await kube.get(
        id('NetworkPolicy', lane.namespace, lane.policy),
        deadline - Date.now(),
        signal
      )
      if (!policy) return false
      try {
        assertPolicy(policy, {
          name: lane.policy,
          namespace: lane.namespace,
          recipe: names.recipe,
          recipeNamespace: config.recipeNamespace,
          lane: lane.lane,
          workload: lane.workload,
          podLabels: lane.pod.metadata.labels,
          uid: lane.uid,
          tuples: ports.map(port => ({
            fqdn: lane.target.fqdn,
            ip: answerIp,
            port,
            protocol: 'TCP',
          })),
          internal:
            lane.lane === 'ui'
              ? internalPorts.map(port => ({
                  namespace: config.recipeNamespace,
                  workload: 'worker',
                  port,
                  protocol: 'TCP',
                }))
              : [],
        })
      } catch {
        return false
      }
      const observation = policyObservation(policy, lane.recoveryAfter)
      if (!observation) return false
      Object.assign(lane, observation)
      lane.uid = policy.metadata.uid
    }
    return true
  }
  await waitFor('baseline-policy-proof', 120_000, (remaining, signal) =>
    policiesMatch([80, 443], [3000, 3001], remaining, signal)
  )
  const services = await kube.list(
    'services',
    config.recipeNamespace,
    `clerum.io/recipe=${names.recipe},clerum.io/workload=worker`
  )
  invariant(
    services.length === 1 &&
      services[0].metadata.ownerReferences?.some(owner => owner.uid === recipe.metadata.uid),
    'WORKER_SERVICE_OWNERSHIP_NOT_PROVEN'
  )
  const workerUrl = `http://${services[0].metadata.name}.${config.recipeNamespace}.svc:3000/mcp`
  const business = async (write = false, internal = true) => {
    for (const [namespace, pod, url, key] of [
      [config.uiNamespace, frontend, `http://${LOOPBACK}:3000/mcp`, 'frontend'],
      [config.recipeNamespace, canary, `http://${LOOPBACK}:3000/mcp`, 'canary'],
      ...(internal ? [[config.uiNamespace, frontend, workerUrl, 'worker']] : []),
    ])
      await kube.exec(
        namespace,
        pod.metadata.name,
        businessCode,
        [url, key, journal.state.runId, write ? 'record' : 'recall'],
        15_000
      )
  }
  const traffic = async (namespace, pod, port, mode = 'tcp') => {
    const output = await kube.exec(
      namespace,
      pod.metadata.name,
      trafficCode,
      [answerIp, String(port), endpointHost, mode],
      10_000
    )
    if (mode === 'https')
      invariant(
        same(JSON.parse(output), { allowed: true, contentVerified: true }),
        'HTTPS_BUSINESS_CONTENT_NOT_PROVEN'
      )
    else invariant(output === 'connected', 'TCP_CONNECTION_NOT_PROVEN')
  }
  const proveTraffic = async removed => {
    // The canary proves the remote port is open at the moment of a negative
    // observation. Only a connection timeout counts as policy denial.
    await traffic(config.recipeNamespace, canary, 80)
    await traffic(config.recipeNamespace, canary, 443, 'https')
    for (const lane of lanes) {
      await traffic(lane.namespace, lane.pod, 443, 'https')
      if (!removed) await traffic(lane.namespace, lane.pod, 80)
      else {
        let denied = false
        try {
          await traffic(lane.namespace, lane.pod, 80)
        } catch (error) {
          denied = error.exitCode === 42
        }
        invariant(denied, 'REVOKED_TRAFFIC_NOT_PROVEN_DENIED')
        await traffic(config.recipeNamespace, canary, 80)
      }
    }
  }
  await proveTraffic(false)
  await business(true)
  // Internal health port is genuinely listening before its declaration is removed.
  invariant(worker.status?.podIP, 'WORKER_POD_ADDRESS_MISSING')
  await kube.exec(
    config.uiNamespace,
    frontend.metadata.name,
    trafficCode,
    [worker.status.podIP, '3001', endpointHost, 'tcp'],
    10_000
  )
  journal.state.phase = 'exercising'
  journal.save()
  console.log('WRC_EGRESS_BASELINE_PROVEN')

  await setAffectedDns('hold')
  const heldBefore = await dnsControl()
  const current = await kube.get(recipeIdentity)
  const reduced = await kube.patch(recipeIdentity, [
    { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },
    { op: 'test', path: '/metadata/resourceVersion', value: current.metadata.resourceVersion },
    {
      op: 'replace',
      path: '/spec/ui/egress/external',
      value: [{ fqdn: targets[0].fqdn, port: 443 }],
    },
    {
      op: 'replace',
      path: '/spec/ui/egress/internal',
      value: [{ workloadRef: 'worker', port: 3000 }],
    },
    {
      op: 'replace',
      path: '/spec/workloads/1/egressBindings',
      value: [{ dns: targets[1].fqdn, port: 443, protocol: 'TCP' }],
    },
  ])
  const countsFor = (state, lane, mode) => {
    const count = state.lanes.find(item => item.lane === lane)?.received?.[mode]
    invariant(Number.isSafeInteger(count) && count >= 0, 'MISSING_UNIQUE_DNS_OBSERVATION')
    return count
  }
  await waitFor('DNS-hold-started', 60_000, async (remaining, signal) => {
    const state = await dnsControl('/state', 'GET', remaining, signal)
    return ['ui', 'worker'].some(
      lane => countsFor(state, lane, 'hold') > countsFor(heldBefore, lane, 'hold')
    )
  })
  // Runtime proves convergence/continuity while the resolver withholds replies.
  // Exact pre/post-DNS ordering and same-attempt races are proved by the
  // deterministic controller tests, not by historical packet counters here.
  await waitFor('contraction-with-resolver-unavailable', 60_000, (remaining, signal) =>
    policiesMatch([443], [3000], remaining, signal)
  )
  await business()
  await proveTraffic(true)
  let internalDenied = false
  try {
    await kube.exec(
      config.uiNamespace,
      frontend.metadata.name,
      trafficCode,
      [worker.status.podIP, '3001', endpointHost, 'tcp'],
      10_000
    )
  } catch (error) {
    internalDenied = error.exitCode === 42
  }
  invariant(internalDenied, 'REMOVED_INTERNAL_PORT_NOT_DENIED')
  console.log('WRC_EGRESS_HELD_CONTINUITY_PROVEN')

  for (const lane of lanes) {
    const policyIdentity = id('NetworkPolicy', lane.namespace, lane.policy)
    const policy = await kube.get(policyIdentity)
    await kube.patch(policyIdentity, [
      { op: 'test', path: '/metadata/uid', value: lane.uid },
      { op: 'test', path: '/metadata/resourceVersion', value: policy.metadata.resourceVersion },
      {
        op: 'add',
        path: '/spec/egress/-',
        value: {
          to: [{ ipBlock: { cidr: `${answerIp}/32` } }],
          ports: [{ port: 80, protocol: 'TCP' }],
        },
      },
    ])
  }
  const failedBefore = await dnsControl()
  await setAffectedDns('servfail')
  await waitFor('both-DNS-lanes-SERVFAIL', 90_000, async (remaining, signal) => {
    const state = await dnsControl('/state', 'GET', remaining, signal)
    return ['ui', 'worker'].every(
      lane => countsFor(state, lane, 'servfail') > countsFor(failedBefore, lane, 'servfail')
    )
  })
  await waitFor('drift-convergence-under-SERVFAIL', 90_000, (remaining, signal) =>
    policiesMatch([443], [3000], remaining, signal)
  )
  await business()
  await proveTraffic(true)
  invariant(
    (await active(names.recipe)) && (await active(names.canary)),
    'RECIPE_AVAILABILITY_LOST'
  )
  console.log('WRC_EGRESS_SERVFAIL_CONTINUITY_PROVEN')

  // A no-op refresh may legitimately avoid writing timestamps until renewal.
  // Bound that wait by the observed stored window instead of demanding churn
  // or mistaking a DNS packet / old active phase for accepted convergence.
  const latestExpiry = Math.max(...lanes.map(lane => lane.latestExpiry))
  invariant(Number.isFinite(latestExpiry), 'MISSING_ACCEPTED_DNS_WINDOW')
  const recoveryBudget = Math.max(90_000, latestExpiry - Date.now() + 90_000)
  invariant(recoveryBudget <= 600_000, 'DNS_WINDOW_EXCEEDS_GATE_RECOVERY_BUDGET')
  for (const lane of lanes) {
    invariant(Number.isFinite(lane.lastAcceptedAt), 'MISSING_ACCEPTED_DNS_OBSERVATION')
    lane.recoveryAfter = lane.lastAcceptedAt
  }
  const recoveredBefore = await dnsControl()
  await setAffectedDns('ok')
  await waitFor('autonomous-DNS-recovery', 180_000, async (remaining, signal) => {
    const state = await dnsControl('/state', 'GET', remaining, signal)
    return ['ui', 'worker'].every(
      lane => countsFor(state, lane, 'ok') > countsFor(recoveredBefore, lane, 'ok')
    )
  })
  const recovered = await kube.get(recipeIdentity)
  invariant(
    recovered.metadata.generation === reduced.metadata.generation,
    'RECOVERY_NEEDED_SPEC_NUDGE'
  )
  await waitFor('accepted-recovered-policy-proof', recoveryBudget, async (remaining, signal) => {
    const deadline = Date.now() + remaining
    return (
      (await policiesMatch([443], [3000], remaining, signal)) &&
      (await active(names.recipe, deadline - Date.now(), signal)) &&
      (await active(names.canary, deadline - Date.now(), signal))
    )
  })
  await business()
  await proveTraffic(true)

  // Exercise the real API's empty-list serialization on a valid A→B edit.
  // Remove the internal route intentionally; no service-continuity claim is
  // made for that explicitly revoked route after this configuration change.
  const migration = await kube.get(recipeIdentity)
  const migrated = await kube.patch(recipeIdentity, [
    { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },
    { op: 'test', path: '/metadata/resourceVersion', value: migration.metadata.resourceVersion },
    {
      op: 'replace',
      path: '/spec/ui/egress',
      value: { external: [{ fqdn: targets[1].fqdn, port: 443 }] },
    },
    {
      op: 'replace',
      path: '/spec/workloads/1/egressBindings',
      value: [{ dns: targets[0].fqdn, port: 443, protocol: 'TCP' }],
    },
  ])
  // Contracting away the worker's last old FQDN legitimately deletes its
  // aggregate policy. Re-pin the recreated identity only after this confirmed
  // intent change and the complete ownership/spec/provenance proof below.
  // The baseline UID remains mandatory throughout the prior fault/race phases.
  acceptPolicyIntentChange(lanes, migration, migrated, recipe.metadata.uid, targets)
  await waitFor('valid-host-migration', 120_000, async (remaining, signal) => {
    const deadline = Date.now() + remaining
    return (
      (await policiesMatch([443], [], remaining, signal)) &&
      (await active(names.recipe, deadline - Date.now(), signal))
    )
  })
  await business(false, false)
  await proveTraffic(true)
  for (const name of [config.wrcDeployment, 'control-api', 'host-context-controller']) {
    invariant(
      deploymentReady(await kube.get(id('Deployment', config.wrcNamespace, name))),
      'SIBLING_DEPLOYMENT_NOT_READY'
    )
  }
  for (const lane of [...lanes, { pod: canary, namespace: config.recipeNamespace }]) {
    const now = (
      await kube.list(
        'pods',
        lane.namespace,
        `clerum.io/recipe=${lane.pod.metadata.labels['clerum.io/recipe']},clerum.io/workload=${lane.pod.metadata.labels['clerum.io/workload']}`
      )
    ).find(pod => pod.metadata.uid === lane.pod.metadata.uid)
    invariant(
      now &&
        !now.metadata.deletionTimestamp &&
        same(
          now.status?.containerStatuses?.map(c => c.restartCount),
          lane.pod.status?.containerStatuses?.map(c => c.restartCount)
        ),
      'FIXTURE_POD_RESTARTED_OR_REPLACED'
    )
  }
  console.log('WRC_EGRESS_AUTONOMOUS_RECOVERY_AND_MIGRATION_PROVEN')
}

async function main(argv) {
  invariant(
    argv.length === 0 || (argv.length === 1 && argv[0] === '--recover'),
    'INVALID_GATE_ARGUMENTS'
  )
  const repository = fs.realpathSync(path.join(__dirname, '../../..'))
  invariant(
    process.env.T2_PROJECT_DIR === repository &&
      process.env.T2_PROFILE === process.env.T2_CONTEXT &&
      process.env.KUBECONTEXT === process.env.T2_CONTEXT,
    'INVALID_GATE_BINDING'
  )
  const binding = {
    repository,
    profile: process.env.T2_PROFILE,
    context: process.env.T2_CONTEXT,
    branch: execFileSync('git', ['branch', '--show-current'], {
      cwd: repository,
      encoding: 'utf8',
      timeout: 5000,
    }).trim(),
    head: execFileSync('git', ['rev-parse', 'HEAD'], {
      cwd: repository,
      encoding: 'utf8',
      timeout: 5000,
    }).trim(),
  }
  const cancellation = new AbortController()
  let signalCode = 0
  const signalHandlers = new Map(
    ['SIGINT', 'SIGTERM'].map(signal => [
      signal,
      () => {
        signalCode ||= signal === 'SIGINT' ? 130 : 143
        const error = new Error('WRC_GATE_INTERRUPTED')
        error.exitCode = signalCode
        cancellation.abort(error)
      },
    ])
  )
  for (const [signal, handler] of signalHandlers) process.on(signal, handler)
  const run = createCommandRunner(repository, cancellation)
  const kube = createKubeClient(repository, binding.context, run)
  try {
    // Protect direct Node invocation as well as the Bash entry point.
    await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {
      timeoutMs: 15_000,
    })
    const file = journalPath(repository, binding.profile)
    if (argv[0] === '--recover') {
      const state = readJournal(file)
      assertJournalBinding(state, binding)
      await restore(kube, new Journal(file, state, true))
      invariant(!signalCode, 'RECOVERY_INTERRUPTED')
      console.log('WRC_EGRESS_RECOVERY_COMPLETE') // Never the interrupted test's PASS.
      return
    }
    invariant(
      process.env.E2E_WRC_EGRESS_FAULT_INJECTION === '1',
      'FAULT_INJECTION_NOT_ACKNOWLEDGED'
    )
    // Direct Node invocation cannot bypass exact-head/image/context verification.
    // Reuse the supported read-only preflight while inheriting the same lease.
    await run(
      [
        'env',
        'T2_PLAN_MODE=false',
        'bash',
        path.join(repository, 'scripts/minikube/t2-preflight.sh'),
      ],
      { timeoutMs: 300_000 }
    )
    const journal = new Journal(file, {
      version: 1,
      binding,
      runId: randomUUID(),
      phase: 'validated',
      resources: [],
      uiCleanup: null,
    })
    const config = {
      repository,
      wrcNamespace: process.env.WRC_NAMESPACE || 'control-plane',
      wrcDeployment: process.env.WRC_DEPLOYMENT || 'workflow-recipes',
      uiNamespace: process.env.SANDBOX_UI_NS || 'sandbox-ui',
      recipeNamespace: process.env.WORKFLOW_RECIPE_NS || 'sandbox-recipes',
    }
    await protectedRun(kube, journal, () => exercise(config, kube, journal), cancellation)
    console.log('WRC_EGRESS_DEGRADATION_E2E_PASS')
  } catch (error) {
    process.stderr.write(
      `${error instanceof AggregateError ? 'WRC_EGRESS_RECOVERY_REQUIRED' : 'WRC_EGRESS_GATE_FAILED'}\n`
    )
    process.exitCode = signalCode || error.exitCode || 1
  } finally {
    for (const [signal, handler] of signalHandlers) process.off(signal, handler)
  }
}

module.exports = {
  restore,
  protectedRun,
  fixtureObjects,
  deploymentReady,
  policyObservation,
  acceptPolicyIntentChange,
  main,
}
if (require.main === module)
  main(process.argv.slice(2)).catch(() => {
    process.exitCode = 1
  })