Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.5 KB
Raw Blame
'use strict'

// Development-only lifecycle for the disruptive WRC gate. The durable journal
// is deliberately outside the transient profile lease: a killed process must
// leave a recovery obligation that later certification cannot silently ignore.
const fs = require('node:fs')
const path = require('node:path')
const { randomUUID } = require('node:crypto')
const { spawn } = require('node:child_process')
const { setTimeout: delay } = require('node:timers/promises')

const RUN_LABEL = 'e2e.clerum.io/run'
const resourceRoutes = new Map([
  ['ConfigMap', ['v1', 'configmaps']],
  ['Service', ['v1', 'services']],
  ['Deployment', ['apps/v1', 'deployments']],
  ['NetworkPolicy', ['networking.k8s.io/v1', 'networkpolicies']],
  ['WorkflowRecipe', ['clerum.io/v1alpha1', 'workflowrecipes']],
])
const validName = value =>
  typeof value === 'string' && /^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/.test(value)
function invariant(value, code) {
  if (!value) throw new Error(code)
}

function journalPath(repository, profile) {
  invariant(validName(profile), 'INVALID_PROFILE')
  invariant(fs.realpathSync(repository) === repository, 'NONCANONICAL_REPOSITORY')
  return path.join(
    repository,
    '.local-notes',
    'infra',
    'runs',
    'fault-injection',
    `wrc-${profile}.json`
  )
}

function safeParents(file, create = false, allowMissing = false) {
  let current = path.parse(file).root
  for (const part of path.dirname(file).slice(current.length).split(path.sep).filter(Boolean)) {
    current = path.join(current, part)
    if (create && !fs.existsSync(current)) fs.mkdirSync(current, { mode: 0o700 })
    const info = fs.lstatSync(current, { throwIfNoEntry: false })
    if (!info && allowMissing) return
    invariant(info, 'MISSING_JOURNAL_PARENT')
    invariant(info.isDirectory() && !info.isSymbolicLink(), 'UNSAFE_JOURNAL_PARENT')
  }
}

function readJournal(file) {
  safeParents(file)
  let descriptor
  try {
    // The local harness runs on POSIX. NOFOLLOW binds validation and reading
    // to one opened file; NONBLOCK prevents a substituted FIFO from hanging
    // before fstat can reject it as a non-regular file.
    descriptor = fs.openSync(
      file,
      fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK
    )
  } catch (error) {
    if (error.code === 'ELOOP') throw new Error('UNSAFE_JOURNAL_FILE', { cause: error })
    throw error
  }
  try {
    const checkedStat = () => {
      const info = fs.fstatSync(descriptor)
      invariant(
        info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0,
        'UNSAFE_JOURNAL_FILE'
      )
      invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')
      return info
    }
    const before = checkedStat()
    // One extra byte detects growth without an unbounded read-to-EOF. Normal
    // journal writers replace atomically; in-place size changes are refused.
    const buffer = Buffer.alloc(before.size + 1)
    let length = 0
    while (length < buffer.length) {
      const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null)
      if (count === 0) break
      length += count
    }
    const after = checkedStat()
    invariant(length === before.size && after.size === before.size, 'JOURNAL_CHANGED_DURING_READ')
    const state = JSON.parse(buffer.subarray(0, length).toString('utf8'))
    invariant(
      state.version === 1 &&
        typeof state.runId === 'string' &&
        /^[a-f0-9-]{36}$/.test(state.runId) &&
        Array.isArray(state.resources),
      'INVALID_JOURNAL'
    )
    return state
  } finally {
    fs.closeSync(descriptor)
  }
}

function assertJournalBinding(state, expected) {
  // HEAD is historical during recovery; stable owner identity must still match.
  for (const key of ['repository', 'branch', 'profile', 'context']) {
    invariant(
      typeof expected[key] === 'string' &&
        expected[key].length > 0 &&
        state.binding?.[key] === expected[key],
      'JOURNAL_OWNER_MISMATCH'
    )
  }
}

class Journal {
  constructor(file, state, existing = false) {
    this.file = file
    this.state = state
    safeParents(file, true)
    if (!existing) {
      const fd = fs.openSync(file, 'wx', 0o600)
      try {
        fs.writeFileSync(fd, JSON.stringify(state))
        fs.fsyncSync(fd)
      } finally {
        fs.closeSync(fd)
      }
    } else readJournal(file)
  }
  save() {
    const current = readJournal(this.file)
    invariant(current.runId === this.state.runId, 'JOURNAL_REPLACED')
    const pending = `${this.file}.${randomUUID()}.new`
    const fd = fs.openSync(pending, 'wx', 0o600)
    try {
      fs.writeFileSync(fd, JSON.stringify(this.state))
      fs.fsyncSync(fd)
    } finally {
      fs.closeSync(fd)
    }
    fs.renameSync(pending, this.file)
  }
  remove() {
    invariant(readJournal(this.file).runId === this.state.runId, 'JOURNAL_REPLACED')
    fs.unlinkSync(this.file)
  }
}

function resourceIdentity(object) {
  const metadata = object?.metadata
  invariant(
    resourceRoutes.has(object?.kind) && validName(metadata?.name) && validName(metadata?.namespace),
    'INVALID_RESOURCE_IDENTITY'
  )
  return { kind: object.kind, name: metadata.name, namespace: metadata.namespace }
}

function resourcePath(identity) {
  const { kind, name, namespace } = resourceIdentity({ kind: identity.kind, metadata: identity })
  const [version, plural] = resourceRoutes.get(kind)
  return `${version === 'v1' ? '/api' : '/apis'}/${version}/namespaces/${namespace}/${plural}/${name}`
}

async function createOwned(kube, journal, object) {
  const identity = resourceIdentity(object)
  invariant(object.metadata.labels?.[RUN_LABEL] === journal.state.runId, 'MISSING_RUN_OWNERSHIP')
  const entry = { ...identity, attempted: true, uid: null }
  journal.state.resources.push(entry)
  journal.save() // Record uncertain acknowledgement BEFORE issuing the request.
  const created = await kube.create(object)
  invariant(
    created?.metadata?.uid && created.metadata.labels?.[RUN_LABEL] === journal.state.runId,
    'UNPROVEN_CREATED_RESOURCE'
  )
  invariant(
    JSON.stringify(resourceIdentity(created)) === JSON.stringify(identity),
    'CREATED_IDENTITY_MISMATCH'
  )
  entry.uid = created.metadata.uid
  journal.save()
  return created
}

async function deleteOwned(kube, journal, entry) {
  const live = await kube.get(entry)
  if (!live) return // Only the client's authoritative NotFound produces null.
  invariant(
    entry.uid &&
      live.metadata?.uid === entry.uid &&
      live.metadata.labels?.[RUN_LABEL] === journal.state.runId,
    'UNPROVEN_CLEANUP_OWNERSHIP'
  )
  invariant(live.metadata.resourceVersion, 'MISSING_DELETE_VERSION')
  try {
    await kube.delete(entry, { uid: entry.uid, resourceVersion: live.metadata.resourceVersion })
  } catch (error) {
    if (await kube.get(entry)) throw error
  }
  await waitFor('owned-resource-absence', 90_000, async (remaining, signal) => {
    const current = await kube.get(entry, remaining, signal)
    if (!current) return true
    invariant(current.metadata?.uid === entry.uid, 'RESOURCE_RECREATED_DURING_CLEANUP')
    return false
  })
}

async function waitFor(label, timeoutMs, predicate, options = {}) {
  const now = options.now ?? Date.now
  const sleep = options.sleep ?? delay
  const deadline = now() + timeoutMs
  const timeoutError = new Error(`TIMEOUT:${label}`)
  timeoutError.exitCode = 124
  while (now() < deadline) {
    if (options.signal?.aborted) throw options.signal.reason
    const remaining = deadline - now()
    // Predicates pass this remaining budget into all child I/O. Race also bounds
    // a misbehaving predicate; cancellation must terminate its actual process.
    const controller = new AbortController()
    let timer
    const cancel = () => controller.abort(options.signal.reason)
    options.signal?.addEventListener('abort', cancel, { once: true })
    const cancelled = new Promise((_, reject) => {
      controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {
        once: true,
      })
    })
    const expired = new Promise((_, reject) => {
      timer = setTimeout(() => {
        controller.abort(timeoutError)
        reject(timeoutError)
      }, remaining)
    })
    let ready
    try {
      ready = await Promise.race([predicate(remaining, controller.signal), expired, cancelled])
    } finally {
      clearTimeout(timer)
      options.signal?.removeEventListener('abort', cancel)
    }
    if (options.signal?.aborted) throw options.signal.reason
    if (ready && now() < deadline) return
    if (now() >= deadline) break
    await sleep(Math.min(500, deadline - now()))
  }
  throw timeoutError
}

function createCommandRunner(repository, cancellation) {
  return (args, { input, timeoutMs = 30_000, signal, cleanup = false } = {}) =>
    new Promise((resolve, reject) => {
      if (!cleanup && cancellation.signal.aborted) return reject(cancellation.signal.reason)
      const effectiveSignal = cleanup
        ? signal
        : AbortSignal.any([cancellation.signal, ...(signal ? [signal] : [])])
      if (effectiveSignal?.aborted) return reject(effectiveSignal.reason)
      const timeoutSeconds = Math.max(1, Math.ceil(timeoutMs / 1000))
      const child = spawn(
        process.execPath,
        [
          path.join(repository, 'scripts/minikube/run-with-deadline.mjs'),
          '--timeout-seconds',
          String(timeoutSeconds),
          '--kill-grace-seconds',
          '2',
          '--heartbeat-seconds',
          '20',
          '--label',
          'wrc-egress-operation',
          '--',
          ...args,
        ],
        { stdio: ['pipe', 'pipe', 'pipe'], env: process.env }
      )
      const started = Date.now()
      const stdout = []
      let bytes = 0
      // Never echo arbitrary API bodies, pod output, or access configuration.
      child.stderr.resume()
      const abort = () => child.kill('SIGTERM')
      effectiveSignal?.addEventListener('abort', abort, { once: true })
      // The reused process-group supervisor accepts seconds; this parent timer
      // enforces the remaining millisecond phase budget without granting another
      // rounded-up second in which a command could still perform work.
      let timedOut = false
      const timer = setTimeout(() => {
        timedOut = true
        abort()
      }, timeoutMs)
      child.stdout.on('data', chunk => {
        bytes += chunk.length
        if (bytes > 4 * 1024 * 1024) abort()
        else stdout.push(chunk)
      })
      child.once('error', error => {
        clearTimeout(timer)
        effectiveSignal?.removeEventListener('abort', abort)
        reject(error)
      })
      child.once('close', (code, signalName) => {
        clearTimeout(timer)
        effectiveSignal?.removeEventListener('abort', abort)
        if (
          timedOut ||
          code !== 0 ||
          signalName ||
          effectiveSignal?.aborted ||
          bytes > 4 * 1024 * 1024 ||
          Date.now() - started > timeoutMs
        ) {
          const error = new Error('WRC_GATE_OPERATION_FAILED')
          error.exitCode =
            timedOut || Date.now() - started > timeoutMs
              ? 124
              : effectiveSignal?.reason?.exitCode || code || 1
          reject(error)
        } else resolve(Buffer.concat(stdout).toString('utf8'))
      })
      child.stdin.on('error', () => {})
      child.stdin.end(input ?? '')
    })
}

function createKubeClient(repository, context, run) {
  let cleanup = false
  const pending = new Set()
  const readBudget = timeoutMs => {
    const budget = timeoutMs ?? 30_000
    if (!Number.isFinite(budget) || budget <= 0) {
      const error = new Error('TIMEOUT:wrc-read')
      error.exitCode = 124
      throw error
    }
    return Math.min(30_000, budget)
  }
  const command = async (args, input, timeoutMs = 30_000, signal) => {
    const operation = run(
      [
        'kubectl',
        `--context=${context}`,
        `--request-timeout=${Math.max(1, Math.ceil(timeoutMs / 1000))}s`,
        ...args,
      ],
      { input, timeoutMs, signal, cleanup }
    )
    pending.add(operation)
    try {
      return await operation
    } finally {
      pending.delete(operation)
    }
  }
  const mutation = async (args, input) => {
    await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {
      timeoutMs: 15_000,
      cleanup,
    })
    return command(args, input)
  }
  return {
    async setCleanup(value) {
      cleanup = value
      // A polling deadline can abort a GET before its process-group supervisor
      // has finished reaping it. Drain those bounded operations before cleanup
      // mutates resources or releases the profile's ownership obligation.
      if (value) await Promise.allSettled([...pending])
    },
    async get(identity, timeoutMs, signal) {
      resourceIdentity({ kind: identity.kind, metadata: identity })
      const text = await command(
        [
          'get',
          identity.kind,
          identity.name,
          '-n',
          identity.namespace,
          '--ignore-not-found=true',
          '-o',
          'json',
        ],
        undefined,
        readBudget(timeoutMs),
        signal
      )
      return text.trim() === '' ? null : JSON.parse(text)
    },
    async list(kind, namespace, selector, timeoutMs, signal) {
      return JSON.parse(
        await command(
          ['get', kind, '-n', namespace, '-l', selector, '-o', 'json'],
          undefined,
          readBudget(timeoutMs),
          signal
        )
      ).items
    },
    async create(object) {
      resourceIdentity(object)
      return JSON.parse(await mutation(['create', '-f', '-', '-o', 'json'], JSON.stringify(object)))
    },
    async patch(identity, patch) {
      resourceIdentity({ kind: identity.kind, metadata: identity })
      return JSON.parse(
        await mutation([
          'patch',
          identity.kind,
          identity.name,
          '-n',
          identity.namespace,
          '--type=json',
          '-p',
          JSON.stringify(patch),
          '-o',
          'json',
        ])
      )
    },
    async delete(identity, preconditions) {
      // kubectl RawDelete accepts a DeleteOptions request body via stdin.
      return mutation(
        ['delete', `--raw=${resourcePath(identity)}`, '-f', '-'],
        JSON.stringify({
          apiVersion: 'v1',
          kind: 'DeleteOptions',
          preconditions,
          propagationPolicy: 'Foreground',
        })
      )
    },
    async exec(namespace, pod, code, args = [], timeoutMs = 20_000, signal) {
      return command(
        ['exec', pod, '-n', namespace, '--', 'node', '-e', code, ...args],
        undefined,
        timeoutMs,
        signal
      )
    },
    async rollout(namespace, name, timeoutMs = 180_000) {
      return command(
        [
          'rollout',
          'status',
          `deployment/${name}`,
          '-n',
          namespace,
          `--timeout=${Math.ceil(timeoutMs / 1000)}s`,
        ],
        undefined,
        timeoutMs
      )
    },
  }
}

module.exports = {
  Journal,
  journalPath,
  readJournal,
  assertJournalBinding,
  createOwned,
  deleteOwned,
  resourceIdentity,
  resourcePath,
  waitFor,
  createCommandRunner,
  createKubeClient,
  invariant,
  RUN_LABEL,
}

if (require.main === module) {
  const [repository, profile] = process.argv.slice(2)
  try {
    const file = journalPath(repository, profile)
    safeParents(file, false, true)
    if (fs.existsSync(file) || fs.lstatSync(file, { throwIfNoEntry: false })) {
      throw new Error('WRC_EGRESS_RECOVERY_REQUIRED')
    }
  } catch (error) {
    process.stderr.write(
      'WRC_EGRESS_RECOVERY_REQUIRED: resolve the owned gate journal before certification\n'
    )
    process.exitCode = 1
  }
}