Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.8 KB
Raw Blame
#!/usr/bin/env node
/**
 * Phases run by scripts/dev/gfs-store-rollback-check.sh, one Node process each.
 * The orchestrator passes every path; this file never guesses one.
 *
 *   legacy-crash   <oldDist> <host> <out>   the dev store (74e0d81d9) writes a
 *                                           ledger, then the process is killed
 *   legacy-restart <oldDist> <host> <out>   the dev Host startup (bootstrapGfsRuntime)
 *                                           reopens that volume and stops cleanly
 *   new-state      <newDist> <host> <out>   the #1028 store retires the dev store
 *                                           and leaves its own state, then is killed
 *   old-on-new     <oldDist> <host> <out>   rollback: the dev Host starts on the
 *                                           #1028 state and completes a download
 *   roll-forward   <newDist> <host> <out>   the #1028 store starts on the
 *                                           post-rollback volume
 *   manifest       <host> <out>             every entry under <host>: type, mode, size, sha256
 *   compare        <before> <after> <out>   entries removed or changed between two manifests
 *   write-fixture  <crashedHost> <restartedHost> <crashOut> <restartOut> <commit> <fixtureDir> <out>
 *                                           copies the dev-store snapshots into the fixture
 *
 * Every phase writes <out> as JSON with `assertions` and `failed`; the
 * orchestrator refuses a phase with zero assertions. A failed assertion throws.
 */
'use strict'

const { createHash, randomUUID } = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')

let assertions = 0
const results = {}

function check(condition, label, detail) {
  assertions += 1
  if (!condition) {
    console.error(`ASSERT_FAIL ${label} ${JSON.stringify(detail ?? null)}`)
    throw new Error(`assertion failed: ${label}`)
  }
  console.log(`ASSERT_OK ${label}`)
}

function writeOut(out, failed) {
  fs.writeFileSync(out, JSON.stringify({ assertions, failed, ...results }, null, 2))
}

function sha256(bytes) {
  return createHash('sha256').update(bytes).digest('hex')
}

function source(index) {
  const resourceId = index.toString(16).padStart(32, '0')
  return {
    kind: 'gfs',
    drive: 'main',
    resourceId,
    gfsUri: `gfs://main/${resourceId}`,
    name: `rollback-${index}.bin`,
    version: 1,
  }
}

function callerRoot(host, key) {
  const root = path.join(host, 'users', key)
  fs.mkdirSync(root, { recursive: true, mode: 0o700 })
  return root
}

/** Admits a transfer and writes its partial file the way the GFS client does. */
async function startTransfer(store, host, caller, index) {
  const root = callerRoot(host, caller)
  const bytes = Buffer.alloc(64, index)
  const transfer = await store.createTransfer({
    callerIdentity: caller,
    callerWorkspacePath: root,
    source: source(index),
    sizeBytes: bytes.byteLength,
    expiresAt: new Date(Date.now() + 60 * 60_000).toISOString(),
  })
  fs.writeFileSync(path.join(root, transfer.partialPath), bytes)
  return { transfer, bytes, root }
}

async function completeDownload(store, host, caller, index) {
  const { transfer, bytes, root } = await startTransfer(store, host, caller, index)
  const receipt = await store.publish(transfer.id, caller, sha256(bytes))
  return { receipt, bytes, root }
}

function listHost(host) {
  return fs.readdirSync(host).sort()
}

/** A crash: no close(), no drain. The kernel releases the writer lock. */
function crash(out) {
  writeOut(out, 0)
  process.kill(process.pid, 'SIGKILL')
}

const phases = {
  async 'legacy-crash'([dist, host, out]) {
    const { GfsDownloadStore } = require(path.join(dist, 'internalTools/gfsDownloadStore.js'))
    fs.mkdirSync(host, { recursive: true, mode: 0o700 })
    const store = new GfsDownloadStore(host)
    await store.initialize()
    check(store.isAvailable(), 'legacy-crash: dev store is available on a new volume')

    const completedA = await completeDownload(store, host, 'caller-dev-a', 1)
    const missingA = await completeDownload(store, host, 'caller-dev-a', 2)
    const completedB = await completeDownload(store, host, 'caller-dev-b', 3)
    // A shell command rewrites a published copy with the Host UID; the next
    // reuse check finds the digest changed and marks the record missing.
    fs.writeFileSync(
      path.join(missingA.root, missingA.receipt.path),
      Buffer.alloc(missingA.bytes.byteLength, 0xee)
    )
    const reused = await store.reusableReceipt('caller-dev-a', source(2), missingA.bytes.byteLength)
    check(reused === undefined, 'legacy-crash: a rewritten copy is not reused')
    check(
      store.debugRecord(missingA.receipt.id)?.state === 'missing',
      'legacy-crash: the rewritten copy is recorded missing',
      store.debugRecord(missingA.receipt.id)
    )
    const transferring = await startTransfer(store, host, 'caller-dev-a', 4)

    // A processing lease that runs out before the crash, then one that does not.
    const expired = await store.processingLeaseProvider('caller-dev-b').acquireProcessingLease({
      durationMs: 1_500,
    })
    await new Promise(resolve => setTimeout(resolve, 2_000))
    check(Date.parse(expired.expiresAt) < Date.now(), 'legacy-crash: the first lease has expired')
    const live = await store.processingLeaseProvider('caller-dev-a').acquireProcessingLease({
      durationMs: 3_600_000,
    })
    check(Date.parse(live.expiresAt) > Date.now(), 'legacy-crash: the second lease is live')

    const ledger = JSON.parse(
      fs.readFileSync(path.join(host, '.gfs-download-store', 'ledger-v1.json'), 'utf8')
    )
    const states = Object.fromEntries(
      Object.values(ledger.records).map(record => [record.id, record.state])
    )
    check(
      Object.keys(ledger.processingLeases).length === 2,
      'legacy-crash: the ledger holds both processing leases',
      ledger.processingLeases
    )
    check(
      states[completedA.receipt.id] === 'completed' &&
        states[completedB.receipt.id] === 'completed' &&
        states[missingA.receipt.id] === 'missing' &&
        states[transferring.transfer.id] === 'transferring',
      'legacy-crash: the ledger holds completed, missing and transferring records',
      states
    )
    results.generatedAtMs = Date.now()
    results.records = states
    results.leases = { expired: expired.leaseId, live: live.leaseId }
    crash(out)
  },

  async 'legacy-restart'([dist, host, out]) {
    const { bootstrapGfsRuntime } = require(path.join(dist, 'gfsRuntime.js'))
    const before = JSON.parse(
      fs.readFileSync(path.join(host, '.gfs-download-store', 'ledger-v1.json'), 'utf8')
    )
    const runtime = await bootstrapGfsRuntime(host, { retryDelayMs: 25, maxRetryAttempts: 1 })
    const available = runtime.store.isAvailable()
    // The #1019 state: leases of the killed writer are inherited executors, so
    // the dev Host quarantines every record and disables managed operations.
    check(available === false, 'legacy-restart: the dev Host is unavailable on its own crash state')
    const states = Object.fromEntries(
      Object.keys(before.records).map(id => [id, runtime.store.debugRecord(id)?.state])
    )
    check(
      Object.values(states).length === 4 && Object.values(states).every(s => s === 'quarantined'),
      'legacy-restart: every record is quarantined',
      states
    )
    await runtime.stop()
    const after = JSON.parse(
      fs.readFileSync(path.join(host, '.gfs-download-store', 'ledger-v1.json'), 'utf8')
    )
    check(
      Object.keys(after.processingLeases).length === 2,
      'legacy-restart: the inherited leases stay in the ledger',
      after.processingLeases
    )
    results.oldStoreAvailable = available
    results.records = states
  },

  async 'new-state'([dist, host, out]) {
    // Fault injection for this whole phase: every removal of a retired dev
    // store or of a replaced .gfs-downloads fails with EBUSY, as a busy mount
    // would (the store retries EACCES/EPERM itself, and every sweep retries
    // the removal), so both leftovers are on the volume when the process dies.
    const fsp = require('node:fs/promises')
    const prefixes = ['.gfs-download-store.retired-', '.gfs-downloads.trash-']
    const injected = new Set()
    const realRm = fsp.rm
    fsp.rm = async (target, options) => {
      const name = path.basename(String(target))
      const prefix = prefixes.find(candidate => name.startsWith(candidate))
      if (prefix !== undefined) {
        injected.add(prefix)
        throw Object.assign(new Error('injected rm failure'), { code: 'EBUSY' })
      }
      return realRm(target, options)
    }
    const { GfsDownloadStore } = require(path.join(dist, 'internalTools/gfsDownloadStore.js'))
    const store = new GfsDownloadStore(host)
    await store.initialize()
    check(store.isAvailable(), 'new-state: the #1028 store is available on the dev state')
    check(
      !listHost(host).includes('.gfs-download-store'),
      'new-state: the dev store directory was retired',
      listHost(host)
    )
    const retired = listHost(host).filter(n => n.startsWith('.gfs-download-store.retired-'))
    check(retired.length === 1, 'new-state: one retired dev store is left on disk', listHost(host))

    const published = await completeDownload(store, host, 'caller-new-a', 11)
    await completeDownload(store, host, 'caller-new-b', 12)
    const downloadsB = path.join(host, 'users', 'caller-new-b', '.gfs-downloads')
    // A shell command makes the caller's .gfs-downloads non-private; the next
    // admission replaces it, and the injected failures leave the trash behind.
    fs.chmodSync(downloadsB, 0o755)
    let replacementError
    try {
      await startTransfer(store, host, 'caller-new-b', 13)
    } catch (error) {
      replacementError = error
    }
    const trash = fs
      .readdirSync(path.join(host, 'users', 'caller-new-b'))
      .filter(n => n.startsWith('.gfs-downloads.trash-'))
    check(trash.length === 1, 'new-state: one .gfs-downloads.trash-* is left on disk', trash)
    const incomplete = await startTransfer(store, host, 'caller-new-a', 14)
    check(
      fs.existsSync(path.join(incomplete.root, incomplete.transfer.partialPath)),
      'new-state: an incomplete transfer is on disk'
    )
    check(injected.size === 2, 'new-state: both injected failures fired', [...injected])
    check(
      listHost(host).filter(n => n.startsWith('.gfs-download-store.retired-')).length === 1 &&
        fs
          .readdirSync(path.join(host, 'users', 'caller-new-b'))
          .filter(n => n.startsWith('.gfs-downloads.trash-')).length === 1,
      'new-state: both leftovers are still on disk at the crash'
    )
    results.published = {
      caller: 'caller-new-a',
      path: published.receipt.path,
      id: published.receipt.id,
    }
    results.replacementError = replacementError
      ? { name: replacementError.name, code: replacementError.code }
      : null
    crash(out)
  },

  async 'old-on-new'([dist, host, out, newStateResult]) {
    const previous = JSON.parse(fs.readFileSync(newStateResult, 'utf8'))
    const { bootstrapGfsRuntime } = require(path.join(dist, 'gfsRuntime.js'))
    const runtime = await bootstrapGfsRuntime(host, { retryDelayMs: 25, maxRetryAttempts: 1 })
    const store = runtime.store
    check(store.isAvailable(), 'old-on-new: the dev Host store is available on the #1028 state')
    check(
      fs.existsSync(path.join(host, '.gfs-download-store', 'ledger-v1.json')),
      'old-on-new: the dev store created a new ledger'
    )
    const { receipt, bytes } = await completeDownload(store, host, previous.published.caller, 21)
    const read = await store.readManagedFile(receipt.path, previous.published.caller)
    check(read.equals(bytes), 'old-on-new: the dev store serves its own new download')
    let foreign
    try {
      await store.readManagedFile(previous.published.path, previous.published.caller)
    } catch (error) {
      foreign = error
    }
    check(
      foreign !== undefined && foreign.code === 'caller_mismatch',
      'old-on-new: a copy published by #1028 is unknown to the dev store (caller_mismatch)',
      foreign && { name: foreign.name, code: foreign.code }
    )
    await store.cleanupExpired()
    check(store.isAvailable(), 'old-on-new: the dev store is available after its sweep')
    await runtime.stop()
    results.oldDownload = { id: receipt.id, path: receipt.path }
  },

  async 'roll-forward'([dist, host, out]) {
    const { GfsDownloadStore } = require(path.join(dist, 'internalTools/gfsDownloadStore.js'))
    const store = new GfsDownloadStore(host)
    await store.initialize()
    check(store.isAvailable(), 'roll-forward: the #1028 store is available after a rollback')
    const leftovers = listHost(host).filter(n => n.startsWith('.gfs-download-store'))
    check(
      leftovers.length === 0,
      'roll-forward: the dev store and every retired tree are gone',
      listHost(host)
    )
    const trash = fs
      .readdirSync(path.join(host, 'users', 'caller-new-b'))
      .filter(n => n.startsWith('.gfs-downloads.trash-'))
    check(trash.length === 0, 'roll-forward: the .gfs-downloads.trash-* is gone', trash)
    await completeDownload(store, host, 'caller-new-a', 31)
    await store.close()
    results.hostEntries = listHost(host)
  },

  async manifest([host, out]) {
    const entries = {}
    const walk = relative => {
      const absolute = path.join(host, relative)
      const info = fs.lstatSync(absolute)
      const entry = { mode: (info.mode & 0o777).toString(8) }
      if (info.isDirectory()) {
        entry.type = 'dir'
        for (const name of fs.readdirSync(absolute).sort()) walk(path.join(relative, name))
      } else if (info.isFile()) {
        entry.type = 'file'
        entry.size = info.size
        entry.sha256 = sha256(fs.readFileSync(absolute))
      } else entry.type = 'other'
      if (relative !== '') entries[relative] = entry
    }
    walk('')
    check(Object.keys(entries).length > 0, 'manifest: the host root is not empty')
    results.entries = entries
  },

  /**
   * Copies the two dev-store snapshots into the committed fixture directory.
   * writer.lock names the device and inode of writer-v2.sqlite on the
   * generating machine; both are written as "0" and the test loader rewrites
   * them to the identity of the copied database. Nothing else is changed.
   */
  async 'write-fixture'([
    crashedHost,
    restartedHost,
    crashResult,
    restartResult,
    commit,
    fixture,
    out,
  ]) {
    const crash = JSON.parse(fs.readFileSync(crashResult, 'utf8'))
    const restart = JSON.parse(fs.readFileSync(restartResult, 'utf8'))
    const modes = {}
    for (const [name, host] of [
      ['crashed', crashedHost],
      ['restarted', restartedHost],
    ]) {
      const target = path.join(fixture, name)
      fs.rmSync(target, { recursive: true, force: true })
      fs.cpSync(host, target, { recursive: true })
      const fence = path.join(target, '.gfs-download-store', 'writer.lock')
      const parsed = JSON.parse(fs.readFileSync(fence, 'utf8'))
      check(
        parsed.schemaVersion === 2 && 'databaseDevice' in parsed && 'databaseInode' in parsed,
        `write-fixture: ${name} has a v2 writer fence`,
        parsed
      )
      fs.writeFileSync(
        fence,
        JSON.stringify({ ...parsed, databaseDevice: '0', databaseInode: '0' })
      )
      const walk = relative => {
        const info = fs.lstatSync(path.join(host, relative))
        if (relative !== '') modes[path.join(name, relative)] = (info.mode & 0o777).toString(8)
        if (info.isDirectory())
          for (const entry of fs.readdirSync(path.join(host, relative)).sort())
            walk(path.join(relative, entry))
      }
      walk('')
      for (const file of Object.keys(modes).filter(m => m.startsWith(`${name}/`))) {
        const absolute = path.join(fixture, file)
        if (!fs.lstatSync(absolute).isFile()) continue
        const text = fs.readFileSync(absolute).toString('latin1')
        check(
          !text.includes(crashedHost) && !text.includes(restartedHost) && !text.includes('/Users/'),
          `write-fixture: ${file} holds no absolute path`
        )
      }
    }
    fs.writeFileSync(
      path.join(fixture, 'generation.json'),
      `${JSON.stringify(
        {
          commit,
          generatedAtMs: crash.generatedAtMs,
          crashed: { records: crash.records, leases: crash.leases },
          restarted: { oldStoreAvailable: restart.oldStoreAvailable, records: restart.records },
          modes,
        },
        null,
        2
      )}\n`
    )
    results.files = Object.keys(modes).length
  },

  async compare([before, after, out]) {
    const a = JSON.parse(fs.readFileSync(before, 'utf8')).entries
    const b = JSON.parse(fs.readFileSync(after, 'utf8')).entries
    const removed = Object.keys(a).filter(name => !(name in b))
    const changed = Object.keys(a).filter(
      name => name in b && JSON.stringify(a[name]) !== JSON.stringify(b[name])
    )
    const added = Object.keys(b).filter(name => !(name in a))
    results.removed = removed
    results.changed = changed.map(name => ({ name, before: a[name], after: b[name] }))
    results.added = added
    check(Object.keys(a).length > 0, 'compare: the #1028 state has entries')
    check(
      removed.length === 0,
      'compare: the dev store removed nothing the #1028 store left',
      removed
    )
    check(
      changed.length === 0,
      'compare: the dev store changed nothing the #1028 store left',
      results.changed
    )
  },
}

async function main() {
  const [phase, ...args] = process.argv.slice(2)
  const run = phases[phase]
  if (!run) throw new Error(`unknown phase ${phase}`)
  const out = phase === 'manifest' ? args[1] : phase === 'write-fixture' ? args[6] : args[2]
  if (out === undefined) throw new Error(`phase ${phase}: missing output path`)
  results.phase = phase
  results.runId = randomUUID()
  try {
    await run(args)
  } catch (error) {
    console.error(error)
    writeOut(out, 1)
    process.exit(1)
  }
  writeOut(out, 0)
  process.exit(0)
}

main()