Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
6.4 KB
Raw Blame
/**
 * Copies a Host root written by the real ledger-based GFS download store of
 * dev 74e0d81d9 into a test's Host root, so restart tests can prove the
 * filesystem store never blocks on it.
 *
 * The snapshots in ./dev-gfs-store-74e0d81d9 are generated by
 * scripts/dev/gfs-store-rollback-check.sh --write-fixture, which runs that
 * commit's store (see the README there). Two normalisations are undone here:
 * writer.lock is rewritten to name the copied writer-v2.sqlite, and every
 * ledger timestamp moves forward by the time elapsed since generation, so the
 * live lease is live and the expired one expired.
 *
 * Synchronous `node:fs` only: tests that mock `node:fs/promises` to observe the
 * store never count the fixture's own calls.
 */
import * as fs from 'node:fs'
import * as path from 'node:path'

/**
 * `crashed`: completed, missing and transferring records, an expired and a
 * live processing lease, the writer killed. `restarted`: the same volume after
 * the dev Host restarted on it: every record quarantined and the store
 * unavailable (#1019).
 */
export type DevSnapshot = 'crashed' | 'restarted'

export interface DevLease {
  leaseId: string
  callerIdentity: string
  recordIds: string[]
  acquiredAt: string
  expiresAt: string
  writerSessionId: string
}

interface DevRecord {
  id: string
  callerIdentity: string
  directory: string
  createdAt: string
  expiresAt: string
  state: string
}

interface DevLedger {
  schemaVersion: 1
  records: Record<string, DevRecord>
  processingLeases: Record<string, DevLease>
  retentionOwners?: Record<string, unknown>
}

interface Generation {
  commit: string
  generatedAtMs: number
  crashed: { records: Record<string, string>; leases: { expired: string; live: string } }
  restarted: { oldStoreAvailable: boolean; records: Record<string, string> }
  modes: Record<string, string>
}

export interface DevStore {
  /** `<root>/.gfs-download-store`. */
  storeRoot: string
  /** The ledger-v1.json text as written into the Host root (timestamps shifted). */
  ledgerText: string
  ledger: DevLedger
  /** Record states the dev store left in this snapshot, by record id. */
  recordStates: Record<string, string>
  leases: { expired: DevLease; live: DevLease }
  /** Whether the dev store was available on this exact state (`restarted` only). */
  oldStoreAvailable: boolean | undefined
  /** Absolute `input-<uuid>` directories copied under the caller roots. */
  userDirectories: string[]
  /** Bytes of every source and source.partial file copied. */
  userBytes: number
  /** The caller whose root holds completed, missing and transferring copies. */
  callerIdentity: string
  callerRoot: string
}

export const DEV_STORE_COMMIT = '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e'
const FIXTURE = path.join(__dirname, 'dev-gfs-store-74e0d81d9')
const CALLER = 'caller-dev-a'

function readGeneration(): Generation {
  const generation = JSON.parse(
    fs.readFileSync(path.join(FIXTURE, 'generation.json'), 'utf8')
  ) as Generation
  if (generation.commit !== DEV_STORE_COMMIT)
    throw new Error(
      `dev store fixture: generated from ${generation.commit}, not ${DEV_STORE_COMMIT}`
    )
  return generation
}

function shifted(timestamp: string, offsetMs: number): string {
  const value = Date.parse(timestamp)
  if (!Number.isFinite(value)) throw new Error(`dev store fixture: bad timestamp ${timestamp}`)
  return new Date(value + offsetMs).toISOString()
}

export function loadDevStore(root: string, snapshot: DevSnapshot): DevStore {
  const generation = readGeneration()
  const source = path.join(FIXTURE, snapshot)
  fs.mkdirSync(root, { recursive: true, mode: 0o700 })
  // Copy entry by entry: from Node 24.22, cpSync with errorOnExist rejects an
  // existing destination root with ERR_FS_CP_EEXIST even when it is empty.
  for (const entry of fs.readdirSync(source))
    fs.cpSync(path.join(source, entry), path.join(root, entry), {
      recursive: true,
      errorOnExist: true,
      force: false,
    })

  // Git keeps no modes: restore the ones the dev store wrote, deepest first so
  // a 0700 directory never blocks its children.
  const modes = Object.entries(generation.modes)
    .filter(([name]) => name.startsWith(`${snapshot}/`))
    .sort(([a], [b]) => b.length - a.length)
  if (modes.length === 0) throw new Error(`dev store fixture: no modes for ${snapshot}`)
  for (const [name, mode] of modes)
    fs.chmodSync(path.join(root, name.slice(snapshot.length + 1)), Number.parseInt(mode, 8))

  const storeRoot = path.join(root, '.gfs-download-store')
  const database = fs.statSync(path.join(storeRoot, 'writer-v2.sqlite'), { bigint: true })
  const fencePath = path.join(storeRoot, 'writer.lock')
  const fence = JSON.parse(fs.readFileSync(fencePath, 'utf8')) as Record<string, unknown>
  fs.writeFileSync(
    fencePath,
    JSON.stringify({
      ...fence,
      databaseDevice: String(database.dev),
      databaseInode: String(database.ino),
    })
  )

  const ledgerPath = path.join(storeRoot, 'ledger-v1.json')
  const ledger = JSON.parse(fs.readFileSync(ledgerPath, 'utf8')) as DevLedger
  const offsetMs = Date.now() - generation.generatedAtMs
  for (const record of Object.values(ledger.records)) {
    record.createdAt = shifted(record.createdAt, offsetMs)
    record.expiresAt = shifted(record.expiresAt, offsetMs)
  }
  for (const lease of Object.values(ledger.processingLeases)) {
    lease.acquiredAt = shifted(lease.acquiredAt, offsetMs)
    lease.expiresAt = shifted(lease.expiresAt, offsetMs)
  }
  const ledgerText = JSON.stringify(ledger)
  fs.writeFileSync(ledgerPath, ledgerText)

  const expired = ledger.processingLeases[generation.crashed.leases.expired]
  const live = ledger.processingLeases[generation.crashed.leases.live]
  if (expired === undefined || live === undefined)
    throw new Error(`dev store fixture: ${snapshot} lacks the generated leases`)

  const userDirectories: string[] = []
  let userBytes = 0
  for (const record of Object.values(ledger.records)) {
    const directory = path.join(root, record.directory)
    userDirectories.push(directory)
    for (const name of fs.readdirSync(directory))
      userBytes += fs.statSync(path.join(directory, name)).size
  }

  return {
    storeRoot,
    ledgerText,
    ledger,
    recordStates: generation[snapshot].records,
    leases: { expired, live },
    oldStoreAvailable:
      snapshot === 'restarted' ? generation.restarted.oldStoreAvailable : undefined,
    userDirectories,
    userBytes,
    callerIdentity: CALLER,
    callerRoot: path.join(root, 'users', CALLER),
  }
}