Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

28.3 KB
Raw Blame
import { createHash } from "node:crypto";
import { gzipSync } from "node:zlib";
import type { Deployment, DeploymentVersion } from "./deploy-store.ts";
import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";
import { readTree } from "./deploy-fs.ts";
import { makeTar } from "../sandbox/tar.ts";
import { sleep } from "../util/async.ts";
import { swallow, swallowAs } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import {
  appBoot,
  appGone,
  appLogs,
  appReporter,
  recentAppChecks,
  waitAppReady,
  type AppErrorSink,
  type Exec,
} from "./shared-deploy-provider.ts";

const MACHINES_API_BASE_URL = "https://api.machines.dev/v1";
const DEFAULT_REGION = "lhr";
const APP_PORT = 8080;
const APP_DIR = "/app";
const BUNDLE_GUEST_PATH = "/app.tar.gz";
const MAX_BUNDLE_BASE64_BYTES = 2_000_000;
const MAX_BUNDLE_SOURCE_BYTES = 20_000_000;
const GUEST = { cpu_kind: "shared", cpus: 1, memory_mb: 512 };
const MACHINE_START_TIMEOUT_MS = 90_000;
const APP_READY_TIMEOUT_MS = 90_000;
const POLL_INTERVAL_MS = 1_000;
const API_TIMEOUT_MS = 30_000;

interface FlyApp {
  name: string;
  network: string;
  organization?: { slug?: string };
}

interface FlyIpAssignment {
  ip: string;
  network?: { name?: string; org_slug?: string };
}

interface FlyNetworkPolicy {
  netpolSelector?: { all?: boolean };
  rules?: Array<{
    action?: string;
    direction?: string;
    ports?: Array<{ protocol?: string; port?: number }>;
  }>;
}

interface FlyMachineExitEvent {
  exit_code?: number;
  oom_killed?: boolean;
}

export interface FlyMachine {
  id: string;
  state: string;
  config?: FlyMachineConfig;
  checks?: Array<{ name?: string; status?: string; output?: string }>;
  events?: Array<{ request?: { exit_event?: FlyMachineExitEvent } }>;
}

export interface FlyMachineConfig {
  image: string;
  metadata?: Record<string, string>;
  mounts?: Array<{ volume: string; path: string }>;
  env: Record<string, string>;
  guest: { cpu_kind: string; cpus: number; memory_mb: number };
  files: Array<{ guest_path: string; raw_value: string }>;
  services: Array<{
    protocol: "tcp";
    internal_port: number;
    autostop: "suspend";
    autostart: true;
    min_machines_running: number;
    ports: Array<{ port: number }>;
    checks: Array<{ type: "tcp"; interval: string; timeout: string; grace_period: string }>;
  }>;
  init: { exec: string[] };
}

interface FlyMachinesApi {
  startMachine(appName: string, machineId: string): Promise<void>;
  hasVolumes(appName: string): Promise<boolean>;
  ensureVolume(appName: string, region: string, sizeGb: number, name: string): Promise<string>;
  updateMachine(appName: string, machineId: string, config: FlyMachineConfig): Promise<void>;
  ensureApp(appName: string, orgSlug: string): Promise<void>;
  ensurePrivateIngress(appName: string, provisioned: boolean, orgSlug: string): Promise<void>;
  assertOwnedApp(appName: string, orgSlug: string): Promise<boolean>;
  deleteApp(appName: string): Promise<void>;
  listMachines(appName: string): Promise<FlyMachine[]>;
  createMachine(
    appName: string,
    input: { region: string; config: FlyMachineConfig; skip_service_registration: true },
  ): Promise<FlyMachine>;
  getMachine(appName: string, machineId: string): Promise<FlyMachine | null>;
  setCordon(appName: string, machineId: string, cordoned: boolean): Promise<void>;
  destroyMachine(appName: string, machineId: string): Promise<void>;
  exec(appName: string, machineId: string, script: string, timeoutSec: number): ReturnType<Exec>;
}

interface FlyApiResponse {
  ok: boolean;
  status: number;
  text: string;
}

function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch }): FlyMachinesApi {
  const fetchImpl = opts.fetchImpl ?? fetch;
  const request = async (
    method: string,
    path: string,
    body?: unknown,
    timeoutMs = API_TIMEOUT_MS,
  ): Promise<FlyApiResponse> => {
    const res = await fetchImpl(`${MACHINES_API_BASE_URL}${path}`, {
      method,
      headers: {
        authorization: `Bearer ${opts.token}`,
        ...(body === undefined ? {} : { "content-type": "application/json" }),
      },
      ...(body === undefined ? {} : { body: JSON.stringify(body) }),
      signal: AbortSignal.timeout(timeoutMs),
    });
    return { ok: res.ok, status: res.status, text: await res.text() };
  };
  const failure = (what: string, r: FlyApiResponse): Error =>
    new Error(`fly ${what}: http ${r.status} ${r.text.slice(0, 200)}`);
  const parse = <T>(what: string, r: FlyApiResponse): T => {
    try {
      return JSON.parse(r.text) as T;
    } catch {
      throw new Error(`fly ${what}: unreadable response: ${r.text.slice(0, 200)}`);
    }
  };
  const app = (appName: string): string => `/apps/${encodeURIComponent(appName)}`;
  const machine = (appName: string, machineId: string): string =>
    `${app(appName)}/machines/${encodeURIComponent(machineId)}`;
  const getApp = async (appName: string): Promise<FlyApp | null> => {
    const r = await request("GET", app(appName));
    if (r.status === 404) return null;
    if (!r.ok) throw failure(`read app ${appName}`, r);
    return parse<FlyApp>(`read app ${appName}`, r);
  };
  const assertOwned = (appName: string, orgSlug: string, found: FlyApp): void => {
    if (found.name !== appName || found.organization?.slug !== orgSlug || found.network !== appName) {
      throw new Error(
        `fly app ${appName} already exists but is not the isolated app owned by this deployment; choose another FLY_DEPLOY_APP_PREFIX`,
      );
    }
  };
  return {
    async startMachine(appName, machineId): Promise<void> {
      const started = await request("POST", `${machine(appName, machineId)}/start`);
      if (!started.ok) throw failure(`start machine ${machineId}`, started);
    },
    async hasVolumes(appName): Promise<boolean> {
      const listed = await request("GET", `${app(appName)}/volumes`);
      if (!listed.ok) throw failure(`list volumes in ${appName}`, listed);
      return parse<unknown[]>(`list volumes in ${appName}`, listed).length > 0;
    },
    async ensureVolume(appName, region, sizeGb, name): Promise<string> {
      const path = `${app(appName)}/volumes`;
      const listed = await request("GET", path);
      if (!listed.ok) throw failure(`list volumes in ${appName}`, listed);
      const volumes = parse<Array<{ id: string; name: string; region: string }>>(
        `list volumes in ${appName}`,
        listed,
      ).filter((volume) => name === "qm_data" || volume.name === name);
      if (volumes.length) {
        if (volumes.length !== 1 || volumes[0]!.name !== name || volumes[0]!.region !== region) {
          throw new Error(`fly app ${appName} has unexpected volumes; reconcile storage before deploying`);
        }
        return volumes[0]!.id;
      }
      const created = await request("POST", path, {
        name,
        region,
        size_gb: sizeGb,
        encrypted: true,
        snapshot_retention: 5,
        auto_backup_enabled: true,
        compute: GUEST,
      });
      if (!created.ok) throw failure(`create volume in ${appName}`, created);
      return parse<{ id: string }>(`create volume in ${appName}`, created).id;
    },
    async updateMachine(appName, machineId, config): Promise<void> {
      const updated = await request("POST", machine(appName, machineId), { config, skip_launch: false });
      if (!updated.ok) throw failure(`update machine ${machineId}`, updated);
      const state = parse<FlyMachine>(`update machine ${machineId}`, updated).state;
      if (state === "stopped" || state === "suspended") {
        const started = await request("POST", `${machine(appName, machineId)}/start`);
        if (!started.ok) throw failure(`start updated machine ${machineId}`, started);
      }
    },
    async ensureApp(appName, orgSlug): Promise<void> {
      const existing = await getApp(appName);
      if (existing) {
        assertOwned(appName, orgSlug, existing);
        return;
      }
      const r = await request("POST", "/apps", { app_name: appName, org_slug: orgSlug, network: appName });
      if (r.ok) return;
      const found = await getApp(appName).catch((e: unknown) => {
        throw new Error(failure(`create app ${appName}`, r).message, { cause: e });
      });
      if (!found) throw failure(`create app ${appName}`, r);
      assertOwned(appName, orgSlug, found);
    },
    async ensurePrivateIngress(appName, provisioned, orgSlug): Promise<void> {
      const listed = await request("GET", `${app(appName)}/ip_assignments`);
      if (!listed.ok) throw failure(`list IP assignments for ${appName}`, listed);
      const ips = parse<{ ips: FlyIpAssignment[] }>(`list IP assignments for ${appName}`, listed).ips;
      if (ips.some((entry) => !entry.ip.toLowerCase().startsWith("fdaa:"))) {
        throw new Error(`fly app ${appName} has a public IP assignment; refusing to expose the published app`);
      }
      if (provisioned && ips.length) {
        if (
          ips.some(
            ({ network }) =>
              !network?.name || network.name === "default" || network.name === appName || network.org_slug !== orgSlug,
          )
        ) {
          throw new Error(`Fly shared app ${appName} requires ingress on a separate private network in ${orgSlug}`);
        }
        const response = await request("GET", `${app(appName)}/network_policies`);
        if (!response.ok) throw failure(`read network policies for ${appName}`, response);
        const policies = parse<FlyNetworkPolicy[]>(`read network policies for ${appName}`, response);
        const ingress = (policy: FlyNetworkPolicy) =>
          policy.rules?.filter((rule) => rule.direction === "ingress") ?? [];
        if (
          !policies.some(
            (policy) =>
              policy.netpolSelector?.all === true &&
              Object.keys(policy.netpolSelector).length === 1 &&
              ingress(policy).length > 0,
          ) ||
          policies.some((policy) =>
            ingress(policy).some(
              (rule) =>
                rule.action !== "allow" ||
                !rule.ports?.length ||
                rule.ports.some((port) => port.protocol !== "tcp" || port.port !== 22),
            ),
          )
        ) {
          throw new Error(`Fly shared app ${appName} requires all-machine ingress restricted to TCP port 22`);
        }
      }
      if (ips.length) return;
      if (provisioned) throw new Error(`Fly shared app ${appName} needs private ingress provisioned before publishing`);
      const assigned = await request("POST", `${app(appName)}/ip_assignments`, { type: "private_v6" });
      if (!assigned.ok) throw failure(`allocate private ingress for ${appName}`, assigned);
    },
    async assertOwnedApp(appName, orgSlug): Promise<boolean> {
      const found = await getApp(appName);
      if (!found) return false;
      assertOwned(appName, orgSlug, found);
      return true;
    },
    async deleteApp(appName): Promise<void> {
      const r = await request("DELETE", `${app(appName)}?force=true`);
      if (r.ok || r.status === 404) return;
      throw failure(`delete app ${appName}`, r);
    },
    async listMachines(appName): Promise<FlyMachine[]> {
      const r = await request("GET", `${app(appName)}/machines`);
      if (r.status === 404) return [];
      if (!r.ok) throw failure(`list machines in ${appName}`, r);
      return parse<FlyMachine[]>(`list machines in ${appName}`, r);
    },
    async createMachine(appName, input): Promise<FlyMachine> {
      const r = await request("POST", `${app(appName)}/machines`, input);
      if (!r.ok) throw failure(`create machine in ${appName}`, r);
      return parse<FlyMachine>(`create machine in ${appName}`, r);
    },
    async getMachine(appName, machineId): Promise<FlyMachine | null> {
      const r = await request("GET", machine(appName, machineId));
      if (r.status === 404) return null;
      if (!r.ok) throw failure(`get machine ${machineId}`, r);
      return parse<FlyMachine>(`get machine ${machineId}`, r);
    },
    async setCordon(appName, machineId, cordoned): Promise<void> {
      const action = cordoned ? "cordon" : "uncordon";
      const r = await request("POST", `${machine(appName, machineId)}/${action}`);
      if (r.ok || (cordoned && r.status === 404)) return;
      throw failure(`${action} machine ${machineId}`, r);
    },
    async destroyMachine(appName, machineId): Promise<void> {
      const r = await request("DELETE", `${machine(appName, machineId)}?force=true`);
      if (r.ok || r.status === 404) return;
      throw failure(`destroy machine ${machineId}`, r);
    },
    async exec(appName, machineId, script, timeoutSec) {
      const what = `exec in machine ${machineId}`;
      const r = await request(
        "POST",
        `${machine(appName, machineId)}/exec`,
        { cmd: `sh -c ${shq(script)}`, timeout: timeoutSec },
        (timeoutSec + 10) * 1000,
      );
      if (!r.ok) throw failure(what, r);
      const out = parse<{ exit_code?: number; stdout?: string; stderr?: string }>(what, r);
      return { code: out.exit_code ?? 0, stdout: out.stdout ?? "", stderr: out.stderr ?? "" };
    },
  };
}

function exitDetail(machine: FlyMachine | null): string {
  const exit = machine?.events?.find((e) => e.request?.exit_event)?.request?.exit_event;
  if (!exit) return "; the machine reported no exit event";
  const parts = [`exit code ${exit.exit_code ?? "unknown"}`];
  if (exit.oom_killed) parts.push("killed for running out of memory");
  return `; last machine exit event: ${parts.join(", ")}`;
}

export interface FlyDeployProviderOptions {
  token: string;
  appPrefix: string;
  sharedAppName?: string;
  portStore?: DurableMap<string>;
  privateTransport?: { ensure(): Promise<number> };
  baseImage: string;
  org: string;
  region?: string;
  dataVolumeSizeGb?: number;
  configStore?: DurableMap<FlyMachineConfig>;
  machineStartTimeoutMs?: number;
  appReadyTimeoutMs?: number;
  pollIntervalMs?: number;
  fetchImpl?: typeof fetch;
  onError?: AppErrorSink;
}

export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployProvider {
  const region = opts.region ?? DEFAULT_REGION;
  const api = createFlyMachinesApi({
    token: opts.token,
    ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
  });
  const machineStartTimeoutMs = opts.machineStartTimeoutMs ?? MACHINE_START_TIMEOUT_MS;
  const appReadyTimeoutMs = opts.appReadyTimeoutMs ?? APP_READY_TIMEOUT_MS;
  const pollIntervalMs = opts.pollIntervalMs ?? POLL_INTERVAL_MS;
  const seconds = (ms: number): string => `${Math.round(ms / 1000)}s`;
  const execIn =
    (appName: string, machineId: string): Exec =>
    (script, timeoutSec) =>
      api.exec(appName, machineId, script, timeoutSec);
  const checked = recentAppChecks();

  function ensureConfigured(): void {
    if (
      opts.sharedAppName &&
      (opts.sharedAppName.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.sharedAppName))
    )
      throw new Error("Fly shared app name must be a lowercase DNS label no longer than 63 characters");
    if (opts.sharedAppName && (!opts.portStore || !opts.dataVolumeSizeGb || !opts.configStore))
      throw new Error("Fly shared apps require persistent port assignments and durable storage");
    if (opts.dataVolumeSizeGb && !opts.configStore)
      throw new Error("Fly durable storage requires a persistent configuration store");
    if (
      opts.dataVolumeSizeGb !== undefined &&
      (!Number.isInteger(opts.dataVolumeSizeGb) || opts.dataVolumeSizeGb < 1)
    ) {
      throw new Error("FLY_DEPLOY_DATA_VOLUME_SIZE_GB must be a positive integer");
    }
    if (!opts.token) throw new Error("FLY_DEPLOY_API_TOKEN not set (DEPLOY_PROVIDER=fly)");
    if (!opts.appPrefix && !opts.sharedAppName) throw new Error("FLY_DEPLOY_APP_PREFIX not set (DEPLOY_PROVIDER=fly)");
    if (!opts.baseImage) throw new Error("FLY_DEPLOY_BASE_IMAGE not set (DEPLOY_PROVIDER=fly)");
    if (!opts.org) throw new Error("FLY_ORG not set (DEPLOY_PROVIDER=fly)");
    if (
      !opts.sharedAppName &&
      (opts.appPrefix.length > 26 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.appPrefix))
    ) {
      throw new Error("FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters");
    }
  }

  const appNameFor = (d: Deployment): string => opts.sharedAppName ?? `${opts.appPrefix}-${d.id}`;
  const endpointFor = async (d: Deployment, port: number): Promise<DeployEndpoint> => ({
    host: `${appNameFor(d)}.flycast`,
    port,
    ...(opts.privateTransport ? { socksProxyPort: await opts.privateTransport.ensure() } : {}),
  });
  const volumeNameFor = (d: Deployment): string =>
    opts.sharedAppName ? `qm_${createHash("sha256").update(d.id).digest("hex").slice(0, 24)}` : "qm_data";
  const deploymentMachines = async (d: Deployment): Promise<FlyMachine[]> => {
    const machines = await api.listMachines(appNameFor(d));
    return opts.sharedAppName
      ? machines.filter((machine) => machine.config?.metadata?.qm_deployment_id === d.id)
      : machines;
  };
  async function portFor(d: Deployment): Promise<number> {
    if (!opts.sharedAppName) return APP_PORT;
    const first = createHash("sha256").update(d.id).digest().readUInt32BE(0) % 45536;
    for (let offset = 0; offset < 45536; offset++) {
      const port = 20000 + ((first + offset) % 45536);
      if ((await opts.portStore!.putIfAbsent(`${opts.sharedAppName}:${port}`, d.id)) === d.id) return port;
    }
    throw new Error("Fly shared app has no available deployment ports");
  }

  async function bundleBase64(version: DeploymentVersion): Promise<string> {
    const tree = await readTree(version.snapshotDir, { tolerateMissing: true });
    const encoded = gzipSync(await makeTar(tree)).toString("base64");
    if (encoded.length > MAX_BUNDLE_BASE64_BYTES) {
      throw new Error(
        `the app bundle is too large for the Fly deploy provider: ${encoded.length} bytes once packed and encoded, ` +
          `maximum ${MAX_BUNDLE_BASE64_BYTES} bytes — publish fewer or smaller files`,
      );
    }
    return encoded;
  }

  const machineConfig = (
    d: Deployment,
    version: DeploymentVersion,
    bundle: string,
    port: number,
  ): FlyMachineConfig => ({
    image: opts.baseImage,
    ...(opts.sharedAppName ? { metadata: { qm_deployment_id: d.id } } : {}),
    env: { ...version.env, PORT: String(APP_PORT) },
    guest: GUEST,
    files: [{ guest_path: BUNDLE_GUEST_PATH, raw_value: bundle }],
    services: [
      {
        protocol: "tcp",
        internal_port: APP_PORT,
        autostop: "suspend",
        autostart: true,
        min_machines_running: d.alwaysOn ? 1 : 0,
        ports: [{ port }],
        checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],
      },
    ],
    init: {
      exec: [
        "/bin/sh",
        "-lc",
        appBoot(
          `cd ${APP_DIR}; ${version.entrypoint}`,
          APP_PORT,
          `mkdir -p ${APP_DIR} && tar -xzf ${BUNDLE_GUEST_PATH} -C ${APP_DIR}`,
        ),
      ],
    },
  });

  async function waitStarted(appName: string, machineId: string): Promise<void> {
    const deadline = Date.now() + machineStartTimeoutMs;
    let last: FlyMachine | null = null;
    let requestedStart = false;
    while (Date.now() < deadline) {
      last = await api.getMachine(appName, machineId);
      if (last?.state === "started") return;
      if (!requestedStart && (last?.state === "stopped" || last?.state === "suspended")) {
        requestedStart = true;
        await api.startMachine(appName, machineId);
      }
      await sleep(pollIntervalMs);
    }
    throw new Error(
      `fly machine ${machineId} never reached state "started" within ${seconds(machineStartTimeoutMs)} ` +
        `(last state: ${last?.state ?? "unknown"})${exitDetail(last)}`,
    );
  }

  async function waitServing(
    d: Deployment,
    env: Record<string, string> | undefined,
    appName: string,
    machineId: string,
  ): Promise<void> {
    await waitAppReady(execIn(appName, machineId), {
      appPort: APP_PORT,
      windowSec: Math.ceil(appReadyTimeoutMs / 1000),
      report: appReporter(opts.onError, d, env),
    });
    const deadline = Date.now() + appReadyTimeoutMs;
    let machine: FlyMachine | null = null;
    while (Date.now() < deadline) {
      machine = await api.getMachine(appName, machineId);
      if (machine?.state === "started" && machine.checks?.some((check) => check.status === "passing")) return;
      await sleep(pollIntervalMs);
    }
    machine = await api
      .getMachine(appName, machineId)
      .catch(swallowAs<FlyMachine | null>("fly-deploy: read machine after readiness timeout", machine));
    const why =
      machine && machine.state !== "started"
        ? `the version's entrypoint exited without binding port ${APP_PORT} (fly machine ${machineId} is ${machine.state})`
        : `the app never listened on port ${APP_PORT} within ${seconds(appReadyTimeoutMs)}`;
    throw new Error(`${why}${exitDetail(machine)}`);
  }

  return {
    profile: {
      managedScaleToZero: !!opts.sharedAppName,
      maxAppBytes: MAX_BUNDLE_SOURCE_BYTES,
      ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}),
    },

    async resolveEndpoint(d, version) {
      if (!checked.has(d.id)) {
        ensureConfigured();
        const machines = await deploymentMachines(d).catch(
          swallowAs<FlyMachine[]>("fly-deploy: list app machines", []),
        );
        const appName = appNameFor(d);
        const report = appReporter(opts.onError, d, version.env);
        if (
          machines.length === 1 &&
          machines[0]!.state === "started" &&
          (await appGone(execIn(appName, machines[0]!.id), { appPort: APP_PORT, report }))
        )
          return null;
        checked.set(d.id, true);
      }
      if (!opts.privateTransport) return d.endpoint;
      return endpointFor(d, await portFor(d));
    },

    async logs(d, logOpts): Promise<string | null> {
      ensureConfigured();
      const machines = await deploymentMachines(d);
      if (machines.length !== 1 || machines[0]!.state !== "started") return null;
      return appLogs(execIn(appNameFor(d), machines[0]!.id), logOpts.tailLines);
    },

    async setAlwaysOn(d, alwaysOn): Promise<void> {
      ensureConfigured();
      const appName = appNameFor(d);
      if (!(await api.assertOwnedApp(appName, opts.org))) throw new Error(`fly app ${appName} is missing`);
      const machines = await deploymentMachines(d);
      if (machines.length !== 1) throw new Error(`fly app ${appName} requires one machine to change always-on`);
      const machine = await api.getMachine(appName, machines[0]!.id);
      if (!machine?.config) throw new Error(`fly app ${appName} has no machine configuration`);
      const accepted = machine.config.mounts?.length
        ? ((await opts.configStore?.get(d.id)) ?? machine.config)
        : machine.config;
      const config = {
        ...accepted,
        services: accepted.services.map((service) => ({ ...service, min_machines_running: alwaysOn ? 1 : 0 })),
      };
      try {
        await api.updateMachine(appName, machine.id, config);
        await waitStarted(appName, machine.id);
        await waitServing(d, config.env, appName, machine.id);
        await api.setCordon(appName, machine.id, false);
        if (opts.configStore) await opts.configStore.put(d.id, config);
      } catch (error) {
        await api.updateMachine(appName, machine.id, accepted);
        await waitStarted(appName, machine.id);
        await waitServing(d, accepted.env, appName, machine.id);
        if (opts.configStore) await opts.configStore.put(d.id, accepted);
        throw error;
      }
    },

    async apply(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint> {
      ensureConfigured();
      const appName = appNameFor(d);
      const bundle = await bundleBase64(version);
      if (opts.sharedAppName) {
        if (!(await api.assertOwnedApp(appName, opts.org)))
          throw new Error(`Fly shared app ${appName} must be provisioned before publishing`);
      } else {
        await api.ensureApp(appName, opts.org);
      }
      await api.ensurePrivateIngress(appName, Boolean(opts.sharedAppName), opts.org);
      const stale = await deploymentMachines(d);
      const port = await portFor(d);
      if (!opts.dataVolumeSizeGb && (await api.hasVolumes(appName))) {
        throw new Error(
          `fly app ${appName} has durable storage; restore FLY_DEPLOY_DATA_VOLUME_SIZE_GB before deploying`,
        );
      }
      if (opts.dataVolumeSizeGb) {
        if (stale.length > 1) throw new Error(`fly app ${appName} has multiple machines; reconcile before deploying`);
        const previous = stale[0] ? await api.getMachine(appName, stale[0].id) : null;
        if (stale.length && !previous?.config?.mounts?.some((mount) => mount.path === "/data")) {
          throw new Error(`fly app ${appName} requires an explicit migration from ephemeral storage`);
        }
        const volume = await api.ensureVolume(appName, region, opts.dataVolumeSizeGb, volumeNameFor(d));
        if (previous && (previous.config!.mounts!.length !== 1 || previous.config!.mounts![0]!.volume !== volume)) {
          throw new Error(`fly app ${appName} has an unexpected volume attachment`);
        }
        const config = machineConfig(d, version, bundle, port);
        config.env.DATA_DIR = "/data";
        config.mounts = [{ volume, path: "/data" }];
        if (previous) {
          const accepted = await opts.configStore!.get(d.id);
          try {
            await api.updateMachine(appName, previous.id, config);
            await waitStarted(appName, previous.id);
            await waitServing(d, config.env, appName, previous.id);
            await api.setCordon(appName, previous.id, false);
            await opts.configStore!.put(d.id, config);
          } catch (error) {
            if (accepted) {
              await api.updateMachine(appName, previous.id, accepted);
              await waitStarted(appName, previous.id);
              await waitServing(d, accepted.env, appName, previous.id);
              await api.setCordon(appName, previous.id, false);
              await opts.configStore!.put(d.id, accepted);
            }
            throw error;
          }
        } else {
          const created = await api.createMachine(appName, { region, config, skip_service_registration: true });
          try {
            await waitStarted(appName, created.id);
            await waitServing(d, config.env, appName, created.id);
            await api.setCordon(appName, created.id, false);
            await opts.configStore!.put(d.id, config);
          } catch (error) {
            await api
              .destroyMachine(appName, created.id)
              .catch((cleanupError) =>
                swallow("fly-deploy: remove unhealthy machine while retaining its data", cleanupError),
              );
            throw error;
          }
        }
        return endpointFor(d, port);
      }
      const machine = await api.createMachine(appName, {
        region,
        config: machineConfig(d, version, bundle, port),
        skip_service_registration: true,
      });
      try {
        await waitStarted(appName, machine.id);
        await waitServing(d, version.env, appName, machine.id);
      } catch (error) {
        await api
          .destroyMachine(appName, machine.id)
          .catch((cleanupError) => swallow("fly-deploy: remove unhealthy replacement", cleanupError));
        throw error;
      }
      try {
        for (const previous of stale) {
          await api.setCordon(appName, previous.id, true);
        }
        await api.setCordon(appName, machine.id, false);
      } catch (error) {
        for (const previous of stale) {
          await api
            .setCordon(appName, previous.id, false)
            .catch((rollbackError) => swallow("fly-deploy: restore previous machine routing", rollbackError));
        }
        await api
          .destroyMachine(appName, machine.id)
          .catch((cleanupError) => swallow("fly-deploy: remove failed replacement", cleanupError));
        throw error;
      }
      for (const previous of stale) {
        await api
          .destroyMachine(appName, previous.id)
          .catch((cleanupError) => swallow("fly-deploy: remove cordoned machine", cleanupError));
      }
      return endpointFor(d, port);
    },

    async destroy(d: Deployment): Promise<void> {
      ensureConfigured();
      const appName = appNameFor(d);
      if (!(await api.assertOwnedApp(appName, opts.org))) return;
      if (opts.dataVolumeSizeGb || (await api.hasVolumes(appName))) {
        for (const machine of await deploymentMachines(d)) await api.destroyMachine(appName, machine.id);
        return;
      }
      await api.deleteApp(appName);
    },
  };
}