Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Add a Fly deploy provider so publish works on fly targets (#585)

kai-jmi090f490b5076
diff --git a/src/config.ts b/src/config.ts
index 8714a3fcd..4d1fb16e5 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -37,7 +37,7 @@ export interface Config {
securityPosture: SecurityPosture;
sandboxBackend: "aws" | "local" | "sprites" | "smolmachines";
sandboxSecondaryBackend?: "aws" | "local" | "sprites" | "smolmachines";
- deployProvider: "docker" | "aws";
+ deployProvider: "docker" | "aws" | "fly";
egressServiceHosts?: string[];
brandingDefault?: OrgBranding;
modelId?: string;
@@ -148,6 +148,7 @@ export interface Config {
spritesSandbox: SpritesSandboxEnv;
smolmachinesSandbox: SmolmachinesSandboxEnv;
awsDeploy: AwsDeployEnv;
+ flyDeploy: FlyDeployEnv;
}
export function configuredModelForHarness(config: Config, harness: string): string | undefined {
@@ -410,6 +411,24 @@ function awsDeployEnv(env: NodeJS.ProcessEnv): AwsDeployEnv {
};
}
+interface FlyDeployEnv {
+ token: string;
+ appPrefix: string;
+ baseImage: string;
+ org: string;
+ region?: string;
+}
+
+function flyDeployEnv(env: NodeJS.ProcessEnv): FlyDeployEnv {
+ return {
+ token: env.FLY_DEPLOY_API_TOKEN ?? "",
+ appPrefix: env.FLY_DEPLOY_APP_PREFIX ?? "",
+ baseImage: env.FLY_DEPLOY_BASE_IMAGE ?? "",
+ org: env.FLY_ORG ?? "",
+ ...(env.FLY_REGION ? { region: env.FLY_REGION } : {}),
+ };
+}
+
const DEFAULT_ORG_ID = "default-org";
export function orgId(): string {
@@ -671,7 +690,12 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
}
const publicApiUrl = env.PUBLIC_API_URL ?? env.AGENT_API_URL;
const publicUrl = env.PUBLIC_WEB_URL ?? publicApiUrl;
- const deployProvider: "aws" | "docker" = env.DEPLOY_PROVIDER === "aws" ? "aws" : "docker";
+ const deployProvider = env.DEPLOY_PROVIDER ?? "docker";
+ if (deployProvider !== "aws" && deployProvider !== "docker" && deployProvider !== "fly") {
+ throw new Error(
+ `DEPLOY_PROVIDER=${JSON.stringify(deployProvider)} is not recognized (expected aws, docker, or fly)`,
+ );
+ }
let runStore: "memory" | "postgres" = env.SESSION_STORE === "postgres" ? "postgres" : "memory";
if (env.RUN_STORE === "memory" || env.RUN_STORE === "postgres") runStore = env.RUN_STORE;
const providerBaseUrls = providerBaseUrlsFromEnv(env);
@@ -894,5 +918,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
spritesSandbox: spritesSandboxEnv(env),
smolmachinesSandbox: smolmachinesSandboxEnv(env),
awsDeploy: awsDeployEnv(env),
+ flyDeploy: flyDeployEnv(env),
};
}
diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.ts
index 674e9badb..7075a396f 100644
--- a/src/deploy/deploy-service.ts
+++ b/src/deploy/deploy-service.ts
@@ -167,7 +167,13 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
allPaths,
});
} else {
- endpoint = await deps.provider.apply(d, version);
+ let materialized = version;
+ if (version.commit) {
+ const files = await deps.deployStore.filesOf(id, version.version);
+ if (files == null) throw new Error(`cannot materialize deployment ${id} version ${version.version}`);
+ materialized = { ...version, snapshotDir: await snapshotFiles(deps.deployDir, files) };
+ }
+ endpoint = await deps.provider.apply(d, materialized);
}
if (endpoint.image && endpoint.image !== version.image) {
await deps.deployStore.setVersionImage(id, version.version, endpoint.image);
diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.ts
new file mode 100644
index 000000000..f29147e0e
--- /dev/null
+++ b/src/deploy/fly-deploy-provider.ts
@@ -0,0 +1,354 @@
+import { gzipSync } from "node:zlib";
+import type { Deployment, DeploymentVersion } from "./deploy-store.ts";
+import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";
+import { readTree } from "./deploy-fs.ts";
+import { makeTar } from "../sandbox/tar.ts";
+import { sleep } from "../util/async.ts";
+import { swallow, swallowAs } from "../util/errors.ts";
+import { shq } from "../util/shell.ts";
+
+const MACHINES_API_BASE_URL = "https://api.machines.dev/v1";
+const DEFAULT_REGION = "lhr";
+const APP_PORT = 8080;
+const APP_DIR = "/app";
+const BUNDLE_GUEST_PATH = "/app.tar.gz";
+const MAX_BUNDLE_BASE64_BYTES = 2_000_000;
+const MAX_BUNDLE_SOURCE_BYTES = 20_000_000;
+const GUEST = { cpu_kind: "shared", cpus: 1, memory_mb: 512 };
+const MACHINE_START_TIMEOUT_MS = 90_000;
+const APP_READY_TIMEOUT_MS = 90_000;
+const POLL_INTERVAL_MS = 1_000;
+const API_TIMEOUT_MS = 30_000;
+const APP_ALREADY_TAKEN = /already\s+(exists|been\s+taken)|name\s+is\s+taken/i;
+
+interface FlyApp {
+ name: string;
+ network: string;
+ organization?: { slug?: string };
+}
+
+interface FlyIpAssignment {
+ ip: string;
+}
+
+interface FlyMachineExitEvent {
+ exit_code?: number;
+ oom_killed?: boolean;
+}
+
+export interface FlyMachine {
+ id: string;
+ state: string;
+ checks?: Array<{ name?: string; status?: string; output?: string }>;
+ events?: Array<{ request?: { exit_event?: FlyMachineExitEvent } }>;
+}
+
+export interface FlyMachineConfig {
+ image: string;
+ env: Record<string, string>;
+ guest: { cpu_kind: string; cpus: number; memory_mb: number };
+ files: Array<{ guest_path: string; raw_value: string }>;
+ services: Array<{
+ protocol: "tcp";
+ internal_port: number;
+ ports: Array<{ port: number }>;
+ checks: Array<{ type: "tcp"; interval: string; timeout: string; grace_period: string }>;
+ }>;
+ init: { exec: string[] };
+}
+
+interface FlyMachinesApi {
+ ensureApp(appName: string, orgSlug: string): Promise<void>;
+ ensurePrivateIngress(appName: string): Promise<void>;
+ assertOwnedApp(appName: string, orgSlug: string): Promise<boolean>;
+ deleteApp(appName: string): Promise<void>;
+ listMachines(appName: string): Promise<FlyMachine[]>;
+ createMachine(
+ appName: string,
+ input: { region: string; config: FlyMachineConfig; skip_service_registration: true },
+ ): Promise<FlyMachine>;
+ getMachine(appName: string, machineId: string): Promise<FlyMachine | null>;
+ setCordon(appName: string, machineId: string, cordoned: boolean): Promise<void>;
+ destroyMachine(appName: string, machineId: string): Promise<void>;
+}
+
+interface FlyApiResponse {
+ ok: boolean;
+ status: number;
+ text: string;
+}
+
+function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch }): FlyMachinesApi {
+ const fetchImpl = opts.fetchImpl ?? fetch;
+ const request = async (method: string, path: string, body?: unknown): Promise<FlyApiResponse> => {
+ const res = await fetchImpl(`${MACHINES_API_BASE_URL}${path}`, {
+ method,
+ headers: {
+ authorization: `Bearer ${opts.token}`,
+ ...(body === undefined ? {} : { "content-type": "application/json" }),
+ },
+ ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+ signal: AbortSignal.timeout(API_TIMEOUT_MS),
+ });
+ return { ok: res.ok, status: res.status, text: await res.text() };
+ };
+ const failure = (what: string, r: FlyApiResponse): Error =>
+ new Error(`fly ${what}: http ${r.status} ${r.text.slice(0, 200)}`);
+ const parse = <T>(what: string, r: FlyApiResponse): T => {
+ try {
+ return JSON.parse(r.text) as T;
+ } catch {
+ throw new Error(`fly ${what}: unreadable response: ${r.text.slice(0, 200)}`);
+ }
+ };
+ const app = (appName: string): string => `/apps/${encodeURIComponent(appName)}`;
+ const machine = (appName: string, machineId: string): string =>
+ `${app(appName)}/machines/${encodeURIComponent(machineId)}`;
+ const getApp = async (appName: string): Promise<FlyApp | null> => {
+ const r = await request("GET", app(appName));
+ if (r.status === 404) return null;
+ if (!r.ok) throw failure(`read app ${appName}`, r);
+ return parse<FlyApp>(`read app ${appName}`, r);
+ };
+ const assertOwned = (appName: string, orgSlug: string, found: FlyApp): void => {
+ if (found.name !== appName || found.organization?.slug !== orgSlug || found.network !== appName) {
+ throw new Error(
+ `fly app ${appName} already exists but is not the isolated app owned by this deployment; choose another FLY_DEPLOY_APP_PREFIX`,
+ );
+ }
+ };
+ return {
+ async ensureApp(appName, orgSlug): Promise<void> {
+ const r = await request("POST", "/apps", { app_name: appName, org_slug: orgSlug, network: appName });
+ if (r.ok) return;
+ if (r.status !== 409 && r.status !== 422 && !APP_ALREADY_TAKEN.test(r.text)) {
+ throw failure(`create app ${appName}`, r);
+ }
+ const found = await getApp(appName);
+ if (!found) throw failure(`create app ${appName}`, r);
+ assertOwned(appName, orgSlug, found);
+ },
+ async ensurePrivateIngress(appName): Promise<void> {
+ const listed = await request("GET", `${app(appName)}/ip_assignments`);
+ if (!listed.ok) throw failure(`list IP assignments for ${appName}`, listed);
+ const ips = parse<{ ips: FlyIpAssignment[] }>(`list IP assignments for ${appName}`, listed).ips;
+ if (ips.some((entry) => !entry.ip.toLowerCase().startsWith("fdaa:"))) {
+ throw new Error(`fly app ${appName} has a public IP assignment; refusing to expose the published app`);
+ }
+ if (ips.length) return;
+ const assigned = await request("POST", `${app(appName)}/ip_assignments`, { type: "private_v6" });
+ if (!assigned.ok) throw failure(`allocate private ingress for ${appName}`, assigned);
+ },
+ async assertOwnedApp(appName, orgSlug): Promise<boolean> {
+ const found = await getApp(appName);
+ if (!found) return false;
+ assertOwned(appName, orgSlug, found);
+ return true;
+ },
+ async deleteApp(appName): Promise<void> {
+ const r = await request("DELETE", `${app(appName)}?force=true`);
+ if (r.ok || r.status === 404) return;
+ throw failure(`delete app ${appName}`, r);
+ },
+ async listMachines(appName): Promise<FlyMachine[]> {
+ const r = await request("GET", `${app(appName)}/machines`);
+ if (r.status === 404) return [];
+ if (!r.ok) throw failure(`list machines in ${appName}`, r);
+ return parse<FlyMachine[]>(`list machines in ${appName}`, r);
+ },
+ async createMachine(appName, input): Promise<FlyMachine> {
+ const r = await request("POST", `${app(appName)}/machines`, input);
+ if (!r.ok) throw failure(`create machine in ${appName}`, r);
+ return parse<FlyMachine>(`create machine in ${appName}`, r);
+ },
+ async getMachine(appName, machineId): Promise<FlyMachine | null> {
+ const r = await request("GET", machine(appName, machineId));
+ if (r.status === 404) return null;
+ if (!r.ok) throw failure(`get machine ${machineId}`, r);
+ return parse<FlyMachine>(`get machine ${machineId}`, r);
+ },
+ async setCordon(appName, machineId, cordoned): Promise<void> {
+ const action = cordoned ? "cordon" : "uncordon";
+ const r = await request("POST", `${machine(appName, machineId)}/${action}`);
+ if (r.ok || (cordoned && r.status === 404)) return;
+ throw failure(`${action} machine ${machineId}`, r);
+ },
+ async destroyMachine(appName, machineId): Promise<void> {
+ const r = await request("DELETE", `${machine(appName, machineId)}?force=true`);
+ if (r.ok || r.status === 404) return;
+ throw failure(`destroy machine ${machineId}`, r);
+ },
+ };
+}
+
+function exitDetail(machine: FlyMachine | null): string {
+ const exit = machine?.events?.find((e) => e.request?.exit_event)?.request?.exit_event;
+ if (!exit) return "; the machine reported no exit event";
+ const parts = [`exit code ${exit.exit_code ?? "unknown"}`];
+ if (exit.oom_killed) parts.push("killed for running out of memory");
+ return `; last machine exit event: ${parts.join(", ")}`;
+}
+
+export interface FlyDeployProviderOptions {
+ token: string;
+ appPrefix: string;
+ baseImage: string;
+ org: string;
+ region?: string;
+ machineStartTimeoutMs?: number;
+ appReadyTimeoutMs?: number;
+ pollIntervalMs?: number;
+ fetchImpl?: typeof fetch;
+}
+
+export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployProvider {
+ const region = opts.region ?? DEFAULT_REGION;
+ const api = createFlyMachinesApi({
+ token: opts.token,
+ ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
+ });
+ const machineStartTimeoutMs = opts.machineStartTimeoutMs ?? MACHINE_START_TIMEOUT_MS;
+ const appReadyTimeoutMs = opts.appReadyTimeoutMs ?? APP_READY_TIMEOUT_MS;
+ const pollIntervalMs = opts.pollIntervalMs ?? POLL_INTERVAL_MS;
+ const seconds = (ms: number): string => `${Math.round(ms / 1000)}s`;
+
+ function ensureConfigured(): void {
+ if (!opts.token) throw new Error("FLY_DEPLOY_API_TOKEN not set (DEPLOY_PROVIDER=fly)");
+ if (!opts.appPrefix) throw new Error("FLY_DEPLOY_APP_PREFIX not set (DEPLOY_PROVIDER=fly)");
+ if (!opts.baseImage) throw new Error("FLY_DEPLOY_BASE_IMAGE not set (DEPLOY_PROVIDER=fly)");
+ if (!opts.org) throw new Error("FLY_ORG not set (DEPLOY_PROVIDER=fly)");
+ if (opts.appPrefix.length > 26 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.appPrefix)) {
+ throw new Error("FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters");
+ }
+ }
+
+ const appNameFor = (d: Deployment): string => `${opts.appPrefix}-${d.id}`;
+
+ async function bundleBase64(version: DeploymentVersion): Promise<string> {
+ const tree = await readTree(version.snapshotDir, { tolerateMissing: true });
+ const sourceBytes = tree.reduce((total, file) => total + file.data.byteLength, 0);
+ if (sourceBytes > MAX_BUNDLE_SOURCE_BYTES) {
+ throw new Error(
+ `the app source is too large for the Fly deploy provider: ${sourceBytes} bytes, maximum ${MAX_BUNDLE_SOURCE_BYTES} bytes`,
+ );
+ }
+ const encoded = gzipSync(await makeTar(tree)).toString("base64");
+ if (encoded.length > MAX_BUNDLE_BASE64_BYTES) {
+ throw new Error(
+ `the app bundle is too large for the Fly deploy provider: ${encoded.length} bytes once packed and encoded, ` +
+ `maximum ${MAX_BUNDLE_BASE64_BYTES} bytes — publish fewer or smaller files`,
+ );
+ }
+ return encoded;
+ }
+
+ const machineConfig = (version: DeploymentVersion, bundle: string): FlyMachineConfig => ({
+ image: opts.baseImage,
+ env: { ...version.env, PORT: String(APP_PORT) },
+ guest: GUEST,
+ files: [{ guest_path: BUNDLE_GUEST_PATH, raw_value: bundle }],
+ services: [
+ {
+ protocol: "tcp",
+ internal_port: APP_PORT,
+ ports: [{ port: APP_PORT }],
+ checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],
+ },
+ ],
+ init: {
+ exec: [
+ "/bin/sh",
+ "-lc",
+ `mkdir -p ${APP_DIR} && tar -xzf ${BUNDLE_GUEST_PATH} -C ${APP_DIR} && cd ${APP_DIR} && exec sh -lc ${shq(version.entrypoint)}`,
+ ],
+ },
+ });
+
+ async function waitStarted(appName: string, machineId: string): Promise<void> {
+ const deadline = Date.now() + machineStartTimeoutMs;
+ let last: FlyMachine | null = null;
+ while (Date.now() < deadline) {
+ last = await api.getMachine(appName, machineId);
+ if (last?.state === "started") return;
+ await sleep(pollIntervalMs);
+ }
+ throw new Error(
+ `fly machine ${machineId} never reached state "started" within ${seconds(machineStartTimeoutMs)} ` +
+ `(last state: ${last?.state ?? "unknown"})${exitDetail(last)}`,
+ );
+ }
+
+ async function waitAppReady(appName: string, machineId: string): Promise<void> {
+ const deadline = Date.now() + appReadyTimeoutMs;
+ let machine: FlyMachine | null = null;
+ while (Date.now() < deadline) {
+ machine = await api.getMachine(appName, machineId);
+ if (machine?.state === "started" && machine.checks?.some((check) => check.status === "passing")) return;
+ await sleep(pollIntervalMs);
+ }
+ machine = await api
+ .getMachine(appName, machineId)
+ .catch(swallowAs<FlyMachine | null>("fly-deploy: read machine after readiness timeout", machine));
+ const why =
+ machine && machine.state !== "started"
+ ? `the version's entrypoint exited without binding port ${APP_PORT} (fly machine ${machineId} is ${machine.state})`
+ : `the app never listened on port ${APP_PORT} within ${seconds(appReadyTimeoutMs)}`;
+ throw new Error(`${why}${exitDetail(machine)}`);
+ }
+
+ return {
+ profile: { managedScaleToZero: false },
+
+ async apply(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint> {
+ ensureConfigured();
+ const appName = appNameFor(d);
+ const bundle = await bundleBase64(version);
+ await api.ensureApp(appName, opts.org);
+ await api.ensurePrivateIngress(appName);
+ const stale = await api.listMachines(appName);
+ const machine = await api.createMachine(appName, {
+ region,
+ config: machineConfig(version, bundle),
+ skip_service_registration: true,
+ });
+ try {
+ await waitStarted(appName, machine.id);
+ await waitAppReady(appName, machine.id);
+ } catch (error) {
+ await api
+ .destroyMachine(appName, machine.id)
+ .catch((cleanupError) => swallow("fly-deploy: remove unhealthy replacement", cleanupError));
+ throw error;
+ }
+ try {
+ for (const previous of stale) {
+ await api.setCordon(appName, previous.id, true);
+ }
+ await api.setCordon(appName, machine.id, false);
+ } catch (error) {
+ for (const previous of stale) {
+ await api
+ .setCordon(appName, previous.id, false)
+ .catch((rollbackError) => swallow("fly-deploy: restore previous machine routing", rollbackError));
+ }
+ await api
+ .destroyMachine(appName, machine.id)
+ .catch((cleanupError) => swallow("fly-deploy: remove failed replacement", cleanupError));
+ throw error;
+ }
+ for (const previous of stale) {
+ await api
+ .destroyMachine(appName, previous.id)
+ .catch((cleanupError) => swallow("fly-deploy: remove cordoned machine", cleanupError));
+ }
+ return { host: `${appName}.flycast`, port: APP_PORT };
+ },
+
+ async destroy(d: Deployment): Promise<void> {
+ ensureConfigured();
+ const appName = appNameFor(d);
+ if (!(await api.assertOwnedApp(appName, opts.org))) return;
+ await api.deleteApp(appName);
+ },
+ };
+}
diff --git a/src/wiring.ts b/src/wiring.ts
index f49b80a81..ae1239e01 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -77,6 +77,7 @@ import { createWebhookReceiver, type WebhookReceiver } from "./webhooks/webhook-
import { createDeployStore, type Deployment } from "./deploy/deploy-store.ts";
import { createDockerDeployProvider } from "./deploy/docker-deploy-provider.ts";
import { createAwsDeployProvider, type StoredDeployBody } from "./deploy/aws-deploy-provider.ts";
+import { createFlyDeployProvider } from "./deploy/fly-deploy-provider.ts";
import type { DeployProvider } from "./deploy/deploy-provider.ts";
import { createDeployService } from "./deploy/deploy-service.ts";
import {
@@ -869,17 +870,19 @@ export function buildApp(
: {}),
},
});
- const deployProvider: DeployProvider =
- config.deployProvider === "aws"
- ? createAwsDeployProvider({
- ...config.awsDeploy,
- ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket
- ? { dataBucket: config.awsSandbox.s3Bucket }
- : {}),
- advisoryLock,
- store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),
- })
- : createDockerDeployProvider();
+ const deployProvider: DeployProvider = ((): DeployProvider => {
+ if (config.deployProvider === "aws")
+ return createAwsDeployProvider({
+ ...config.awsDeploy,
+ ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket
+ ? { dataBucket: config.awsSandbox.s3Bucket }
+ : {}),
+ advisoryLock,
+ store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),
+ });
+ if (config.deployProvider === "fly") return createFlyDeployProvider(config.flyDeploy);
+ return createDockerDeployProvider();
+ })();
if (config.deployProvider === "aws" && !config.awsDeploy.dataBucket && !config.awsSandbox.s3Bucket) {
console.warn(
"[wiring] aws deploy: no data bucket resolved (AWS_DEPLOY_DATA_BUCKET unset, sandbox is not aws) — deployed apps have NO durable /data",
diff --git a/test/config.test.ts b/test/config.test.ts
index 78f042aaa..6a85c9789 100644
--- a/test/config.test.ts
+++ b/test/config.test.ts
@@ -46,6 +46,12 @@ test("store kinds default to memory and accept postgres", () => {
);
});
+test("deploy provider defaults to docker and rejects unknown values", () => {
+ assert.equal(loadConfig({}).deployProvider, "docker");
+ assert.equal(loadConfig({ DEPLOY_PROVIDER: "fly", FLY_DEPLOY_API_TOKEN: "test-token" }).deployProvider, "fly");
+ assert.throws(() => loadConfig({ DEPLOY_PROVIDER: "flly" }), /DEPLOY_PROVIDER="flly" is not recognized/);
+});
+
test("production and unauthenticated-core escape hatch are parsed once", () => {
assert.throws(() => loadConfig({ NODE_ENV: "production" }), /missing or insecure required core secrets/);
assert.equal(loadConfig(productionEnv).production, true);
diff --git a/test/deploy-store.test.ts b/test/deploy-store.test.ts
index 1d89df00b..c7e85c400 100644
--- a/test/deploy-store.test.ts
+++ b/test/deploy-store.test.ts
@@ -78,6 +78,40 @@ test("addVersionFromCommit registers a pushed commit as a new version inheriting
assert.equal((await s.get(d.id))!.versions.length, 2);
});
+test("a provider without reconcile receives durable Git contents after a push", async () => {
+ const root = mkdtempSync(join(tmpdir(), "from-commit-apply-"));
+ const deployStore = createDeployStore({ git: { repoRoot: join(root, "repos") } });
+ const applied: string[] = [];
+ const deploy = createDeployService({
+ deployStore,
+ provider: {
+ profile: { managedScaleToZero: false },
+ apply: async (_deployment, version) => {
+ applied.push(readFileSync(join(version.snapshotDir, "server.js"), "utf8"));
+ return { host: "127.0.0.1", port: 8080 };
+ },
+ destroy: async () => {},
+ },
+ deployDir: join(root, "snapshots"),
+ auditLog: { record() {}, events: async () => [], tail: async () => [] },
+ acl: createAclStore(),
+ });
+ const d = await deploy.deploy({
+ ownerScopeId: scopeId("personal", "U1"),
+ createdBy: "U1",
+ entrypoint: "node server.js",
+ files: [{ path: "server.js", data: "v1" }],
+ });
+ const work = mkdtempSync(join(tmpdir(), "from-commit-apply-work-"));
+ const repo = await deploy.gitRepoPath(d.id);
+ execFileSync("git", ["clone", "--quiet", repo!, work]);
+ writeFileSync(join(work, "server.js"), "v2");
+ execFileSync("git", ["-c", "user.name=T", "-c", "user.email=t@t", "commit", "-aqm", "v2"], { cwd: work });
+ execFileSync("git", ["push", "--quiet", repo!, "HEAD:current"], { cwd: work });
+ await deploy.pushGit(d.id, async () => ({ result: true, ok: true }));
+ assert.deepEqual(applied, ["v1", "v2"]);
+});
+
test("homeDir (resident-auth snapshot) round-trips through create + addVersion", async () => {
const s = createDeployStore();
const d = await s.create({
diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.ts
new file mode 100644
index 000000000..d3d01d304
--- /dev/null
+++ b/test/fly-deploy-provider.test.ts
@@ -0,0 +1,431 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { randomBytes } from "node:crypto";
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { gunzipSync } from "node:zlib";
+import {
+ createFlyDeployProvider,
+ type FlyDeployProviderOptions,
+ type FlyMachine,
+ type FlyMachineConfig,
+} from "../src/deploy/fly-deploy-provider.ts";
+import { parseTar } from "../src/sandbox/tar.ts";
+import type { Deployment, DeploymentVersion } from "../src/deploy/deploy-store.ts";
+import { scopeId } from "../src/types.ts";
+
+const TOKEN = "FlyV1-test-token";
+const PREFIX = "qm-d";
+const IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a";
+const ORG = "acme";
+const ID = "550e8400-e29b-41d4-a716-446655440000";
+const APP = `${PREFIX}-${ID}`;
+const OWNED_APP = { name: APP, network: APP, organization: { slug: ORG } };
+
+interface FlyCall {
+ method: string;
+ path: string;
+ query: string;
+ auth: string | null;
+ body: Record<string, unknown> | undefined;
+}
+
+interface FakeFlyOptions {
+ createAppStatus?: number;
+ createAppBody?: string;
+ existingApp?: { name: string; network: string; organization: { slug: string } };
+ ips?: string[];
+ deleteAppStatus?: number;
+ existingMachines?: string[];
+ states?: string[];
+ checkStates?: string[];
+ events?: FlyMachine["events"];
+ destroyMachineStatus?: number;
+ cordonStatus?: number;
+ cordonFailsAfterApply?: boolean;
+ uncordonStatus?: number;
+}
+
+function fakeFly(opts: FakeFlyOptions = {}) {
+ const calls: FlyCall[] = [];
+ const machines = new Map<string, string>();
+ for (const id of opts.existingMachines ?? []) machines.set(id, "started");
+ const states = [...(opts.states ?? ["started"])];
+ const checkStates = [...(opts.checkStates ?? ["passing"])];
+ const nextState = (): string => (states.length > 1 ? states.shift()! : (states[0] ?? "started"));
+ const nextCheck = (): string => (checkStates.length > 1 ? checkStates.shift()! : (checkStates[0] ?? "passing"));
+ const ips = [...(opts.ips ?? [])];
+ const cordoned = new Set<string>();
+ let app = opts.existingApp;
+ let created = 0;
+ const fetchImpl = (async (input: string | URL, init: RequestInit = {}): Promise<Response> => {
+ const url = new URL(String(input));
+ const method = init.method ?? "GET";
+ const segments = url.pathname.split("/").filter(Boolean);
+ calls.push({
+ method,
+ path: url.pathname,
+ query: url.search,
+ auth: new Headers(init.headers).get("authorization"),
+ body: init.body ? (JSON.parse(String(init.body)) as Record<string, unknown>) : undefined,
+ });
+ const json = (status: number, payload: unknown): Response =>
+ new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } });
+ if (method === "POST" && segments.length === 2) {
+ const body = JSON.parse(String(init.body)) as { app_name: string; network: string; org_slug: string };
+ if ((opts.createAppStatus ?? 201) < 300) {
+ app = { name: body.app_name, network: body.network, organization: { slug: body.org_slug } };
+ }
+ return new Response(opts.createAppBody ?? "{}", { status: opts.createAppStatus ?? 201 });
+ }
+ if (method === "GET" && segments.length === 3) return app ? json(200, app) : json(404, { error: "not found" });
+ if (method === "DELETE" && segments.length === 3) {
+ app = undefined;
+ return new Response("{}", { status: opts.deleteAppStatus ?? 202 });
+ }
+ if (segments[3] === "ip_assignments" && method === "GET") return json(200, { ips: ips.map((ip) => ({ ip })) });
+ if (segments[3] === "ip_assignments" && method === "POST") {
+ ips.push("fdaa:1:2:3::1");
+ return json(201, { ip: ips[0] });
+ }
+ if (method === "GET" && segments.length === 4)
+ return json(
+ 200,
+ [...machines.keys()].map((id) => ({ id, state: machines.get(id) })),
+ );
+ if (method === "POST" && segments.length === 4) {
+ const id = `machine-${++created}`;
+ machines.set(id, "created");
+ return json(200, { id, state: "created" });
+ }
+ const machineId = segments[4] ?? "";
+ if (method === "GET" && segments.length === 5) {
+ if (!machines.has(machineId)) return json(404, { error: "not found" });
+ const state = nextState();
+ machines.set(machineId, state);
+ return json(200, {
+ id: machineId,
+ state,
+ checks: [{ name: "app", status: nextCheck() }],
+ ...(opts.events ? { events: opts.events } : {}),
+ });
+ }
+ if (method === "POST" && segments.length === 6 && segments[5] === "cordon") {
+ if ((opts.cordonStatus ?? 200) < 300) cordoned.add(machineId);
+ if (opts.cordonFailsAfterApply) throw new Error("connection lost after cordon");
+ return json(opts.cordonStatus ?? 200, {});
+ }
+ if (method === "POST" && segments.length === 6 && segments[5] === "uncordon") {
+ const status = machineId === "machine-1" ? (opts.uncordonStatus ?? 200) : 200;
+ if (status < 300) cordoned.delete(machineId);
+ return json(status, {});
+ }
+ if (method === "DELETE" && segments.length === 5) {
+ if ((opts.destroyMachineStatus ?? 200) < 300) {
+ machines.delete(machineId);
+ cordoned.delete(machineId);
+ }
+ return json(opts.destroyMachineStatus ?? 200, {});
+ }
+ return json(500, { error: `unexpected ${method} ${url.pathname}` });
+ }) as unknown as typeof fetch;
+ return { fetchImpl, calls, machines, cordoned };
+}
+
+function provider(fetchImpl: typeof fetch, extra: Partial<FlyDeployProviderOptions> = {}) {
+ return createFlyDeployProvider({
+ token: TOKEN,
+ appPrefix: PREFIX,
+ baseImage: IMAGE,
+ org: ORG,
+ fetchImpl,
+ pollIntervalMs: 1,
+ machineStartTimeoutMs: 200,
+ appReadyTimeoutMs: 200,
+ ...extra,
+ });
+}
+
+function deployment(id: string): Deployment {
+ return {
+ id,
+ ownerScopeId: scopeId("personal", "U1"),
+ createdBy: "U1",
+ currentVersion: 1,
+ status: "stopped",
+ endpoint: null,
+ versions: [],
+ };
+}
+
+function snapshot(files: Record<string, string | Uint8Array>): string {
+ const dir = mkdtempSync(join(tmpdir(), "fly-deploy-"));
+ for (const [path, contents] of Object.entries(files)) {
+ const target = join(dir, path);
+ mkdirSync(dirname(target), { recursive: true });
+ writeFileSync(target, contents);
+ }
+ return dir;
+}
+
+function version(snapshotDir: string, over: Partial<DeploymentVersion> = {}): DeploymentVersion {
+ return { version: 1, createdAt: 0, entrypoint: "node server.js", snapshotDir, env: { API_KEY: "secret" }, ...over };
+}
+
+const machineCreate = (
+ calls: FlyCall[],
+): { region: string; config: FlyMachineConfig; skip_service_registration: true } =>
+ calls.find((c) => c.method === "POST" && c.path.endsWith("/machines"))!.body as unknown as {
+ region: string;
+ config: FlyMachineConfig;
+ skip_service_registration: true;
+ };
+
+test("apply: creates an isolated Fly app, injects the snapshot, and returns private Flycast ingress", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ const endpoint = await provider(fetchImpl).apply(
+ deployment(ID),
+ version(snapshot({ "index.html": "<h1>hi</h1>", "lib/util.js": "export const x = 1;" })),
+ );
+
+ assert.deepEqual(endpoint, { host: `${APP}.flycast`, port: 8080 });
+ assert.equal(endpoint.tls, undefined, "Flycast dialing is plaintext — the proxy must not attempt TLS");
+ assert.equal(endpoint.httpVersion, undefined, "the core proxy defaults to HTTP/1.1");
+
+ const createApp = calls.find((c) => c.method === "POST" && c.path === "/v1/apps")!;
+ assert.deepEqual(createApp.body, { app_name: APP, org_slug: ORG, network: APP });
+ assert.equal(createApp.auth, `Bearer ${TOKEN}`);
+ assert.deepEqual(calls.find((c) => c.method === "POST" && c.path.endsWith("/ip_assignments"))!.body, {
+ type: "private_v6",
+ });
+
+ const { region, config, skip_service_registration } = machineCreate(calls);
+ assert.equal(region, "lhr");
+ assert.equal(skip_service_registration, true);
+ assert.equal(config.image, IMAGE);
+ assert.deepEqual(config.env, { API_KEY: "secret", PORT: "8080" });
+ assert.deepEqual(config.guest, { cpu_kind: "shared", cpus: 1, memory_mb: 512 });
+ assert.deepEqual(config.services, [
+ {
+ protocol: "tcp",
+ internal_port: 8080,
+ ports: [{ port: 8080 }],
+ checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],
+ },
+ ]);
+ assert.equal(config.files[0]!.guest_path, "/app.tar.gz");
+ assert.deepEqual(config.init.exec.slice(0, 2), ["/bin/sh", "-lc"]);
+ assert.match(config.init.exec[2]!, /tar -xzf \/app\.tar\.gz -C \/app/);
+ assert.match(config.init.exec[2]!, /exec sh -lc 'node server\.js'/);
+
+ const unpacked = await parseTar(gunzipSync(Buffer.from(config.files[0]!.raw_value, "base64")));
+ assert.deepEqual(
+ unpacked.map((f) => f.path).sort(),
+ ["index.html", "lib/util.js"],
+ "the whole snapshot tree rides in the machine file",
+ );
+ assert.equal(unpacked.find((f) => f.path === "index.html")!.data.toString("utf8"), "<h1>hi</h1>");
+});
+
+test("apply: refuses a pre-existing public IP before creating a machine", async () => {
+ const unsafe = fakeFly({ createAppStatus: 422, existingApp: OWNED_APP, ips: ["2a09:8280:1::1"] });
+ await assert.rejects(
+ provider(unsafe.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /public IP assignment; refusing to expose/,
+ );
+ assert.ok(!unsafe.calls.some((c) => c.path.endsWith("/machines")), "an unsafe app never gets a machine");
+});
+
+test("apply: an app that already exists is not an error", async () => {
+ for (const over of [
+ { createAppStatus: 409, existingApp: OWNED_APP },
+ { createAppStatus: 422, createAppBody: '{"error":"Name has already been taken"}', existingApp: OWNED_APP },
+ ]) {
+ const { fetchImpl } = fakeFly(over);
+ const endpoint = await provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));
+ assert.deepEqual(endpoint, { host: `${APP}.flycast`, port: 8080 });
+ }
+});
+
+test("apply: a name conflict is reused only when its organization and isolated network match", async () => {
+ for (const existingApp of [
+ { ...OWNED_APP, network: "default" },
+ { ...OWNED_APP, organization: { slug: "someone-else" } },
+ ]) {
+ const { fetchImpl, calls } = fakeFly({ createAppStatus: 422, existingApp });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /not the isolated app owned by this deployment/,
+ );
+ assert.ok(!calls.some((c) => c.path.endsWith("/machines")));
+ }
+});
+
+test("apply: a rejected app creation still surfaces", async () => {
+ const { fetchImpl } = fakeFly({ createAppStatus: 401, createAppBody: '{"error":"unauthorized"}' });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /create app .*http 401.*unauthorized/,
+ );
+});
+
+test("apply: the previous version stays up until its healthy replacement is ready", async () => {
+ const { fetchImpl, calls, machines } = fakeFly({ existingMachines: ["machine-old"] });
+ await provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));
+
+ const destroyIndex = calls.findIndex((c) => c.method === "DELETE" && c.path.endsWith("/machines/machine-old"));
+ const createIndex = calls.findIndex((c) => c.method === "POST" && c.path.endsWith("/machines"));
+ const cordonIndex = calls.findIndex((c) => c.path.endsWith("/machines/machine-old/cordon"));
+ const uncordonIndex = calls.findIndex((c) => c.path.endsWith("/machines/machine-1/uncordon"));
+ assert.ok(destroyIndex >= 0, "the stale machine is destroyed");
+ assert.ok(createIndex < cordonIndex && cordonIndex < uncordonIndex && uncordonIndex < destroyIndex);
+ assert.equal(calls[destroyIndex]!.query, "?force=true");
+ assert.deepEqual([...machines.keys()], ["machine-1"]);
+});
+
+test("apply: a failed stale cleanup leaves the old machine cordoned, never mixed into traffic", async () => {
+ const fake = fakeFly({ existingMachines: ["machine-old"], destroyMachineStatus: 500 });
+ const logged = console.warn;
+ console.warn = () => {};
+ try {
+ await provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));
+ } finally {
+ console.warn = logged;
+ }
+ assert.deepEqual([...fake.machines.keys()], ["machine-old", "machine-1"]);
+ assert.deepEqual([...fake.cordoned], ["machine-old"]);
+});
+
+test("apply: a failed cutover restores the old route and removes the replacement", async () => {
+ for (const status of [404, 500]) {
+ const fake = fakeFly({ existingMachines: ["machine-old"], uncordonStatus: status });
+ await assert.rejects(
+ provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ new RegExp(`uncordon machine machine-1.*http ${status}`),
+ );
+ assert.deepEqual([...fake.machines.keys()], ["machine-old"]);
+ assert.deepEqual([...fake.cordoned], []);
+ }
+});
+
+test("apply: rollback restores an old route when the cordon response is lost", async () => {
+ const fake = fakeFly({ existingMachines: ["machine-old"], cordonFailsAfterApply: true });
+ await assert.rejects(
+ provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /connection lost after cordon/,
+ );
+ assert.deepEqual([...fake.machines.keys()], ["machine-old"]);
+ assert.deepEqual([...fake.cordoned], []);
+});
+
+test("apply: an app bundle over the machine-file cap is refused with its actual and maximum size", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ const big = snapshot({ "blob.bin": randomBytes(1_800_000) });
+ await assert.rejects(provider(fetchImpl).apply(deployment(ID), version(big)), (e: Error) => {
+ assert.match(e.message, /app bundle is too large for the Fly deploy provider/);
+ assert.match(e.message, /maximum 2000000 bytes/);
+ assert.match(e.message, /^the app bundle is too large for the Fly deploy provider: \d{7,} bytes/);
+ return true;
+ });
+ assert.deepEqual(calls, [], "nothing is created on Fly for a bundle that could never be injected");
+});
+
+test("apply: highly compressible source cannot bypass the unpacked-size cap", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ const big = snapshot({ "zeros.bin": Buffer.alloc(20_000_001) });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(big)),
+ /app source is too large.*20000001 bytes, maximum 20000000 bytes/,
+ );
+ assert.deepEqual(calls, []);
+});
+
+test("apply: a machine that never starts reports its last state", async () => {
+ const { fetchImpl, machines } = fakeFly({ existingMachines: ["machine-old"], states: ["created"] });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /never reached state "started" within 0s \(last state: created\)/,
+ );
+ assert.deepEqual(
+ [...machines.keys()],
+ ["machine-old"],
+ "a failed replacement is removed without touching the live version",
+ );
+});
+
+test("apply: an entrypoint that exits without binding the port reports why, with the machine's exit event", async () => {
+ const { fetchImpl } = fakeFly({
+ states: ["started", "stopped"],
+ checkStates: ["critical"],
+ events: [{ request: { exit_event: { exit_code: 127, oom_killed: false } } }],
+ });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /entrypoint exited without binding port 8080 \(fly machine machine-1 is stopped\).*exit code 127/s,
+ );
+});
+
+test("apply: an app that stays up but never passes its service check reports the readiness window", async () => {
+ const { fetchImpl } = fakeFly({ checkStates: ["critical"] });
+ await assert.rejects(
+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /never listened on port 8080 within 0s; the machine reported no exit event/,
+ );
+});
+
+test("destroy: deletes the whole Fly app and tolerates one that is already gone", async () => {
+ const { fetchImpl, calls } = fakeFly({ existingApp: OWNED_APP });
+ await provider(fetchImpl).destroy(deployment(ID));
+ assert.deepEqual(
+ calls.map((c) => `${c.method} ${c.path}`),
+ [`GET /v1/apps/${APP}`, `DELETE /v1/apps/${APP}`],
+ );
+ assert.equal(calls.at(-1)!.query, "?force=true");
+
+ const gone = fakeFly({ deleteAppStatus: 404 });
+ await provider(gone.fetchImpl).destroy(deployment(ID));
+});
+
+test("destroy: refuses a mismatched app and surfaces a failed deletion", async () => {
+ const mismatched = fakeFly({ existingApp: { ...OWNED_APP, network: "default" } });
+ await assert.rejects(provider(mismatched.fetchImpl).destroy(deployment(ID)), /not the isolated app owned/);
+ assert.ok(!mismatched.calls.some((c) => c.method === "DELETE"));
+
+ const { fetchImpl } = fakeFly({ existingApp: OWNED_APP, deleteAppStatus: 500 });
+ await assert.rejects(provider(fetchImpl).destroy(deployment(ID)), /delete app .*http 500/);
+});
+
+test("profile: the core keeps managing idle TTL because 6PN dialing cannot wake a stopped machine", () => {
+ const { fetchImpl } = fakeFly();
+ assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false });
+});
+
+test("missing fly configuration fails at the point of use with the env var that is missing", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ const missing: Array<[Partial<FlyDeployProviderOptions>, RegExp]> = [
+ [{ token: "" }, /FLY_DEPLOY_API_TOKEN not set \(DEPLOY_PROVIDER=fly\)/],
+ [{ appPrefix: "" }, /FLY_DEPLOY_APP_PREFIX not set \(DEPLOY_PROVIDER=fly\)/],
+ [{ baseImage: "" }, /FLY_DEPLOY_BASE_IMAGE not set \(DEPLOY_PROVIDER=fly\)/],
+ [{ org: "" }, /FLY_ORG not set \(DEPLOY_PROVIDER=fly\)/],
+ ];
+ for (const [over, expected] of missing) {
+ await assert.rejects(
+ provider(fetchImpl, over).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ expected,
+ );
+ await assert.rejects(provider(fetchImpl, over).destroy(deployment(ID)), expected);
+ }
+ assert.deepEqual(calls, [], "a misconfigured provider never reaches Fly");
+});
+
+test("an invalid app prefix fails before reaching Fly", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ for (const appPrefix of ["Bad_Prefix", "a".repeat(27)]) {
+ await assert.rejects(
+ provider(fetchImpl, { appPrefix }).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters/,
+ );
+ }
+ assert.deepEqual(calls, []);
+});