Multiplayer agent harness for work.
Add a Fly deploy provider so publish works on fly targets (#585)
kai-jmi090f490b5076
diff --git a/src/config.ts b/src/config.tsindex 8714a3fcd..4d1fb16e5 100644--- a/src/config.ts+++ b/src/config.ts@@ -37,7 +37,7 @@ export interface Config {securityPosture: SecurityPosture;sandboxBackend: "aws" | "local" | "sprites" | "smolmachines";sandboxSecondaryBackend?: "aws" | "local" | "sprites" | "smolmachines";- deployProvider: "docker" | "aws";+ deployProvider: "docker" | "aws" | "fly";egressServiceHosts?: string[];brandingDefault?: OrgBranding;modelId?: string;@@ -148,6 +148,7 @@ export interface Config {spritesSandbox: SpritesSandboxEnv;smolmachinesSandbox: SmolmachinesSandboxEnv;awsDeploy: AwsDeployEnv;+ flyDeploy: FlyDeployEnv;}export function configuredModelForHarness(config: Config, harness: string): string | undefined {@@ -410,6 +411,24 @@ function awsDeployEnv(env: NodeJS.ProcessEnv): AwsDeployEnv {};}+interface FlyDeployEnv {+ token: string;+ appPrefix: string;+ baseImage: string;+ org: string;+ region?: string;+}++function flyDeployEnv(env: NodeJS.ProcessEnv): FlyDeployEnv {+ return {+ token: env.FLY_DEPLOY_API_TOKEN ?? "",+ appPrefix: env.FLY_DEPLOY_APP_PREFIX ?? "",+ baseImage: env.FLY_DEPLOY_BASE_IMAGE ?? "",+ org: env.FLY_ORG ?? "",+ ...(env.FLY_REGION ? { region: env.FLY_REGION } : {}),+ };+}+const DEFAULT_ORG_ID = "default-org";export function orgId(): string {@@ -671,7 +690,12 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {}const publicApiUrl = env.PUBLIC_API_URL ?? env.AGENT_API_URL;const publicUrl = env.PUBLIC_WEB_URL ?? publicApiUrl;- const deployProvider: "aws" | "docker" = env.DEPLOY_PROVIDER === "aws" ? "aws" : "docker";+ const deployProvider = env.DEPLOY_PROVIDER ?? "docker";+ if (deployProvider !== "aws" && deployProvider !== "docker" && deployProvider !== "fly") {+ throw new Error(+ `DEPLOY_PROVIDER=${JSON.stringify(deployProvider)} is not recognized (expected aws, docker, or fly)`,+ );+ }let runStore: "memory" | "postgres" = env.SESSION_STORE === "postgres" ? "postgres" : "memory";if (env.RUN_STORE === "memory" || env.RUN_STORE === "postgres") runStore = env.RUN_STORE;const providerBaseUrls = providerBaseUrlsFromEnv(env);@@ -894,5 +918,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {spritesSandbox: spritesSandboxEnv(env),smolmachinesSandbox: smolmachinesSandboxEnv(env),awsDeploy: awsDeployEnv(env),+ flyDeploy: flyDeployEnv(env),};}diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.tsindex 674e9badb..7075a396f 100644--- a/src/deploy/deploy-service.ts+++ b/src/deploy/deploy-service.ts@@ -167,7 +167,13 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {allPaths,});} else {- endpoint = await deps.provider.apply(d, version);+ let materialized = version;+ if (version.commit) {+ const files = await deps.deployStore.filesOf(id, version.version);+ if (files == null) throw new Error(`cannot materialize deployment ${id} version ${version.version}`);+ materialized = { ...version, snapshotDir: await snapshotFiles(deps.deployDir, files) };+ }+ endpoint = await deps.provider.apply(d, materialized);}if (endpoint.image && endpoint.image !== version.image) {await deps.deployStore.setVersionImage(id, version.version, endpoint.image);diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.tsnew file mode 100644index 000000000..f29147e0e--- /dev/null+++ b/src/deploy/fly-deploy-provider.ts@@ -0,0 +1,354 @@+import { gzipSync } from "node:zlib";+import type { Deployment, DeploymentVersion } from "./deploy-store.ts";+import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";+import { readTree } from "./deploy-fs.ts";+import { makeTar } from "../sandbox/tar.ts";+import { sleep } from "../util/async.ts";+import { swallow, swallowAs } from "../util/errors.ts";+import { shq } from "../util/shell.ts";++const MACHINES_API_BASE_URL = "https://api.machines.dev/v1";+const DEFAULT_REGION = "lhr";+const APP_PORT = 8080;+const APP_DIR = "/app";+const BUNDLE_GUEST_PATH = "/app.tar.gz";+const MAX_BUNDLE_BASE64_BYTES = 2_000_000;+const MAX_BUNDLE_SOURCE_BYTES = 20_000_000;+const GUEST = { cpu_kind: "shared", cpus: 1, memory_mb: 512 };+const MACHINE_START_TIMEOUT_MS = 90_000;+const APP_READY_TIMEOUT_MS = 90_000;+const POLL_INTERVAL_MS = 1_000;+const API_TIMEOUT_MS = 30_000;+const APP_ALREADY_TAKEN = /already\s+(exists|been\s+taken)|name\s+is\s+taken/i;++interface FlyApp {+ name: string;+ network: string;+ organization?: { slug?: string };+}++interface FlyIpAssignment {+ ip: string;+}++interface FlyMachineExitEvent {+ exit_code?: number;+ oom_killed?: boolean;+}++export interface FlyMachine {+ id: string;+ state: string;+ checks?: Array<{ name?: string; status?: string; output?: string }>;+ events?: Array<{ request?: { exit_event?: FlyMachineExitEvent } }>;+}++export interface FlyMachineConfig {+ image: string;+ env: Record<string, string>;+ guest: { cpu_kind: string; cpus: number; memory_mb: number };+ files: Array<{ guest_path: string; raw_value: string }>;+ services: Array<{+ protocol: "tcp";+ internal_port: number;+ ports: Array<{ port: number }>;+ checks: Array<{ type: "tcp"; interval: string; timeout: string; grace_period: string }>;+ }>;+ init: { exec: string[] };+}++interface FlyMachinesApi {+ ensureApp(appName: string, orgSlug: string): Promise<void>;+ ensurePrivateIngress(appName: string): Promise<void>;+ assertOwnedApp(appName: string, orgSlug: string): Promise<boolean>;+ deleteApp(appName: string): Promise<void>;+ listMachines(appName: string): Promise<FlyMachine[]>;+ createMachine(+ appName: string,+ input: { region: string; config: FlyMachineConfig; skip_service_registration: true },+ ): Promise<FlyMachine>;+ getMachine(appName: string, machineId: string): Promise<FlyMachine | null>;+ setCordon(appName: string, machineId: string, cordoned: boolean): Promise<void>;+ destroyMachine(appName: string, machineId: string): Promise<void>;+}++interface FlyApiResponse {+ ok: boolean;+ status: number;+ text: string;+}++function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch }): FlyMachinesApi {+ const fetchImpl = opts.fetchImpl ?? fetch;+ const request = async (method: string, path: string, body?: unknown): Promise<FlyApiResponse> => {+ const res = await fetchImpl(`${MACHINES_API_BASE_URL}${path}`, {+ method,+ headers: {+ authorization: `Bearer ${opts.token}`,+ ...(body === undefined ? {} : { "content-type": "application/json" }),+ },+ ...(body === undefined ? {} : { body: JSON.stringify(body) }),+ signal: AbortSignal.timeout(API_TIMEOUT_MS),+ });+ return { ok: res.ok, status: res.status, text: await res.text() };+ };+ const failure = (what: string, r: FlyApiResponse): Error =>+ new Error(`fly ${what}: http ${r.status} ${r.text.slice(0, 200)}`);+ const parse = <T>(what: string, r: FlyApiResponse): T => {+ try {+ return JSON.parse(r.text) as T;+ } catch {+ throw new Error(`fly ${what}: unreadable response: ${r.text.slice(0, 200)}`);+ }+ };+ const app = (appName: string): string => `/apps/${encodeURIComponent(appName)}`;+ const machine = (appName: string, machineId: string): string =>+ `${app(appName)}/machines/${encodeURIComponent(machineId)}`;+ const getApp = async (appName: string): Promise<FlyApp | null> => {+ const r = await request("GET", app(appName));+ if (r.status === 404) return null;+ if (!r.ok) throw failure(`read app ${appName}`, r);+ return parse<FlyApp>(`read app ${appName}`, r);+ };+ const assertOwned = (appName: string, orgSlug: string, found: FlyApp): void => {+ if (found.name !== appName || found.organization?.slug !== orgSlug || found.network !== appName) {+ throw new Error(+ `fly app ${appName} already exists but is not the isolated app owned by this deployment; choose another FLY_DEPLOY_APP_PREFIX`,+ );+ }+ };+ return {+ async ensureApp(appName, orgSlug): Promise<void> {+ const r = await request("POST", "/apps", { app_name: appName, org_slug: orgSlug, network: appName });+ if (r.ok) return;+ if (r.status !== 409 && r.status !== 422 && !APP_ALREADY_TAKEN.test(r.text)) {+ throw failure(`create app ${appName}`, r);+ }+ const found = await getApp(appName);+ if (!found) throw failure(`create app ${appName}`, r);+ assertOwned(appName, orgSlug, found);+ },+ async ensurePrivateIngress(appName): Promise<void> {+ const listed = await request("GET", `${app(appName)}/ip_assignments`);+ if (!listed.ok) throw failure(`list IP assignments for ${appName}`, listed);+ const ips = parse<{ ips: FlyIpAssignment[] }>(`list IP assignments for ${appName}`, listed).ips;+ if (ips.some((entry) => !entry.ip.toLowerCase().startsWith("fdaa:"))) {+ throw new Error(`fly app ${appName} has a public IP assignment; refusing to expose the published app`);+ }+ if (ips.length) return;+ const assigned = await request("POST", `${app(appName)}/ip_assignments`, { type: "private_v6" });+ if (!assigned.ok) throw failure(`allocate private ingress for ${appName}`, assigned);+ },+ async assertOwnedApp(appName, orgSlug): Promise<boolean> {+ const found = await getApp(appName);+ if (!found) return false;+ assertOwned(appName, orgSlug, found);+ return true;+ },+ async deleteApp(appName): Promise<void> {+ const r = await request("DELETE", `${app(appName)}?force=true`);+ if (r.ok || r.status === 404) return;+ throw failure(`delete app ${appName}`, r);+ },+ async listMachines(appName): Promise<FlyMachine[]> {+ const r = await request("GET", `${app(appName)}/machines`);+ if (r.status === 404) return [];+ if (!r.ok) throw failure(`list machines in ${appName}`, r);+ return parse<FlyMachine[]>(`list machines in ${appName}`, r);+ },+ async createMachine(appName, input): Promise<FlyMachine> {+ const r = await request("POST", `${app(appName)}/machines`, input);+ if (!r.ok) throw failure(`create machine in ${appName}`, r);+ return parse<FlyMachine>(`create machine in ${appName}`, r);+ },+ async getMachine(appName, machineId): Promise<FlyMachine | null> {+ const r = await request("GET", machine(appName, machineId));+ if (r.status === 404) return null;+ if (!r.ok) throw failure(`get machine ${machineId}`, r);+ return parse<FlyMachine>(`get machine ${machineId}`, r);+ },+ async setCordon(appName, machineId, cordoned): Promise<void> {+ const action = cordoned ? "cordon" : "uncordon";+ const r = await request("POST", `${machine(appName, machineId)}/${action}`);+ if (r.ok || (cordoned && r.status === 404)) return;+ throw failure(`${action} machine ${machineId}`, r);+ },+ async destroyMachine(appName, machineId): Promise<void> {+ const r = await request("DELETE", `${machine(appName, machineId)}?force=true`);+ if (r.ok || r.status === 404) return;+ throw failure(`destroy machine ${machineId}`, r);+ },+ };+}++function exitDetail(machine: FlyMachine | null): string {+ const exit = machine?.events?.find((e) => e.request?.exit_event)?.request?.exit_event;+ if (!exit) return "; the machine reported no exit event";+ const parts = [`exit code ${exit.exit_code ?? "unknown"}`];+ if (exit.oom_killed) parts.push("killed for running out of memory");+ return `; last machine exit event: ${parts.join(", ")}`;+}++export interface FlyDeployProviderOptions {+ token: string;+ appPrefix: string;+ baseImage: string;+ org: string;+ region?: string;+ machineStartTimeoutMs?: number;+ appReadyTimeoutMs?: number;+ pollIntervalMs?: number;+ fetchImpl?: typeof fetch;+}++export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployProvider {+ const region = opts.region ?? DEFAULT_REGION;+ const api = createFlyMachinesApi({+ token: opts.token,+ ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),+ });+ const machineStartTimeoutMs = opts.machineStartTimeoutMs ?? MACHINE_START_TIMEOUT_MS;+ const appReadyTimeoutMs = opts.appReadyTimeoutMs ?? APP_READY_TIMEOUT_MS;+ const pollIntervalMs = opts.pollIntervalMs ?? POLL_INTERVAL_MS;+ const seconds = (ms: number): string => `${Math.round(ms / 1000)}s`;++ function ensureConfigured(): void {+ if (!opts.token) throw new Error("FLY_DEPLOY_API_TOKEN not set (DEPLOY_PROVIDER=fly)");+ if (!opts.appPrefix) throw new Error("FLY_DEPLOY_APP_PREFIX not set (DEPLOY_PROVIDER=fly)");+ if (!opts.baseImage) throw new Error("FLY_DEPLOY_BASE_IMAGE not set (DEPLOY_PROVIDER=fly)");+ if (!opts.org) throw new Error("FLY_ORG not set (DEPLOY_PROVIDER=fly)");+ if (opts.appPrefix.length > 26 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.appPrefix)) {+ throw new Error("FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters");+ }+ }++ const appNameFor = (d: Deployment): string => `${opts.appPrefix}-${d.id}`;++ async function bundleBase64(version: DeploymentVersion): Promise<string> {+ const tree = await readTree(version.snapshotDir, { tolerateMissing: true });+ const sourceBytes = tree.reduce((total, file) => total + file.data.byteLength, 0);+ if (sourceBytes > MAX_BUNDLE_SOURCE_BYTES) {+ throw new Error(+ `the app source is too large for the Fly deploy provider: ${sourceBytes} bytes, maximum ${MAX_BUNDLE_SOURCE_BYTES} bytes`,+ );+ }+ const encoded = gzipSync(await makeTar(tree)).toString("base64");+ if (encoded.length > MAX_BUNDLE_BASE64_BYTES) {+ throw new Error(+ `the app bundle is too large for the Fly deploy provider: ${encoded.length} bytes once packed and encoded, ` ++ `maximum ${MAX_BUNDLE_BASE64_BYTES} bytes — publish fewer or smaller files`,+ );+ }+ return encoded;+ }++ const machineConfig = (version: DeploymentVersion, bundle: string): FlyMachineConfig => ({+ image: opts.baseImage,+ env: { ...version.env, PORT: String(APP_PORT) },+ guest: GUEST,+ files: [{ guest_path: BUNDLE_GUEST_PATH, raw_value: bundle }],+ services: [+ {+ protocol: "tcp",+ internal_port: APP_PORT,+ ports: [{ port: APP_PORT }],+ checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],+ },+ ],+ init: {+ exec: [+ "/bin/sh",+ "-lc",+ `mkdir -p ${APP_DIR} && tar -xzf ${BUNDLE_GUEST_PATH} -C ${APP_DIR} && cd ${APP_DIR} && exec sh -lc ${shq(version.entrypoint)}`,+ ],+ },+ });++ async function waitStarted(appName: string, machineId: string): Promise<void> {+ const deadline = Date.now() + machineStartTimeoutMs;+ let last: FlyMachine | null = null;+ while (Date.now() < deadline) {+ last = await api.getMachine(appName, machineId);+ if (last?.state === "started") return;+ await sleep(pollIntervalMs);+ }+ throw new Error(+ `fly machine ${machineId} never reached state "started" within ${seconds(machineStartTimeoutMs)} ` ++ `(last state: ${last?.state ?? "unknown"})${exitDetail(last)}`,+ );+ }++ async function waitAppReady(appName: string, machineId: string): Promise<void> {+ const deadline = Date.now() + appReadyTimeoutMs;+ let machine: FlyMachine | null = null;+ while (Date.now() < deadline) {+ machine = await api.getMachine(appName, machineId);+ if (machine?.state === "started" && machine.checks?.some((check) => check.status === "passing")) return;+ await sleep(pollIntervalMs);+ }+ machine = await api+ .getMachine(appName, machineId)+ .catch(swallowAs<FlyMachine | null>("fly-deploy: read machine after readiness timeout", machine));+ const why =+ machine && machine.state !== "started"+ ? `the version's entrypoint exited without binding port ${APP_PORT} (fly machine ${machineId} is ${machine.state})`+ : `the app never listened on port ${APP_PORT} within ${seconds(appReadyTimeoutMs)}`;+ throw new Error(`${why}${exitDetail(machine)}`);+ }++ return {+ profile: { managedScaleToZero: false },++ async apply(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint> {+ ensureConfigured();+ const appName = appNameFor(d);+ const bundle = await bundleBase64(version);+ await api.ensureApp(appName, opts.org);+ await api.ensurePrivateIngress(appName);+ const stale = await api.listMachines(appName);+ const machine = await api.createMachine(appName, {+ region,+ config: machineConfig(version, bundle),+ skip_service_registration: true,+ });+ try {+ await waitStarted(appName, machine.id);+ await waitAppReady(appName, machine.id);+ } catch (error) {+ await api+ .destroyMachine(appName, machine.id)+ .catch((cleanupError) => swallow("fly-deploy: remove unhealthy replacement", cleanupError));+ throw error;+ }+ try {+ for (const previous of stale) {+ await api.setCordon(appName, previous.id, true);+ }+ await api.setCordon(appName, machine.id, false);+ } catch (error) {+ for (const previous of stale) {+ await api+ .setCordon(appName, previous.id, false)+ .catch((rollbackError) => swallow("fly-deploy: restore previous machine routing", rollbackError));+ }+ await api+ .destroyMachine(appName, machine.id)+ .catch((cleanupError) => swallow("fly-deploy: remove failed replacement", cleanupError));+ throw error;+ }+ for (const previous of stale) {+ await api+ .destroyMachine(appName, previous.id)+ .catch((cleanupError) => swallow("fly-deploy: remove cordoned machine", cleanupError));+ }+ return { host: `${appName}.flycast`, port: APP_PORT };+ },++ async destroy(d: Deployment): Promise<void> {+ ensureConfigured();+ const appName = appNameFor(d);+ if (!(await api.assertOwnedApp(appName, opts.org))) return;+ await api.deleteApp(appName);+ },+ };+}diff --git a/src/wiring.ts b/src/wiring.tsindex f49b80a81..ae1239e01 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -77,6 +77,7 @@ import { createWebhookReceiver, type WebhookReceiver } from "./webhooks/webhook-import { createDeployStore, type Deployment } from "./deploy/deploy-store.ts";import { createDockerDeployProvider } from "./deploy/docker-deploy-provider.ts";import { createAwsDeployProvider, type StoredDeployBody } from "./deploy/aws-deploy-provider.ts";+import { createFlyDeployProvider } from "./deploy/fly-deploy-provider.ts";import type { DeployProvider } from "./deploy/deploy-provider.ts";import { createDeployService } from "./deploy/deploy-service.ts";import {@@ -869,17 +870,19 @@ export function buildApp(: {}),},});- const deployProvider: DeployProvider =- config.deployProvider === "aws"- ? createAwsDeployProvider({- ...config.awsDeploy,- ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket- ? { dataBucket: config.awsSandbox.s3Bucket }- : {}),- advisoryLock,- store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),- })- : createDockerDeployProvider();+ const deployProvider: DeployProvider = ((): DeployProvider => {+ if (config.deployProvider === "aws")+ return createAwsDeployProvider({+ ...config.awsDeploy,+ ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket+ ? { dataBucket: config.awsSandbox.s3Bucket }+ : {}),+ advisoryLock,+ store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),+ });+ if (config.deployProvider === "fly") return createFlyDeployProvider(config.flyDeploy);+ return createDockerDeployProvider();+ })();if (config.deployProvider === "aws" && !config.awsDeploy.dataBucket && !config.awsSandbox.s3Bucket) {console.warn("[wiring] aws deploy: no data bucket resolved (AWS_DEPLOY_DATA_BUCKET unset, sandbox is not aws) — deployed apps have NO durable /data",diff --git a/test/config.test.ts b/test/config.test.tsindex 78f042aaa..6a85c9789 100644--- a/test/config.test.ts+++ b/test/config.test.ts@@ -46,6 +46,12 @@ test("store kinds default to memory and accept postgres", () => {);});+test("deploy provider defaults to docker and rejects unknown values", () => {+ assert.equal(loadConfig({}).deployProvider, "docker");+ assert.equal(loadConfig({ DEPLOY_PROVIDER: "fly", FLY_DEPLOY_API_TOKEN: "test-token" }).deployProvider, "fly");+ assert.throws(() => loadConfig({ DEPLOY_PROVIDER: "flly" }), /DEPLOY_PROVIDER="flly" is not recognized/);+});+test("production and unauthenticated-core escape hatch are parsed once", () => {assert.throws(() => loadConfig({ NODE_ENV: "production" }), /missing or insecure required core secrets/);assert.equal(loadConfig(productionEnv).production, true);diff --git a/test/deploy-store.test.ts b/test/deploy-store.test.tsindex 1d89df00b..c7e85c400 100644--- a/test/deploy-store.test.ts+++ b/test/deploy-store.test.ts@@ -78,6 +78,40 @@ test("addVersionFromCommit registers a pushed commit as a new version inheritingassert.equal((await s.get(d.id))!.versions.length, 2);});+test("a provider without reconcile receives durable Git contents after a push", async () => {+ const root = mkdtempSync(join(tmpdir(), "from-commit-apply-"));+ const deployStore = createDeployStore({ git: { repoRoot: join(root, "repos") } });+ const applied: string[] = [];+ const deploy = createDeployService({+ deployStore,+ provider: {+ profile: { managedScaleToZero: false },+ apply: async (_deployment, version) => {+ applied.push(readFileSync(join(version.snapshotDir, "server.js"), "utf8"));+ return { host: "127.0.0.1", port: 8080 };+ },+ destroy: async () => {},+ },+ deployDir: join(root, "snapshots"),+ auditLog: { record() {}, events: async () => [], tail: async () => [] },+ acl: createAclStore(),+ });+ const d = await deploy.deploy({+ ownerScopeId: scopeId("personal", "U1"),+ createdBy: "U1",+ entrypoint: "node server.js",+ files: [{ path: "server.js", data: "v1" }],+ });+ const work = mkdtempSync(join(tmpdir(), "from-commit-apply-work-"));+ const repo = await deploy.gitRepoPath(d.id);+ execFileSync("git", ["clone", "--quiet", repo!, work]);+ writeFileSync(join(work, "server.js"), "v2");+ execFileSync("git", ["-c", "user.name=T", "-c", "user.email=t@t", "commit", "-aqm", "v2"], { cwd: work });+ execFileSync("git", ["push", "--quiet", repo!, "HEAD:current"], { cwd: work });+ await deploy.pushGit(d.id, async () => ({ result: true, ok: true }));+ assert.deepEqual(applied, ["v1", "v2"]);+});+test("homeDir (resident-auth snapshot) round-trips through create + addVersion", async () => {const s = createDeployStore();const d = await s.create({diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.tsnew file mode 100644index 000000000..d3d01d304--- /dev/null+++ b/test/fly-deploy-provider.test.ts@@ -0,0 +1,431 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { randomBytes } from "node:crypto";+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import { gunzipSync } from "node:zlib";+import {+ createFlyDeployProvider,+ type FlyDeployProviderOptions,+ type FlyMachine,+ type FlyMachineConfig,+} from "../src/deploy/fly-deploy-provider.ts";+import { parseTar } from "../src/sandbox/tar.ts";+import type { Deployment, DeploymentVersion } from "../src/deploy/deploy-store.ts";+import { scopeId } from "../src/types.ts";++const TOKEN = "FlyV1-test-token";+const PREFIX = "qm-d";+const IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a";+const ORG = "acme";+const ID = "550e8400-e29b-41d4-a716-446655440000";+const APP = `${PREFIX}-${ID}`;+const OWNED_APP = { name: APP, network: APP, organization: { slug: ORG } };++interface FlyCall {+ method: string;+ path: string;+ query: string;+ auth: string | null;+ body: Record<string, unknown> | undefined;+}++interface FakeFlyOptions {+ createAppStatus?: number;+ createAppBody?: string;+ existingApp?: { name: string; network: string; organization: { slug: string } };+ ips?: string[];+ deleteAppStatus?: number;+ existingMachines?: string[];+ states?: string[];+ checkStates?: string[];+ events?: FlyMachine["events"];+ destroyMachineStatus?: number;+ cordonStatus?: number;+ cordonFailsAfterApply?: boolean;+ uncordonStatus?: number;+}++function fakeFly(opts: FakeFlyOptions = {}) {+ const calls: FlyCall[] = [];+ const machines = new Map<string, string>();+ for (const id of opts.existingMachines ?? []) machines.set(id, "started");+ const states = [...(opts.states ?? ["started"])];+ const checkStates = [...(opts.checkStates ?? ["passing"])];+ const nextState = (): string => (states.length > 1 ? states.shift()! : (states[0] ?? "started"));+ const nextCheck = (): string => (checkStates.length > 1 ? checkStates.shift()! : (checkStates[0] ?? "passing"));+ const ips = [...(opts.ips ?? [])];+ const cordoned = new Set<string>();+ let app = opts.existingApp;+ let created = 0;+ const fetchImpl = (async (input: string | URL, init: RequestInit = {}): Promise<Response> => {+ const url = new URL(String(input));+ const method = init.method ?? "GET";+ const segments = url.pathname.split("/").filter(Boolean);+ calls.push({+ method,+ path: url.pathname,+ query: url.search,+ auth: new Headers(init.headers).get("authorization"),+ body: init.body ? (JSON.parse(String(init.body)) as Record<string, unknown>) : undefined,+ });+ const json = (status: number, payload: unknown): Response =>+ new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } });+ if (method === "POST" && segments.length === 2) {+ const body = JSON.parse(String(init.body)) as { app_name: string; network: string; org_slug: string };+ if ((opts.createAppStatus ?? 201) < 300) {+ app = { name: body.app_name, network: body.network, organization: { slug: body.org_slug } };+ }+ return new Response(opts.createAppBody ?? "{}", { status: opts.createAppStatus ?? 201 });+ }+ if (method === "GET" && segments.length === 3) return app ? json(200, app) : json(404, { error: "not found" });+ if (method === "DELETE" && segments.length === 3) {+ app = undefined;+ return new Response("{}", { status: opts.deleteAppStatus ?? 202 });+ }+ if (segments[3] === "ip_assignments" && method === "GET") return json(200, { ips: ips.map((ip) => ({ ip })) });+ if (segments[3] === "ip_assignments" && method === "POST") {+ ips.push("fdaa:1:2:3::1");+ return json(201, { ip: ips[0] });+ }+ if (method === "GET" && segments.length === 4)+ return json(+ 200,+ [...machines.keys()].map((id) => ({ id, state: machines.get(id) })),+ );+ if (method === "POST" && segments.length === 4) {+ const id = `machine-${++created}`;+ machines.set(id, "created");+ return json(200, { id, state: "created" });+ }+ const machineId = segments[4] ?? "";+ if (method === "GET" && segments.length === 5) {+ if (!machines.has(machineId)) return json(404, { error: "not found" });+ const state = nextState();+ machines.set(machineId, state);+ return json(200, {+ id: machineId,+ state,+ checks: [{ name: "app", status: nextCheck() }],+ ...(opts.events ? { events: opts.events } : {}),+ });+ }+ if (method === "POST" && segments.length === 6 && segments[5] === "cordon") {+ if ((opts.cordonStatus ?? 200) < 300) cordoned.add(machineId);+ if (opts.cordonFailsAfterApply) throw new Error("connection lost after cordon");+ return json(opts.cordonStatus ?? 200, {});+ }+ if (method === "POST" && segments.length === 6 && segments[5] === "uncordon") {+ const status = machineId === "machine-1" ? (opts.uncordonStatus ?? 200) : 200;+ if (status < 300) cordoned.delete(machineId);+ return json(status, {});+ }+ if (method === "DELETE" && segments.length === 5) {+ if ((opts.destroyMachineStatus ?? 200) < 300) {+ machines.delete(machineId);+ cordoned.delete(machineId);+ }+ return json(opts.destroyMachineStatus ?? 200, {});+ }+ return json(500, { error: `unexpected ${method} ${url.pathname}` });+ }) as unknown as typeof fetch;+ return { fetchImpl, calls, machines, cordoned };+}++function provider(fetchImpl: typeof fetch, extra: Partial<FlyDeployProviderOptions> = {}) {+ return createFlyDeployProvider({+ token: TOKEN,+ appPrefix: PREFIX,+ baseImage: IMAGE,+ org: ORG,+ fetchImpl,+ pollIntervalMs: 1,+ machineStartTimeoutMs: 200,+ appReadyTimeoutMs: 200,+ ...extra,+ });+}++function deployment(id: string): Deployment {+ return {+ id,+ ownerScopeId: scopeId("personal", "U1"),+ createdBy: "U1",+ currentVersion: 1,+ status: "stopped",+ endpoint: null,+ versions: [],+ };+}++function snapshot(files: Record<string, string | Uint8Array>): string {+ const dir = mkdtempSync(join(tmpdir(), "fly-deploy-"));+ for (const [path, contents] of Object.entries(files)) {+ const target = join(dir, path);+ mkdirSync(dirname(target), { recursive: true });+ writeFileSync(target, contents);+ }+ return dir;+}++function version(snapshotDir: string, over: Partial<DeploymentVersion> = {}): DeploymentVersion {+ return { version: 1, createdAt: 0, entrypoint: "node server.js", snapshotDir, env: { API_KEY: "secret" }, ...over };+}++const machineCreate = (+ calls: FlyCall[],+): { region: string; config: FlyMachineConfig; skip_service_registration: true } =>+ calls.find((c) => c.method === "POST" && c.path.endsWith("/machines"))!.body as unknown as {+ region: string;+ config: FlyMachineConfig;+ skip_service_registration: true;+ };++test("apply: creates an isolated Fly app, injects the snapshot, and returns private Flycast ingress", async () => {+ const { fetchImpl, calls } = fakeFly();+ const endpoint = await provider(fetchImpl).apply(+ deployment(ID),+ version(snapshot({ "index.html": "<h1>hi</h1>", "lib/util.js": "export const x = 1;" })),+ );++ assert.deepEqual(endpoint, { host: `${APP}.flycast`, port: 8080 });+ assert.equal(endpoint.tls, undefined, "Flycast dialing is plaintext — the proxy must not attempt TLS");+ assert.equal(endpoint.httpVersion, undefined, "the core proxy defaults to HTTP/1.1");++ const createApp = calls.find((c) => c.method === "POST" && c.path === "/v1/apps")!;+ assert.deepEqual(createApp.body, { app_name: APP, org_slug: ORG, network: APP });+ assert.equal(createApp.auth, `Bearer ${TOKEN}`);+ assert.deepEqual(calls.find((c) => c.method === "POST" && c.path.endsWith("/ip_assignments"))!.body, {+ type: "private_v6",+ });++ const { region, config, skip_service_registration } = machineCreate(calls);+ assert.equal(region, "lhr");+ assert.equal(skip_service_registration, true);+ assert.equal(config.image, IMAGE);+ assert.deepEqual(config.env, { API_KEY: "secret", PORT: "8080" });+ assert.deepEqual(config.guest, { cpu_kind: "shared", cpus: 1, memory_mb: 512 });+ assert.deepEqual(config.services, [+ {+ protocol: "tcp",+ internal_port: 8080,+ ports: [{ port: 8080 }],+ checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],+ },+ ]);+ assert.equal(config.files[0]!.guest_path, "/app.tar.gz");+ assert.deepEqual(config.init.exec.slice(0, 2), ["/bin/sh", "-lc"]);+ assert.match(config.init.exec[2]!, /tar -xzf \/app\.tar\.gz -C \/app/);+ assert.match(config.init.exec[2]!, /exec sh -lc 'node server\.js'/);++ const unpacked = await parseTar(gunzipSync(Buffer.from(config.files[0]!.raw_value, "base64")));+ assert.deepEqual(+ unpacked.map((f) => f.path).sort(),+ ["index.html", "lib/util.js"],+ "the whole snapshot tree rides in the machine file",+ );+ assert.equal(unpacked.find((f) => f.path === "index.html")!.data.toString("utf8"), "<h1>hi</h1>");+});++test("apply: refuses a pre-existing public IP before creating a machine", async () => {+ const unsafe = fakeFly({ createAppStatus: 422, existingApp: OWNED_APP, ips: ["2a09:8280:1::1"] });+ await assert.rejects(+ provider(unsafe.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /public IP assignment; refusing to expose/,+ );+ assert.ok(!unsafe.calls.some((c) => c.path.endsWith("/machines")), "an unsafe app never gets a machine");+});++test("apply: an app that already exists is not an error", async () => {+ for (const over of [+ { createAppStatus: 409, existingApp: OWNED_APP },+ { createAppStatus: 422, createAppBody: '{"error":"Name has already been taken"}', existingApp: OWNED_APP },+ ]) {+ const { fetchImpl } = fakeFly(over);+ const endpoint = await provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));+ assert.deepEqual(endpoint, { host: `${APP}.flycast`, port: 8080 });+ }+});++test("apply: a name conflict is reused only when its organization and isolated network match", async () => {+ for (const existingApp of [+ { ...OWNED_APP, network: "default" },+ { ...OWNED_APP, organization: { slug: "someone-else" } },+ ]) {+ const { fetchImpl, calls } = fakeFly({ createAppStatus: 422, existingApp });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /not the isolated app owned by this deployment/,+ );+ assert.ok(!calls.some((c) => c.path.endsWith("/machines")));+ }+});++test("apply: a rejected app creation still surfaces", async () => {+ const { fetchImpl } = fakeFly({ createAppStatus: 401, createAppBody: '{"error":"unauthorized"}' });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /create app .*http 401.*unauthorized/,+ );+});++test("apply: the previous version stays up until its healthy replacement is ready", async () => {+ const { fetchImpl, calls, machines } = fakeFly({ existingMachines: ["machine-old"] });+ await provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));++ const destroyIndex = calls.findIndex((c) => c.method === "DELETE" && c.path.endsWith("/machines/machine-old"));+ const createIndex = calls.findIndex((c) => c.method === "POST" && c.path.endsWith("/machines"));+ const cordonIndex = calls.findIndex((c) => c.path.endsWith("/machines/machine-old/cordon"));+ const uncordonIndex = calls.findIndex((c) => c.path.endsWith("/machines/machine-1/uncordon"));+ assert.ok(destroyIndex >= 0, "the stale machine is destroyed");+ assert.ok(createIndex < cordonIndex && cordonIndex < uncordonIndex && uncordonIndex < destroyIndex);+ assert.equal(calls[destroyIndex]!.query, "?force=true");+ assert.deepEqual([...machines.keys()], ["machine-1"]);+});++test("apply: a failed stale cleanup leaves the old machine cordoned, never mixed into traffic", async () => {+ const fake = fakeFly({ existingMachines: ["machine-old"], destroyMachineStatus: 500 });+ const logged = console.warn;+ console.warn = () => {};+ try {+ await provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" })));+ } finally {+ console.warn = logged;+ }+ assert.deepEqual([...fake.machines.keys()], ["machine-old", "machine-1"]);+ assert.deepEqual([...fake.cordoned], ["machine-old"]);+});++test("apply: a failed cutover restores the old route and removes the replacement", async () => {+ for (const status of [404, 500]) {+ const fake = fakeFly({ existingMachines: ["machine-old"], uncordonStatus: status });+ await assert.rejects(+ provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ new RegExp(`uncordon machine machine-1.*http ${status}`),+ );+ assert.deepEqual([...fake.machines.keys()], ["machine-old"]);+ assert.deepEqual([...fake.cordoned], []);+ }+});++test("apply: rollback restores an old route when the cordon response is lost", async () => {+ const fake = fakeFly({ existingMachines: ["machine-old"], cordonFailsAfterApply: true });+ await assert.rejects(+ provider(fake.fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /connection lost after cordon/,+ );+ assert.deepEqual([...fake.machines.keys()], ["machine-old"]);+ assert.deepEqual([...fake.cordoned], []);+});++test("apply: an app bundle over the machine-file cap is refused with its actual and maximum size", async () => {+ const { fetchImpl, calls } = fakeFly();+ const big = snapshot({ "blob.bin": randomBytes(1_800_000) });+ await assert.rejects(provider(fetchImpl).apply(deployment(ID), version(big)), (e: Error) => {+ assert.match(e.message, /app bundle is too large for the Fly deploy provider/);+ assert.match(e.message, /maximum 2000000 bytes/);+ assert.match(e.message, /^the app bundle is too large for the Fly deploy provider: \d{7,} bytes/);+ return true;+ });+ assert.deepEqual(calls, [], "nothing is created on Fly for a bundle that could never be injected");+});++test("apply: highly compressible source cannot bypass the unpacked-size cap", async () => {+ const { fetchImpl, calls } = fakeFly();+ const big = snapshot({ "zeros.bin": Buffer.alloc(20_000_001) });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(big)),+ /app source is too large.*20000001 bytes, maximum 20000000 bytes/,+ );+ assert.deepEqual(calls, []);+});++test("apply: a machine that never starts reports its last state", async () => {+ const { fetchImpl, machines } = fakeFly({ existingMachines: ["machine-old"], states: ["created"] });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /never reached state "started" within 0s \(last state: created\)/,+ );+ assert.deepEqual(+ [...machines.keys()],+ ["machine-old"],+ "a failed replacement is removed without touching the live version",+ );+});++test("apply: an entrypoint that exits without binding the port reports why, with the machine's exit event", async () => {+ const { fetchImpl } = fakeFly({+ states: ["started", "stopped"],+ checkStates: ["critical"],+ events: [{ request: { exit_event: { exit_code: 127, oom_killed: false } } }],+ });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /entrypoint exited without binding port 8080 \(fly machine machine-1 is stopped\).*exit code 127/s,+ );+});++test("apply: an app that stays up but never passes its service check reports the readiness window", async () => {+ const { fetchImpl } = fakeFly({ checkStates: ["critical"] });+ await assert.rejects(+ provider(fetchImpl).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /never listened on port 8080 within 0s; the machine reported no exit event/,+ );+});++test("destroy: deletes the whole Fly app and tolerates one that is already gone", async () => {+ const { fetchImpl, calls } = fakeFly({ existingApp: OWNED_APP });+ await provider(fetchImpl).destroy(deployment(ID));+ assert.deepEqual(+ calls.map((c) => `${c.method} ${c.path}`),+ [`GET /v1/apps/${APP}`, `DELETE /v1/apps/${APP}`],+ );+ assert.equal(calls.at(-1)!.query, "?force=true");++ const gone = fakeFly({ deleteAppStatus: 404 });+ await provider(gone.fetchImpl).destroy(deployment(ID));+});++test("destroy: refuses a mismatched app and surfaces a failed deletion", async () => {+ const mismatched = fakeFly({ existingApp: { ...OWNED_APP, network: "default" } });+ await assert.rejects(provider(mismatched.fetchImpl).destroy(deployment(ID)), /not the isolated app owned/);+ assert.ok(!mismatched.calls.some((c) => c.method === "DELETE"));++ const { fetchImpl } = fakeFly({ existingApp: OWNED_APP, deleteAppStatus: 500 });+ await assert.rejects(provider(fetchImpl).destroy(deployment(ID)), /delete app .*http 500/);+});++test("profile: the core keeps managing idle TTL because 6PN dialing cannot wake a stopped machine", () => {+ const { fetchImpl } = fakeFly();+ assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false });+});++test("missing fly configuration fails at the point of use with the env var that is missing", async () => {+ const { fetchImpl, calls } = fakeFly();+ const missing: Array<[Partial<FlyDeployProviderOptions>, RegExp]> = [+ [{ token: "" }, /FLY_DEPLOY_API_TOKEN not set \(DEPLOY_PROVIDER=fly\)/],+ [{ appPrefix: "" }, /FLY_DEPLOY_APP_PREFIX not set \(DEPLOY_PROVIDER=fly\)/],+ [{ baseImage: "" }, /FLY_DEPLOY_BASE_IMAGE not set \(DEPLOY_PROVIDER=fly\)/],+ [{ org: "" }, /FLY_ORG not set \(DEPLOY_PROVIDER=fly\)/],+ ];+ for (const [over, expected] of missing) {+ await assert.rejects(+ provider(fetchImpl, over).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ expected,+ );+ await assert.rejects(provider(fetchImpl, over).destroy(deployment(ID)), expected);+ }+ assert.deepEqual(calls, [], "a misconfigured provider never reaches Fly");+});++test("an invalid app prefix fails before reaching Fly", async () => {+ const { fetchImpl, calls } = fakeFly();+ for (const appPrefix of ["Bad_Prefix", "a".repeat(27)]) {+ await assert.rejects(+ provider(fetchImpl, { appPrefix }).apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),+ /FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters/,+ );+ }+ assert.deepEqual(calls, []);+});