Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Add durable shared Fly publishing with private scale-to-zero routing (#1216)

Regan Bell5a3a4dc37746
diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts
index 8c914dd76..95baa0d4c 100644
--- a/cli/src/backends/fly.ts
+++ b/cli/src/backends/fly.ts
@@ -1464,8 +1464,13 @@ export function verifyLocalFlyTokens(config: QmConfig, secrets: ReadonlyMap<stri
}
step(`${name}: live authorization ok`);
};
- if (config.flyOrg && config.env.core?.DEPLOY_PROVIDER === "fly") {
- verify("FLY_DEPLOY_API_TOKEN", ["apps", "list", "-o", config.flyOrg, "--json"], `organization ${config.flyOrg}`);
+ if (config.env.core?.DEPLOY_PROVIDER === "fly") {
+ const sharedApp = config.env.core.FLY_DEPLOY_SHARED_APP_NAME?.trim();
+ if (sharedApp) {
+ verify("FLY_DEPLOY_API_TOKEN", ["machines", "list", "-a", sharedApp, "--json"], `application ${sharedApp}`);
+ } else if (config.flyOrg) {
+ verify("FLY_DEPLOY_API_TOKEN", ["apps", "list", "-o", config.flyOrg, "--json"], `organization ${config.flyOrg}`);
+ }
}
}
diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts
index a158ad5e4..bc8c83806 100644
--- a/cli/src/secrets.ts
+++ b/cli/src/secrets.ts
@@ -116,8 +116,21 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [
name: "FLY_DEPLOY_API_TOKEN",
service: "core",
required: { when: { kind: "env-equals", service: "core", name: "DEPLOY_PROVIDER", value: "fly" } },
- description: "Fly organization token used only by qm's opt-in per-deployment app publisher.",
- generate: "fly tokens create org -o <fly-org> -x 8760h",
+ description: "Fly publisher token: app-scoped for FLY_DEPLOY_SHARED_APP_NAME, organization-scoped otherwise.",
+ },
+ {
+ name: "FLY_DEPLOY_WIREGUARD_PEERS",
+ service: "core",
+ required: {
+ when: {
+ kind: "all",
+ conditions: [
+ { kind: "env-equals", service: "core", name: "DEPLOY_PROVIDER", value: "fly" },
+ { kind: "env-present", service: "core", name: "FLY_DEPLOY_SHARED_APP_NAME" },
+ ],
+ },
+ },
+ description: "JSON array of distinct private ingress WireGuard peer configurations for shared Fly publishing.",
},
{
name: "PORTER_DEPLOY_API_TOKEN",
diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tf
index 1b82a18e5..5f966f1ac 100644
--- a/cli/templates/aws/main.tf
+++ b/cli/templates/aws/main.tf
@@ -730,7 +730,7 @@ resource "aws_iam_role_policy" "task_objects" {
},
{
Effect = "Allow"
- Action = ["ecs:GetTaskProtection", "ecs:UpdateTaskProtection"]
+ Action = ["ecs:GetTaskProtection", "ecs:UpdateTaskProtection", "ecs:DescribeTasks"]
Resource = "arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task/${var.cluster_name}/*"
},
{
diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.ts
index 6f8836817..3ddc06f93 100644
--- a/cli/test/fly-sandbox.test.ts
+++ b/cli/test/fly-sandbox.test.ts
@@ -13,6 +13,7 @@ import {
flyLiveSessionCommand,
flyS3ProbeCommand,
flyUp,
+ verifyLocalFlyTokens,
} from "../src/backends/fly.ts";
import type { ResolvedPlugin } from "../src/plugins.ts";
@@ -1170,3 +1171,27 @@ test("fly secrets push stages a secretEnv alias under its declared env name on i
rmSync(dir, { recursive: true, force: true });
}
});
+
+test("shared publisher authorization uses app-scoped access without organization listing", () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-fly-app-token-"));
+ const fake = fakeFly(dir, 'if (a !== "machines list -a acme-apps --json") process.exit(1);');
+ try {
+ const { config } = loadConfigAt(join(repoRoot, "deploy", "stacks", "acme", "qm.config.jsonc"));
+ verifyLocalFlyTokens(
+ {
+ ...config,
+ env: {
+ core: {
+ DEPLOY_PROVIDER: "fly",
+ FLY_DEPLOY_SHARED_APP_NAME: "acme-apps",
+ },
+ },
+ },
+ new Map([["FLY_DEPLOY_API_TOKEN", "app-scoped-test-token"]]),
+ );
+ assert.equal(readFileSync(fake.log, "utf8").trim(), "machines list -a acme-apps --json");
+ } finally {
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/cli/test/secrets.test.ts b/cli/test/secrets.test.ts
index 016103c22..131c24bee 100644
--- a/cli/test/secrets.test.ts
+++ b/cli/test/secrets.test.ts
@@ -400,3 +400,22 @@ test("scope-selected providers require both Modal and Sprites credentials", () =
for (const name of ["SPRITES_TOKEN", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"])
assert.ok(required.includes(name), name);
});
+
+test("shared Fly publishing requires private peers only when selected", () => {
+ for (const provider of ["fly", "aws"]) {
+ for (const shared of [false, true]) {
+ const config = makeConfig({
+ target: "aws",
+ env: {
+ core: {
+ DEPLOY_PROVIDER: provider,
+ ...(shared ? { FLY_DEPLOY_SHARED_APP_NAME: "acme-apps" } : {}),
+ },
+ },
+ });
+ const peer = computedSecrets(config).find((secret) => secret.name === "FLY_DEPLOY_WIREGUARD_PEERS");
+ assert.equal(Boolean(peer?.required), provider === "fly" && shared);
+ if (provider === "fly" && shared) assert.doesNotMatch(renderEnvExample(config), /tokens create org/);
+ }
+ }
+});
diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.ts
index 6f2ccdb24..1d3ddbd80 100644
--- a/cli/test/terraform.test.ts
+++ b/cli/test/terraform.test.ts
@@ -620,7 +620,7 @@ test("AWS module reuses account OIDC, guards account and passes configured task
/Sid\s*= "ManageDeploymentLayers"[\s\S]*"s3:GetObject", "s3:PutObject"[\s\S]*deployment\/layers\/\*/,
);
assert.match(mainTf, /Sid\s*= "InspectGithubOidcProvider"[\s\S]*iam:GetOpenIDConnectProvider/);
- assert.match(mainTf, /"ecs:GetTaskProtection", "ecs:UpdateTaskProtection"/);
+ assert.match(mainTf, /"ecs:GetTaskProtection", "ecs:UpdateTaskProtection", "ecs:DescribeTasks"/);
assert.match(mainTf, /task\/\$\{var\.cluster_name\}\/\*/);
assert.match(mainTf, /"lambda:RunMicrovm"[\s\S]*"lambda:CreateMicrovmAuthToken"/);
assert.match(mainTf, /"lambda:ListMicrovmImages"/);
diff --git a/deploy/core/Dockerfile b/deploy/core/Dockerfile
index 5785ce496..b811dc97d 100644
--- a/deploy/core/Dockerfile
+++ b/deploy/core/Dockerfile
@@ -1,7 +1,14 @@
+FROM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS wireproxy-build
+
+RUN apk add --no-cache go \
+ && CGO_ENABLED=0 GOBIN=/out go install github.com/windtf/wireproxy/cmd/[email protected]
+
FROM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd
RUN apk add --no-cache ca-certificates curl git git-daemon docker-cli
+COPY --from=wireproxy-build /out/wireproxy /usr/local/bin/wireproxy
+
WORKDIR /app
COPY package.json package-lock.json ./
diff --git a/docs/qa/fly-published-apps.md b/docs/qa/fly-published-apps.md
new file mode 100644
index 000000000..4b2d12d4d
--- /dev/null
+++ b/docs/qa/fly-published-apps.md
@@ -0,0 +1,90 @@
+# Fly published application acceptance
+
+Status: final-image HTTP/runtime and private-network qualification passed. Company Bookface/browser qualification remains outstanding; existing deployments are not approved for migration yet. Automatic suspension is verified with variable timing, not a fixed deadline. See the final entries below for the immutable image and measured results.
+
+The Fly adapter must retain QM's application access controls and durable application data while delegating idle suspension and request-driven wake-up to Fly Proxy. Existing deployments remain on their current provider until the following checks pass against the candidate source and a real QM instance.
+
+| Area | Required live evidence |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
+| Publishing | Publish through QM, open the resulting URL, verify actual application responses |
+| Authentication | Signed-out, unrelated-user and unrelated-company denial; authorized-user success; direct-origin bypass denied |
+| Idle and wake | Observe automatic suspension with zero active Machines, then wake through the normal QM URL; repeat with concurrent first requests |
+| Cold start | Force a stopped state, then verify request-driven boot and data recovery |
+| Persistence | Write SQLite and file data, verify after suspend, stop, update, rollback and replacement |
+| Updates | Publish v2 and roll back to v1 without losing data; failed entrypoint must preserve the working version |
+| Transport | Streaming responses, WebSockets, uploads, cookies, redirects and proxy headers |
+| Lifecycle | Always-on enable/disable, explicit stop/restart, deletion, core restart and overlapping core instances |
+| Isolation | Company credentials cannot inspect, modify or delete another company's apps; application code cannot reach sibling networks |
+| Capacity | Bounded concurrent publishes and wake-ups; retry partial failures without orphaning resources |
+
+Record candidate commit, image digest, timestamps, HTTP assertions, machine states, data checks and cleanup in private receipts. Unit tests and direct provider calls are supporting evidence, not end-to-end acceptance.
+
+The following entries record the implementation and QA chronologically; later entries resolve earlier blockers.
+
+The prototype adds Fly service autostop/suspend, autostart and a zero minimum. The application reaper must not delete provider-managed sleeping apps. AWS-to-Fly company-isolated private connectivity remains unresolved.
+
+Observed live results from the initial prototype:
+
+- Publish v1 succeeded with private Flycast ingress.
+- A tunnel on the application's custom network could not reach its Flycast address allocated on the organization network. Using a tunnel on the matching organization network returned HTTP 200. Production connectivity still needs a company-isolated path from AWS.
+- Forced suspension was confirmed through the Machines API. A normal request through Flycast resumed it in 0.37 seconds, preserving a file counter.
+- Replacement publish v2 succeeded, but the file counter reset from 1 to 0. This is a failing acceptance check; the existing ephemeral filesystem cannot provide durable application data across replacement.
+- Automatic idle suspension was observed after about five minutes. A request resumed the app in 0.83 seconds and preserved the counter. This used a private Flycast tunnel, not the complete QM request path.
+
+Set `FLY_DEPLOY_DATA_VOLUME_SIZE_GB` to a positive integer to opt into persistent `/data`. The prototype creates one encrypted volume per application, updates the attached Machine in place, and retains the volume on archive. It refuses to silently convert existing ephemeral applications or downgrade a volume-backed app when configuration is missing. Volumes are host-local; this is not replicated storage or high availability. Machine replacement must retain the existing volume, and host-loss recovery needs separate qualification.
+
+The direct live volume fixture passed file and SQLite persistence after an update, failed-entrypoint rollback, explicit version rollback, cold stop/request wake, and archive/recreation. Follow-up tests passed both always-on toggles and recovery after confirming a broken configuration had stopped, then recreating the provider. The fixture uses a private file-backed configuration store; application wiring uses Postgres, which still needs full QM qualification. Unit tests cover accepted-create response loss, missing volume configuration, failed configuration persistence, rollback after provider recreation, and always-on changes.
+
+The first live always-on toggle from suspension failed: updating the Machine configuration left it stopped. Treat configuration acceptance and a running application as separate conditions. The candidate now requests a start when either the update response or subsequent readiness polling reports stopped/suspended. The successful retry verified the toggle, live data, and persisted rollback recovery. Durable mode requires Postgres in application wiring; an in-memory configuration map does not qualify restart recovery.
+
+A possible credential arrangement is one pre-created Fly app per company with an app-scoped token and multiple deployment Machines. The proposed routing uses a distinct raw TCP external port for each deployment. This requires live wake/routing tests and durable port ownership before it is accepted. No company credentials or production provider settings have changed for this prototype.
+
+An app-scoped token passed direct Machines API operations but failed `flyctl proxy`'s organization lookup. Do not distribute an organization-wide token to work around this. Network policies and a separately provisioned private tunnel are being evaluated for company-isolated ingress. Policies do not filter Fly Proxy traffic, so direct-ingress restrictions alone cannot establish isolation. The API also rejects an ingress rule with no allowed ports; do not assume an empty rule means deny-all.
+
+The live network fixture established direct private-IP access to port 8080, then denied that access with an ingress policy allowing only managed SSH on port 22. Flycast continued serving the fixture and its data. A separately provisioned custom-network WireGuard peer and userspace wireproxy tunnel also returned HTTP 200 without a Fly API token at runtime. This was one local tunnel, not an AWS core deployment.
+
+Design review identified two further requirements. Prefer a distinct raw TCP external port per deployment over the HTTP force-instance header: an untrusted app can emit Fly replay instructions when the HTTP handler is enabled. Port assignments need durable collision-free ownership. Also give simultaneously running core tunnels distinct WireGuard peers; sharing one peer causes endpoint roaming between replicas. Exclusive peer assignment and blue-green overlap remain unimplemented and unqualified.
+
+A live two-Machine fixture in the same Fly app passed six raw TCP routing assertions: plain responses, `fly-replay` headers, and replay JSON each stayed on the assigned deployment port, both with and without a forged `fly-force-instance-id` request header targeting the sibling Machine. Replay instructions were returned as ordinary bytes; sibling application data was not returned. The initial requests ran before the new service was ready and received empty replies; after a successful plain-response baseline, all six assertions passed. This supports raw TCP port routing, but does not qualify port allocation, concurrent wake-up, QM authentication, or overlapping core tunnels.
+
+The same two Machines were then confirmed suspended and received 16 concurrent requests split evenly between their raw TCP ports. All 16 returned the correct application response, both Machines started, and the durable fixture retained its file and SQLite counters. Requests completed in 0.56–0.67 seconds; total request wall time was 0.675 seconds. This is a two-Machine private-tunnel measurement, not a fleet throughput estimate or a full QM cold-start measurement.
+
+The provider now has an optional shared-app mode requiring a durable configuration store, durable port claims, and persistent volumes. Machine metadata selects the owning deployment; each deployment keeps its volume and port across archive and restore. Tests cover sibling preservation through update, always-on, archive and restore, plus an occupied initial port and provider recreation. Set `FLY_DEPLOY_SHARED_APP_NAME` to select this mode; wiring supplies Postgres-backed port and configuration maps. The shared app and private ingress must already exist. Private transport integration remains unfinished, and this mode has not been exercised through QM.
+
+Independent review found no additional sibling resource deletion or port collision defect, but rejected a database lease alone as fencing for WireGuard peer reuse: a paused core can leave its tunnel child alive after the database releases the lock. Peer reuse must wait for authoritative previous-task termination or equivalent fencing. Acceptance must include a paused core plus server-side database-session termination. Shared provisioning must also establish and verify the separate ingress network and direct-Machine ingress restrictions; the provider's private-IP check alone does not establish this boundary.
+
+The shared provider itself passed a live two-deployment fixture: parallel publish assigned distinct ports, a file/SQLite write in one deployment left its sibling unchanged, archiving that deployment left the sibling serving, and provider recreation restored the original port and persisted data. This fixture uses file-backed maps and the existing local tunnel; Postgres-backed QM and AWS task lifecycle remain unqualified.
+
+The peer-claim prototype now uses sticky ECS task ARN ownership, exact STOPPED verification, and atomic compare-and-swap on reclamation. Unit tests cover concurrent tasks, stale observations, missing records and lookup errors. Independent review accepted the cross-task fencing premise for a tunnel contained in a Fargate task, but requires one manager per task and trusted task metadata. Integration must persist STOPPED evidence promptly or replace stranded peers because ECS task history is temporary. This claim helper is not yet connected to a running tunnel or fleet rollout scripts.
+
+Peer claims can now retain confirmed termination evidence and reuse it after ECS history disappears; tests ensure that evidence is cleared on reassignment and cannot mark a replacement owner stopped. The collector still needs runtime scheduling and rollout recovery integration. The local tunnel lifecycle passed two start/serve/stop cycles against the live shared provider, preserving file and SQLite data and rejecting an occupied listener. This does not yet prove task identity, singleton behavior across process failure, or ECS rollout handoff.
+
+QM HTTP deployment proxy and agent fetch now support the local SOCKS tunnel. A live route fixture reached the shared Fly app and preserved data: unsigned requests returned 401, unrelated actors 404, owners 200, and the ordinary proxy 200. Authorization decisions were stubbed, so this proves transport and route enforcement, not Bookface or real scope ACL acceptance. The affected proxy/fetch and peer-claim tests passed (29 tests).
+
+Runtime wiring now constructs a lazy Fargate tunnel manager from `FLY_DEPLOY_WIREGUARD_PEERS`, requires private connectivity for shared-app mode, and persists SOCKS routing on deployment endpoints. Task-termination collection runs while the tunnel manager is active. The core Dockerfile builds wireproxy v1.1.2 in a separate stage; that image has not yet been built or deployed. Hostname resolution through the private tunnel passed the live route fixture. ECS startup, task failure and deployment overlap still need live qualification.
+
+Runtime review fixes now have tests: concurrent requests restart a confirmed-dead tunnel once; termination collection starts with core runtime even without application traffic; and peer claim IDs derive from the actual X25519 public identity, preventing duplicate private-key identities or label changes from bypassing ownership. The initial local core-image build is in progress; the final source must be rebuilt before image qualification.
+
+The local ARM64 core image built successfully after the review fixes (`sha256:3b5e80d716c48902f06d580bf73fdef0116d29b84e8dd547cb215be1fb208b4f`). Its wireproxy executable runs. The pinned v1.1.2 source reports its upstream hard-coded `1.0.8-dev` banner when built without release linker flags; source/module provenance, rather than that banner, identifies this build. This image is not yet an ECS candidate or a production release.
+
+A standalone ARM64 Fargate task passed private transport QA with source `ce4b53eaad246ab2733fdf98ef160c24ceda8ff6` and ECR digest `sha256:15fdf763ed76e0e6376fec44087a20da405d027c934dcaf438451b85e6c94144`. It verified its exact running ECS task record, started wireproxy inside Fargate, resolved the private Flycast hostname and read the expected file/SQLite data. It exited with code 0, reached STOPPED, and its task definition was deregistered. This used the tunnel helper directly, not the full QM runtime or shared Postgres claims; overlapping core-task ownership remains to qualify.
+
+The live Fargate overlap fixture passed against disposable Postgres. An observer confirmed core A was SIGSTOP-paused while its tunnel remained in the task; B terminated A's database sessions, acquired a different peer and served requests while A remained RUNNING. After ECS confirmed A STOPPED, C reused A's peer and served requests while B continued on its original peer. The first attempted pause ran Node as PID 1 and did not pause; it was discarded and repeated with ECS init enabled and an independent process-state observer. Full QM login/ACL and published-app acceptance still remain.
+
+The deployment contract now requires `FLY_DEPLOY_WIREGUARD_PEERS` when shared Fly publishing is selected, in both CLI secret discovery and runtime validation. AWS core task roles gain cluster-scoped `ecs:DescribeTasks` for peer reclamation. The Fly token check uses the shared application's Machines list, avoiding an organization-list permission requirement for app-scoped credentials. All 155 affected config, secret, Terraform and Fly CLI tests passed; typecheck and lint passed. These source changes have not been applied to company infrastructure or included in the earlier candidate image.
+
+A full core runtime in Fargate passed HTTP API acceptance using the immutable candidate above and a fresh disposable Postgres sidecar. The test invoked real `src/index.ts`, published through `/v1/deployments`, and exercised persisted ACLs: signed-out access returned 401, an unrelated user received 403, and that user could not archive the app. Owner access, file/SQLite writes, a 1 MiB upload, two separate SSE chunks, redirects and cookies passed. Publishing v2 and archive/restore preserved both counters. The successful run took 54.6 seconds from the QA process start, excluding ECS provisioning/image pull. The first run incorrectly expected unrelated-user denial to be 404; its 403 response was valid and the assertion was corrected before the successful rerun. Both runs archived their test apps and their exact orphan volumes were removed. This test used signed test identities, not Bookface; browser login, cross-company isolation, WebSockets and idle/wake through QM remain acceptance work.
+
+Independent comparison with deployed source `208086ab` found WebSockets unsupported on the existing published-app path: neither core nor portal has an upgrade handler, and AWS app subdomains still enter core's authentication/proxy path. The Fly candidate does not remove an existing WebSocket path. Record this as a product limitation rather than a Fly regression; raw provider WebSocket support alone would not establish QM support. The QA app-scoped token also passed own-app Machine listing (200) and was denied a separate app's Machine listing (403); this is a control-plane credential check, not proof of cross-company network isolation.
+
+The full runtime idle test observed the exact deployment Machine suspended before eight concurrent authenticated QM reads; all succeeded with intact file/SQLite counters in 763 ms. With no further application requests, automatic suspension was observed after 324.3 seconds, and the next authenticated QM request resumed it in 622 ms with both counters intact. This used the actual Fargate core proxy and Postgres-backed provider wiring. A separate local dev-instance browser check on branch source loaded the signed-in web Apps and admin Apps surfaces; it used a disposable local database and development identity, not Bookface or the Fly backend. Local Docker disk exhaustion was worked around with a QA-only tmpfs database, and that dev instance was stopped after inspection.
+
+The published-code network probe passed with positive controls: the trusted ingress served the target app and the attacking Machine reached its own listener, while that Machine could not connect to its sibling's private address, the private ingress Flycast address, or the original organization-network Flycast address. A second QA company then used its own app network, separate ingress network, private Flycast address, app-scoped token and WireGuard peer. Both trusted ingresses served their own canary; A's ingress could not reach B's Flycast address and vice versa. Code executing in A could reach its own listener but neither B's private Machine nor B's ingress. Both publisher tokens received 403 for the other app's Machines API. The canaries contained no company data or application credentials.
+
+Final review fixes preserve the latest ephemeral publication when always-on changes, retain legacy idle cleanup for existing standalone Fly apps, and persist changed always-on settings only after successful version application. Regression tests cover failed redeploy and rollback in both setting directions. Shared publishing now verifies dedicated nondefault ingress networks and all-Machine TCP22-only policies; narrowed selectors are rejected. Fleet provisioning additionally binds the exact company ingress network, rejects existing published Machines pending explicit migration, and validates that trusted ingress contains only a recoverable initializer.
+
+After removing the QA app's obsolete default-network Flycast address, the trusted ingress still served version 2 with file and SQLite counters intact. Published-code probes could reach their own loopback listener but could not reach a sibling Machine or the trusted ingress address. This verifies address cleanup independently of final-image acceptance. Bookface/browser qualification and the rebuilt final candidate remain outstanding.
+
+The rebuilt image from source `6d51832edfa442019ee03005c2f0c8d0c3b73c35` (`sha256:e7a431a6232b533fadedd8a43fb6b374cfdeef0835b178d78a9ed4b3cc68dce4`) passed publishing, persisted ACL checks, upload/streaming and eight concurrent requests after forced suspension (871 ms). It failed the automatic-suspension assertion: the Machine remained running for the entire 360-second observation window. The task exited 1; its test Machine, exact disposable volume and task definition were cleaned up. Preserve this failed acceptance result; the earlier successful idle measurement does not qualify the rebuilt candidate. Investigate idle timing and active connections before activation.
+
+A diagnostic run of the same immutable image extended observation without changing the production runtime. The disposable app added passive connection-count logging. Automatic suspension occurred after 536,868 ms (8m57s), and the subsequent request woke it in 987 ms with data intact. Update and archive/restore persistence passed afterward. ECS independently records both containers exiting 0 and the exact candidate digest. Twelve captured application-connection samples were zero; they cover part of the idle interval rather than continuous observation. The Machine, its exact volume, and the task definition were cleaned up. Independent review checked the success and earlier failure receipts. This establishes eventual automatic suspension and request-driven wake, with variable timing; it does not establish an upper bound or erase the earlier six-minute failure. A separate single-Machine comparison in a separate prepared company app passed both trusted peers and observed automatic suspension after 376,319 ms (6m16s). The comparison Machine was deleted, and live inventory confirmed the app empty afterward. This simpler fixture supports timing variability; it does not replace the full-runtime checks or establish a deadline.
diff --git a/package-lock.json b/package-lock.json
index 6760a0a13..63f37cfc7 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -12,6 +12,7 @@
"@anthropic-ai/claude-agent-sdk": "0.3.211",
"@anthropic-ai/tokenizer": "^0.0.4",
"@aws-crypto/sha256-js": "^5.2.0",
+ "@aws-sdk/client-ecs": "3.1096.0",
"@aws-sdk/client-s3": "^3.1061.0",
"@aws-sdk/client-secrets-manager": "^3.1089.0",
"@aws-sdk/client-sts": "^3.1075.0",
@@ -38,6 +39,7 @@
"pg": "^8.13.1",
"pg-boss": "^12.26.1",
"porter-sandbox": "^0.1.41",
+ "socks-proxy-agent": "10.1.0",
"tar-stream": "^3.2.0",
"typebox": "^1.1.38",
"undici": "^8.9.0",
@@ -375,6 +377,25 @@
"node": ">=20.0.0"
}
},
+ "node_modules/@aws-sdk/client-ecs": {
+ "version": "3.1096.0",
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-ecs/-/client-ecs-3.1096.0.tgz",
+ "integrity": "sha512-4jWCTCyfh9LMSjvEy3bQEIgWKelgZCPav/mHRZzDqhdcTu5EnNNA+Dh+Y5LzyPRbd+8PLLI+HxFnxsV/+igukQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@aws-sdk/core": "^3.977.1",
+ "@aws-sdk/credential-provider-node": "^3.972.73",
+ "@aws-sdk/types": "^3.974.2",
+ "@smithy/core": "^3.29.8",
+ "@smithy/fetch-http-handler": "^5.6.10",
+ "@smithy/node-http-handler": "^4.9.10",
+ "@smithy/types": "^4.16.1",
+ "tslib": "^2.6.2"
+ },
+ "engines": {
+ "node": ">=20.0.0"
+ }
+ },
"node_modules/@aws-sdk/client-s3": {
"version": "3.1096.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1096.0.tgz",
@@ -9899,6 +9920,16 @@
"url": "https://github.com/chalk/slice-ansi?sponsor=1"
}
},
+ "node_modules/smart-buffer": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
+ "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 6.0.0",
+ "npm": ">= 3.0.0"
+ }
+ },
"node_modules/smol-toml": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
@@ -9911,6 +9942,43 @@
"url": "https://github.com/sponsors/cyyynthia"
}
},
+ "node_modules/socks": {
+ "version": "2.8.10",
+ "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.10.tgz",
+ "integrity": "sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==",
+ "license": "MIT",
+ "dependencies": {
+ "ip-address": "^10.1.1",
+ "smart-buffer": "^4.2.0"
+ },
+ "engines": {
+ "node": ">= 10.0.0",
+ "npm": ">= 3.0.0"
+ }
+ },
+ "node_modules/socks-proxy-agent": {
+ "version": "10.1.0",
+ "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-10.1.0.tgz",
+ "integrity": "sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==",
+ "license": "MIT",
+ "dependencies": {
+ "agent-base": "9.0.0",
+ "debug": "^4.3.4",
+ "socks": "^2.8.3"
+ },
+ "engines": {
+ "node": ">= 20"
+ }
+ },
+ "node_modules/socks-proxy-agent/node_modules/agent-base": {
+ "version": "9.0.0",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-9.0.0.tgz",
+ "integrity": "sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 20"
+ }
+ },
"node_modules/sonic-boom": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
diff --git a/package.json b/package.json
index c116a59d6..93e7f76ea 100644
--- a/package.json
+++ b/package.json
@@ -56,6 +56,7 @@
"@anthropic-ai/claude-agent-sdk": "0.3.211",
"@anthropic-ai/tokenizer": "^0.0.4",
"@aws-crypto/sha256-js": "^5.2.0",
+ "@aws-sdk/client-ecs": "3.1096.0",
"@aws-sdk/client-s3": "^3.1061.0",
"@aws-sdk/client-secrets-manager": "^3.1089.0",
"@aws-sdk/client-sts": "^3.1075.0",
@@ -82,6 +83,7 @@
"pg": "^8.13.1",
"pg-boss": "^12.26.1",
"porter-sandbox": "^0.1.41",
+ "socks-proxy-agent": "10.1.0",
"tar-stream": "^3.2.0",
"typebox": "^1.1.38",
"undici": "^8.9.0",
diff --git a/src/api/routes/deployments.ts b/src/api/routes/deployments.ts
index 0a962438c..c141cb807 100644
--- a/src/api/routes/deployments.ts
+++ b/src/api/routes/deployments.ts
@@ -1,3 +1,4 @@
+import { SocksProxyAgent } from "socks-proxy-agent";
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
import {
@@ -30,6 +31,12 @@ import { principalDestination } from "../../reach/reach.ts";
import { portalSessionSub } from "../../deploy/viewer-session.ts";
import { proxyHeaders } from "../../util/http-proxy.ts";
+function deploymentProxyAgent(port?: number): { agent?: SocksProxyAgent } {
+ if (port === undefined) return {};
+ if (!Number.isInteger(port) || port < 1024 || port > 65535) throw new Error("Invalid deployment SOCKS port");
+ return { agent: new SocksProxyAgent(`socks5h://127.0.0.1:${port}`, { keepAlive: false }) };
+}
+
function isDeployInput(b: unknown): b is DeployInput {
return (
isObj(b) &&
@@ -538,15 +545,25 @@ async function proxyReach(
return;
}
const htmlNav = wantsWarmingPage(req, method);
- const up = requestFn({ hostname: host, port, path: subPath + url.search, method, headers }, (upRes) => {
- up.setTimeout(0);
- markUpstreamUp(upstreamKey);
- upRes.on("error", () => res.destroy());
- armThrottleShield(upstreamKey, upRes.statusCode ?? 0, upRes);
- const headers = gatewaySafeResponseHeaders(upRes.headers, opts?.sandbox ?? false);
- res.writeHead(upRes.statusCode ?? 502, headers);
- upRes.pipe(res);
- });
+ const up = requestFn(
+ {
+ hostname: host,
+ port,
+ path: subPath + url.search,
+ method,
+ headers,
+ ...deploymentProxyAgent(reach.endpoint.socksProxyPort),
+ },
+ (upRes) => {
+ up.setTimeout(0);
+ markUpstreamUp(upstreamKey);
+ upRes.on("error", () => res.destroy());
+ armThrottleShield(upstreamKey, upRes.statusCode ?? 0, upRes);
+ const headers = gatewaySafeResponseHeaders(upRes.headers, opts?.sandbox ?? false);
+ res.writeHead(upRes.statusCode ?? 502, headers);
+ upRes.pipe(res);
+ },
+ );
const dialMs = warmingDialTimeoutMs(
upstreamKey,
htmlNav,
@@ -623,7 +640,14 @@ function deploymentFetchHttp1(
const { host, port, tls, proxyHeaders } = endpoint.endpoint;
const requestFn = tls ? httpsRequest : httpRequest;
const request = requestFn(
- { hostname: host, port, path, method: "GET", headers: { ...proxyHeaders, "accept-encoding": "identity" } },
+ {
+ hostname: host,
+ port,
+ path,
+ method: "GET",
+ headers: { ...proxyHeaders, "accept-encoding": "identity" },
+ ...deploymentProxyAgent(endpoint.endpoint.socksProxyPort),
+ },
async (response) => {
clearTimeout(timeout);
try {
diff --git a/src/config.ts b/src/config.ts
index e926583d9..b19174fdf 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -742,18 +742,28 @@ function awsDeployEnv(env: NodeJS.ProcessEnv): AwsDeployEnv {
interface FlyDeployEnv {
token: string;
appPrefix: string;
+ sharedAppName?: string;
+ wireguardPeers?: string;
+ metadataUri?: string;
baseImage: string;
org: string;
region?: string;
+ dataVolumeSizeGb?: number;
}
function flyDeployEnv(env: NodeJS.ProcessEnv): FlyDeployEnv {
return {
token: env.FLY_DEPLOY_API_TOKEN ?? "",
appPrefix: env.FLY_DEPLOY_APP_PREFIX ?? "",
+ ...(env.FLY_DEPLOY_SHARED_APP_NAME ? { sharedAppName: env.FLY_DEPLOY_SHARED_APP_NAME } : {}),
+ ...(env.FLY_DEPLOY_WIREGUARD_PEERS ? { wireguardPeers: env.FLY_DEPLOY_WIREGUARD_PEERS } : {}),
+ ...(env.ECS_CONTAINER_METADATA_URI_V4 ? { metadataUri: env.ECS_CONTAINER_METADATA_URI_V4 } : {}),
baseImage: env.FLY_DEPLOY_BASE_IMAGE ?? "",
org: env.FLY_ORG ?? "",
...(env.FLY_REGION ? { region: env.FLY_REGION } : {}),
+ ...(env.FLY_DEPLOY_DATA_VOLUME_SIZE_GB !== undefined
+ ? { dataVolumeSizeGb: Number(env.FLY_DEPLOY_DATA_VOLUME_SIZE_GB) }
+ : {}),
};
}
diff --git a/src/deploy/deploy-provider.ts b/src/deploy/deploy-provider.ts
index e9b41c035..bd0563bc5 100644
--- a/src/deploy/deploy-provider.ts
+++ b/src/deploy/deploy-provider.ts
@@ -20,6 +20,7 @@ export interface DeployProvider {
apply(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint>;
reconcile?(d: Deployment, version: DeploymentVersion, input: DeployReconcileInput): Promise<DeployEndpoint>;
destroy(d: Deployment): Promise<void>;
+ setAlwaysOn?(d: Deployment, alwaysOn: boolean): Promise<void>;
resolveEndpoint?(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint | null>;
/** Recent output from the running app (entrypoint stdout+stderr), newest last. */
logs?(d: Deployment, opts: { tailLines: number }): Promise<string | null>;
diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.ts
index 8fbfe00e1..9f55bb1f9 100644
--- a/src/deploy/deploy-service.ts
+++ b/src/deploy/deploy-service.ts
@@ -66,11 +66,17 @@ export interface DeployService {
deploy(input: DeployInput): Promise<Deployment>;
redeploy(
id: string,
- input: { entrypoint: string; files: DeployFile[]; homeFiles?: DeployFile[]; env?: Record<string, string> },
+ input: {
+ entrypoint: string;
+ files: DeployFile[];
+ homeFiles?: DeployFile[];
+ env?: Record<string, string>;
+ alwaysOn?: boolean;
+ },
): Promise<Deployment>;
getDeployment(idOrName: string): Promise<Deployment | null>;
listDeployments(): Promise<Deployment[]>;
- rollbackDeployment(id: string, version: number): Promise<void>;
+ rollbackDeployment(id: string, version: number, options?: { alwaysOn?: boolean }): Promise<void>;
archiveDeployment(id: string): Promise<void>;
restoreDeployment(id: string, actorId?: string): Promise<Deployment>;
renameDeployment(id: string, name: string): Promise<Deployment>;
@@ -160,8 +166,10 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
id: string,
version: DeploymentVersion,
fromVersion?: number,
+ alwaysOn?: boolean,
): Promise<DeployEndpoint> => {
- const d = (await deps.deployStore.get(id))!;
+ const stored = (await deps.deployStore.get(id))!;
+ const d = alwaysOn === undefined ? stored : { ...stored, alwaysOn };
const deploymentEnv = await deps.deploymentEnv?.(d);
if (deploymentEnv && Object.keys(deploymentEnv).length)
version = { ...version, env: { ...version.env, ...deploymentEnv } };
@@ -185,6 +193,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
}
endpoint = await deps.provider.apply(d, materialized);
}
+ if (alwaysOn !== undefined) await deps.deployStore.setAlwaysOn(id, alwaysOn);
if (endpoint.image && endpoint.image !== version.image) {
await deps.deployStore.setVersionImage(id, version.version, endpoint.image);
}
@@ -389,7 +398,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
});
const d = await deps.deployStore.get(id);
if (!d) throw new Error(`unknown deployment: ${id}`);
- const endpoint = await applyVersion(id, v, before?.appliedVersion ?? before?.currentVersion);
+ const endpoint = await applyVersion(id, v, before?.appliedVersion ?? before?.currentVersion, input.alwaysOn);
await markVersionRunning(id, v.version, endpoint);
deps.auditLog.record({
at: Date.now(),
@@ -410,14 +419,14 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
return deps.deployStore.list();
},
- async rollbackDeployment(id, version) {
+ async rollbackDeployment(id, version, options) {
return withDeployLock(id, async () => {
const before = await deps.deployStore.get(id);
await deps.deployStore.setCurrentVersion(id, version);
const v = await deps.deployStore.versionOf(id, version);
const d = await deps.deployStore.get(id);
if (!d || !v) return;
- const endpoint = await applyVersion(id, v, before?.appliedVersion ?? before?.currentVersion);
+ const endpoint = await applyVersion(id, v, before?.appliedVersion ?? before?.currentVersion, options?.alwaysOn);
await markVersionRunning(id, v.version, endpoint);
deps.auditLog.record({
at: Date.now(),
@@ -513,6 +522,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
return withDeployLock(id, async () => {
const d = await deps.deployStore.get(id);
if (!d) throw new Error(`unknown deployment: ${id}`);
+ if (d.status === "running" && Boolean(d.alwaysOn) !== alwaysOn) await deps.provider.setAlwaysOn?.(d, alwaysOn);
await deps.deployStore.setAlwaysOn(id, alwaysOn);
deps.auditLog.record({
at: Date.now(),
@@ -645,18 +655,18 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
resource: existing.id,
scopeLabel: existing.ownerScopeId,
});
- if (input.alwaysOn !== undefined) await deps.deployStore.setAlwaysOn(existing.id, input.alwaysOn);
if ((input.entrypoint !== undefined || input.files !== undefined) && input.files) {
const entrypoint = requiredEntrypoint(input.entrypoint, existing);
await this.redeploy(existing.id, {
entrypoint,
files: input.files,
+ ...(input.alwaysOn !== undefined ? { alwaysOn: input.alwaysOn } : {}),
...(input.homeFiles ? { homeFiles: input.homeFiles } : {}),
...(input.env ? { env: input.env } : {}),
});
if (input.defaultAudience)
await reconcileDefaultAudience((await deps.deployStore.get(existing.id))!, input.defaultAudience, false);
- }
+ } else if (input.alwaysOn !== undefined) await this.setDeploymentAlwaysOn(existing.id, input.alwaysOn);
if (input.share?.length) await issueShares((await deps.deployStore.get(existing.id))!, createdBy, input.share);
return (await deps.deployStore.get(existing.id))!;
}
@@ -667,8 +677,11 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
if (!existing) throw new Error(`no deployment named: ${input.name}`);
if (!(await canManage(existing, ownerScopeId, createdBy, input.createdInScope)))
throw new Error(`not authorized to manage deployment: ${input.name}`);
- if (input.alwaysOn !== undefined) await deps.deployStore.setAlwaysOn(existing.id, input.alwaysOn);
- await this.rollbackDeployment(existing.id, input.rollbackTo);
+ await this.rollbackDeployment(
+ existing.id,
+ input.rollbackTo,
+ input.alwaysOn !== undefined ? { alwaysOn: input.alwaysOn } : undefined,
+ );
if (input.share?.length) await issueShares((await deps.deployStore.get(existing.id))!, createdBy, input.share);
return (await deps.deployStore.get(existing.id))!;
}
@@ -684,11 +697,11 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
) {
throw new Error(`deployment name taken: ${input.name}`);
}
- if (input.alwaysOn !== undefined) await deps.deployStore.setAlwaysOn(existing.id, input.alwaysOn);
const entrypoint = requiredEntrypoint(input.entrypoint, existing);
d = await this.redeploy(existing.id, {
entrypoint,
files,
+ ...(input.alwaysOn !== undefined ? { alwaysOn: input.alwaysOn } : {}),
...(input.homeFiles ? { homeFiles: input.homeFiles } : {}),
...(input.env ? { env: input.env } : {}),
});
diff --git a/src/deploy/deploy-store.ts b/src/deploy/deploy-store.ts
index 88abb9625..9fea7aa53 100644
--- a/src/deploy/deploy-store.ts
+++ b/src/deploy/deploy-store.ts
@@ -32,6 +32,7 @@ export interface DeployEndpoint {
tls?: boolean;
httpVersion?: "1.1" | "2";
proxyHeaders?: Record<string, string>;
+ socksProxyPort?: number;
}
export function publicUrlOf(endpoint: DeployEndpoint | null | undefined): string | undefined {
diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.ts
index f29147e0e..5899d2de7 100644
--- a/src/deploy/fly-deploy-provider.ts
+++ b/src/deploy/fly-deploy-provider.ts
@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
import { gzipSync } from "node:zlib";
import type { Deployment, DeploymentVersion } from "./deploy-store.ts";
import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";
@@ -6,6 +7,7 @@ import { makeTar } from "../sandbox/tar.ts";
import { sleep } from "../util/async.ts";
import { swallow, swallowAs } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
+import type { DurableMap } from "../persistence/durable-map.ts";
const MACHINES_API_BASE_URL = "https://api.machines.dev/v1";
const DEFAULT_REGION = "lhr";
@@ -29,6 +31,16 @@ interface FlyApp {
interface FlyIpAssignment {
ip: string;
+ network?: { name?: string; org_slug?: string };
+}
+
+interface FlyNetworkPolicy {
+ netpolSelector?: { all?: boolean };
+ rules?: Array<{
+ action?: string;
+ direction?: string;
+ ports?: Array<{ protocol?: string; port?: number }>;
+ }>;
}
interface FlyMachineExitEvent {
@@ -39,18 +51,24 @@ interface FlyMachineExitEvent {
export interface FlyMachine {
id: string;
state: string;
+ config?: FlyMachineConfig;
checks?: Array<{ name?: string; status?: string; output?: string }>;
events?: Array<{ request?: { exit_event?: FlyMachineExitEvent } }>;
}
export interface FlyMachineConfig {
image: string;
+ metadata?: Record<string, string>;
+ mounts?: Array<{ volume: string; path: string }>;
env: Record<string, string>;
guest: { cpu_kind: string; cpus: number; memory_mb: number };
files: Array<{ guest_path: string; raw_value: string }>;
services: Array<{
protocol: "tcp";
internal_port: number;
+ autostop: "suspend";
+ autostart: true;
+ min_machines_running: number;
ports: Array<{ port: number }>;
checks: Array<{ type: "tcp"; interval: string; timeout: string; grace_period: string }>;
}>;
@@ -58,8 +76,12 @@ export interface FlyMachineConfig {
}
interface FlyMachinesApi {
+ startMachine(appName: string, machineId: string): Promise<void>;
+ hasVolumes(appName: string): Promise<boolean>;
+ ensureVolume(appName: string, region: string, sizeGb: number, name: string): Promise<string>;
+ updateMachine(appName: string, machineId: string, config: FlyMachineConfig): Promise<void>;
ensureApp(appName: string, orgSlug: string): Promise<void>;
- ensurePrivateIngress(appName: string): Promise<void>;
+ ensurePrivateIngress(appName: string, provisioned: boolean, orgSlug: string): Promise<void>;
assertOwnedApp(appName: string, orgSlug: string): Promise<boolean>;
deleteApp(appName: string): Promise<void>;
listMachines(appName: string): Promise<FlyMachine[]>;
@@ -118,7 +140,56 @@ function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch })
}
};
return {
+ async startMachine(appName, machineId): Promise<void> {
+ const started = await request("POST", `${machine(appName, machineId)}/start`);
+ if (!started.ok) throw failure(`start machine ${machineId}`, started);
+ },
+ async hasVolumes(appName): Promise<boolean> {
+ const listed = await request("GET", `${app(appName)}/volumes`);
+ if (!listed.ok) throw failure(`list volumes in ${appName}`, listed);
+ return parse<unknown[]>(`list volumes in ${appName}`, listed).length > 0;
+ },
+ async ensureVolume(appName, region, sizeGb, name): Promise<string> {
+ const path = `${app(appName)}/volumes`;
+ const listed = await request("GET", path);
+ if (!listed.ok) throw failure(`list volumes in ${appName}`, listed);
+ const volumes = parse<Array<{ id: string; name: string; region: string }>>(
+ `list volumes in ${appName}`,
+ listed,
+ ).filter((volume) => name === "qm_data" || volume.name === name);
+ if (volumes.length) {
+ if (volumes.length !== 1 || volumes[0]!.name !== name || volumes[0]!.region !== region) {
+ throw new Error(`fly app ${appName} has unexpected volumes; reconcile storage before deploying`);
+ }
+ return volumes[0]!.id;
+ }
+ const created = await request("POST", path, {
+ name,
+ region,
+ size_gb: sizeGb,
+ encrypted: true,
+ snapshot_retention: 5,
+ auto_backup_enabled: true,
+ compute: GUEST,
+ });
+ if (!created.ok) throw failure(`create volume in ${appName}`, created);
+ return parse<{ id: string }>(`create volume in ${appName}`, created).id;
+ },
+ async updateMachine(appName, machineId, config): Promise<void> {
+ const updated = await request("POST", machine(appName, machineId), { config, skip_launch: false });
+ if (!updated.ok) throw failure(`update machine ${machineId}`, updated);
+ const state = parse<FlyMachine>(`update machine ${machineId}`, updated).state;
+ if (state === "stopped" || state === "suspended") {
+ const started = await request("POST", `${machine(appName, machineId)}/start`);
+ if (!started.ok) throw failure(`start updated machine ${machineId}`, started);
+ }
+ },
async ensureApp(appName, orgSlug): Promise<void> {
+ const existing = await getApp(appName);
+ if (existing) {
+ assertOwned(appName, orgSlug, existing);
+ return;
+ }
const r = await request("POST", "/apps", { app_name: appName, org_slug: orgSlug, network: appName });
if (r.ok) return;
if (r.status !== 409 && r.status !== 422 && !APP_ALREADY_TAKEN.test(r.text)) {
@@ -128,14 +199,48 @@ function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch })
if (!found) throw failure(`create app ${appName}`, r);
assertOwned(appName, orgSlug, found);
},
- async ensurePrivateIngress(appName): Promise<void> {
+ async ensurePrivateIngress(appName, provisioned, orgSlug): Promise<void> {
const listed = await request("GET", `${app(appName)}/ip_assignments`);
if (!listed.ok) throw failure(`list IP assignments for ${appName}`, listed);
const ips = parse<{ ips: FlyIpAssignment[] }>(`list IP assignments for ${appName}`, listed).ips;
if (ips.some((entry) => !entry.ip.toLowerCase().startsWith("fdaa:"))) {
throw new Error(`fly app ${appName} has a public IP assignment; refusing to expose the published app`);
}
+ if (provisioned && ips.length) {
+ if (
+ ips.some(
+ ({ network }) =>
+ !network?.name || network.name === "default" || network.name === appName || network.org_slug !== orgSlug,
+ )
+ ) {
+ throw new Error(`Fly shared app ${appName} requires ingress on a separate private network in ${orgSlug}`);
+ }
+ const response = await request("GET", `${app(appName)}/network_policies`);
+ if (!response.ok) throw failure(`read network policies for ${appName}`, response);
+ const policies = parse<FlyNetworkPolicy[]>(`read network policies for ${appName}`, response);
+ const ingress = (policy: FlyNetworkPolicy) =>
+ policy.rules?.filter((rule) => rule.direction === "ingress") ?? [];
+ if (
+ !policies.some(
+ (policy) =>
+ policy.netpolSelector?.all === true &&
+ Object.keys(policy.netpolSelector).length === 1 &&
+ ingress(policy).length > 0,
+ ) ||
+ policies.some((policy) =>
+ ingress(policy).some(
+ (rule) =>
+ rule.action !== "allow" ||
+ !rule.ports?.length ||
+ rule.ports.some((port) => port.protocol !== "tcp" || port.port !== 22),
+ ),
+ )
+ ) {
+ throw new Error(`Fly shared app ${appName} requires all-machine ingress restricted to TCP port 22`);
+ }
+ }
if (ips.length) return;
+ if (provisioned) throw new Error(`Fly shared app ${appName} needs private ingress provisioned before publishing`);
const assigned = await request("POST", `${app(appName)}/ip_assignments`, { type: "private_v6" });
if (!assigned.ok) throw failure(`allocate private ingress for ${appName}`, assigned);
},
@@ -192,9 +297,14 @@ function exitDetail(machine: FlyMachine | null): string {
export interface FlyDeployProviderOptions {
token: string;
appPrefix: string;
+ sharedAppName?: string;
+ portStore?: DurableMap<string>;
+ privateTransport?: { ensure(): Promise<number> };
baseImage: string;
org: string;
region?: string;
+ dataVolumeSizeGb?: number;
+ configStore?: DurableMap<FlyMachineConfig>;
machineStartTimeoutMs?: number;
appReadyTimeoutMs?: number;
pollIntervalMs?: number;
@@ -213,16 +323,56 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
const seconds = (ms: number): string => `${Math.round(ms / 1000)}s`;
function ensureConfigured(): void {
+ if (
+ opts.sharedAppName &&
+ (opts.sharedAppName.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.sharedAppName))
+ )
+ throw new Error("Fly shared app name must be a lowercase DNS label no longer than 63 characters");
+ if (opts.sharedAppName && (!opts.portStore || !opts.dataVolumeSizeGb || !opts.configStore))
+ throw new Error("Fly shared apps require persistent port assignments and durable storage");
+ if (opts.dataVolumeSizeGb && !opts.configStore)
+ throw new Error("Fly durable storage requires a persistent configuration store");
+ if (
+ opts.dataVolumeSizeGb !== undefined &&
+ (!Number.isInteger(opts.dataVolumeSizeGb) || opts.dataVolumeSizeGb < 1)
+ ) {
+ throw new Error("FLY_DEPLOY_DATA_VOLUME_SIZE_GB must be a positive integer");
+ }
if (!opts.token) throw new Error("FLY_DEPLOY_API_TOKEN not set (DEPLOY_PROVIDER=fly)");
- if (!opts.appPrefix) throw new Error("FLY_DEPLOY_APP_PREFIX not set (DEPLOY_PROVIDER=fly)");
+ if (!opts.appPrefix && !opts.sharedAppName) throw new Error("FLY_DEPLOY_APP_PREFIX not set (DEPLOY_PROVIDER=fly)");
if (!opts.baseImage) throw new Error("FLY_DEPLOY_BASE_IMAGE not set (DEPLOY_PROVIDER=fly)");
if (!opts.org) throw new Error("FLY_ORG not set (DEPLOY_PROVIDER=fly)");
- if (opts.appPrefix.length > 26 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.appPrefix)) {
+ if (
+ !opts.sharedAppName &&
+ (opts.appPrefix.length > 26 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(opts.appPrefix))
+ ) {
throw new Error("FLY_DEPLOY_APP_PREFIX must be a lowercase DNS label no longer than 26 characters");
}
}
- const appNameFor = (d: Deployment): string => `${opts.appPrefix}-${d.id}`;
+ const appNameFor = (d: Deployment): string => opts.sharedAppName ?? `${opts.appPrefix}-${d.id}`;
+ const endpointFor = async (d: Deployment, port: number): Promise<DeployEndpoint> => ({
+ host: `${appNameFor(d)}.flycast`,
+ port,
+ ...(opts.privateTransport ? { socksProxyPort: await opts.privateTransport.ensure() } : {}),
+ });
+ const volumeNameFor = (d: Deployment): string =>
+ opts.sharedAppName ? `qm_${createHash("sha256").update(d.id).digest("hex").slice(0, 24)}` : "qm_data";
+ const deploymentMachines = async (d: Deployment): Promise<FlyMachine[]> => {
+ const machines = await api.listMachines(appNameFor(d));
+ return opts.sharedAppName
+ ? machines.filter((machine) => machine.config?.metadata?.qm_deployment_id === d.id)
+ : machines;
+ };
+ async function portFor(d: Deployment): Promise<number> {
+ if (!opts.sharedAppName) return APP_PORT;
+ const first = createHash("sha256").update(d.id).digest().readUInt32BE(0) % 45536;
+ for (let offset = 0; offset < 45536; offset++) {
+ const port = 20000 + ((first + offset) % 45536);
+ if ((await opts.portStore!.putIfAbsent(`${opts.sharedAppName}:${port}`, d.id)) === d.id) return port;
+ }
+ throw new Error("Fly shared app has no available deployment ports");
+ }
async function bundleBase64(version: DeploymentVersion): Promise<string> {
const tree = await readTree(version.snapshotDir, { tolerateMissing: true });
@@ -242,8 +392,14 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
return encoded;
}
- const machineConfig = (version: DeploymentVersion, bundle: string): FlyMachineConfig => ({
+ const machineConfig = (
+ d: Deployment,
+ version: DeploymentVersion,
+ bundle: string,
+ port: number,
+ ): FlyMachineConfig => ({
image: opts.baseImage,
+ ...(opts.sharedAppName ? { metadata: { qm_deployment_id: d.id } } : {}),
env: { ...version.env, PORT: String(APP_PORT) },
guest: GUEST,
files: [{ guest_path: BUNDLE_GUEST_PATH, raw_value: bundle }],
@@ -251,7 +407,10 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
{
protocol: "tcp",
internal_port: APP_PORT,
- ports: [{ port: APP_PORT }],
+ autostop: "suspend",
+ autostart: true,
+ min_machines_running: d.alwaysOn ? 1 : 0,
+ ports: [{ port }],
checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],
},
],
@@ -267,9 +426,14 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
async function waitStarted(appName: string, machineId: string): Promise<void> {
const deadline = Date.now() + machineStartTimeoutMs;
let last: FlyMachine | null = null;
+ let requestedStart = false;
while (Date.now() < deadline) {
last = await api.getMachine(appName, machineId);
if (last?.state === "started") return;
+ if (!requestedStart && (last?.state === "stopped" || last?.state === "suspended")) {
+ requestedStart = true;
+ await api.startMachine(appName, machineId);
+ }
await sleep(pollIntervalMs);
}
throw new Error(
@@ -297,18 +461,114 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
}
return {
- profile: { managedScaleToZero: false },
+ profile: { managedScaleToZero: !!opts.sharedAppName, ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}) },
+
+ async resolveEndpoint(d) {
+ if (!opts.privateTransport) return d.endpoint;
+ ensureConfigured();
+ return endpointFor(d, await portFor(d));
+ },
+
+ async setAlwaysOn(d, alwaysOn): Promise<void> {
+ ensureConfigured();
+ const appName = appNameFor(d);
+ if (!(await api.assertOwnedApp(appName, opts.org))) throw new Error(`fly app ${appName} is missing`);
+ const machines = await deploymentMachines(d);
+ if (machines.length !== 1) throw new Error(`fly app ${appName} requires one machine to change always-on`);
+ const machine = await api.getMachine(appName, machines[0]!.id);
+ if (!machine?.config) throw new Error(`fly app ${appName} has no machine configuration`);
+ const accepted = machine.config.mounts?.length
+ ? ((await opts.configStore?.get(d.id)) ?? machine.config)
+ : machine.config;
+ const config = {
+ ...accepted,
+ services: accepted.services.map((service) => ({ ...service, min_machines_running: alwaysOn ? 1 : 0 })),
+ };
+ try {
+ await api.updateMachine(appName, machine.id, config);
+ await waitStarted(appName, machine.id);
+ await waitAppReady(appName, machine.id);
+ await api.setCordon(appName, machine.id, false);
+ if (opts.configStore) await opts.configStore.put(d.id, config);
+ } catch (error) {
+ await api.updateMachine(appName, machine.id, accepted);
+ await waitStarted(appName, machine.id);
+ await waitAppReady(appName, machine.id);
+ if (opts.configStore) await opts.configStore.put(d.id, accepted);
+ throw error;
+ }
+ },
async apply(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint> {
ensureConfigured();
const appName = appNameFor(d);
const bundle = await bundleBase64(version);
- await api.ensureApp(appName, opts.org);
- await api.ensurePrivateIngress(appName);
- const stale = await api.listMachines(appName);
+ if (opts.sharedAppName) {
+ if (!(await api.assertOwnedApp(appName, opts.org)))
+ throw new Error(`Fly shared app ${appName} must be provisioned before publishing`);
+ } else {
+ await api.ensureApp(appName, opts.org);
+ }
+ await api.ensurePrivateIngress(appName, Boolean(opts.sharedAppName), opts.org);
+ const stale = await deploymentMachines(d);
+ const port = await portFor(d);
+ if (!opts.dataVolumeSizeGb && (await api.hasVolumes(appName))) {
+ throw new Error(
+ `fly app ${appName} has durable storage; restore FLY_DEPLOY_DATA_VOLUME_SIZE_GB before deploying`,
+ );
+ }
+ if (opts.dataVolumeSizeGb) {
+ if (stale.length > 1) throw new Error(`fly app ${appName} has multiple machines; reconcile before deploying`);
+ const previous = stale[0] ? await api.getMachine(appName, stale[0].id) : null;
+ if (stale.length && !previous?.config?.mounts?.some((mount) => mount.path === "/data")) {
+ throw new Error(`fly app ${appName} requires an explicit migration from ephemeral storage`);
+ }
+ const volume = await api.ensureVolume(appName, region, opts.dataVolumeSizeGb, volumeNameFor(d));
+ if (previous && (previous.config!.mounts!.length !== 1 || previous.config!.mounts![0]!.volume !== volume)) {
+ throw new Error(`fly app ${appName} has an unexpected volume attachment`);
+ }
+ const config = machineConfig(d, version, bundle, port);
+ config.env.DATA_DIR = "/data";
+ config.mounts = [{ volume, path: "/data" }];
+ if (previous) {
+ const accepted = await opts.configStore!.get(d.id);
+ try {
+ await api.updateMachine(appName, previous.id, config);
+ await waitStarted(appName, previous.id);
+ await waitAppReady(appName, previous.id);
+ await api.setCordon(appName, previous.id, false);
+ await opts.configStore!.put(d.id, config);
+ } catch (error) {
+ if (accepted) {
+ await api.updateMachine(appName, previous.id, accepted);
+ await waitStarted(appName, previous.id);
+ await waitAppReady(appName, previous.id);
+ await api.setCordon(appName, previous.id, false);
+ await opts.configStore!.put(d.id, accepted);
+ }
+ throw error;
+ }
+ } else {
+ const created = await api.createMachine(appName, { region, config, skip_service_registration: true });
+ try {
+ await waitStarted(appName, created.id);
+ await waitAppReady(appName, created.id);
+ await api.setCordon(appName, created.id, false);
+ await opts.configStore!.put(d.id, config);
+ } catch (error) {
+ await api
+ .destroyMachine(appName, created.id)
+ .catch((cleanupError) =>
+ swallow("fly-deploy: remove unhealthy machine while retaining its data", cleanupError),
+ );
+ throw error;
+ }
+ }
+ return endpointFor(d, port);
+ }
const machine = await api.createMachine(appName, {
region,
- config: machineConfig(version, bundle),
+ config: machineConfig(d, version, bundle, port),
skip_service_registration: true,
});
try {
@@ -341,13 +601,17 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
.destroyMachine(appName, previous.id)
.catch((cleanupError) => swallow("fly-deploy: remove cordoned machine", cleanupError));
}
- return { host: `${appName}.flycast`, port: APP_PORT };
+ return endpointFor(d, port);
},
async destroy(d: Deployment): Promise<void> {
ensureConfigured();
const appName = appNameFor(d);
if (!(await api.assertOwnedApp(appName, opts.org))) return;
+ if (opts.dataVolumeSizeGb || (await api.hasVolumes(appName))) {
+ for (const machine of await deploymentMachines(d)) await api.destroyMachine(appName, machine.id);
+ return;
+ }
await api.deleteApp(appName);
},
};
diff --git a/src/deploy/fly-peer-claims.ts b/src/deploy/fly-peer-claims.ts
new file mode 100644
index 000000000..cb4c8fc48
--- /dev/null
+++ b/src/deploy/fly-peer-claims.ts
@@ -0,0 +1,65 @@
+import { ECSClient, DescribeTasksCommand, type DescribeTasksCommandOutput } from "@aws-sdk/client-ecs";
+import type { DurableMap } from "../persistence/durable-map.ts";
+
+export interface FlyPeerClaim {
+ taskArn: string;
+ stoppedConfirmedAt?: string;
+}
+
+export async function claimFlyPeer(opts: {
+ peerIds: readonly string[];
+ taskArn: string;
+ claims: DurableMap<FlyPeerClaim>;
+ taskStopped: (taskArn: string) => Promise<boolean>;
+}): Promise<string> {
+ if (!opts.taskArn || !opts.peerIds.length || new Set(opts.peerIds).size !== opts.peerIds.length)
+ throw new Error("Fly peer claims require a task identity and distinct peer IDs");
+ if (!opts.claims.update) throw new Error("Fly peer claims require atomic ownership updates");
+ const desired = { taskArn: opts.taskArn };
+ for (const id of opts.peerIds) {
+ if ((await opts.claims.get(id))?.taskArn === opts.taskArn) return id;
+ }
+ for (const id of opts.peerIds) {
+ if ((await opts.claims.putIfAbsent(id, desired)).taskArn === opts.taskArn) return id;
+ }
+ for (const id of opts.peerIds) {
+ const previous = await opts.claims.get(id);
+ if (!previous || (!previous.stoppedConfirmedAt && !(await opts.taskStopped(previous.taskArn)))) continue;
+ const assigned = await opts.claims.update(id, (current) =>
+ current.taskArn === previous.taskArn ? desired : current,
+ );
+ if (assigned?.taskArn === opts.taskArn) return id;
+ }
+ throw new Error("No Fly peer is available; previous owners must be confirmed stopped before reuse");
+}
+
+export function createEcsTaskStopped(opts: {
+ cluster: string;
+ describe?: (command: DescribeTasksCommand) => Promise<DescribeTasksCommandOutput>;
+}): (taskArn: string) => Promise<boolean> {
+ const client = opts.describe ? undefined : new ECSClient({});
+ const describe = opts.describe ?? ((command: DescribeTasksCommand) => client!.send(command));
+ return async (taskArn) => {
+ const result = await describe(new DescribeTasksCommand({ cluster: opts.cluster, tasks: [taskArn] }));
+ if (result.failures?.length) return false;
+ return (
+ result.tasks?.length === 1 && result.tasks[0]?.taskArn === taskArn && result.tasks[0]?.lastStatus === "STOPPED"
+ );
+ };
+}
+
+export async function recordStoppedFlyPeers(opts: {
+ peerIds: readonly string[];
+ claims: DurableMap<FlyPeerClaim>;
+ taskStopped: (taskArn: string) => Promise<boolean>;
+}): Promise<void> {
+ if (!opts.claims.update) throw new Error("Fly peer claims require atomic ownership updates");
+ for (const id of opts.peerIds) {
+ const previous = await opts.claims.get(id);
+ if (!previous || previous.stoppedConfirmedAt || !(await opts.taskStopped(previous.taskArn))) continue;
+ const stoppedConfirmedAt = new Date().toISOString();
+ await opts.claims.update(id, (current) =>
+ current.taskArn === previous.taskArn ? { ...current, stoppedConfirmedAt } : current,
+ );
+ }
+}
diff --git a/src/deploy/fly-private-tunnel.ts b/src/deploy/fly-private-tunnel.ts
new file mode 100644
index 000000000..d2e127fc0
--- /dev/null
+++ b/src/deploy/fly-private-tunnel.ts
@@ -0,0 +1,86 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createConnection, createServer } from "node:net";
+import { sleep } from "../util/async.ts";
+
+export interface FlyPrivateTunnel {
+ stop(): Promise<void>;
+ isAlive(): boolean;
+}
+
+export async function startFlyPrivateTunnel(opts: {
+ executable: string;
+ wireguardConfig: string;
+ port: number;
+}): Promise<FlyPrivateTunnel> {
+ if (!Number.isInteger(opts.port) || opts.port < 1024 || opts.port > 65535)
+ throw new Error("Fly tunnel requires a valid local port");
+ if (
+ /^\s*\[(?:Socks5|HTTP|TCPClientTunnel|TCPServerTunnel|UDPClientTunnel|UDPServerTunnel)\]/im.test(
+ opts.wireguardConfig,
+ )
+ )
+ throw new Error("Fly peer configuration must not contain forwarding listeners");
+ await new Promise<void>((resolve, reject) => {
+ const listener = createServer();
+ listener.once("error", reject);
+ listener.listen(opts.port, "127.0.0.1", () => listener.close((error) => (error ? reject(error) : resolve())));
+ });
+ const directory = await mkdtemp(join(tmpdir(), "qm-fly-tunnel-"));
+ const path = join(directory, "wireproxy.conf");
+ let child: ChildProcess | undefined;
+ let exited: Promise<void> = Promise.resolve();
+ let dead = false;
+ const stop = async () => {
+ if (child && !dead) {
+ child.kill("SIGTERM");
+ const kill = setTimeout(() => {
+ if (!dead) child!.kill("SIGKILL");
+ }, 2000);
+ await exited;
+ clearTimeout(kill);
+ }
+ await rm(directory, { recursive: true, force: true });
+ };
+ try {
+ await writeFile(path, `${opts.wireguardConfig}\n[Socks5]\nBindAddress = 127.0.0.1:${opts.port}\n`, { mode: 0o600 });
+ child = spawn(opts.executable, ["-c", path], { stdio: "ignore" });
+ exited = new Promise<void>((resolve) => {
+ child!.once("exit", () => {
+ dead = true;
+ resolve();
+ });
+ child!.once("error", () => {
+ dead = true;
+ resolve();
+ });
+ });
+ for (let i = 0; i < 100; i++) {
+ if (dead) throw new Error("Fly private tunnel exited before becoming ready");
+ const listening = await new Promise<boolean>((resolve) => {
+ const socket = createConnection({ host: "127.0.0.1", port: opts.port });
+ socket.setTimeout(100);
+ socket.once("connect", () => {
+ socket.destroy();
+ resolve(true);
+ });
+ socket.once("error", () => {
+ socket.destroy();
+ resolve(false);
+ });
+ socket.once("timeout", () => {
+ socket.destroy();
+ resolve(false);
+ });
+ });
+ if (listening && !dead) return { stop, isAlive: () => !dead };
+ await sleep(100);
+ }
+ throw new Error("Fly private tunnel did not become ready");
+ } catch (error) {
+ await stop();
+ throw error;
+ }
+}
diff --git a/src/deploy/fly-tunnel-manager.ts b/src/deploy/fly-tunnel-manager.ts
new file mode 100644
index 000000000..643da771a
--- /dev/null
+++ b/src/deploy/fly-tunnel-manager.ts
@@ -0,0 +1,132 @@
+import { createHash, createPrivateKey, createPublicKey } from "node:crypto";
+import type { DurableMap } from "../persistence/durable-map.ts";
+import { createSweeper } from "../util/sweeper.ts";
+import { claimFlyPeer, createEcsTaskStopped, recordStoppedFlyPeers, type FlyPeerClaim } from "./fly-peer-claims.ts";
+import { startFlyPrivateTunnel, type FlyPrivateTunnel } from "./fly-private-tunnel.ts";
+
+export interface FlyWireguardPeer {
+ id: string;
+ config: string;
+}
+
+export function parseFlyWireguardPeers(raw: string): FlyWireguardPeer[] {
+ let value: unknown;
+ try {
+ value = JSON.parse(raw);
+ } catch {
+ throw new Error("FLY_DEPLOY_WIREGUARD_PEERS must be a JSON array");
+ }
+ if (
+ !Array.isArray(value) ||
+ value.length < 2 ||
+ value.some((p) => !p || typeof p.id !== "string" || !p.id || typeof p.config !== "string" || !p.config) ||
+ new Set(value.map((p) => p.id)).size !== value.length
+ )
+ throw new Error("Fly private connectivity requires at least two distinct WireGuard peers");
+ const peers = value.map((peer) => {
+ const keys = [...peer.config.matchAll(/^\s*PrivateKey\s*=\s*([A-Za-z0-9+/]+=*)\s*$/gm)];
+ if (keys.length !== 1 || Buffer.from(keys[0]![1]!, "base64").length !== 32)
+ throw new Error("Each Fly WireGuard peer must contain one valid private key");
+ return {
+ id: createHash("sha256")
+ .update(
+ createPublicKey(
+ createPrivateKey({
+ key: Buffer.concat([
+ Buffer.from("302e020100300506032b656e04220420", "hex"),
+ Buffer.from(keys[0]![1]!, "base64"),
+ ]),
+ format: "der",
+ type: "pkcs8",
+ }),
+ ).export({ format: "der", type: "spki" }),
+ )
+ .digest("hex"),
+ config: peer.config,
+ };
+ });
+ if (new Set(peers.map((peer) => peer.id)).size !== peers.length)
+ throw new Error("Fly WireGuard peers must have distinct identities");
+ return peers;
+}
+
+export function createFlyTunnelManager(opts: {
+ peers: FlyWireguardPeer[];
+ claims: DurableMap<FlyPeerClaim>;
+ metadataUri: string;
+ executable: string;
+ port: number;
+}) {
+ let starting: Promise<number> | undefined;
+ let tunnel: FlyPrivateTunnel | undefined;
+ let stopped = false;
+ let collecting = false;
+
+ const sweeper = createSweeper(
+ async () => {
+ if (collecting || stopped) return;
+ collecting = true;
+ try {
+ const { shared } = await context();
+ await recordStoppedFlyPeers(shared);
+ } finally {
+ collecting = false;
+ }
+ },
+ 60_000,
+ { label: "fly-peer-termination", immediate: true },
+ );
+ async function context() {
+ const uri = new URL(opts.metadataUri);
+ if (uri.protocol !== "http:" || uri.hostname !== "169.254.170.2")
+ throw new Error("Fly private connectivity requires ECS task metadata");
+ const response = await fetch(`${uri.href.replace(/\/$/, "")}/task`, { signal: AbortSignal.timeout(5000) });
+ if (!response.ok) throw new Error("Unable to read ECS task identity for Fly private connectivity");
+ const metadata = (await response.json()) as { TaskARN?: string; Cluster?: string; LaunchType?: string };
+ if (metadata.LaunchType !== "FARGATE" || !metadata.TaskARN || !metadata.Cluster)
+ throw new Error("Fly private connectivity requires a Fargate task identity");
+ const taskStopped = createEcsTaskStopped({ cluster: metadata.Cluster });
+ const shared = { claims: opts.claims, peerIds: opts.peers.map((p) => p.id), taskStopped };
+ return { shared, taskArn: metadata.TaskARN };
+ }
+ async function start(): Promise<number> {
+ const previous = tunnel;
+ tunnel = undefined;
+ await previous?.stop();
+ const { shared, taskArn } = await context();
+ const peerId = await claimFlyPeer({ ...shared, taskArn });
+ if (stopped) throw new Error("Fly tunnel manager is stopped");
+ tunnel = await startFlyPrivateTunnel({
+ executable: opts.executable,
+ port: opts.port,
+ wireguardConfig: opts.peers.find((p) => p.id === peerId)!.config,
+ });
+ if (stopped) {
+ await tunnel.stop();
+ throw new Error("Fly tunnel manager is stopped");
+ }
+ sweeper.start();
+ return opts.port;
+ }
+ return {
+ monitor(): void {
+ if (!stopped) sweeper.start();
+ },
+ async ensure(): Promise<number> {
+ if (stopped) throw new Error("Fly tunnel manager is stopped");
+ if (tunnel && !tunnel.isAlive()) starting = undefined;
+ if (!starting)
+ starting = start().catch((error) => {
+ starting = undefined;
+ throw error;
+ });
+ return starting;
+ },
+ async stop(): Promise<void> {
+ stopped = true;
+ sweeper.stop();
+ await starting?.catch(() => {});
+ await tunnel?.stop();
+ },
+ };
+}
diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.ts
index 421b8b746..78798361f 100644
--- a/src/deployment/secret-schema.ts
+++ b/src/deployment/secret-schema.ts
@@ -11,6 +11,7 @@ type SecretGate =
| "porter"
| "agent37"
| "porter-deploy"
+ | "fly-shared-deploy"
| "fly-deploy"
| "aws-deploy-gate"
| "google-oauth"
@@ -45,6 +46,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [
{ name: "MODAL_TOKEN_SECRET", requiredWhen: "modal" },
{ name: "PORTER_DEPLOY_API_TOKEN", requiredWhen: ["porter", "porter-deploy"] },
{ name: "FLY_DEPLOY_API_TOKEN", requiredWhen: "fly-deploy" },
+ { name: "FLY_DEPLOY_WIREGUARD_PEERS", requiredWhen: "fly-shared-deploy" },
{ name: "AWS_DEPLOY_GATE_SECRET", requiredWhen: "aws-deploy-gate" },
{ name: "GOOGLE_OAUTH_CLIENT_SECRET", requiredWhen: "google-oauth" },
{ name: "DROPBOX_OAUTH_CLIENT_SECRET", requiredWhen: "dropbox-oauth" },
@@ -70,6 +72,7 @@ const GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => b
porter: (env) => sandboxBackendSelected(env, "porter"),
agent37: (env) => sandboxBackendSelected(env, "agent37"),
"porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter",
+ "fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()),
"fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly",
"aws-deploy-gate": (env) => Boolean(env.AWS_DEPLOY_APPS_DOMAIN || env.DEPLOY_APPS_DOMAIN),
"google-oauth": (env) => Boolean(env.GOOGLE_OAUTH_CLIENT_ID),
diff --git a/src/wiring.ts b/src/wiring.ts
index 6c41b95f5..0d0372565 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -1,3 +1,5 @@
+import { createFlyTunnelManager, parseFlyWireguardPeers } from "./deploy/fly-tunnel-manager.ts";
+import type { FlyPeerClaim } from "./deploy/fly-peer-claims.ts";
import { createMemoryEventBus } from "./util/event-bus.ts";
import { createPostgresNotifyBus } from "./persistence/postgres-notify-bus.ts";
import { emitRunText, type RunStreamEvent } from "./runs/run-stream-events.ts";
@@ -140,7 +142,7 @@ import { viewerIdentityKey } from "./deploy/access-token.ts";
import { deploymentCredentialSlugs } from "./deploy/deployment-credentials.ts";
import { createDockerDeployProvider } from "./deploy/docker-deploy-provider.ts";
import { createAwsDeployProvider, type StoredDeployBody } from "./deploy/aws-deploy-provider.ts";
-import { createFlyDeployProvider } from "./deploy/fly-deploy-provider.ts";
+import { createFlyDeployProvider, type FlyMachineConfig } from "./deploy/fly-deploy-provider.ts";
import { createPorterDeployProvider, type StoredPorterDeployBody } from "./deploy/porter-deploy-provider.ts";
import type { DeployProvider } from "./deploy/deploy-provider.ts";
import { createDeployService } from "./deploy/deploy-service.ts";
@@ -1389,10 +1391,28 @@ export function buildApp(
advisoryLock,
store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),
});
+ if (config.deployProvider === "fly" && config.flyDeploy.sharedAppName && !config.flyDeploy.wireguardPeers)
+ throw new Error("Fly shared apps require FLY_DEPLOY_WIREGUARD_PEERS for private connectivity");
+ const flyTunnel =
+ config.deployProvider === "fly" && config.flyDeploy.wireguardPeers
+ ? createFlyTunnelManager({
+ peers: parseFlyWireguardPeers(config.flyDeploy.wireguardPeers),
+ claims: artifactMap<FlyPeerClaim>("fly_peer_claims"),
+ metadataUri: config.flyDeploy.metadataUri ?? "",
+ executable: "wireproxy",
+ port: 18096,
+ })
+ : undefined;
const buildDeployProvider: Record<Config["deployProvider"], () => DeployProvider> = {
aws: buildAwsDeploy,
docker: createDockerDeployProvider,
- fly: () => createFlyDeployProvider(config.flyDeploy),
+ fly: () =>
+ createFlyDeployProvider({
+ ...config.flyDeploy,
+ ...(flyTunnel ? { privateTransport: flyTunnel } : {}),
+ configStore: artifactMap<FlyMachineConfig>("fly_deploy_configs"),
+ portStore: artifactMap<string>("fly_deploy_ports"),
+ }),
porter: () =>
createPorterDeployProvider({
...config.porterDeploy,
@@ -1400,6 +1420,13 @@ export function buildApp(
store: artifactMap<StoredPorterDeployBody>("porter_deploy_bodies"),
}),
};
+ if (
+ config.deployProvider === "fly" &&
+ (config.flyDeploy.dataVolumeSizeGb || config.flyDeploy.sharedAppName || config.flyDeploy.wireguardPeers) &&
+ !config.databaseUrl
+ ) {
+ throw new Error("Fly durable application storage requires DATABASE_URL for persistent rollback configuration");
+ }
const deployProvider: DeployProvider = buildDeployProvider[config.deployProvider]();
if (config.deployProvider === "aws" && !config.awsDeploy.dataBucket && !config.awsSandbox.s3Bucket) {
console.warn(
@@ -2130,6 +2157,7 @@ export function buildApp(
: null;
const runtime: Runtime = {
start() {
+ flyTunnel?.monitor();
if (!config.backgroundWorkEnabled) return;
for (const w of workers) w.start();
reaper.start();
@@ -2178,6 +2206,7 @@ export function buildApp(
void runStreamEvents.close?.();
await harness.turns.close?.();
await tasks.close?.();
+ await flyTunnel?.stop();
},
};
diff --git a/test/config.test.ts b/test/config.test.ts
index 7289e7c47..2c22469f6 100644
--- a/test/config.test.ts
+++ b/test/config.test.ts
@@ -737,3 +737,15 @@ test("sandbox scope defaults parse exact scope kinds and reject malformed mappin
])
assert.throws(() => loadConfig({ ...credentials, SANDBOX_SCOPE_BACKENDS: value }));
});
+
+test("Fly shared application name is passed to the deployment provider", () => {
+ const config = loadConfig({
+ DEPLOY_PROVIDER: "fly",
+ FLY_DEPLOY_SHARED_APP_NAME: "qm-example-apps",
+ FLY_DEPLOY_WIREGUARD_PEERS: "[]",
+ FLY_DEPLOY_API_TOKEN: "test-token",
+ FLY_DEPLOY_DATA_VOLUME_SIZE_GB: "1",
+ });
+ assert.equal(config.flyDeploy.sharedAppName, "qm-example-apps");
+ assert.equal(config.flyDeploy.dataVolumeSizeGb, 1);
+});
diff --git a/test/deploy-always-on.test.ts b/test/deploy-always-on.test.ts
index 9301d3c43..171e2bb61 100644
--- a/test/deploy-always-on.test.ts
+++ b/test/deploy-always-on.test.ts
@@ -10,17 +10,26 @@ import type { DeployProvider } from "../src/deploy/deploy-provider.ts";
import type { DeployEndpoint } from "../src/deploy/deploy-store.ts";
import { scopeId } from "../src/types.ts";
-function svc(opts: { managedScaleToZero?: boolean; resolve?: () => Promise<DeployEndpoint | null> } = {}) {
+function svc(
+ opts: {
+ managedScaleToZero?: boolean;
+ resolve?: () => Promise<DeployEndpoint | null>;
+ setAlwaysOn?: DeployProvider["setAlwaysOn"];
+ apply?: DeployProvider["apply"];
+ } = {},
+) {
const deployStore = createDeployStore();
let applies = 0;
let resolves = 0;
const provider: DeployProvider = {
profile: { managedScaleToZero: opts.managedScaleToZero ?? true },
- apply: async () => {
+ apply: async (d, version) => {
applies++;
+ if (opts.apply) return opts.apply(d, version);
return { host: "127.0.0.1", port: 5000 };
},
destroy: async () => {},
+ ...(opts.setAlwaysOn ? { setAlwaysOn: opts.setAlwaysOn } : {}),
...(opts.resolve
? {
resolveEndpoint: async () => {
@@ -51,6 +60,44 @@ function svc(opts: { managedScaleToZero?: boolean; resolve?: () => Promise<Deplo
const owner = { ownerScopeId: scopeId("personal", "U1"), createdBy: "U1" };
+for (const operation of ["redeploy", "rollback"] as const) {
+ for (const initial of [false, true]) {
+ test(`failed ${operation} preserves always-on=${initial} and permits a subsequent settings retry`, async () => {
+ let reject = false;
+ const seen: Array<boolean | undefined> = [];
+ const toggled: boolean[] = [];
+ const { deploy, deployStore } = svc({
+ apply: async (d) => {
+ seen.push(d.alwaysOn);
+ if (reject) throw new Error("publication failed");
+ return { host: "127.0.0.1", port: 5000 };
+ },
+ setAlwaysOn: async (_d, enabled) => {
+ toggled.push(enabled);
+ },
+ });
+ const d = await deploy.deploy({ ...owner, entrypoint: "first", files: [], alwaysOn: initial });
+ await deploy.redeploy(d.id, { entrypoint: "second", files: [] });
+ const accepted = (await deployStore.get(d.id))!;
+ reject = true;
+ await assert.rejects(
+ operation === "redeploy"
+ ? deploy.redeploy(d.id, { entrypoint: "third", files: [], alwaysOn: !initial })
+ : deploy.rollbackDeployment(d.id, 1, { alwaysOn: !initial }),
+ /publication failed/,
+ );
+ assert.equal(seen.at(-1), !initial);
+ const after = (await deployStore.get(d.id))!;
+ assert.equal(!!after.alwaysOn, initial);
+ assert.equal(after.appliedVersion, accepted.appliedVersion);
+ assert.deepEqual(after.endpoint, accepted.endpoint);
+ await deploy.setDeploymentAlwaysOn(d.id, !initial);
+ assert.deepEqual(toggled, [!initial]);
+ assert.equal((await deployStore.get(d.id))!.alwaysOn, !initial);
+ });
+ }
+}
+
test("setDeploymentAlwaysOn flips the flag on and off, and persists", async () => {
const { deploy, deployStore } = svc();
const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
@@ -185,3 +232,38 @@ test("keepAlwaysOnWarm relaunches when the platform has torn the app down", asyn
assert.ok(ctx.applies > appliesBefore, "expected a re-apply when no live endpoint exists");
assert.equal((await deployStore.get(d.id))!.status, "running");
});
+
+test("always-on reaches the provider before persisting and retains the flag on failure", async () => {
+ const seen: boolean[] = [];
+ let reject = false;
+ const { deploy, deployStore } = svc({
+ setAlwaysOn: async (_d, enabled) => {
+ if (reject) throw new Error("provider unavailable");
+ seen.push(enabled);
+ },
+ });
+ const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
+ await deploy.setDeploymentAlwaysOn(d.id, true);
+ assert.deepEqual(seen, [true]);
+ reject = true;
+ await assert.rejects(deploy.setDeploymentAlwaysOn(d.id, false), /provider unavailable/);
+ assert.equal((await deployStore.get(d.id))!.alwaysOn, true);
+ await deploy.archiveDeployment(d.id);
+ await deploy.setDeploymentAlwaysOn(d.id, false);
+ assert.equal((await deployStore.get(d.id))!.alwaysOn, false);
+ assert.deepEqual(seen, [true]);
+});
+
+test("repair publication and rollback apply always-on with the version instead of restarting the old app", async () => {
+ const { deploy, deployStore } = svc({
+ setAlwaysOn: async () => {
+ throw new Error("old runtime is broken");
+ },
+ });
+ const d = await deploy.deployOrUpdate({ ...owner, name: "repair-app", entrypoint: "first", files: [] });
+ await deploy.deployOrUpdate({ ...owner, name: "repair-app", entrypoint: "fixed", files: [], alwaysOn: true });
+ assert.equal((await deployStore.get(d.id))!.alwaysOn, true);
+ await deploy.deployOrUpdate({ ...owner, name: "repair-app", rollbackTo: 1, alwaysOn: false });
+ assert.equal((await deployStore.get(d.id))!.alwaysOn, false);
+ assert.equal((await deployStore.get(d.id))!.appliedVersion, 1);
+});
diff --git a/test/deploy-core-image.test.ts b/test/deploy-core-image.test.ts
index 8f5ca2b39..809b286b4 100644
--- a/test/deploy-core-image.test.ts
+++ b/test/deploy-core-image.test.ts
@@ -27,6 +27,13 @@ test("core deploy image includes git", () => {
);
for (const line of dockerfile.split("\n").filter((candidate) => candidate.startsWith("COPY "))) {
const sources = line.trim().split(/\s+/).slice(1, -1);
+ if (sources[0]?.startsWith("--from=")) {
+ const stage = sources[0].slice("--from=".length);
+ const stages = [...dockerfile.matchAll(/^FROM\s+\S+\s+AS\s+(\S+)$/gm)].map((match) => match[1]);
+ assert.ok(stages.includes(stage), `core Dockerfile COPY references an undeclared stage: ${stage}`);
+ assert.ok(sources.length > 1, "stage COPY must include a source path");
+ continue;
+ }
for (const source of sources) {
assert.equal(existsSync(join(repoRoot, source)), true, `core Dockerfile COPY source does not exist: ${source}`);
}
diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.ts
index d3d01d304..cc1e35cf1 100644
--- a/test/fly-deploy-provider.test.ts
+++ b/test/fly-deploy-provider.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
@@ -14,6 +14,7 @@ import {
import { parseTar } from "../src/sandbox/tar.ts";
import type { Deployment, DeploymentVersion } from "../src/deploy/deploy-store.ts";
import { scopeId } from "../src/types.ts";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
const TOKEN = "FlyV1-test-token";
const PREFIX = "qm-d";
@@ -32,10 +33,14 @@ interface FlyCall {
}
interface FakeFlyOptions {
+ updateLeavesStopped?: boolean;
+ loseFirstCreateResponse?: boolean;
createAppStatus?: number;
createAppBody?: string;
existingApp?: { name: string; network: string; organization: { slug: string } };
ips?: string[];
+ ingressNetwork?: string;
+ networkPolicies?: unknown[];
deleteAppStatus?: number;
existingMachines?: string[];
states?: string[];
@@ -50,6 +55,8 @@ interface FakeFlyOptions {
function fakeFly(opts: FakeFlyOptions = {}) {
const calls: FlyCall[] = [];
const machines = new Map<string, string>();
+ const configs = new Map<string, FlyMachineConfig>();
+ const volumes: Array<{ id: string; name: string; region: string }> = [];
for (const id of opts.existingMachines ?? []) machines.set(id, "started");
const states = [...(opts.states ?? ["started"])];
const checkStates = [...(opts.checkStates ?? ["passing"])];
@@ -84,22 +91,53 @@ function fakeFly(opts: FakeFlyOptions = {}) {
app = undefined;
return new Response("{}", { status: opts.deleteAppStatus ?? 202 });
}
- if (segments[3] === "ip_assignments" && method === "GET") return json(200, { ips: ips.map((ip) => ({ ip })) });
+ if (segments[3] === "ip_assignments" && method === "GET")
+ return json(200, {
+ ips: ips.map((ip) => ({ ip, network: { name: opts.ingressNetwork ?? "trusted-ingress", org_slug: ORG } })),
+ });
+ if (segments[3] === "network_policies" && method === "GET")
+ return json(
+ 200,
+ opts.networkPolicies ?? [
+ {
+ netpolSelector: { all: true },
+ rules: [{ action: "allow", direction: "ingress", ports: [{ protocol: "tcp", port: 22 }] }],
+ },
+ ],
+ );
if (segments[3] === "ip_assignments" && method === "POST") {
ips.push("fdaa:1:2:3::1");
return json(201, { ip: ips[0] });
}
+ if (segments[3] === "volumes" && method === "GET") return json(200, volumes);
+ if (segments[3] === "volumes" && method === "POST") {
+ const body = JSON.parse(String(init.body));
+ const volume = { id: `volume-${volumes.length + 1}`, name: body.name, region: body.region };
+ volumes.push(volume);
+ return json(201, volume);
+ }
if (method === "GET" && segments.length === 4)
return json(
200,
- [...machines.keys()].map((id) => ({ id, state: machines.get(id) })),
+ [...machines.keys()].map((id) => ({ id, state: machines.get(id), config: configs.get(id) })),
);
if (method === "POST" && segments.length === 4) {
const id = `machine-${++created}`;
machines.set(id, "created");
+ configs.set(id, JSON.parse(String(init.body)).config);
+ cordoned.add(id);
+ if (opts.loseFirstCreateResponse && created === 1) throw new Error("create response lost");
return json(200, { id, state: "created" });
}
const machineId = segments[4] ?? "";
+ if (method === "POST" && segments.length === 5) {
+ configs.set(machineId, JSON.parse(String(init.body)).config);
+ return json(200, { id: machineId, state: opts.updateLeavesStopped ? "stopped" : "started" });
+ }
+ if (method === "POST" && segments.length === 6 && segments[5] === "start") {
+ machines.set(machineId, "started");
+ return json(200, { id: machineId, state: "started" });
+ }
if (method === "GET" && segments.length === 5) {
if (!machines.has(machineId)) return json(404, { error: "not found" });
const state = nextState();
@@ -107,7 +145,8 @@ function fakeFly(opts: FakeFlyOptions = {}) {
return json(200, {
id: machineId,
state,
- checks: [{ name: "app", status: nextCheck() }],
+ config: configs.get(machineId),
+ checks: [{ name: "app", status: configs.get(machineId)?.env.BROKEN === "1" ? "failing" : nextCheck() }],
...(opts.events ? { events: opts.events } : {}),
});
}
@@ -130,7 +169,7 @@ function fakeFly(opts: FakeFlyOptions = {}) {
}
return json(500, { error: `unexpected ${method} ${url.pathname}` });
}) as unknown as typeof fetch;
- return { fetchImpl, calls, machines, cordoned };
+ return { fetchImpl, calls, machines, cordoned, configs, volumes };
}
function provider(fetchImpl: typeof fetch, extra: Partial<FlyDeployProviderOptions> = {}) {
@@ -139,6 +178,7 @@ function provider(fetchImpl: typeof fetch, extra: Partial<FlyDeployProviderOptio
appPrefix: PREFIX,
baseImage: IMAGE,
org: ORG,
+ configStore: createMemoryMap<FlyMachineConfig>(),
fetchImpl,
pollIntervalMs: 1,
machineStartTimeoutMs: 200,
@@ -210,6 +250,9 @@ test("apply: creates an isolated Fly app, injects the snapshot, and returns priv
{
protocol: "tcp",
internal_port: 8080,
+ autostop: "suspend",
+ autostart: true,
+ min_machines_running: 0,
ports: [{ port: 8080 }],
checks: [{ type: "tcp", interval: "2s", timeout: "1s", grace_period: "1s" }],
},
@@ -228,6 +271,14 @@ test("apply: creates an isolated Fly app, injects the snapshot, and returns priv
assert.equal(unpacked.find((f) => f.path === "index.html")!.data.toString("utf8"), "<h1>hi</h1>");
});
+test("apply: keeps an always-on deployment running under Fly managed lifecycle", async () => {
+ const { fetchImpl, calls } = fakeFly();
+ await provider(fetchImpl).apply({ ...deployment(ID), alwaysOn: true }, version(snapshot({ "server.js": "" })));
+ const { config } = machineCreate(calls);
+ assert.equal(config.services[0]!.min_machines_running, 1);
+ assert.equal(config.services[0]!.autostart, true);
+});
+
test("apply: refuses a pre-existing public IP before creating a machine", async () => {
const unsafe = fakeFly({ createAppStatus: 422, existingApp: OWNED_APP, ips: ["2a09:8280:1::1"] });
await assert.rejects(
@@ -379,7 +430,7 @@ test("destroy: deletes the whole Fly app and tolerates one that is already gone"
await provider(fetchImpl).destroy(deployment(ID));
assert.deepEqual(
calls.map((c) => `${c.method} ${c.path}`),
- [`GET /v1/apps/${APP}`, `DELETE /v1/apps/${APP}`],
+ [`GET /v1/apps/${APP}`, `GET /v1/apps/${APP}/volumes`, `DELETE /v1/apps/${APP}`],
);
assert.equal(calls.at(-1)!.query, "?force=true");
@@ -396,9 +447,10 @@ test("destroy: refuses a mismatched app and surfaces a failed deletion", async (
await assert.rejects(provider(fetchImpl).destroy(deployment(ID)), /delete app .*http 500/);
});
-test("profile: the core keeps managing idle TTL because 6PN dialing cannot wake a stopped machine", () => {
+test("profile: shared Flycast manages suspension while existing standalone apps retain idle cleanup", () => {
const { fetchImpl } = fakeFly();
assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false });
+ assert.deepEqual(provider(fetchImpl, { sharedAppName: "company-app" }).profile, { managedScaleToZero: true });
});
test("missing fly configuration fails at the point of use with the env var that is missing", async () => {
@@ -429,3 +481,312 @@ test("an invalid app prefix fails before reaching Fly", async () => {
}
assert.deepEqual(calls, []);
});
+
+test("durable apply retains one volume across updates, rollback, archive and recreation", async () => {
+ const fake = fakeFly();
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1, appReadyTimeoutMs: 10 });
+ const d = deployment(ID);
+ const first = version(snapshot({ "server.js": "first" }));
+ await deploy.apply(d, first);
+ const original = structuredClone(fake.configs.get("machine-1")!);
+ assert.deepEqual(original.mounts, [{ volume: "volume-1", path: "/data" }]);
+ assert.equal(original.env.DATA_DIR, "/data");
+ assert.equal(deploy.profile.dataDir, "/data");
+ await deploy.apply(d, version(snapshot({ "server.js": "second" }), { version: 2 }));
+ assert.deepEqual([...fake.machines.keys()], ["machine-1"]);
+ assert.deepEqual(fake.configs.get("machine-1")!.mounts, original.mounts);
+ const working = structuredClone(fake.configs.get("machine-1")!);
+ await assert.rejects(
+ deploy.apply(d, version(snapshot({ "server.js": "broken" }), { env: { BROKEN: "1" } })),
+ /never listened/,
+ );
+ assert.deepEqual(fake.configs.get("machine-1"), working);
+ await deploy.destroy(d);
+ assert.equal(fake.machines.size, 0);
+ assert.equal(fake.volumes.length, 1);
+ assert.equal(
+ fake.calls.some((c) => c.method === "DELETE" && c.path === `/v1/apps/${APP}`),
+ false,
+ );
+ await deploy.apply(d, first);
+ assert.deepEqual(fake.configs.get("machine-2")!.mounts, original.mounts);
+ assert.equal(fake.volumes.length, 1);
+});
+
+test("durable apply refuses to replace an existing ephemeral machine", async () => {
+ const fake = fakeFly({ existingMachines: ["old"] });
+ await assert.rejects(
+ provider(fake.fetchImpl, { dataVolumeSizeGb: 1 }).apply(
+ deployment(ID),
+ version(snapshot({ "server.js": "first" })),
+ ),
+ /explicit migration/,
+ );
+ assert.deepEqual([...fake.machines.keys()], ["old"]);
+ assert.equal(fake.volumes.length, 0);
+});
+
+test("durable retry restores routing after an accepted create loses its response", async () => {
+ const fake = fakeFly({ loseFirstCreateResponse: true });
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1 });
+ const d = deployment(ID),
+ v = version(snapshot({ "server.js": "app" }));
+ await assert.rejects(deploy.apply(d, v), /create response lost/);
+ assert.equal(fake.cordoned.has("machine-1"), true);
+ await deploy.apply(d, v);
+ assert.equal(fake.cordoned.has("machine-1"), false);
+ assert.equal(fake.machines.size, 1);
+});
+
+test("missing volume configuration cannot downgrade or destroy durable storage", async () => {
+ const fake = fakeFly();
+ const d = deployment(ID),
+ v = version(snapshot({ "server.js": "app" }));
+ await provider(fake.fetchImpl, { dataVolumeSizeGb: 1 }).apply(d, v);
+ const missing = provider(fake.fetchImpl);
+ await assert.rejects(missing.apply(d, v), /restore FLY_DEPLOY_DATA_VOLUME_SIZE_GB/);
+ assert.equal(fake.machines.size, 1);
+ await missing.destroy(d);
+ assert.equal(fake.volumes.length, 1);
+ assert.equal(
+ fake.calls.some((c) => c.method === "DELETE" && c.path === `/v1/apps/${APP}`),
+ false,
+ );
+});
+
+test("durable rollback uses the accepted configuration after a provider restart", async () => {
+ const fake = fakeFly();
+ const configStore = createMemoryMap<FlyMachineConfig>();
+ const opts = { dataVolumeSizeGb: 1, configStore, appReadyTimeoutMs: 10 };
+ const d = deployment(ID),
+ good = version(snapshot({ "server.js": "good" }));
+ await provider(fake.fetchImpl, opts).apply(d, good);
+ const accepted = structuredClone(fake.configs.get("machine-1")!);
+ fake.configs.set("machine-1", { ...accepted, env: { BROKEN: "1" } });
+ const restarted = provider(fake.fetchImpl, opts);
+ await assert.rejects(
+ restarted.apply(d, version(snapshot({ "server.js": "bad" }), { env: { BROKEN: "1" } })),
+ /never listened/,
+ );
+ assert.deepEqual(fake.configs.get("machine-1"), accepted);
+});
+
+test("always-on toggles update an existing machine without replacing its data", async () => {
+ const fake = fakeFly();
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1 });
+ const d = deployment(ID);
+ await deploy.apply(d, version(snapshot({ "server.js": "app" })));
+ for (const alwaysOn of [true, false]) {
+ await deploy.setAlwaysOn!(d, alwaysOn);
+ assert.equal(fake.configs.get("machine-1")!.services[0]!.min_machines_running, alwaysOn ? 1 : 0);
+ assert.deepEqual(fake.configs.get("machine-1")!.mounts, [{ volume: "volume-1", path: "/data" }]);
+ assert.equal(fake.machines.size, 1);
+ }
+});
+
+test("always-on toggles preserve the latest ephemeral publication after a prior toggle", async () => {
+ const fake = fakeFly();
+ const deploy = provider(fake.fetchImpl);
+ const d = deployment(ID);
+ await deploy.apply(d, version(snapshot({ "server.js": "first" })));
+ await deploy.setAlwaysOn!(d, true);
+ await deploy.apply({ ...d, alwaysOn: true }, version(snapshot({ "server.js": "second" }), { version: 2 }));
+ const current = structuredClone(fake.configs.get("machine-2")!);
+ await deploy.setAlwaysOn!(d, false);
+ assert.deepEqual(fake.configs.get("machine-2"), {
+ ...current,
+ services: current.services.map((service) => ({ ...service, min_machines_running: 0 })),
+ });
+});
+
+test("an update that leaves the machine stopped explicitly starts it", async () => {
+ const fake = fakeFly({ updateLeavesStopped: true });
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1 });
+ const d = deployment(ID),
+ v = version(snapshot({ "server.js": "app" }));
+ await deploy.apply(d, v);
+ await deploy.setAlwaysOn!(d, true);
+ assert.ok(fake.calls.some((c) => c.method === "POST" && c.path.endsWith("/machines/machine-1/start")));
+});
+
+test("configuration-store failure rolls back an updated machine", async () => {
+ const fake = fakeFly();
+ const configStore = createMemoryMap<FlyMachineConfig>();
+ const put = configStore.put.bind(configStore);
+ let writes = 0;
+ configStore.put = async (id, value) => {
+ if (++writes === 2) throw new Error("store unavailable");
+ await put(id, value);
+ };
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1, configStore });
+ const d = deployment(ID),
+ v = version(snapshot({ "server.js": "first" }));
+ await deploy.apply(d, v);
+ const accepted = structuredClone(fake.configs.get("machine-1"));
+ await assert.rejects(deploy.apply(d, version(snapshot({ "server.js": "second" }))), /store unavailable/);
+ assert.deepEqual(fake.configs.get("machine-1"), accepted);
+ assert.deepEqual(await configStore.get(d.id), accepted);
+});
+
+test("always-on uses the accepted configuration after an interrupted update", async () => {
+ const fake = fakeFly();
+ const deploy = provider(fake.fetchImpl, { dataVolumeSizeGb: 1 });
+ const d = deployment(ID);
+ await deploy.apply(d, version(snapshot({ "server.js": "good" })));
+ const accepted = structuredClone(fake.configs.get("machine-1")!);
+ fake.configs.set("machine-1", { ...accepted, env: { BROKEN: "1" } });
+ await deploy.setAlwaysOn!(d, true);
+ assert.deepEqual(fake.configs.get("machine-1")!.env, accepted.env);
+});
+
+test("shared app updates and archives only the owning deployment", async () => {
+ const fake = fakeFly({
+ existingApp: { name: "company-app", network: "company-app", organization: { slug: ORG } },
+ ips: ["fdaa:1:2:3::1"],
+ });
+ const portStore = createMemoryMap<string>();
+ const configStore = createMemoryMap<FlyMachineConfig>();
+ const options = { sharedAppName: "company-app", portStore, configStore, dataVolumeSizeGb: 1 };
+ const deploy = provider(fake.fetchImpl, options);
+ const a = deployment(ID),
+ b = deployment("other-deployment");
+ const v = version(snapshot({ "server.js": "good" }));
+ const ea = await deploy.apply(a, v),
+ eb = await deploy.apply(b, v);
+ assert.equal(ea.host, eb.host);
+ assert.notEqual(ea.port, eb.port);
+ assert.equal(fake.volumes.length, 2);
+ const sibling = structuredClone(fake.configs.get("machine-2"));
+ assert.deepEqual(await provider(fake.fetchImpl, options).apply(a, v), ea);
+ await deploy.setAlwaysOn!(a, true);
+ assert.deepEqual(fake.configs.get("machine-2"), sibling);
+ await deploy.destroy(a);
+ assert.deepEqual([...fake.machines.keys()], ["machine-2"]);
+ assert.equal(fake.volumes.length, 2);
+ assert.equal(
+ fake.calls.some((call) => call.method === "DELETE" && call.path === "/v1/apps/company-app"),
+ false,
+ );
+ assert.deepEqual(await deploy.apply(a, v), ea);
+ assert.equal(fake.volumes.length, 2);
+});
+
+test("shared app refuses missing durable port ownership before API calls", async () => {
+ const fake = fakeFly();
+ await assert.rejects(
+ provider(fake.fetchImpl, { sharedAppName: "company-app", dataVolumeSizeGb: 1 }).apply(
+ deployment(ID),
+ version(snapshot({})),
+ ),
+ /persistent port assignments/,
+ );
+ assert.equal(fake.calls.length, 0);
+});
+
+test("shared ports preserve an existing owner's claim and survive provider recreation", async () => {
+ const fake = fakeFly({
+ existingApp: { name: "company-app", network: "company-app", organization: { slug: ORG } },
+ ips: ["fdaa:1:2:3::1"],
+ });
+ const portStore = createMemoryMap<string>();
+ const first = 20000 + (createHash("sha256").update(ID).digest().readUInt32BE(0) % 45536);
+ await portStore.put(`company-app:${first}`, "existing-owner");
+ const options = {
+ sharedAppName: "company-app",
+ portStore,
+ configStore: createMemoryMap<FlyMachineConfig>(),
+ dataVolumeSizeGb: 1,
+ };
+ const d = deployment(ID),
+ v = version(snapshot({ "server.js": "good" }));
+ const endpoint = await provider(fake.fetchImpl, options).apply(d, v);
+ assert.notEqual(endpoint.port, first);
+ assert.equal(await portStore.get(`company-app:${first}`), "existing-owner");
+ assert.deepEqual(await provider(fake.fetchImpl, options).apply(d, v), endpoint);
+});
+
+test("shared publishing requires pre-provisioned app and ingress without creating either", async () => {
+ for (const existing of [false, true]) {
+ const fake = fakeFly(
+ existing ? { existingApp: { name: "company-app", network: "company-app", organization: { slug: ORG } } } : {},
+ );
+ const deploy = provider(fake.fetchImpl, {
+ appPrefix: "",
+ sharedAppName: "company-app",
+ portStore: createMemoryMap<string>(),
+ dataVolumeSizeGb: 1,
+ });
+ await assert.rejects(deploy.apply(deployment(ID), version(snapshot({}))), /provisioned before publishing/);
+ assert.equal(
+ fake.calls.some((call) => call.method === "POST"),
+ false,
+ );
+ }
+});
+
+test("private transport is restored when resolving a persisted endpoint", async () => {
+ const fake = fakeFly();
+ let connections = 0;
+ const deploy = provider(fake.fetchImpl, {
+ privateTransport: {
+ ensure: async () => {
+ connections++;
+ return 18096;
+ },
+ },
+ });
+ const d = deployment(ID);
+ const endpoint = await deploy.apply(d, version(snapshot({ "server.js": "app" })));
+ assert.equal(endpoint.socksProxyPort, 18096);
+ assert.deepEqual(await deploy.resolveEndpoint!({ ...d, endpoint }, version(snapshot({}))), endpoint);
+ assert.equal(connections, 2);
+});
+
+test("shared publishing rejects missing or permissive ingress isolation before creating resources", async () => {
+ for (const overrides of [
+ { ingressNetwork: "" },
+ { ingressNetwork: "default" },
+ { ingressNetwork: "company-app" },
+ { networkPolicies: [] },
+ {
+ networkPolicies: [
+ {
+ netpolSelector: { all: true, metadata: { role: "not-qm" } },
+ rules: [{ action: "allow", direction: "ingress", ports: [{ protocol: "tcp", port: 22 }] }],
+ },
+ ],
+ },
+ {
+ networkPolicies: [
+ {
+ netpolSelector: { all: false },
+ rules: [{ action: "allow", direction: "ingress", ports: [{ protocol: "tcp", port: 22 }] }],
+ },
+ ],
+ },
+ {
+ networkPolicies: [
+ {
+ netpolSelector: { all: true },
+ rules: [{ action: "allow", direction: "ingress", ports: [{ protocol: "tcp", port: 8080 }] }],
+ },
+ ],
+ },
+ ]) {
+ const fake = fakeFly({
+ existingApp: { name: "company-app", network: "company-app", organization: { slug: ORG } },
+ ips: ["fdaa:1:2:3::1"],
+ ...overrides,
+ });
+ const deploy = provider(fake.fetchImpl, {
+ sharedAppName: "company-app",
+ portStore: createMemoryMap<string>(),
+ dataVolumeSizeGb: 1,
+ });
+ await assert.rejects(
+ deploy.apply(deployment(ID), version(snapshot({ "index.html": "hi" }))),
+ /requires.*(private network|ingress restricted)/,
+ );
+ assert.ok(fake.calls.every((call) => call.method === "GET"));
+ }
+});
diff --git a/test/fly-peer-claims.test.ts b/test/fly-peer-claims.test.ts
new file mode 100644
index 000000000..841d154c1
--- /dev/null
+++ b/test/fly-peer-claims.test.ts
@@ -0,0 +1,108 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ claimFlyPeer,
+ createEcsTaskStopped,
+ recordStoppedFlyPeers,
+ type FlyPeerClaim,
+} from "../src/deploy/fly-peer-claims.ts";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
+
+test("overlapping tasks get distinct peers and reconnect to their existing claim", async () => {
+ const claims = createMemoryMap<FlyPeerClaim>();
+ const shared = { claims, peerIds: ["a", "b"], taskStopped: async () => false };
+ const peers = await Promise.all(["old", "new"].map((taskArn) => claimFlyPeer({ ...shared, taskArn })));
+ assert.notEqual(peers[0], peers[1]);
+ assert.equal(await claimFlyPeer({ ...shared, taskArn: "old" }), peers[0]);
+ await assert.rejects(claimFlyPeer({ ...shared, taskArn: "third" }), /confirmed stopped/);
+});
+
+test("only confirmed stopped tasks relinquish a peer", async () => {
+ const claims = createMemoryMap<FlyPeerClaim>();
+ await claims.put("a", { taskArn: "old" });
+ const opts = { claims, peerIds: ["a"], taskArn: "new" };
+ await assert.rejects(claimFlyPeer({ ...opts, taskStopped: async () => false }), /confirmed stopped/);
+ await assert.rejects(
+ claimFlyPeer({
+ ...opts,
+ taskStopped: async () => {
+ throw new Error("AWS unavailable");
+ },
+ }),
+ /AWS unavailable/,
+ );
+ assert.deepEqual(await claims.get("a"), { taskArn: "old" });
+ assert.equal(await claimFlyPeer({ ...opts, taskStopped: async () => true }), "a");
+ assert.deepEqual(await claims.get("a"), { taskArn: "new" });
+});
+
+test("a stale stopped-task observation cannot replace a newly assigned owner", async () => {
+ const claims = createMemoryMap<FlyPeerClaim>();
+ await claims.put("a", { taskArn: "old" });
+ await assert.rejects(
+ claimFlyPeer({
+ claims,
+ peerIds: ["a"],
+ taskArn: "late",
+ taskStopped: async () => {
+ await claims.put("a", { taskArn: "winner" });
+ return true;
+ },
+ }),
+ /confirmed stopped/,
+ );
+ assert.deepEqual(await claims.get("a"), { taskArn: "winner" });
+});
+
+test("ECS termination requires the exact task and STOPPED status without failures", async () => {
+ for (const [response, expected] of [
+ [{ tasks: [{ taskArn: "owner", lastStatus: "STOPPED" }] }, true],
+ [{ tasks: [{ taskArn: "owner", lastStatus: "RUNNING", desiredStatus: "STOPPED" }] }, false],
+ [{ tasks: [{ taskArn: "other", lastStatus: "STOPPED" }] }, false],
+ [{ tasks: [] }, false],
+ [{ failures: [{ arn: "owner", reason: "MISSING" }] }, false],
+ ] satisfies Array<[Partial<import("@aws-sdk/client-ecs").DescribeTasksCommandOutput>, boolean]>) {
+ const stopped = createEcsTaskStopped({
+ cluster: "company",
+ describe: async (command) => {
+ assert.deepEqual(command.input, { cluster: "company", tasks: ["owner"] });
+ return {
+ $metadata: {},
+ ...structuredClone(response),
+ } as import("@aws-sdk/client-ecs").DescribeTasksCommandOutput;
+ },
+ });
+ assert.equal(await stopped("owner"), expected);
+ }
+});
+
+test("persisted termination evidence permits reuse after ECS history disappears", async () => {
+ const claims = createMemoryMap<FlyPeerClaim>();
+ await claims.put("a", { taskArn: "old" });
+ await recordStoppedFlyPeers({ claims, peerIds: ["a"], taskStopped: async () => true });
+ assert.ok((await claims.get("a"))?.stoppedConfirmedAt);
+ const peer = await claimFlyPeer({
+ claims,
+ peerIds: ["a"],
+ taskArn: "new",
+ taskStopped: async () => {
+ throw new Error("historical lookup must not be needed");
+ },
+ });
+ assert.equal(peer, "a");
+ assert.deepEqual(await claims.get("a"), { taskArn: "new" });
+});
+
+test("termination evidence is never copied to a replacement task", async () => {
+ const claims = createMemoryMap<FlyPeerClaim>();
+ await claims.put("a", { taskArn: "old" });
+ await recordStoppedFlyPeers({
+ claims,
+ peerIds: ["a"],
+ taskStopped: async () => {
+ await claims.put("a", { taskArn: "new" });
+ return true;
+ },
+ });
+ assert.deepEqual(await claims.get("a"), { taskArn: "new" });
+});
diff --git a/test/fly-tunnel-manager.test.ts b/test/fly-tunnel-manager.test.ts
new file mode 100644
index 000000000..27a8c9081
--- /dev/null
+++ b/test/fly-tunnel-manager.test.ts
@@ -0,0 +1,130 @@
+import { test, mock } from "node:test";
+import assert from "node:assert/strict";
+import { ECSClient } from "@aws-sdk/client-ecs";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
+import type { FlyPeerClaim } from "../src/deploy/fly-peer-claims.ts";
+
+let launches = 0;
+let live = false;
+let stopCalls = 0;
+mock.module("../src/deploy/fly-private-tunnel.ts", {
+ namedExports: {
+ startFlyPrivateTunnel: async () => {
+ launches++;
+ live = true;
+ return {
+ isAlive: () => live,
+ stop: async () => {
+ live = false;
+ stopCalls++;
+ },
+ };
+ },
+ },
+});
+const { createFlyTunnelManager } = await import("../src/deploy/fly-tunnel-manager.ts");
+
+test("concurrent requests share startup and restart a confirmed dead tunnel once", async () => {
+ launches = 0;
+ stopCalls = 0;
+ live = false;
+ const fetchMock = mock.method(
+ globalThis,
+ "fetch",
+ async () => new Response(JSON.stringify({ TaskARN: "task", Cluster: "cluster", LaunchType: "FARGATE" })),
+ );
+ const ecsMock = mock.method(ECSClient.prototype, "send", async () => ({
+ tasks: [{ taskArn: "task", lastStatus: "RUNNING" }],
+ }));
+ const manager = createFlyTunnelManager({
+ peers: [
+ { id: "a", config: "a" },
+ { id: "b", config: "b" },
+ ],
+ claims: createMemoryMap<FlyPeerClaim>(),
+ metadataUri: "http://169.254.170.2/v4/test",
+ executable: "wireproxy",
+ port: 18096,
+ });
+ try {
+ assert.deepEqual(await Promise.all(Array.from({ length: 8 }, () => manager.ensure())), Array(8).fill(18096));
+ assert.equal(launches, 1);
+ live = false;
+ await Promise.all(Array.from({ length: 8 }, () => manager.ensure()));
+ assert.equal(launches, 2);
+ assert.equal(stopCalls, 1);
+ await manager.stop();
+ assert.equal(stopCalls, 2);
+ await assert.rejects(manager.ensure(), /stopped/);
+ } finally {
+ await manager.stop();
+ fetchMock.mock.restore();
+ ecsMock.mock.restore();
+ }
+});
+
+test("monitor records stopped predecessors before any app request or tunnel startup", async () => {
+ launches = 0;
+ const fetchMock = mock.method(
+ globalThis,
+ "fetch",
+ async () => new Response(JSON.stringify({ TaskARN: "new", Cluster: "cluster", LaunchType: "FARGATE" })),
+ );
+ const ecsMock = mock.method(ECSClient.prototype, "send", async () => ({
+ tasks: [{ taskArn: "old", lastStatus: "STOPPED" }],
+ }));
+ const claims = createMemoryMap<FlyPeerClaim>();
+ await claims.put("a", { taskArn: "old" });
+ let recorded!: () => void;
+ const ready = new Promise<void>((resolve) => {
+ recorded = resolve;
+ });
+ const update = claims.update!.bind(claims);
+ claims.update = async (id, fn) => {
+ const value = await update(id, fn);
+ recorded();
+ return value;
+ };
+ const manager = createFlyTunnelManager({
+ peers: [
+ { id: "a", config: "a" },
+ { id: "b", config: "b" },
+ ],
+ claims,
+ metadataUri: "http://169.254.170.2/v4/test",
+ executable: "wireproxy",
+ port: 18096,
+ });
+ try {
+ manager.monitor();
+ await ready;
+ assert.ok((await claims.get("a"))?.stoppedConfirmedAt);
+ assert.equal(launches, 0);
+ } finally {
+ await manager.stop();
+ fetchMock.mock.restore();
+ ecsMock.mock.restore();
+ }
+});
+
+test("peer identity is stable across labels and duplicate keys are rejected", async () => {
+ const { parseFlyWireguardPeers } = await import("../src/deploy/fly-tunnel-manager.ts");
+ const key = (n: number) => `[Interface]\nPrivateKey = ${Buffer.alloc(32, n).toString("base64")}\n`;
+ const peers = [
+ { id: "a", config: key(1) },
+ { id: "b", config: key(2) },
+ ];
+ const first = parseFlyWireguardPeers(JSON.stringify(peers));
+ const renamed = parseFlyWireguardPeers(JSON.stringify(peers.map((p) => ({ ...p, id: p.id + "-renamed" }))));
+ assert.deepEqual(first, renamed);
+ assert.throws(
+ () =>
+ parseFlyWireguardPeers(
+ JSON.stringify([
+ { id: "a", config: key(1) },
+ { id: "b", config: key(1) },
+ ]),
+ ),
+ /distinct identities/,
+ );
+});
diff --git a/test/secret-schema-drift.test.ts b/test/secret-schema-drift.test.ts
index b54fcc694..6ceab52bd 100644
--- a/test/secret-schema-drift.test.ts
+++ b/test/secret-schema-drift.test.ts
@@ -121,3 +121,10 @@ test("both porter roles share PORTER_DEPLOY_API_TOKEN", () => {
[],
);
});
+
+test("shared Fly publishing validates its private peer secret", () => {
+ const env = { DEPLOY_PROVIDER: "fly", FLY_DEPLOY_API_TOKEN: "token", FLY_DEPLOY_SHARED_APP_NAME: "acme-apps" };
+ assert.deepEqual(validateCoreSecretEnv(env), ["FLY_DEPLOY_WIREGUARD_PEERS"]);
+ assert.deepEqual(validateCoreSecretEnv({ ...env, FLY_DEPLOY_WIREGUARD_PEERS: "peer-configs" }), []);
+ assert.deepEqual(validateCoreSecretEnv({ ...env, DEPLOY_PROVIDER: "aws" }), []);
+});