Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Replace Fly ssh console with machine exec for internal transports (#1407)

Sina Matianebc442a4a9f8
diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts
index 95baa0d4c..753133190 100644
--- a/cli/src/backends/fly.ts
+++ b/cli/src/backends/fly.ts
@@ -70,26 +70,89 @@ const flyServiceCtx = (config: QmConfig, appPrefix: string, deployAppPrefix: str
const FLY_RESPONSE = "QM_LAYER_RESPONSE=";
const FLY_REMOTE_ERROR = "QM_LAYER_ERROR=";
const FLY_REQUEST_TIMEOUT_MS = 120_000;
+const FLY_EXEC_TIMEOUT_SEC = 120;
+const FLY_APP_NOT_FOUND = /could not find app|app not found/i;
+const FLY_MACHINES_API = "https://api.machines.dev/v1";
-function flyRequest(config: QmConfig, method: "GET" | "PUT", body: string): { status: number; body: string } {
+function classifyFlyReachability(app: string, detail: string): never {
+ if (FLY_APP_NOT_FOUND.test(detail)) throw new CliError(`Fly app ${app} not found: ${detail}`);
+ throw new CoreUnreachableError(`could not reach the Fly core: ${detail}`);
+}
+
+function flyCoreMachineId(app: string): string {
+ let raw: string;
+ try {
+ raw = fly(["status", "-a", app, "--json"]);
+ } catch (error) {
+ classifyFlyReachability(app, errMessage(error));
+ }
+ if (FLY_APP_NOT_FOUND.test(raw)) classifyFlyReachability(app, raw);
+ let parsed: { Machines?: Array<{ id?: string; ID?: string; state?: string }> };
+ try {
+ parsed = JSON.parse(raw) as typeof parsed;
+ } catch {
+ throw new CoreUnreachableError(`could not reach the Fly core: fly status returned invalid JSON`);
+ }
+ const machine = parsed.Machines?.find((entry) => entry.state === "started") ?? parsed.Machines?.[0];
+ const machineId = machine?.id ?? machine?.ID;
+ if (!machineId) throw new CoreUnreachableError(`could not reach the Fly core: no machine on ${app}`);
+ return machineId;
+}
+
+function flyApiAuthorization(): string {
+ try {
+ const token = process.env.FLY_API_TOKEN?.trim() || fly(["auth", "token"]).trim();
+ if (!token) throw new Error("Fly API token is missing");
+ return token.startsWith("FlyV1") || token.startsWith("Bearer ") ? token : `Bearer ${token}`;
+ } catch (error) {
+ throw new CoreUnreachableError(`could not reach the Fly core: ${errMessage(error)}`);
+ }
+}
+
+function flyExecOutput(payload: unknown): string {
+ const { stdout, stderr } = (payload ?? {}) as { stdout?: unknown; stderr?: unknown };
+ return [stdout, stderr].filter((value): value is string => typeof value === "string" && value !== "").join("\n");
+}
+
+async function flyRequest(
+ config: QmConfig,
+ method: "GET" | "PUT",
+ body: string,
+): Promise<{ status: number; body: string }> {
const app = `${appPrefixOf(config)}-core`;
const script = `const fs=require("node:fs"),{createHmac}=require("node:crypto");const fail=error=>{const code=error&&(error.cause&&error.cause.code||error.code);console.log(${JSON.stringify(FLY_REMOTE_ERROR)}+JSON.stringify({message:error&&error.message?error.message:String(error),...(typeof code==="string"?{code}:{})}))};try{const method=${JSON.stringify(method)},path="/v1/deployment-layer",body=fs.readFileSync(0,"utf8"),timestamp=Math.floor(Date.now()/1000),canonical=method+"\\n"+path+"\\n"+body,secret=process.env.CORE_SIGNING_SECRET;if(!secret)throw new Error("CORE_SIGNING_SECRET is not set on core");const signature=createHmac("sha256",secret).update("v0:"+timestamp+":"+canonical).digest("hex");fetch("http://127.0.0.1:"+(process.env.PORT||8080)+path,{method,headers:{"content-type":"application/json","x-timestamp":String(timestamp),"x-signature":"v0="+signature},...(method==="PUT"?{body}: {})}).then(async response=>console.log(${JSON.stringify(FLY_RESPONSE)}+JSON.stringify({status:response.status,body:await response.text()}))).catch(fail)}catch(error){fail(error)}`;
- const encoded = Buffer.from(script).toString("base64");
- const command = `node -e "eval(Buffer.from('${encoded}','base64').toString())"`;
- let output: string;
+ const machineId = flyCoreMachineId(app);
+ let response: Response;
+ let text: string;
try {
- output = execFileSync(flyBin(), ["ssh", "console", "-a", app, "-C", command], {
- encoding: "utf8",
- input: body,
- stdio: ["pipe", "pipe", "pipe"],
- timeout: FLY_REQUEST_TIMEOUT_MS,
- });
+ response = await fetch(
+ `${FLY_MACHINES_API}/apps/${encodeURIComponent(app)}/machines/${encodeURIComponent(machineId)}/exec`,
+ {
+ method: "POST",
+ headers: {
+ authorization: flyApiAuthorization(),
+ "content-type": "application/json",
+ },
+ body: JSON.stringify({
+ command: ["node", "-e", script],
+ stdin: body,
+ timeout: FLY_EXEC_TIMEOUT_SEC,
+ }),
+ signal: AbortSignal.timeout(FLY_REQUEST_TIMEOUT_MS),
+ },
+ );
+ text = await response.text();
} catch (error) {
- const detail = error as { stdout?: string; stderr?: string; message?: string };
- const text = `${detail.stderr ?? ""}${detail.stdout ?? ""}`.trim() || detail.message || "fly ssh failed";
- if (/could not find app|app not found/i.test(text)) throw new CliError(`Fly app ${app} not found: ${text}`);
- throw new CoreUnreachableError(`could not reach the Fly core: ${text}`);
+ throw new CoreUnreachableError(`could not reach the Fly core: ${errMessage(error)}`);
}
+ if (!response.ok) classifyFlyReachability(app, text || `HTTP ${response.status}`);
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ throw new CoreUnreachableError(`could not reach the Fly core: unparseable machine exec response`);
+ }
+ const output = flyExecOutput(parsed);
const remoteError = output.split("\n").find((value) => value.startsWith(FLY_REMOTE_ERROR));
if (remoteError) {
const detail = JSON.parse(remoteError.slice(FLY_REMOTE_ERROR.length)) as { message?: string; code?: string };
@@ -104,9 +167,8 @@ function flyRequest(config: QmConfig, method: "GET" | "PUT", body: string): { st
return JSON.parse(line.slice(FLY_RESPONSE.length)) as { status: number; body: string };
}
-/** Deployment-layer transport for Fly: a signed request executed on the core VM over fly ssh. */
export const flyDeploymentLayerTransport: DeploymentLayerTransport = (opts) =>
- Promise.resolve(flyRequest(opts.config, opts.method, opts.body));
+ flyRequest(opts.config, opts.method, opts.body);
import { doctorCommon, localDoctorSecrets, requireFlyAuth } from "./doctor.ts";
@@ -387,8 +449,6 @@ function writeDerived(ctx: FlyCtx, service: ServiceName): string {
return path;
}
-const FLY_APP_NOT_FOUND = /app not found|Could not find App/i;
-
function secretNames(app: string): Set<string> | undefined {
const out = fly(["secrets", "list", "-a", app], { allow: FLY_APP_NOT_FOUND });
if (FLY_APP_NOT_FOUND.test(out)) return undefined;
@@ -468,7 +528,7 @@ export function flyS3ProbeCommand(): string {
}
function flyS3RoundTrip(app: string, machineId: string): void {
- fly(["ssh", "console", "-a", app, "--machine", machineId, "--command", flyS3ProbeCommand(), "--quiet"]);
+ fly(["machine", "exec", "-a", app, machineId, flyS3ProbeCommand(), "--timeout", String(FLY_EXEC_TIMEOUT_SEC)]);
}
export function flyLiveSessionCommand(): string {
@@ -476,7 +536,7 @@ export function flyLiveSessionCommand(): string {
}
function flyLiveSession(app: string, machineId: string): void {
- fly(["ssh", "console", "-a", app, "--machine", machineId, "--command", flyLiveSessionCommand(), "--quiet"]);
+ fly(["machine", "exec", "-a", app, machineId, flyLiveSessionCommand(), "--timeout", String(FLY_EXEC_TIMEOUT_SEC)]);
}
function flyOrgApps(flyOrg: string): Set<string> {
diff --git a/cli/test/deployment-layer.test.ts b/cli/test/deployment-layer.test.ts
index 13b75a73d..7a3c8d811 100644
--- a/cli/test/deployment-layer.test.ts
+++ b/cli/test/deployment-layer.test.ts
@@ -610,6 +610,57 @@ function fakeFly(dir: string, body: string): string {
return bin;
}
+function fakeFlyStatus(dir: string, extra = ""): string {
+ return fakeFly(
+ dir,
+ `
+const args = process.argv.slice(2);
+${extra}
+if (args[0] === "status") {
+ console.log(
+ JSON.stringify({
+ Machines: [
+ { id: "machine-stale", state: "stopped" },
+ { id: "machine-core", state: "started" },
+ ],
+ }),
+ );
+ process.exit(0);
+}
+process.exit(1);
+`,
+ );
+}
+
+interface CapturedFlyExec {
+ url: string;
+ authorization: string;
+ body: { command?: unknown; stdin?: unknown; timeout?: unknown };
+}
+
+async function withFlyMachineExec<T>(stdout: string, fn: (captured: CapturedFlyExec[]) => Promise<T>): Promise<T> {
+ const captured: CapturedFlyExec[] = [];
+ const previous = globalThis.fetch;
+ globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
+ const url = typeof input === "string" || input instanceof URL ? String(input) : input.url;
+ const headers = new Headers(init?.headers);
+ captured.push({
+ url,
+ authorization: headers.get("authorization") ?? "",
+ body: init?.body ? (JSON.parse(String(init.body)) as CapturedFlyExec["body"]) : {},
+ });
+ return new Response(JSON.stringify({ stdout, exit_code: 0 }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ });
+ }) as typeof fetch;
+ try {
+ return await fn(captured);
+ } finally {
+ globalThis.fetch = previous;
+ }
+}
+
function flySyncOpts(dir: string, allowUnavailable?: boolean): Parameters<typeof syncDeploymentLayer>[0] {
return {
config: makeConfig("http://example.invalid"),
@@ -625,24 +676,35 @@ test("fly sync succeeds on the response marker, piping the exact bundle over std
const dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));
try {
writeLayer(dir);
- const stdinLog = join(dir, "stdin.log");
const argsLog = join(dir, "args.log");
- const bin = fakeFly(
+ const bin = fakeFlyStatus(
dir,
- [
- `fs.writeFileSync(${JSON.stringify(argsLog)}, JSON.stringify(process.argv.slice(2)));`,
- `fs.writeFileSync(${JSON.stringify(stdinLog)}, fs.readFileSync(0, "utf8"));`,
- `console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 200, body: JSON.stringify({ version: 7, contentHash: "abcdef123456" }) }));`,
- ].join("\n"),
+ `fs.writeFileSync(${JSON.stringify(argsLog)}, JSON.stringify(process.argv.slice(2)));`,
);
- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir)));
- assert.equal(
- readFileSync(stdinLog, "utf8"),
- JSON.stringify(deploymentLayerBundle(join(dir, "sandbox"))),
- "the full bundle reaches the remote script's stdin",
+ const bundle = JSON.stringify(deploymentLayerBundle(join(dir, "sandbox")));
+ await withFlyMachineExec(
+ `QM_LAYER_RESPONSE=${JSON.stringify({ status: 200, body: JSON.stringify({ version: 7, contentHash: "abcdef123456" }) })}`,
+ async (captured) => {
+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir)));
+ assert.equal(captured.length, 1);
+ const request = captured[0]!;
+ assert.equal(request.url, "https://api.machines.dev/v1/apps/acme-core/machines/machine-core/exec");
+ assert.equal(request.authorization, "Bearer test-token");
+ assert.equal(request.body.stdin, bundle, "the full bundle reaches the remote script's stdin");
+ assert.equal(request.body.timeout, 120);
+ assert.ok(Array.isArray(request.body.command));
+ assert.deepEqual((request.body.command as string[]).slice(0, 2), ["node", "-e"]);
+ const remoteScript = (request.body.command as string[])[2] ?? "";
+ assert.match(remoteScript, /createHmac\("sha256"/, "the signing script is passed to node -e directly");
+ assert.doesNotMatch(remoteScript, /eval\(/, "no eval indirection");
+ assert.deepEqual(JSON.parse(readFileSync(argsLog, "utf8")) as string[], [
+ "status",
+ "-a",
+ "acme-core",
+ "--json",
+ ]);
+ },
);
- const args = JSON.parse(readFileSync(argsLog, "utf8")) as string[];
- assert.deepEqual(args.slice(0, 4), ["ssh", "console", "-a", "acme-core"]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
@@ -654,11 +716,11 @@ test("a 202 degraded response is accepted and warns with the core's persisted-bu
t.mock.method(console, "warn", (...parts: unknown[]) => void warnings.push(parts.join(" ")));
try {
writeLayer(dir);
- const bin = fakeFly(
- dir,
- `console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 202, body: JSON.stringify({ status: "degraded", version: 8, contentHash: "abc", durable: true, message: "skill collision" }) }));`,
+ const bin = fakeFlyStatus(dir);
+ await withFlyMachineExec(
+ `QM_LAYER_RESPONSE=${JSON.stringify({ status: 202, body: JSON.stringify({ status: "degraded", version: 8, contentHash: "abc", durable: true, message: "skill collision" }) })}`,
+ () => withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir))),
);
- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir)));
assert.ok(warnings.some((line) => /persisted but only partially applied: skill collision/.test(line)));
} finally {
rmSync(dir, { recursive: true, force: true });
@@ -669,15 +731,16 @@ test("a remote-script error (signing secret missing on core) is NOT deferrable a
const dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));
try {
writeLayer(dir);
- const bin = fakeFly(
- dir,
- `console.log('QM_LAYER_ERROR=' + JSON.stringify({ message: "CORE_SIGNING_SECRET is not set on core" }));`,
- );
- await withEnv({ FLY_BIN: bin }, () =>
- assert.rejects(
- () => syncDeploymentLayer(flySyncOpts(dir, true)),
- /could not sync deployment layer: .*CORE_SIGNING_SECRET is not set on core/,
- ),
+ const bin = fakeFlyStatus(dir);
+ await withFlyMachineExec(
+ `QM_LAYER_ERROR=${JSON.stringify({ message: "CORE_SIGNING_SECRET is not set on core" })}`,
+ () =>
+ withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () =>
+ assert.rejects(
+ () => syncDeploymentLayer(flySyncOpts(dir, true)),
+ /could not sync deployment layer: .*CORE_SIGNING_SECRET is not set on core/,
+ ),
+ ),
);
} finally {
rmSync(dir, { recursive: true, force: true });
@@ -688,13 +751,15 @@ test("a remote connection failure (core process down inside the VM) IS deferrabl
const dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));
try {
writeLayer(dir);
- const bin = fakeFly(
- dir,
- `console.log('QM_LAYER_ERROR=' + JSON.stringify({ message: "fetch failed", code: "ECONNREFUSED" }));`,
- );
- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir, true)));
- await withEnv({ FLY_BIN: bin }, () =>
- assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir)), /could not sync deployment layer/),
+ const bin = fakeFlyStatus(dir);
+ await withFlyMachineExec(
+ `QM_LAYER_ERROR=${JSON.stringify({ message: "fetch failed", code: "ECONNREFUSED" })}`,
+ async () => {
+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir, true)));
+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () =>
+ assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir)), /could not sync deployment layer/),
+ );
+ },
);
} finally {
rmSync(dir, { recursive: true, force: true });
@@ -764,9 +829,11 @@ test("a fly-ssh transport failure defers under allowUnavailable, but a missing a
try {
writeLayer(dir);
const transport = fakeFly(dir, `console.error("Error: tunnel unavailable"); process.exit(1);`);
- await withEnv({ FLY_BIN: transport }, () => syncDeploymentLayer(flySyncOpts(dir, true)));
+ await withEnv({ FLY_BIN: transport, FLY_API_TOKEN: "test-token" }, () =>
+ syncDeploymentLayer(flySyncOpts(dir, true)),
+ );
const missingApp = fakeFly(dir, `console.error("Error: Could not find App 'acme-core'"); process.exit(1);`);
- await withEnv({ FLY_BIN: missingApp }, () =>
+ await withEnv({ FLY_BIN: missingApp, FLY_API_TOKEN: "test-token" }, () =>
assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir, true)), /Fly app acme-core not found/),
);
} finally {
diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.ts
index 3ddc06f93..5e895b7ea 100644
--- a/cli/test/fly-sandbox.test.ts
+++ b/cli/test/fly-sandbox.test.ts
@@ -593,7 +593,7 @@ else console.log("ok");`,
}
assert.match(
calls,
- /ssh console -a acme-core --machine machine-core .* --quiet/,
+ /machine exec -a acme-core machine-core .* --timeout 120/,
"live readiness proves S3 from the running core",
);
} finally {
@@ -640,7 +640,7 @@ else console.log("ok");`,
);
assert.doesNotMatch(
readFileSync(fake.log, "utf8"),
- /ssh console/,
+ /machine exec/,
"an unowned core never receives the storage probe",
);
} finally {
@@ -836,7 +836,7 @@ test("fly live readiness fails when core cannot round-trip durable object storag
if (a.startsWith("apps list")) console.log(JSON.stringify([{ Name: "acme-core" }]));
else if (a.startsWith("status")) console.log(JSON.stringify({ Machines: [{ id: "machine-core", state: "started", region: "sjc", config: { image: "registry.fly.io/app@sha256:abc", env: ${JSON.stringify(env)} } }] }));
else if (a.startsWith("checks list")) console.log(JSON.stringify({ machine: [{ status: "passing" }] }));
-else if (a.startsWith("ssh console")) { console.error("AccessDenied"); process.exit(1); }
+else if (a.startsWith("machine exec")) { console.error("AccessDenied"); process.exit(1); }
else console.log("ok");`,
);
try {
diff --git a/cli/test/fly-timing.test.ts b/cli/test/fly-timing.test.ts
index 24888ffc3..3cbaca990 100644
--- a/cli/test/fly-timing.test.ts
+++ b/cli/test/fly-timing.test.ts
@@ -45,8 +45,6 @@ if (args[0] === "apps" && args[1] === "create") {
]));
} else if (args[0] === "deploy") {
console.log("deployed");
-} else if (args[0] === "ssh" && args[1] === "console") {
- console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 200, body: JSON.stringify({ version: 1, contentHash: "0123456789abcdef" }) }));
} else {
console.error("unexpected fake fly command: " + cmd);
process.exit(42);
@@ -78,6 +76,7 @@ test("fly up emits phase timings and appends a GitHub step summary", () => {
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
GITHUB_STEP_SUMMARY: summaryPath,
},
stdio: ["ignore", "pipe", "pipe"],
@@ -123,6 +122,7 @@ test("fly up can deploy a tagged image without consulting the source stack's run
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
},
stdio: ["ignore", "pipe", "pipe"],
@@ -136,7 +136,7 @@ test("fly up can deploy a tagged image without consulting the source stack's run
.split("\n")
.map((line) => JSON.parse(line) as string[]);
assert.equal(
- commands.some((args) => args[0] === "status"),
+ commands.some((args) => args[0] === "status" && args.includes("qm-core")),
false,
);
assert.equal(
@@ -167,6 +167,7 @@ test("fly image-from fails closed when Fly cannot resolve the running tag to a d
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_NO_IMAGE_DIGEST: "1",
},
stdio: ["ignore", "pipe", "pipe"],
@@ -199,6 +200,7 @@ test("fly image-from resolves by machine id during a mixed rollout", () => {
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
FAKE_FLY_MIXED_IMAGES: "1",
},
@@ -236,6 +238,7 @@ test("a fresh Fly deploy stages the direct Managed Postgres URL without attachin
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
FAKE_FLY_FRESH_PG: "1",
},
@@ -280,6 +283,7 @@ test("Fly preserves an existing DATABASE_URL even when a same-name Managed Postg
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
},
stdio: ["ignore", "pipe", "pipe"],
@@ -320,6 +324,7 @@ test("Fly redacts credential-bearing Managed Postgres status failures", () => {
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_FRESH_PG: "1",
FAKE_FLY_STATUS_FAIL: "1",
},
@@ -354,6 +359,7 @@ test("fly up build-only pushes a tagged image without checking runtime deploy se
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
},
stdio: ["ignore", "pipe", "pipe"],
@@ -410,6 +416,7 @@ test("fly up build-only dry-run plans without pushing an image", () => {
env: {
...process.env,
FLY_BIN: fakeFlyBin(dir),
+ FLY_API_TOKEN: "",
FAKE_FLY_LOG: logPath,
},
stdio: ["ignore", "pipe", "pipe"],