Multiplayer agent harness for work.
Replace Fly ssh console with machine exec for internal transports (#1407)
Sina Matianebc442a4a9f8
diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.tsindex 95baa0d4c..753133190 100644--- a/cli/src/backends/fly.ts+++ b/cli/src/backends/fly.ts@@ -70,26 +70,89 @@ const flyServiceCtx = (config: QmConfig, appPrefix: string, deployAppPrefix: strconst FLY_RESPONSE = "QM_LAYER_RESPONSE=";const FLY_REMOTE_ERROR = "QM_LAYER_ERROR=";const FLY_REQUEST_TIMEOUT_MS = 120_000;+const FLY_EXEC_TIMEOUT_SEC = 120;+const FLY_APP_NOT_FOUND = /could not find app|app not found/i;+const FLY_MACHINES_API = "https://api.machines.dev/v1";-function flyRequest(config: QmConfig, method: "GET" | "PUT", body: string): { status: number; body: string } {+function classifyFlyReachability(app: string, detail: string): never {+ if (FLY_APP_NOT_FOUND.test(detail)) throw new CliError(`Fly app ${app} not found: ${detail}`);+ throw new CoreUnreachableError(`could not reach the Fly core: ${detail}`);+}++function flyCoreMachineId(app: string): string {+ let raw: string;+ try {+ raw = fly(["status", "-a", app, "--json"]);+ } catch (error) {+ classifyFlyReachability(app, errMessage(error));+ }+ if (FLY_APP_NOT_FOUND.test(raw)) classifyFlyReachability(app, raw);+ let parsed: { Machines?: Array<{ id?: string; ID?: string; state?: string }> };+ try {+ parsed = JSON.parse(raw) as typeof parsed;+ } catch {+ throw new CoreUnreachableError(`could not reach the Fly core: fly status returned invalid JSON`);+ }+ const machine = parsed.Machines?.find((entry) => entry.state === "started") ?? parsed.Machines?.[0];+ const machineId = machine?.id ?? machine?.ID;+ if (!machineId) throw new CoreUnreachableError(`could not reach the Fly core: no machine on ${app}`);+ return machineId;+}++function flyApiAuthorization(): string {+ try {+ const token = process.env.FLY_API_TOKEN?.trim() || fly(["auth", "token"]).trim();+ if (!token) throw new Error("Fly API token is missing");+ return token.startsWith("FlyV1") || token.startsWith("Bearer ") ? token : `Bearer ${token}`;+ } catch (error) {+ throw new CoreUnreachableError(`could not reach the Fly core: ${errMessage(error)}`);+ }+}++function flyExecOutput(payload: unknown): string {+ const { stdout, stderr } = (payload ?? {}) as { stdout?: unknown; stderr?: unknown };+ return [stdout, stderr].filter((value): value is string => typeof value === "string" && value !== "").join("\n");+}++async function flyRequest(+ config: QmConfig,+ method: "GET" | "PUT",+ body: string,+): Promise<{ status: number; body: string }> {const app = `${appPrefixOf(config)}-core`;const script = `const fs=require("node:fs"),{createHmac}=require("node:crypto");const fail=error=>{const code=error&&(error.cause&&error.cause.code||error.code);console.log(${JSON.stringify(FLY_REMOTE_ERROR)}+JSON.stringify({message:error&&error.message?error.message:String(error),...(typeof code==="string"?{code}:{})}))};try{const method=${JSON.stringify(method)},path="/v1/deployment-layer",body=fs.readFileSync(0,"utf8"),timestamp=Math.floor(Date.now()/1000),canonical=method+"\\n"+path+"\\n"+body,secret=process.env.CORE_SIGNING_SECRET;if(!secret)throw new Error("CORE_SIGNING_SECRET is not set on core");const signature=createHmac("sha256",secret).update("v0:"+timestamp+":"+canonical).digest("hex");fetch("http://127.0.0.1:"+(process.env.PORT||8080)+path,{method,headers:{"content-type":"application/json","x-timestamp":String(timestamp),"x-signature":"v0="+signature},...(method==="PUT"?{body}: {})}).then(async response=>console.log(${JSON.stringify(FLY_RESPONSE)}+JSON.stringify({status:response.status,body:await response.text()}))).catch(fail)}catch(error){fail(error)}`;- const encoded = Buffer.from(script).toString("base64");- const command = `node -e "eval(Buffer.from('${encoded}','base64').toString())"`;- let output: string;+ const machineId = flyCoreMachineId(app);+ let response: Response;+ let text: string;try {- output = execFileSync(flyBin(), ["ssh", "console", "-a", app, "-C", command], {- encoding: "utf8",- input: body,- stdio: ["pipe", "pipe", "pipe"],- timeout: FLY_REQUEST_TIMEOUT_MS,- });+ response = await fetch(+ `${FLY_MACHINES_API}/apps/${encodeURIComponent(app)}/machines/${encodeURIComponent(machineId)}/exec`,+ {+ method: "POST",+ headers: {+ authorization: flyApiAuthorization(),+ "content-type": "application/json",+ },+ body: JSON.stringify({+ command: ["node", "-e", script],+ stdin: body,+ timeout: FLY_EXEC_TIMEOUT_SEC,+ }),+ signal: AbortSignal.timeout(FLY_REQUEST_TIMEOUT_MS),+ },+ );+ text = await response.text();} catch (error) {- const detail = error as { stdout?: string; stderr?: string; message?: string };- const text = `${detail.stderr ?? ""}${detail.stdout ?? ""}`.trim() || detail.message || "fly ssh failed";- if (/could not find app|app not found/i.test(text)) throw new CliError(`Fly app ${app} not found: ${text}`);- throw new CoreUnreachableError(`could not reach the Fly core: ${text}`);+ throw new CoreUnreachableError(`could not reach the Fly core: ${errMessage(error)}`);}+ if (!response.ok) classifyFlyReachability(app, text || `HTTP ${response.status}`);+ let parsed: unknown;+ try {+ parsed = JSON.parse(text);+ } catch {+ throw new CoreUnreachableError(`could not reach the Fly core: unparseable machine exec response`);+ }+ const output = flyExecOutput(parsed);const remoteError = output.split("\n").find((value) => value.startsWith(FLY_REMOTE_ERROR));if (remoteError) {const detail = JSON.parse(remoteError.slice(FLY_REMOTE_ERROR.length)) as { message?: string; code?: string };@@ -104,9 +167,8 @@ function flyRequest(config: QmConfig, method: "GET" | "PUT", body: string): { streturn JSON.parse(line.slice(FLY_RESPONSE.length)) as { status: number; body: string };}-/** Deployment-layer transport for Fly: a signed request executed on the core VM over fly ssh. */export const flyDeploymentLayerTransport: DeploymentLayerTransport = (opts) =>- Promise.resolve(flyRequest(opts.config, opts.method, opts.body));+ flyRequest(opts.config, opts.method, opts.body);import { doctorCommon, localDoctorSecrets, requireFlyAuth } from "./doctor.ts";@@ -387,8 +449,6 @@ function writeDerived(ctx: FlyCtx, service: ServiceName): string {return path;}-const FLY_APP_NOT_FOUND = /app not found|Could not find App/i;-function secretNames(app: string): Set<string> | undefined {const out = fly(["secrets", "list", "-a", app], { allow: FLY_APP_NOT_FOUND });if (FLY_APP_NOT_FOUND.test(out)) return undefined;@@ -468,7 +528,7 @@ export function flyS3ProbeCommand(): string {}function flyS3RoundTrip(app: string, machineId: string): void {- fly(["ssh", "console", "-a", app, "--machine", machineId, "--command", flyS3ProbeCommand(), "--quiet"]);+ fly(["machine", "exec", "-a", app, machineId, flyS3ProbeCommand(), "--timeout", String(FLY_EXEC_TIMEOUT_SEC)]);}export function flyLiveSessionCommand(): string {@@ -476,7 +536,7 @@ export function flyLiveSessionCommand(): string {}function flyLiveSession(app: string, machineId: string): void {- fly(["ssh", "console", "-a", app, "--machine", machineId, "--command", flyLiveSessionCommand(), "--quiet"]);+ fly(["machine", "exec", "-a", app, machineId, flyLiveSessionCommand(), "--timeout", String(FLY_EXEC_TIMEOUT_SEC)]);}function flyOrgApps(flyOrg: string): Set<string> {diff --git a/cli/test/deployment-layer.test.ts b/cli/test/deployment-layer.test.tsindex 13b75a73d..7a3c8d811 100644--- a/cli/test/deployment-layer.test.ts+++ b/cli/test/deployment-layer.test.ts@@ -610,6 +610,57 @@ function fakeFly(dir: string, body: string): string {return bin;}+function fakeFlyStatus(dir: string, extra = ""): string {+ return fakeFly(+ dir,+ `+const args = process.argv.slice(2);+${extra}+if (args[0] === "status") {+ console.log(+ JSON.stringify({+ Machines: [+ { id: "machine-stale", state: "stopped" },+ { id: "machine-core", state: "started" },+ ],+ }),+ );+ process.exit(0);+}+process.exit(1);+`,+ );+}++interface CapturedFlyExec {+ url: string;+ authorization: string;+ body: { command?: unknown; stdin?: unknown; timeout?: unknown };+}++async function withFlyMachineExec<T>(stdout: string, fn: (captured: CapturedFlyExec[]) => Promise<T>): Promise<T> {+ const captured: CapturedFlyExec[] = [];+ const previous = globalThis.fetch;+ globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {+ const url = typeof input === "string" || input instanceof URL ? String(input) : input.url;+ const headers = new Headers(init?.headers);+ captured.push({+ url,+ authorization: headers.get("authorization") ?? "",+ body: init?.body ? (JSON.parse(String(init.body)) as CapturedFlyExec["body"]) : {},+ });+ return new Response(JSON.stringify({ stdout, exit_code: 0 }), {+ status: 200,+ headers: { "content-type": "application/json" },+ });+ }) as typeof fetch;+ try {+ return await fn(captured);+ } finally {+ globalThis.fetch = previous;+ }+}+function flySyncOpts(dir: string, allowUnavailable?: boolean): Parameters<typeof syncDeploymentLayer>[0] {return {config: makeConfig("http://example.invalid"),@@ -625,24 +676,35 @@ test("fly sync succeeds on the response marker, piping the exact bundle over stdconst dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));try {writeLayer(dir);- const stdinLog = join(dir, "stdin.log");const argsLog = join(dir, "args.log");- const bin = fakeFly(+ const bin = fakeFlyStatus(dir,- [- `fs.writeFileSync(${JSON.stringify(argsLog)}, JSON.stringify(process.argv.slice(2)));`,- `fs.writeFileSync(${JSON.stringify(stdinLog)}, fs.readFileSync(0, "utf8"));`,- `console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 200, body: JSON.stringify({ version: 7, contentHash: "abcdef123456" }) }));`,- ].join("\n"),+ `fs.writeFileSync(${JSON.stringify(argsLog)}, JSON.stringify(process.argv.slice(2)));`,);- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir)));- assert.equal(- readFileSync(stdinLog, "utf8"),- JSON.stringify(deploymentLayerBundle(join(dir, "sandbox"))),- "the full bundle reaches the remote script's stdin",+ const bundle = JSON.stringify(deploymentLayerBundle(join(dir, "sandbox")));+ await withFlyMachineExec(+ `QM_LAYER_RESPONSE=${JSON.stringify({ status: 200, body: JSON.stringify({ version: 7, contentHash: "abcdef123456" }) })}`,+ async (captured) => {+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir)));+ assert.equal(captured.length, 1);+ const request = captured[0]!;+ assert.equal(request.url, "https://api.machines.dev/v1/apps/acme-core/machines/machine-core/exec");+ assert.equal(request.authorization, "Bearer test-token");+ assert.equal(request.body.stdin, bundle, "the full bundle reaches the remote script's stdin");+ assert.equal(request.body.timeout, 120);+ assert.ok(Array.isArray(request.body.command));+ assert.deepEqual((request.body.command as string[]).slice(0, 2), ["node", "-e"]);+ const remoteScript = (request.body.command as string[])[2] ?? "";+ assert.match(remoteScript, /createHmac\("sha256"/, "the signing script is passed to node -e directly");+ assert.doesNotMatch(remoteScript, /eval\(/, "no eval indirection");+ assert.deepEqual(JSON.parse(readFileSync(argsLog, "utf8")) as string[], [+ "status",+ "-a",+ "acme-core",+ "--json",+ ]);+ },);- const args = JSON.parse(readFileSync(argsLog, "utf8")) as string[];- assert.deepEqual(args.slice(0, 4), ["ssh", "console", "-a", "acme-core"]);} finally {rmSync(dir, { recursive: true, force: true });}@@ -654,11 +716,11 @@ test("a 202 degraded response is accepted and warns with the core's persisted-but.mock.method(console, "warn", (...parts: unknown[]) => void warnings.push(parts.join(" ")));try {writeLayer(dir);- const bin = fakeFly(- dir,- `console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 202, body: JSON.stringify({ status: "degraded", version: 8, contentHash: "abc", durable: true, message: "skill collision" }) }));`,+ const bin = fakeFlyStatus(dir);+ await withFlyMachineExec(+ `QM_LAYER_RESPONSE=${JSON.stringify({ status: 202, body: JSON.stringify({ status: "degraded", version: 8, contentHash: "abc", durable: true, message: "skill collision" }) })}`,+ () => withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir))),);- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir)));assert.ok(warnings.some((line) => /persisted but only partially applied: skill collision/.test(line)));} finally {rmSync(dir, { recursive: true, force: true });@@ -669,15 +731,16 @@ test("a remote-script error (signing secret missing on core) is NOT deferrable aconst dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));try {writeLayer(dir);- const bin = fakeFly(- dir,- `console.log('QM_LAYER_ERROR=' + JSON.stringify({ message: "CORE_SIGNING_SECRET is not set on core" }));`,- );- await withEnv({ FLY_BIN: bin }, () =>- assert.rejects(- () => syncDeploymentLayer(flySyncOpts(dir, true)),- /could not sync deployment layer: .*CORE_SIGNING_SECRET is not set on core/,- ),+ const bin = fakeFlyStatus(dir);+ await withFlyMachineExec(+ `QM_LAYER_ERROR=${JSON.stringify({ message: "CORE_SIGNING_SECRET is not set on core" })}`,+ () =>+ withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () =>+ assert.rejects(+ () => syncDeploymentLayer(flySyncOpts(dir, true)),+ /could not sync deployment layer: .*CORE_SIGNING_SECRET is not set on core/,+ ),+ ),);} finally {rmSync(dir, { recursive: true, force: true });@@ -688,13 +751,15 @@ test("a remote connection failure (core process down inside the VM) IS deferrablconst dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));try {writeLayer(dir);- const bin = fakeFly(- dir,- `console.log('QM_LAYER_ERROR=' + JSON.stringify({ message: "fetch failed", code: "ECONNREFUSED" }));`,- );- await withEnv({ FLY_BIN: bin }, () => syncDeploymentLayer(flySyncOpts(dir, true)));- await withEnv({ FLY_BIN: bin }, () =>- assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir)), /could not sync deployment layer/),+ const bin = fakeFlyStatus(dir);+ await withFlyMachineExec(+ `QM_LAYER_ERROR=${JSON.stringify({ message: "fetch failed", code: "ECONNREFUSED" })}`,+ async () => {+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () => syncDeploymentLayer(flySyncOpts(dir, true)));+ await withEnv({ FLY_BIN: bin, FLY_API_TOKEN: "test-token" }, () =>+ assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir)), /could not sync deployment layer/),+ );+ },);} finally {rmSync(dir, { recursive: true, force: true });@@ -764,9 +829,11 @@ test("a fly-ssh transport failure defers under allowUnavailable, but a missing atry {writeLayer(dir);const transport = fakeFly(dir, `console.error("Error: tunnel unavailable"); process.exit(1);`);- await withEnv({ FLY_BIN: transport }, () => syncDeploymentLayer(flySyncOpts(dir, true)));+ await withEnv({ FLY_BIN: transport, FLY_API_TOKEN: "test-token" }, () =>+ syncDeploymentLayer(flySyncOpts(dir, true)),+ );const missingApp = fakeFly(dir, `console.error("Error: Could not find App 'acme-core'"); process.exit(1);`);- await withEnv({ FLY_BIN: missingApp }, () =>+ await withEnv({ FLY_BIN: missingApp, FLY_API_TOKEN: "test-token" }, () =>assert.rejects(() => syncDeploymentLayer(flySyncOpts(dir, true)), /Fly app acme-core not found/),);} finally {diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.tsindex 3ddc06f93..5e895b7ea 100644--- a/cli/test/fly-sandbox.test.ts+++ b/cli/test/fly-sandbox.test.ts@@ -593,7 +593,7 @@ else console.log("ok");`,}assert.match(calls,- /ssh console -a acme-core --machine machine-core .* --quiet/,+ /machine exec -a acme-core machine-core .* --timeout 120/,"live readiness proves S3 from the running core",);} finally {@@ -640,7 +640,7 @@ else console.log("ok");`,);assert.doesNotMatch(readFileSync(fake.log, "utf8"),- /ssh console/,+ /machine exec/,"an unowned core never receives the storage probe",);} finally {@@ -836,7 +836,7 @@ test("fly live readiness fails when core cannot round-trip durable object storagif (a.startsWith("apps list")) console.log(JSON.stringify([{ Name: "acme-core" }]));else if (a.startsWith("status")) console.log(JSON.stringify({ Machines: [{ id: "machine-core", state: "started", region: "sjc", config: { image: "registry.fly.io/app@sha256:abc", env: ${JSON.stringify(env)} } }] }));else if (a.startsWith("checks list")) console.log(JSON.stringify({ machine: [{ status: "passing" }] }));-else if (a.startsWith("ssh console")) { console.error("AccessDenied"); process.exit(1); }+else if (a.startsWith("machine exec")) { console.error("AccessDenied"); process.exit(1); }else console.log("ok");`,);try {diff --git a/cli/test/fly-timing.test.ts b/cli/test/fly-timing.test.tsindex 24888ffc3..3cbaca990 100644--- a/cli/test/fly-timing.test.ts+++ b/cli/test/fly-timing.test.ts@@ -45,8 +45,6 @@ if (args[0] === "apps" && args[1] === "create") {]));} else if (args[0] === "deploy") {console.log("deployed");-} else if (args[0] === "ssh" && args[1] === "console") {- console.log('QM_LAYER_RESPONSE=' + JSON.stringify({ status: 200, body: JSON.stringify({ version: 1, contentHash: "0123456789abcdef" }) }));} else {console.error("unexpected fake fly command: " + cmd);process.exit(42);@@ -78,6 +76,7 @@ test("fly up emits phase timings and appends a GitHub step summary", () => {env: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",GITHUB_STEP_SUMMARY: summaryPath,},stdio: ["ignore", "pipe", "pipe"],@@ -123,6 +122,7 @@ test("fly up can deploy a tagged image without consulting the source stack's runenv: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,},stdio: ["ignore", "pipe", "pipe"],@@ -136,7 +136,7 @@ test("fly up can deploy a tagged image without consulting the source stack's run.split("\n").map((line) => JSON.parse(line) as string[]);assert.equal(- commands.some((args) => args[0] === "status"),+ commands.some((args) => args[0] === "status" && args.includes("qm-core")),false,);assert.equal(@@ -167,6 +167,7 @@ test("fly image-from fails closed when Fly cannot resolve the running tag to a denv: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_NO_IMAGE_DIGEST: "1",},stdio: ["ignore", "pipe", "pipe"],@@ -199,6 +200,7 @@ test("fly image-from resolves by machine id during a mixed rollout", () => {env: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,FAKE_FLY_MIXED_IMAGES: "1",},@@ -236,6 +238,7 @@ test("a fresh Fly deploy stages the direct Managed Postgres URL without attachinenv: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,FAKE_FLY_FRESH_PG: "1",},@@ -280,6 +283,7 @@ test("Fly preserves an existing DATABASE_URL even when a same-name Managed Postgenv: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,},stdio: ["ignore", "pipe", "pipe"],@@ -320,6 +324,7 @@ test("Fly redacts credential-bearing Managed Postgres status failures", () => {env: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_FRESH_PG: "1",FAKE_FLY_STATUS_FAIL: "1",},@@ -354,6 +359,7 @@ test("fly up build-only pushes a tagged image without checking runtime deploy seenv: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,},stdio: ["ignore", "pipe", "pipe"],@@ -410,6 +416,7 @@ test("fly up build-only dry-run plans without pushing an image", () => {env: {...process.env,FLY_BIN: fakeFlyBin(dir),+ FLY_API_TOKEN: "",FAKE_FLY_LOG: logPath,},stdio: ["ignore", "pipe", "pipe"],