Multiplayer agent harness for work.
Retry deployment-layer sync when core is still unreachable after up. (#1406)
Sina Matiand4d20acf08b6
diff --git a/cli/src/deployment-layer.ts b/cli/src/deployment-layer.tsindex 03f33c6fa..be9431de0 100644--- a/cli/src/deployment-layer.ts+++ b/cli/src/deployment-layer.ts@@ -3,7 +3,7 @@ import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs";import { join, relative, sep } from "node:path";import type { QmConfig } from "./config.ts";import { CliError, errMessage, step, warn } from "./log.ts";-import { deploymentSecretValue, readEnvFile } from "./util.ts";+import { deploymentSecretValue, readEnvFile, sleep } from "./util.ts";import { parseToolDescriptor } from "./sandbox-layer.ts";interface DeploymentLayerFile {@@ -165,6 +165,9 @@ export const CONNECTIVITY_CODES = new Set(["UND_ERR_SOCKET",]);+export const DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS = 5;+const DEPLOYMENT_LAYER_UNAVAILABLE_BACKOFF_MS = 2_000;+function isCoreUnreachable(error: unknown): boolean {if (error instanceof CoreUnreachableError) return true;if (error instanceof CliError) return false;@@ -244,6 +247,7 @@ export async function syncDeploymentLayer(opts: {sandboxDir: string;envFile?: string;allowUnavailable?: boolean;+ wait?: (ms: number) => Promise<void>;}): Promise<void> {if (!existsSync(opts.sandboxDir)) {step(`deployment layer: skipped (no sandbox directory at ${opts.sandboxDir})`);@@ -301,25 +305,35 @@ export async function syncDeploymentLayerBody(configDir: string;envFile?: string;allowUnavailable?: boolean;+ wait?: (ms: number) => Promise<void>;},body: string,): Promise<DeploymentLayerSyncResult | undefined> {+ const wait = opts.wait ?? sleep;let response: { status: number; body: string };- try {- response = await deploymentLayerRequest({- config: opts.config,- configDir: opts.configDir,- method: "PUT",- body,- transport: opts.transport,- ...(opts.envFile ? { envFile: opts.envFile } : {}),- });- } catch (error) {- if (opts.allowUnavailable && isCoreUnreachable(error)) {- step(`deployment layer: core is not reachable; deployment succeeded and sync is deferred until the next up`);- return;+ for (let attempt = 1; ; attempt++) {+ try {+ response = await deploymentLayerRequest({+ config: opts.config,+ configDir: opts.configDir,+ method: "PUT",+ body,+ transport: opts.transport,+ ...(opts.envFile ? { envFile: opts.envFile } : {}),+ });+ break;+ } catch (error) {+ const retryable = opts.allowUnavailable && isCoreUnreachable(error);+ if (retryable && attempt < DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS) {+ await wait(DEPLOYMENT_LAYER_UNAVAILABLE_BACKOFF_MS);+ continue;+ }+ if (retryable) {+ step(`deployment layer: core is not reachable; deployment succeeded and sync is deferred until the next up`);+ return;+ }+ throw new CliError(`could not sync deployment layer: ${errMessage(error)}`);}- throw new CliError(`could not sync deployment layer: ${errMessage(error)}`);}if (response.status < 200 || response.status >= 300)throw new CliError(`deployment layer sync failed (${response.status}): ${response.body}`);diff --git a/cli/test/deployment-layer.test.ts b/cli/test/deployment-layer.test.tsindex 54ca4d0bc..13b75a73d 100644--- a/cli/test/deployment-layer.test.ts+++ b/cli/test/deployment-layer.test.ts@@ -6,10 +6,13 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { CONFIG_FILENAME, loadConfigInDir, type QmConfig } from "../src/config.ts";import {+ CoreUnreachableError,+ DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS,currentDeploymentLayerState,deploymentLayerBundle,syncDeploymentLayer,httpDeploymentLayerTransport,+ type DeploymentLayerTransport,} from "../src/deployment-layer.ts";import { dockerDeploymentLayerTransport } from "../src/backends/docker.ts";import { flyDeploymentLayerTransport } from "../src/backends/fly.ts";@@ -566,6 +569,7 @@ test("allowUnavailable swallows an unreachable core but NOT a local config errorconfigDir: dir,sandboxDir: join(dir, "sandbox"),allowUnavailable: true,+ wait: async () => {},}),);await withEnv({ CORE_SIGNING_SECRET: SECRET, QM_BASE_PORT: String(port) }, () =>@@ -589,6 +593,7 @@ test("allowUnavailable swallows an unreachable core but NOT a local config errorconfigDir: dir,sandboxDir: join(dir, "sandbox"),allowUnavailable: true,+ wait: async () => {},}),/CORE_SIGNING_SECRET is required/,),@@ -612,6 +617,7 @@ function flySyncOpts(dir: string, allowUnavailable?: boolean): Parameters<typeofconfigDir: dir,sandboxDir: join(dir, "sandbox"),...(allowUnavailable !== undefined ? { allowUnavailable } : {}),+ ...(allowUnavailable ? { wait: async () => {} } : {}),};}@@ -695,6 +701,64 @@ test("a remote connection failure (core process down inside the VM) IS deferrabl}});+test("allowUnavailable retries an unreachable PUT until the core comes back", async (t) => {+ const dir = mkdtempSync(join(tmpdir(), "qm-layer-retry-recover-"));+ const lines: string[] = [];+ t.mock.method(console, "log", (...parts: unknown[]) => void lines.push(parts.join(" ")));+ try {+ writeLayer(dir);+ let calls = 0;+ const transport: DeploymentLayerTransport = async () => {+ calls += 1;+ if (calls < 3) {+ const error = Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" });+ throw error;+ }+ return { status: 200, body: JSON.stringify({ version: 4, contentHash: "abc123def456" }) };+ };+ await syncDeploymentLayer({+ config: makeConfig("http://example.invalid"),+ transport,+ configDir: dir,+ sandboxDir: join(dir, "sandbox"),+ allowUnavailable: true,+ wait: async () => {},+ });+ assert.equal(calls, 3);+ assert.ok(lines.some((line) => /deployment layer: v4 abc123def456/.test(line)));+ assert.equal(lines.filter((line) => /sync is deferred/.test(line)).length, 0);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("allowUnavailable retries then prints the deferred message once when the core stays unreachable", async (t) => {+ const dir = mkdtempSync(join(tmpdir(), "qm-layer-retry-exhaust-"));+ const lines: string[] = [];+ t.mock.method(console, "log", (...parts: unknown[]) => void lines.push(parts.join(" ")));+ try {+ writeLayer(dir);+ let calls = 0;+ const transport: DeploymentLayerTransport = async () => {+ calls += 1;+ throw new CoreUnreachableError("core is not reachable");+ };+ await syncDeploymentLayer({+ config: makeConfig("http://example.invalid"),+ transport,+ configDir: dir,+ sandboxDir: join(dir, "sandbox"),+ allowUnavailable: true,+ wait: async () => {},+ });+ assert.equal(calls, DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS);+ assert.ok(calls > 1);+ assert.equal(lines.filter((line) => /sync is deferred/.test(line)).length, 1);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});+test("a fly-ssh transport failure defers under allowUnavailable, but a missing app never does", async () => {const dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));try {