Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Retry deployment-layer sync when core is still unreachable after up. (#1406)

Sina Matiand4d20acf08b6
diff --git a/cli/src/deployment-layer.ts b/cli/src/deployment-layer.ts
index 03f33c6fa..be9431de0 100644
--- a/cli/src/deployment-layer.ts
+++ b/cli/src/deployment-layer.ts
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs";
import { join, relative, sep } from "node:path";
import type { QmConfig } from "./config.ts";
import { CliError, errMessage, step, warn } from "./log.ts";
-import { deploymentSecretValue, readEnvFile } from "./util.ts";
+import { deploymentSecretValue, readEnvFile, sleep } from "./util.ts";
import { parseToolDescriptor } from "./sandbox-layer.ts";
interface DeploymentLayerFile {
@@ -165,6 +165,9 @@ export const CONNECTIVITY_CODES = new Set([
"UND_ERR_SOCKET",
]);
+export const DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS = 5;
+const DEPLOYMENT_LAYER_UNAVAILABLE_BACKOFF_MS = 2_000;
+
function isCoreUnreachable(error: unknown): boolean {
if (error instanceof CoreUnreachableError) return true;
if (error instanceof CliError) return false;
@@ -244,6 +247,7 @@ export async function syncDeploymentLayer(opts: {
sandboxDir: string;
envFile?: string;
allowUnavailable?: boolean;
+ wait?: (ms: number) => Promise<void>;
}): Promise<void> {
if (!existsSync(opts.sandboxDir)) {
step(`deployment layer: skipped (no sandbox directory at ${opts.sandboxDir})`);
@@ -301,25 +305,35 @@ export async function syncDeploymentLayerBody(
configDir: string;
envFile?: string;
allowUnavailable?: boolean;
+ wait?: (ms: number) => Promise<void>;
},
body: string,
): Promise<DeploymentLayerSyncResult | undefined> {
+ const wait = opts.wait ?? sleep;
let response: { status: number; body: string };
- try {
- response = await deploymentLayerRequest({
- config: opts.config,
- configDir: opts.configDir,
- method: "PUT",
- body,
- transport: opts.transport,
- ...(opts.envFile ? { envFile: opts.envFile } : {}),
- });
- } catch (error) {
- if (opts.allowUnavailable && isCoreUnreachable(error)) {
- step(`deployment layer: core is not reachable; deployment succeeded and sync is deferred until the next up`);
- return;
+ for (let attempt = 1; ; attempt++) {
+ try {
+ response = await deploymentLayerRequest({
+ config: opts.config,
+ configDir: opts.configDir,
+ method: "PUT",
+ body,
+ transport: opts.transport,
+ ...(opts.envFile ? { envFile: opts.envFile } : {}),
+ });
+ break;
+ } catch (error) {
+ const retryable = opts.allowUnavailable && isCoreUnreachable(error);
+ if (retryable && attempt < DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS) {
+ await wait(DEPLOYMENT_LAYER_UNAVAILABLE_BACKOFF_MS);
+ continue;
+ }
+ if (retryable) {
+ step(`deployment layer: core is not reachable; deployment succeeded and sync is deferred until the next up`);
+ return;
+ }
+ throw new CliError(`could not sync deployment layer: ${errMessage(error)}`);
}
- throw new CliError(`could not sync deployment layer: ${errMessage(error)}`);
}
if (response.status < 200 || response.status >= 300)
throw new CliError(`deployment layer sync failed (${response.status}): ${response.body}`);
diff --git a/cli/test/deployment-layer.test.ts b/cli/test/deployment-layer.test.ts
index 54ca4d0bc..13b75a73d 100644
--- a/cli/test/deployment-layer.test.ts
+++ b/cli/test/deployment-layer.test.ts
@@ -6,10 +6,13 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { CONFIG_FILENAME, loadConfigInDir, type QmConfig } from "../src/config.ts";
import {
+ CoreUnreachableError,
+ DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS,
currentDeploymentLayerState,
deploymentLayerBundle,
syncDeploymentLayer,
httpDeploymentLayerTransport,
+ type DeploymentLayerTransport,
} from "../src/deployment-layer.ts";
import { dockerDeploymentLayerTransport } from "../src/backends/docker.ts";
import { flyDeploymentLayerTransport } from "../src/backends/fly.ts";
@@ -566,6 +569,7 @@ test("allowUnavailable swallows an unreachable core but NOT a local config error
configDir: dir,
sandboxDir: join(dir, "sandbox"),
allowUnavailable: true,
+ wait: async () => {},
}),
);
await withEnv({ CORE_SIGNING_SECRET: SECRET, QM_BASE_PORT: String(port) }, () =>
@@ -589,6 +593,7 @@ test("allowUnavailable swallows an unreachable core but NOT a local config error
configDir: dir,
sandboxDir: join(dir, "sandbox"),
allowUnavailable: true,
+ wait: async () => {},
}),
/CORE_SIGNING_SECRET is required/,
),
@@ -612,6 +617,7 @@ function flySyncOpts(dir: string, allowUnavailable?: boolean): Parameters<typeof
configDir: dir,
sandboxDir: join(dir, "sandbox"),
...(allowUnavailable !== undefined ? { allowUnavailable } : {}),
+ ...(allowUnavailable ? { wait: async () => {} } : {}),
};
}
@@ -695,6 +701,64 @@ test("a remote connection failure (core process down inside the VM) IS deferrabl
}
});
+test("allowUnavailable retries an unreachable PUT until the core comes back", async (t) => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-layer-retry-recover-"));
+ const lines: string[] = [];
+ t.mock.method(console, "log", (...parts: unknown[]) => void lines.push(parts.join(" ")));
+ try {
+ writeLayer(dir);
+ let calls = 0;
+ const transport: DeploymentLayerTransport = async () => {
+ calls += 1;
+ if (calls < 3) {
+ const error = Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" });
+ throw error;
+ }
+ return { status: 200, body: JSON.stringify({ version: 4, contentHash: "abc123def456" }) };
+ };
+ await syncDeploymentLayer({
+ config: makeConfig("http://example.invalid"),
+ transport,
+ configDir: dir,
+ sandboxDir: join(dir, "sandbox"),
+ allowUnavailable: true,
+ wait: async () => {},
+ });
+ assert.equal(calls, 3);
+ assert.ok(lines.some((line) => /deployment layer: v4 abc123def456/.test(line)));
+ assert.equal(lines.filter((line) => /sync is deferred/.test(line)).length, 0);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("allowUnavailable retries then prints the deferred message once when the core stays unreachable", async (t) => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-layer-retry-exhaust-"));
+ const lines: string[] = [];
+ t.mock.method(console, "log", (...parts: unknown[]) => void lines.push(parts.join(" ")));
+ try {
+ writeLayer(dir);
+ let calls = 0;
+ const transport: DeploymentLayerTransport = async () => {
+ calls += 1;
+ throw new CoreUnreachableError("core is not reachable");
+ };
+ await syncDeploymentLayer({
+ config: makeConfig("http://example.invalid"),
+ transport,
+ configDir: dir,
+ sandboxDir: join(dir, "sandbox"),
+ allowUnavailable: true,
+ wait: async () => {},
+ });
+ assert.equal(calls, DEPLOYMENT_LAYER_UNAVAILABLE_ATTEMPTS);
+ assert.ok(calls > 1);
+ assert.equal(lines.filter((line) => /sync is deferred/.test(line)).length, 1);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test("a fly-ssh transport failure defers under allowUnavailable, but a missing app never does", async () => {
const dir = mkdtempSync(join(tmpdir(), "qm-layer-fly-"));
try {