Multiplayer agent harness for work.
Resolve sandbox base digests without registry reads and force amd64 base builds (#30)
Regan Bella44e2dd02b08
diff --git a/cli/src/commands/sandbox.ts b/cli/src/commands/sandbox.tsindex bfd13f492..fb5cd512f 100644--- a/cli/src/commands/sandbox.ts+++ b/cli/src/commands/sandbox.ts@@ -405,7 +405,7 @@ export function pinnedByDigest(ref: string): string {return ref;}throw new CliError(- `could not resolve an immutable digest for ${ref} — the image may not exist or is not readable${detail ? ` (${detail})` : ""}`,+ `could not resolve an immutable digest for ${ref} — the image may not exist, or the registry rejects reads with this token (Fly app-scoped deploy tokens cannot read the registry); pass ${imageRepository(ref)}@sha256:<digest> to skip the registry lookup${detail ? ` (${detail})` : ""}`,);}if (ref.includes("@sha256:")) return ref;@@ -414,6 +414,27 @@ export function pinnedByDigest(ref: string): string {return `${ref}@${digest}`;}+function pinnedByPull(ref: string): string {+ runDocker(+ ["pull", "--platform", SANDBOX_RUNTIME_PLATFORM, ref],+ `could not pull ${ref} to resolve its immutable digest — pin the base by digest (${imageRepository(ref)}@sha256:<digest>, via --from or the sandbox/Dockerfile FROM) to skip resolution`,+ );+ let output: string;+ try {+ output = execFileSync("docker", ["image", "inspect", "--format", "{{json .RepoDigests}}", ref], {+ encoding: "utf8",+ stdio: ["ignore", "pipe", "pipe"],+ });+ } catch (error) {+ throw new CliError(`could not read the pulled image ${ref}: ${errMessage(error)}`);+ }+ const repository = imageRepository(ref);+ const digest = (JSON.parse(output) as string[]).find((entry) => entry.split("@")[0] === repository)?.split("@")[1];+ if (!digest || !/^sha256:[a-f0-9]{64}$/.test(digest))+ throw new CliError(`docker did not record an immutable digest for ${ref}`);+ return `${ref}@${digest}`;+}+export function recordSandboxPin(configPath: string, image: string | undefined, base?: string): void {const updates: Record<string, string> = {};if (image) updates["image"] = image;@@ -428,7 +449,7 @@ export function runSandboxPublish(opts: SandboxPublishOpts): { image: string } |const repository = publishedRepository(opts);if (!opts.dryRun) authenticateFlyRegistry(opts, [repository, prepared.base]);if (!opts.dryRun && prepared.base !== "scratch" && !prepared.base.includes("@sha256:")) {- const base = pinnedByDigest(prepared.base);+ const base = pinnedByPull(prepared.base);if (prepared.hasCustom) {const dockerfileBody = replaceDockerfileBase(prepared.dockerfileBody, prepared.base, base);const dockerfilePath = join(mkdtempSync(join(tmpdir(), "qm-sandbox-")), "Dockerfile");diff --git a/cli/test/sandbox-publish.test.ts b/cli/test/sandbox-publish.test.tsindex a47d75e52..1dcc5d113 100644--- a/cli/test/sandbox-publish.test.ts+++ b/cli/test/sandbox-publish.test.ts@@ -47,6 +47,18 @@ if (joined.startsWith("buildx imagetools inspect")) {console.log("Name: " + args[3] + "\\nMediaType: application/vnd.oci.image.index.v1+json\\nDigest: ${BASE_DIGEST}\\n");process.exit(0);}+if (args[0] === "pull") {+ process.exit(0);+}+if (joined.startsWith("image inspect")) {+ const ref = args[args.length - 1];+ const withoutDigest = ref.split("@")[0];+ const slash = withoutDigest.lastIndexOf("/");+ const colon = withoutDigest.lastIndexOf(":");+ const repo = colon > slash ? withoutDigest.slice(0, colon) : withoutDigest;+ console.log(JSON.stringify([repo + "@${BASE_DIGEST}"]));+ process.exit(0);+}if (joined.startsWith("buildx build")) {if (!fs.existsSync(${JSON.stringify(join(dir, "fly-authenticated"))})) process.exit(3);const metadata = args[args.indexOf("--metadata-file") + 1];@@ -141,6 +153,67 @@ test("sandbox publish pins the base, reads the pushed digest, and records the pi}});+test("sandbox publish --from a Fly registry tag pins by pull, never by registry inspect", () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-publish-from-fly-"));+ const priorPath = process.env.PATH;+ const log = console.log,+ warn = console.warn;+ console.log = (): void => {};+ console.warn = console.log;+ try {+ const configPath = join(dir, CONFIG_FILENAME);+ writeFileSync(+ configPath,+ JSON.stringify({+ contract: 1,+ orgId: "acme",+ publicUrl: "http://localhost:8080",+ target: "docker",+ services: ["core"],+ sandbox: { app: "acme-sandboxes" },+ }),+ );+ writeFileSync(join(dir, ".env"), "FLY_SANDBOX_API_TOKEN=FlyV1-scoped\n");+ const toolDir = join(dir, "sandbox", "tools", "t");+ mkdirSync(toolDir, { recursive: true });+ writeFileSync(join(toolDir, "tool.json"), JSON.stringify({ id: "t" }));+ writeFileSync(join(toolDir, "t"), "#!/usr/bin/env bash\necho hi\n");+ chmodSync(join(toolDir, "t"), 0o755);+ const dockerLog = fakeDocker(dir);+ process.env.PATH = `${dir}:${priorPath}`;++ runSandboxPublish({+ sandboxDir: join(dir, "sandbox"),+ config: loadConfigAt(configPath).config,+ configPath,+ from: "registry.fly.io/acme-sandboxes:base",+ });++ const calls = readFileSync(dockerLog, "utf8");+ assert.doesNotMatch(calls, /imagetools inspect/, "app-scoped deploy tokens cannot read the registry");+ assert.match(+ calls,+ /pull --platform linux\/amd64 registry\.fly\.io\/acme-sandboxes:base/,+ "the base tag resolves through docker pull, which the deploy token allows",+ );+ assert.match(+ calls,+ new RegExp(`DOCKERFILE .*FROM registry\\.fly\\.io/acme-sandboxes:base@${BASE_DIGEST}`),+ "the layer builds from the digest-pinned base",+ );+ assert.equal(+ loadConfigAt(configPath).config.sandbox?.baseImage,+ `registry.fly.io/acme-sandboxes:base@${BASE_DIGEST}`,+ "the pull-resolved pin is recorded",+ );+ } finally {+ console.log = log;+ console.warn = warn;+ process.env.PATH = priorPath;+ rmSync(dir, { recursive: true, force: true });+ }+});+test("sandbox publish reuses the recorded base pin for a custom Dockerfile instead of re-resolving the tag", () => {const dir = mkdtempSync(join(tmpdir(), "qm-publish-"));const priorPath = process.env.PATH;@@ -190,11 +263,9 @@ test("sandbox publish reuses the recorded base pin for a custom Dockerfile instewriteFileSync(join(dir, "sandbox", "Dockerfile"), "FROM ghcr.io/acme/base:v2 \\\n AS final\nRUN echo hi\n");runSandboxPublish({ sandboxDir: join(dir, "sandbox"), config: loadConfigAt(configPath).config, configPath });- assert.match(- readFileSync(dockerLog, "utf8"),- /imagetools inspect ghcr\.io\/acme\/base:v2/,- "the bumped tag is resolved fresh",- );+ const rebuild = readFileSync(dockerLog, "utf8");+ assert.match(rebuild, /pull --platform linux\/amd64 ghcr\.io\/acme\/base:v2/, "the bumped tag is resolved fresh");+ assert.doesNotMatch(rebuild, /imagetools inspect/, "publish never inspects the registry");assert.equal(loadConfigAt(configPath).config.sandbox?.baseImage,`ghcr.io/acme/base:v2@${BASE_DIGEST}`,@@ -771,7 +842,7 @@ process.exit(1););assert.throws(() => pinnedByDigest("repo.example/app:v9"),- /could not resolve an immutable digest for repo\.example\/app:v9 .*\(ERROR: manifest unknown\)/,+ /could not resolve an immutable digest for repo\.example\/app:v9 .*pass repo\.example\/app@sha256:<digest> to skip the registry lookup \(ERROR: manifest unknown\)/,"tag-form refs still fail hard — the inspect is what resolves them",);process.env.PATH = emptyPath;diff --git a/fly/README.md b/fly/README.mdindex 94541077e..10a4b6453 100644--- a/fly/README.md+++ b/fly/README.md@@ -129,6 +129,14 @@ Use `npm run deploy:fly-image` rather than bare `fly deploy`: this sandbox app iexec-only, and bare deploy creates default launch machines that are not used by`FlySandbox`.+Sandbox machines run `linux/amd64` only. `npm run deploy:fly-image` builds on Fly's+remote amd64 builder, so it works unchanged from arm64 (Apple Silicon) hosts, where a+local `docker build` produces an arm64 image the machines reject and+`--platform linux/amd64` under qemu emulation is slow and unreliable.+`scripts/local-sandbox-build.sh` follows the same rule: it uses the remote builder when+`FLY_SANDBOX_APP_NAME` is set and otherwise builds locally with+`--platform linux/amd64`.+## Configure the core```bashdiff --git a/scripts/local-sandbox-build.sh b/scripts/local-sandbox-build.shindex 447e4cf96..70e5b5ea7 100755--- a/scripts/local-sandbox-build.sh+++ b/scripts/local-sandbox-build.sh@@ -4,6 +4,7 @@ cd "$(dirname "$0")/.."BASE_TAG="qm-sandbox-base:dev"LOCAL_TAG="${LOCAL_SANDBOX_IMAGE:-qm-sandbox-local:latest}"+PLATFORM="linux/amd64"FINGERPRINT="$(node --input-type=module -e 'const { computeSandboxImageFingerprint } = await import("./src/sandbox/local-sandbox.ts");@@ -12,11 +13,21 @@ if (!fp) { console.error("cannot compute sandbox image fingerprint (missing sourconsole.log(fp);')"-echo "==> building ${BASE_TAG} from fly/Dockerfile"-docker build -f fly/Dockerfile -t "${BASE_TAG}" .+if [[ -n "${FLY_SANDBOX_APP_NAME:-}" ]] && command -v flyctl >/dev/null 2>&1; then+ BASE_REF="registry.fly.io/${FLY_SANDBOX_APP_NAME}:dev"+ echo "==> building ${BASE_REF} from fly/Dockerfile on Fly's remote amd64 builder"+ flyctl deploy --build-only --push --remote-only --image-label dev \+ --app "${FLY_SANDBOX_APP_NAME}" -c fly/fly.toml --dockerfile fly/Dockerfile . --yes+ flyctl auth docker+ docker pull --platform "${PLATFORM}" "${BASE_REF}"+ docker tag "${BASE_REF}" "${BASE_TAG}"+else+ echo "==> building ${BASE_TAG} from fly/Dockerfile (${PLATFORM})"+ docker build --platform "${PLATFORM}" -f fly/Dockerfile -t "${BASE_TAG}" .+fiecho "==> building ${LOCAL_TAG} from local/Dockerfile (fingerprint ${FINGERPRINT})"-docker build -f local/Dockerfile --build-arg "BASE=${BASE_TAG}" \+docker build --platform "${PLATFORM}" -f local/Dockerfile --build-arg "BASE=${BASE_TAG}" \--label "qm.sandbox-fingerprint=${FINGERPRINT}" \-t "${LOCAL_TAG}" .