Multiplayer agent harness for work.
Deploys: per-provider app size cap (AWS 100 MiB) and chunked MicroVM writes (#2423)
Regan Bell7a6d05bdb62b
diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.tsindex 01b87d0a3..3d47d72d8 100644--- a/src/deploy/aws-deploy-provider.ts+++ b/src/deploy/aws-deploy-provider.ts@@ -39,6 +39,7 @@ const DATA_CREDS_REFRESH_MS = 45 * 60_000;const DATA_CREDS_RETRY_MS = 60_000;const failureBackoffStamp = (): number => Date.now() - DATA_CREDS_REFRESH_MS + DATA_CREDS_RETRY_MS;const MAX_DATA_TAR_BYTES = 180 * 1024 * 1024;+const MAX_APP_BYTES = 100 * 1024 * 1024;const READY_PATH = `${APP_DIR}/.qm-ready`;const APP_READY_WINDOW_SEC = 60;const APP_PORT_RELEASE_TICKS = 40;@@ -637,7 +638,12 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP};return {- profile: { managedScaleToZero: true, inPlaceReconcile: true, ...(s3 ? { dataDir: DATA_DIR } : {}) },+ profile: {+ managedScaleToZero: true,+ inPlaceReconcile: true,+ maxAppBytes: MAX_APP_BYTES,+ ...(s3 ? { dataDir: DATA_DIR } : {}),+ },apply: (d, version) => place(d, version),diff --git a/src/deploy/deploy-provider.ts b/src/deploy/deploy-provider.tsindex 5a19bf1cc..129b1a1c9 100644--- a/src/deploy/deploy-provider.ts+++ b/src/deploy/deploy-provider.ts@@ -6,6 +6,7 @@ export interface DeployProfile {managedScaleToZero: boolean;inPlaceReconcile?: boolean;dataDir?: string;+ maxAppBytes?: number;}export interface DeployReconcileInput {diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.tsindex 9a91f6e88..86108374c 100644--- a/src/deploy/deploy-service.ts+++ b/src/deploy/deploy-service.ts@@ -149,6 +149,19 @@ export interface DeployServiceDeps {const NAME_RE = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/;const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+function assertAppSize(sizes: readonly number[], maxBytes: number | undefined): void {+ if (!maxBytes) return;+ const total = sizes.reduce((sum, n) => sum + n, 0);+ if (total > maxBytes) {+ throw new Error(+ `the app is too large to publish: ${total} bytes, maximum ${maxBytes} bytes — publish fewer or smaller files`,+ );+ }+}++const fileSizes = (files: readonly DeployFile[] | undefined): number[] =>+ (files ?? []).map((f) => (typeof f.data === "string" ? Buffer.byteLength(f.data) : f.data.byteLength));+function validateName(name: string): void {if (UUID_RE.test(name)) throw new Error(`invalid deployment name (looks like an id): ${name}`);if (!NAME_RE.test(name)) {@@ -325,6 +338,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {validateName(input.name);if (await deps.deployStore.getByName(input.name)) throw new Error(`deployment name taken: ${input.name}`);}+ assertAppSize(fileSizes(input.files), deps.provider.profile.maxAppBytes);const snapshotDir = await snapshotFiles(deps.deployDir, input.files);const homeDir = input.homeFiles?.length ? await snapshotFiles(deps.deployDir, input.homeFiles) : undefined;const env = input.stampEnv ? { ...input.env, ...input.stampEnv } : input.env;@@ -369,6 +383,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {const before = await deps.deployStore.get(id);if (!before) throw new Error(`unknown deployment: ${id}`);const current = currentVersionOf(before);+ assertAppSize(fileSizes(input.files), deps.provider.profile.maxAppBytes);const snapshotDir = await snapshotFiles(deps.deployDir, input.files);let homeDir = input.homeFiles === undefined ? current?.homeDir : undefined;if (input.homeFiles?.length) homeDir = await snapshotFiles(deps.deployDir, input.homeFiles);@@ -612,6 +627,11 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {ownerScopeId = before.ownerScopeId;const pushed = await deps.deployStore.refOf(id, deployCurrentGitRef);if (!pushed) return result;+ const tree = await deps.deployStore.commitTreeOf(id, pushed);+ assertAppSize(+ tree.map((f) => f.size),+ deps.provider.profile.maxAppBytes,+ );const v = await deps.deployStore.addVersionFromCommit(id, pushed);if (!v) return result;const endpoint = await applyVersion(id, v, before.appliedVersion ?? before.currentVersion);diff --git a/src/deploy/deploy-store.ts b/src/deploy/deploy-store.tsindex 19ba897e0..c631cd3bb 100644--- a/src/deploy/deploy-store.ts+++ b/src/deploy/deploy-store.ts@@ -118,6 +118,7 @@ export interface DeployStore {touch(id: string, at: number): Promise<void>;versionOf(id: string, version: number): Promise<DeploymentVersion | null>;treeOf(id: string, version: number): Promise<DeployGitTreeFile[] | null>;+ commitTreeOf(id: string, commit: string): Promise<DeployGitTreeFile[]>;filesOf(id: string, version: number, paths?: string[]): Promise<DeployGitInputFile[] | null>;diffVersions(id: string, fromVersion: number | undefined, toVersion: number): Promise<DeployGitDiff | null>;bundleOf(id: string, version: number): Promise<Uint8Array | null>;@@ -470,6 +471,7 @@ export function createDeployStore(backing?: DurableMap<Deployment> | DeployStoreconst v = (await backingMap.get(id))?.versions.find((x) => x.version === version);return v?.commit ? git.treeOf(id, v.commit) : null;},+ commitTreeOf: (id, commit) => git.treeOf(id, commit),async filesOf(id, version, paths) {const v = (await backingMap.get(id))?.versions.find((x) => x.version === version);return v?.commit ? git.filesOf(id, v.commit, paths) : null;diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.tsindex b1f67fd7c..201bf4128 100644--- a/src/deploy/fly-deploy-provider.ts+++ b/src/deploy/fly-deploy-provider.ts@@ -410,12 +410,6 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPasync function bundleBase64(version: DeploymentVersion): Promise<string> {const tree = await readTree(version.snapshotDir, { tolerateMissing: true });- const sourceBytes = tree.reduce((total, file) => total + file.data.byteLength, 0);- if (sourceBytes > MAX_BUNDLE_SOURCE_BYTES) {- throw new Error(- `the app source is too large for the Fly deploy provider: ${sourceBytes} bytes, maximum ${MAX_BUNDLE_SOURCE_BYTES} bytes`,- );- }const encoded = gzipSync(await makeTar(tree)).toString("base64");if (encoded.length > MAX_BUNDLE_BASE64_BYTES) {throw new Error(@@ -509,7 +503,11 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP}return {- profile: { managedScaleToZero: !!opts.sharedAppName, ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}) },+ profile: {+ managedScaleToZero: !!opts.sharedAppName,+ maxAppBytes: MAX_BUNDLE_SOURCE_BYTES,+ ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}),+ },async resolveEndpoint(d, version) {if (!checked.has(d.id)) {diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.tsindex 8b6f83d42..54fa964f7 100644--- a/src/sandbox/aws-microvm-api.ts+++ b/src/sandbox/aws-microvm-api.ts@@ -2,7 +2,9 @@ import { SignatureV4 } from "@smithy/signature-v4";import { Sha256 } from "@aws-crypto/sha256-js";import { defaultProvider } from "@aws-sdk/credential-provider-node";import { fetchWithRetry, sleep } from "../util/async.ts";-import { errMessage, swallow, withRequestId } from "../util/errors.ts";+import { errMessage, execFailure, swallow, withRequestId } from "../util/errors.ts";+import { randomBytes } from "node:crypto";+import { shq } from "../util/shell.ts";type CredentialProvider = () => Promise<{accessKeyId: string;@@ -273,6 +275,7 @@ export async function vmFetch(export interface MicrovmClientOptions {agentPort: number;tokenTtlMinutes: number;+ writeChunkBytes?: number;fetchImpl?: typeof fetch;}@@ -301,6 +304,7 @@ export interface MicrovmClient {export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptions): MicrovmClient {const { agentPort, tokenTtlMinutes } = opts;+ const writeChunkBytes = opts.writeChunkBytes ?? 32 * 1024 * 1024;const fetchOpt = opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {};const tokenById = new Map<string, { token: string; expMs: number }>();@@ -342,12 +346,36 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptioreturn { stdout: j.stdout ?? "", stderr: j.stderr ?? "", code: j.code, timedOut: !!j.timedOut };}- async function writeAbs(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {+ async function writeOnce(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {const res = await daemon(id, endpoint, "/write", { path: absPath, b64: Buffer.from(data).toString("base64") });if (res.status !== 200)throw new Error(`microVM write ${absPath} failed (${res.status}): ${res.text.slice(0, 200)}`);}+ async function writeAbs(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {+ if (data.byteLength <= writeChunkBytes) return writeOnce(id, endpoint, absPath, data);+ const stem = `${absPath}.qm-${randomBytes(6).toString("hex")}`;+ const parts: string[] = [];+ try {+ for (let offset = 0; offset < data.byteLength; offset += writeChunkBytes) {+ parts.push(`${stem}.part-${parts.length}`);+ await writeOnce(id, endpoint, parts[parts.length - 1]!, data.subarray(offset, offset + writeChunkBytes));+ }+ const joined = `${stem}.joined`;+ const r = await execRaw(+ id,+ endpoint,+ `cat ${parts.map(shq).join(" ")} > ${shq(joined)} && [ "$(wc -c < ${shq(joined)})" -eq ${data.byteLength} ] && { chmod --reference=${shq(absPath)} ${shq(joined)} 2>/dev/null || true; } && mv -fT ${shq(joined)} ${shq(absPath)}`,+ 300,+ );+ if (r.code !== 0) throw execFailure(`microVM write ${absPath} (joining ${parts.length} parts)`, r);+ } finally {+ await execRaw(id, endpoint, `rm -f ${shq(stem)}.part-* ${shq(stem)}.joined`, 60).catch((e) =>+ swallow("microvm: remove write parts", e),+ );+ }+ }+async function waitDaemon(id: string, endpoint: string): Promise<void> {const started = Date.now();const deadline = started + 60_000;diff --git a/test/aws-microvm-api.test.ts b/test/aws-microvm-api.test.tsindex 30e980733..f07bfeb24 100644--- a/test/aws-microvm-api.test.ts+++ b/test/aws-microvm-api.test.ts@@ -228,3 +228,48 @@ test("runMicrovm retries a throttled create but never an ambiguous server error"await assert.rejects(api2.runMicrovm(run), /-> 502: bad gateway/);assert.equal(broken.calls.length, 1);});++test("writeAbs splits writes above the chunk size into parts, joins them in the MicroVM, and removes the parts", async () => {+ const writes: Array<{ path: string; bytes: number }> = [];+ const execs: string[] = [];+ let failWrite: RegExp | null = null;+ const { fetchImpl } = fakeFetch((rec) => {+ const body = JSON.parse(rec.body ?? "{}") as { path?: string; b64?: string; cmd?: string };+ if (rec.url.endsWith("/write")) {+ if (failWrite?.test(body.path!)) return { status: 500, body: { error: "disk full" } };+ writes.push({ path: body.path!, bytes: Buffer.from(body.b64!, "base64").length });+ }+ if (rec.url.endsWith("/exec")) {+ execs.push(body.cmd!);+ return { status: 200, body: { stdout: "", stderr: "", code: 0 } };+ }+ return { status: 200, body: { ok: true } };+ });+ const api = { createAuthToken: async () => "tok" } as unknown as AwsMicrovmApi;+ const client = createMicrovmClient(api, { agentPort: 8080, tokenTtlMinutes: 30, writeChunkBytes: 4, fetchImpl });+ await client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/small", new Uint8Array(4));+ assert.deepEqual(writes, [{ path: "/app/small", bytes: 4 }]);+ assert.equal(execs.length, 0);+ writes.length = 0;+ await client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/big", new Uint8Array(10));+ const stem = writes[0]!.path.replace(/\.part-0$/, "");+ assert.match(stem, /^\/app\/big\.qm-[0-9a-f]{12}$/);+ assert.deepEqual(writes, [+ { path: `${stem}.part-0`, bytes: 4 },+ { path: `${stem}.part-1`, bytes: 4 },+ { path: `${stem}.part-2`, bytes: 2 },+ ]);+ assert.deepEqual(execs, [+ `cat '${stem}.part-0' '${stem}.part-1' '${stem}.part-2' > '${stem}.joined' && [ "$(wc -c < '${stem}.joined')" -eq 10 ] && { chmod --reference='/app/big' '${stem}.joined' 2>/dev/null || true; } && mv -fT '${stem}.joined' '/app/big'`,+ `rm -f '${stem}'.part-* '${stem}'.joined`,+ ]);+ execs.length = 0;+ writes.length = 0;+ failWrite = /^\/app\/other\.qm-[0-9a-f]{12}\.part-1$/;+ await assert.rejects(+ client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/other", new Uint8Array(10)),+ /part-1 failed \(500\)/,+ );+ const other = writes[0]!.path.replace(/\.part-0$/, "");+ assert.deepEqual(execs, [`rm -f '${other}'.part-* '${other}'.joined`]);+});diff --git a/test/deploy-app-size.test.ts b/test/deploy-app-size.test.tsnew file mode 100644index 000000000..fc4b81c56--- /dev/null+++ b/test/deploy-app-size.test.ts@@ -0,0 +1,70 @@+import test from "node:test";+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createDeployStore } from "../src/deploy/deploy-store.ts";+import { createDeployService } from "../src/deploy/deploy-service.ts";+import { createAclStore } from "../src/acl/acl-store.ts";+import { scopeId } from "../src/types.ts";++function service(maxAppBytes?: number) {+ const dir = mkdtempSync(join(tmpdir(), "deploy-app-size-"));+ let applies = 0;+ const store = createDeployStore({ git: { repoRoot: join(dir, "repos") } });+ const deploy = createDeployService({+ deployStore: store,+ provider: {+ profile: { managedScaleToZero: false, ...(maxAppBytes ? { maxAppBytes } : {}) },+ apply: async () => {+ applies++;+ return { host: "127.0.0.1", port: 1 };+ },+ destroy: async () => {},+ },+ auditLog: { record() {}, events: async () => [], tail: async () => [] },+ acl: createAclStore(),+ deployDir: join(dir, "deploy"),+ });+ return { deploy, store, applies: () => applies };+}++const owner = { ownerScopeId: scopeId("personal", "U1"), createdBy: "U1", entrypoint: "node server.js" };++test("a provider's app size cap rejects oversized publishes and redeploys before anything is applied", async () => {+ const { deploy, applies } = service(10);+ await assert.rejects(+ deploy.deploy({+ ...owner,+ files: [+ { path: "a.bin", data: new Uint8Array(6) },+ { path: "b.txt", data: "hello" },+ ],+ }),+ /^Error: the app is too large to publish: 11 bytes, maximum 10 bytes/,+ );+ assert.equal(applies(), 0);+ const d = await deploy.deploy({ ...owner, files: [{ path: "a.txt", data: "0123456789" }] });+ assert.equal(applies(), 1);+ await assert.rejects(+ deploy.redeploy(d.id, { entrypoint: "node server.js", files: [{ path: "a.txt", data: "0123456789!" }] }),+ /too large to publish: 11 bytes/,+ );+ assert.equal(applies(), 1);+});++test("the cap also blocks an oversized version pushed to the deployment's git repo", async () => {+ const { deploy, store, applies } = service(10);+ const d = await deploy.deploy({ ...owner, files: [{ path: "a.txt", data: "0123456789" }] });+ const work = mkdtempSync(join(tmpdir(), "deploy-app-size-push-"));+ execFileSync("git", ["clone", "--quiet", (await deploy.gitRepoPath(d.id))!, work]);+ writeFileSync(join(work, "a.txt"), "0123456789!");+ execFileSync("git", ["-c", "user.name=T", "-c", "user.email=t@t", "commit", "-aqm", "big"], { cwd: work });+ execFileSync("git", ["push", "--quiet", (await deploy.gitRepoPath(d.id))!, "HEAD:current"], { cwd: work });+ await deploy.pushGit(d.id, async () => ({ result: true, ok: true }));+ assert.equal(applies(), 1);+ const after = (await store.get(d.id))!;+ assert.equal(after.currentVersion, 1);+ assert.equal(after.versions.length, 1);+});diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.tsindex 5a516e88f..b1484cdec 100644--- a/test/fly-deploy-provider.test.ts+++ b/test/fly-deploy-provider.test.ts@@ -387,16 +387,6 @@ test("apply: an app bundle over the machine-file cap is refused with its actualassert.deepEqual(calls, [], "nothing is created on Fly for a bundle that could never be injected");});-test("apply: highly compressible source cannot bypass the unpacked-size cap", async () => {- const { fetchImpl, calls } = fakeFly();- const big = snapshot({ "zeros.bin": Buffer.alloc(20_000_001) });- await assert.rejects(- provider(fetchImpl).apply(deployment(ID), version(big)),- /app source is too large.*20000001 bytes, maximum 20000000 bytes/,- );- assert.deepEqual(calls, []);-});-test("apply: a machine that never starts reports its last state", async () => {const { fetchImpl, machines } = fakeFly({ existingMachines: ["machine-old"], states: ["created"] });await assert.rejects(@@ -487,8 +477,11 @@ test("destroy: refuses a mismatched app and surfaces a failed deletion", async (test("profile: shared Flycast manages suspension while existing standalone apps retain idle cleanup", () => {const { fetchImpl } = fakeFly();- assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false });- assert.deepEqual(provider(fetchImpl, { sharedAppName: "company-app" }).profile, { managedScaleToZero: true });+ assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false, maxAppBytes: 20_000_000 });+ assert.deepEqual(provider(fetchImpl, { sharedAppName: "company-app" }).profile, {+ managedScaleToZero: true,+ maxAppBytes: 20_000_000,+ });});test("missing fly configuration fails at the point of use with the env var that is missing", async () => {