Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Deploys: per-provider app size cap (AWS 100 MiB) and chunked MicroVM writes (#2423)

Regan Bell7a6d05bdb62b
diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.ts
index 01b87d0a3..3d47d72d8 100644
--- a/src/deploy/aws-deploy-provider.ts
+++ b/src/deploy/aws-deploy-provider.ts
@@ -39,6 +39,7 @@ const DATA_CREDS_REFRESH_MS = 45 * 60_000;
const DATA_CREDS_RETRY_MS = 60_000;
const failureBackoffStamp = (): number => Date.now() - DATA_CREDS_REFRESH_MS + DATA_CREDS_RETRY_MS;
const MAX_DATA_TAR_BYTES = 180 * 1024 * 1024;
+const MAX_APP_BYTES = 100 * 1024 * 1024;
const READY_PATH = `${APP_DIR}/.qm-ready`;
const APP_READY_WINDOW_SEC = 60;
const APP_PORT_RELEASE_TICKS = 40;
@@ -637,7 +638,12 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
};
return {
- profile: { managedScaleToZero: true, inPlaceReconcile: true, ...(s3 ? { dataDir: DATA_DIR } : {}) },
+ profile: {
+ managedScaleToZero: true,
+ inPlaceReconcile: true,
+ maxAppBytes: MAX_APP_BYTES,
+ ...(s3 ? { dataDir: DATA_DIR } : {}),
+ },
apply: (d, version) => place(d, version),
diff --git a/src/deploy/deploy-provider.ts b/src/deploy/deploy-provider.ts
index 5a19bf1cc..129b1a1c9 100644
--- a/src/deploy/deploy-provider.ts
+++ b/src/deploy/deploy-provider.ts
@@ -6,6 +6,7 @@ export interface DeployProfile {
managedScaleToZero: boolean;
inPlaceReconcile?: boolean;
dataDir?: string;
+ maxAppBytes?: number;
}
export interface DeployReconcileInput {
diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.ts
index 9a91f6e88..86108374c 100644
--- a/src/deploy/deploy-service.ts
+++ b/src/deploy/deploy-service.ts
@@ -149,6 +149,19 @@ export interface DeployServiceDeps {
const NAME_RE = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/;
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+function assertAppSize(sizes: readonly number[], maxBytes: number | undefined): void {
+ if (!maxBytes) return;
+ const total = sizes.reduce((sum, n) => sum + n, 0);
+ if (total > maxBytes) {
+ throw new Error(
+ `the app is too large to publish: ${total} bytes, maximum ${maxBytes} bytes — publish fewer or smaller files`,
+ );
+ }
+}
+
+const fileSizes = (files: readonly DeployFile[] | undefined): number[] =>
+ (files ?? []).map((f) => (typeof f.data === "string" ? Buffer.byteLength(f.data) : f.data.byteLength));
+
function validateName(name: string): void {
if (UUID_RE.test(name)) throw new Error(`invalid deployment name (looks like an id): ${name}`);
if (!NAME_RE.test(name)) {
@@ -325,6 +338,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
validateName(input.name);
if (await deps.deployStore.getByName(input.name)) throw new Error(`deployment name taken: ${input.name}`);
}
+ assertAppSize(fileSizes(input.files), deps.provider.profile.maxAppBytes);
const snapshotDir = await snapshotFiles(deps.deployDir, input.files);
const homeDir = input.homeFiles?.length ? await snapshotFiles(deps.deployDir, input.homeFiles) : undefined;
const env = input.stampEnv ? { ...input.env, ...input.stampEnv } : input.env;
@@ -369,6 +383,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
const before = await deps.deployStore.get(id);
if (!before) throw new Error(`unknown deployment: ${id}`);
const current = currentVersionOf(before);
+ assertAppSize(fileSizes(input.files), deps.provider.profile.maxAppBytes);
const snapshotDir = await snapshotFiles(deps.deployDir, input.files);
let homeDir = input.homeFiles === undefined ? current?.homeDir : undefined;
if (input.homeFiles?.length) homeDir = await snapshotFiles(deps.deployDir, input.homeFiles);
@@ -612,6 +627,11 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
ownerScopeId = before.ownerScopeId;
const pushed = await deps.deployStore.refOf(id, deployCurrentGitRef);
if (!pushed) return result;
+ const tree = await deps.deployStore.commitTreeOf(id, pushed);
+ assertAppSize(
+ tree.map((f) => f.size),
+ deps.provider.profile.maxAppBytes,
+ );
const v = await deps.deployStore.addVersionFromCommit(id, pushed);
if (!v) return result;
const endpoint = await applyVersion(id, v, before.appliedVersion ?? before.currentVersion);
diff --git a/src/deploy/deploy-store.ts b/src/deploy/deploy-store.ts
index 19ba897e0..c631cd3bb 100644
--- a/src/deploy/deploy-store.ts
+++ b/src/deploy/deploy-store.ts
@@ -118,6 +118,7 @@ export interface DeployStore {
touch(id: string, at: number): Promise<void>;
versionOf(id: string, version: number): Promise<DeploymentVersion | null>;
treeOf(id: string, version: number): Promise<DeployGitTreeFile[] | null>;
+ commitTreeOf(id: string, commit: string): Promise<DeployGitTreeFile[]>;
filesOf(id: string, version: number, paths?: string[]): Promise<DeployGitInputFile[] | null>;
diffVersions(id: string, fromVersion: number | undefined, toVersion: number): Promise<DeployGitDiff | null>;
bundleOf(id: string, version: number): Promise<Uint8Array | null>;
@@ -470,6 +471,7 @@ export function createDeployStore(backing?: DurableMap<Deployment> | DeployStore
const v = (await backingMap.get(id))?.versions.find((x) => x.version === version);
return v?.commit ? git.treeOf(id, v.commit) : null;
},
+ commitTreeOf: (id, commit) => git.treeOf(id, commit),
async filesOf(id, version, paths) {
const v = (await backingMap.get(id))?.versions.find((x) => x.version === version);
return v?.commit ? git.filesOf(id, v.commit, paths) : null;
diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.ts
index b1f67fd7c..201bf4128 100644
--- a/src/deploy/fly-deploy-provider.ts
+++ b/src/deploy/fly-deploy-provider.ts
@@ -410,12 +410,6 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
async function bundleBase64(version: DeploymentVersion): Promise<string> {
const tree = await readTree(version.snapshotDir, { tolerateMissing: true });
- const sourceBytes = tree.reduce((total, file) => total + file.data.byteLength, 0);
- if (sourceBytes > MAX_BUNDLE_SOURCE_BYTES) {
- throw new Error(
- `the app source is too large for the Fly deploy provider: ${sourceBytes} bytes, maximum ${MAX_BUNDLE_SOURCE_BYTES} bytes`,
- );
- }
const encoded = gzipSync(await makeTar(tree)).toString("base64");
if (encoded.length > MAX_BUNDLE_BASE64_BYTES) {
throw new Error(
@@ -509,7 +503,11 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP
}
return {
- profile: { managedScaleToZero: !!opts.sharedAppName, ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}) },
+ profile: {
+ managedScaleToZero: !!opts.sharedAppName,
+ maxAppBytes: MAX_BUNDLE_SOURCE_BYTES,
+ ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}),
+ },
async resolveEndpoint(d, version) {
if (!checked.has(d.id)) {
diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.ts
index 8b6f83d42..54fa964f7 100644
--- a/src/sandbox/aws-microvm-api.ts
+++ b/src/sandbox/aws-microvm-api.ts
@@ -2,7 +2,9 @@ import { SignatureV4 } from "@smithy/signature-v4";
import { Sha256 } from "@aws-crypto/sha256-js";
import { defaultProvider } from "@aws-sdk/credential-provider-node";
import { fetchWithRetry, sleep } from "../util/async.ts";
-import { errMessage, swallow, withRequestId } from "../util/errors.ts";
+import { errMessage, execFailure, swallow, withRequestId } from "../util/errors.ts";
+import { randomBytes } from "node:crypto";
+import { shq } from "../util/shell.ts";
type CredentialProvider = () => Promise<{
accessKeyId: string;
@@ -273,6 +275,7 @@ export async function vmFetch(
export interface MicrovmClientOptions {
agentPort: number;
tokenTtlMinutes: number;
+ writeChunkBytes?: number;
fetchImpl?: typeof fetch;
}
@@ -301,6 +304,7 @@ export interface MicrovmClient {
export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptions): MicrovmClient {
const { agentPort, tokenTtlMinutes } = opts;
+ const writeChunkBytes = opts.writeChunkBytes ?? 32 * 1024 * 1024;
const fetchOpt = opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {};
const tokenById = new Map<string, { token: string; expMs: number }>();
@@ -342,12 +346,36 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptio
return { stdout: j.stdout ?? "", stderr: j.stderr ?? "", code: j.code, timedOut: !!j.timedOut };
}
- async function writeAbs(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {
+ async function writeOnce(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {
const res = await daemon(id, endpoint, "/write", { path: absPath, b64: Buffer.from(data).toString("base64") });
if (res.status !== 200)
throw new Error(`microVM write ${absPath} failed (${res.status}): ${res.text.slice(0, 200)}`);
}
+ async function writeAbs(id: string, endpoint: string, absPath: string, data: Uint8Array): Promise<void> {
+ if (data.byteLength <= writeChunkBytes) return writeOnce(id, endpoint, absPath, data);
+ const stem = `${absPath}.qm-${randomBytes(6).toString("hex")}`;
+ const parts: string[] = [];
+ try {
+ for (let offset = 0; offset < data.byteLength; offset += writeChunkBytes) {
+ parts.push(`${stem}.part-${parts.length}`);
+ await writeOnce(id, endpoint, parts[parts.length - 1]!, data.subarray(offset, offset + writeChunkBytes));
+ }
+ const joined = `${stem}.joined`;
+ const r = await execRaw(
+ id,
+ endpoint,
+ `cat ${parts.map(shq).join(" ")} > ${shq(joined)} && [ "$(wc -c < ${shq(joined)})" -eq ${data.byteLength} ] && { chmod --reference=${shq(absPath)} ${shq(joined)} 2>/dev/null || true; } && mv -fT ${shq(joined)} ${shq(absPath)}`,
+ 300,
+ );
+ if (r.code !== 0) throw execFailure(`microVM write ${absPath} (joining ${parts.length} parts)`, r);
+ } finally {
+ await execRaw(id, endpoint, `rm -f ${shq(stem)}.part-* ${shq(stem)}.joined`, 60).catch((e) =>
+ swallow("microvm: remove write parts", e),
+ );
+ }
+ }
+
async function waitDaemon(id: string, endpoint: string): Promise<void> {
const started = Date.now();
const deadline = started + 60_000;
diff --git a/test/aws-microvm-api.test.ts b/test/aws-microvm-api.test.ts
index 30e980733..f07bfeb24 100644
--- a/test/aws-microvm-api.test.ts
+++ b/test/aws-microvm-api.test.ts
@@ -228,3 +228,48 @@ test("runMicrovm retries a throttled create but never an ambiguous server error"
await assert.rejects(api2.runMicrovm(run), /-> 502: bad gateway/);
assert.equal(broken.calls.length, 1);
});
+
+test("writeAbs splits writes above the chunk size into parts, joins them in the MicroVM, and removes the parts", async () => {
+ const writes: Array<{ path: string; bytes: number }> = [];
+ const execs: string[] = [];
+ let failWrite: RegExp | null = null;
+ const { fetchImpl } = fakeFetch((rec) => {
+ const body = JSON.parse(rec.body ?? "{}") as { path?: string; b64?: string; cmd?: string };
+ if (rec.url.endsWith("/write")) {
+ if (failWrite?.test(body.path!)) return { status: 500, body: { error: "disk full" } };
+ writes.push({ path: body.path!, bytes: Buffer.from(body.b64!, "base64").length });
+ }
+ if (rec.url.endsWith("/exec")) {
+ execs.push(body.cmd!);
+ return { status: 200, body: { stdout: "", stderr: "", code: 0 } };
+ }
+ return { status: 200, body: { ok: true } };
+ });
+ const api = { createAuthToken: async () => "tok" } as unknown as AwsMicrovmApi;
+ const client = createMicrovmClient(api, { agentPort: 8080, tokenTtlMinutes: 30, writeChunkBytes: 4, fetchImpl });
+ await client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/small", new Uint8Array(4));
+ assert.deepEqual(writes, [{ path: "/app/small", bytes: 4 }]);
+ assert.equal(execs.length, 0);
+ writes.length = 0;
+ await client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/big", new Uint8Array(10));
+ const stem = writes[0]!.path.replace(/\.part-0$/, "");
+ assert.match(stem, /^\/app\/big\.qm-[0-9a-f]{12}$/);
+ assert.deepEqual(writes, [
+ { path: `${stem}.part-0`, bytes: 4 },
+ { path: `${stem}.part-1`, bytes: 4 },
+ { path: `${stem}.part-2`, bytes: 2 },
+ ]);
+ assert.deepEqual(execs, [
+ `cat '${stem}.part-0' '${stem}.part-1' '${stem}.part-2' > '${stem}.joined' && [ "$(wc -c < '${stem}.joined')" -eq 10 ] && { chmod --reference='/app/big' '${stem}.joined' 2>/dev/null || true; } && mv -fT '${stem}.joined' '/app/big'`,
+ `rm -f '${stem}'.part-* '${stem}'.joined`,
+ ]);
+ execs.length = 0;
+ writes.length = 0;
+ failWrite = /^\/app\/other\.qm-[0-9a-f]{12}\.part-1$/;
+ await assert.rejects(
+ client.writeAbs("mvm-1", "mvm-1.example.on.aws", "/app/other", new Uint8Array(10)),
+ /part-1 failed \(500\)/,
+ );
+ const other = writes[0]!.path.replace(/\.part-0$/, "");
+ assert.deepEqual(execs, [`rm -f '${other}'.part-* '${other}'.joined`]);
+});
diff --git a/test/deploy-app-size.test.ts b/test/deploy-app-size.test.ts
new file mode 100644
index 000000000..fc4b81c56
--- /dev/null
+++ b/test/deploy-app-size.test.ts
@@ -0,0 +1,70 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createDeployStore } from "../src/deploy/deploy-store.ts";
+import { createDeployService } from "../src/deploy/deploy-service.ts";
+import { createAclStore } from "../src/acl/acl-store.ts";
+import { scopeId } from "../src/types.ts";
+
+function service(maxAppBytes?: number) {
+ const dir = mkdtempSync(join(tmpdir(), "deploy-app-size-"));
+ let applies = 0;
+ const store = createDeployStore({ git: { repoRoot: join(dir, "repos") } });
+ const deploy = createDeployService({
+ deployStore: store,
+ provider: {
+ profile: { managedScaleToZero: false, ...(maxAppBytes ? { maxAppBytes } : {}) },
+ apply: async () => {
+ applies++;
+ return { host: "127.0.0.1", port: 1 };
+ },
+ destroy: async () => {},
+ },
+ auditLog: { record() {}, events: async () => [], tail: async () => [] },
+ acl: createAclStore(),
+ deployDir: join(dir, "deploy"),
+ });
+ return { deploy, store, applies: () => applies };
+}
+
+const owner = { ownerScopeId: scopeId("personal", "U1"), createdBy: "U1", entrypoint: "node server.js" };
+
+test("a provider's app size cap rejects oversized publishes and redeploys before anything is applied", async () => {
+ const { deploy, applies } = service(10);
+ await assert.rejects(
+ deploy.deploy({
+ ...owner,
+ files: [
+ { path: "a.bin", data: new Uint8Array(6) },
+ { path: "b.txt", data: "hello" },
+ ],
+ }),
+ /^Error: the app is too large to publish: 11 bytes, maximum 10 bytes/,
+ );
+ assert.equal(applies(), 0);
+ const d = await deploy.deploy({ ...owner, files: [{ path: "a.txt", data: "0123456789" }] });
+ assert.equal(applies(), 1);
+ await assert.rejects(
+ deploy.redeploy(d.id, { entrypoint: "node server.js", files: [{ path: "a.txt", data: "0123456789!" }] }),
+ /too large to publish: 11 bytes/,
+ );
+ assert.equal(applies(), 1);
+});
+
+test("the cap also blocks an oversized version pushed to the deployment's git repo", async () => {
+ const { deploy, store, applies } = service(10);
+ const d = await deploy.deploy({ ...owner, files: [{ path: "a.txt", data: "0123456789" }] });
+ const work = mkdtempSync(join(tmpdir(), "deploy-app-size-push-"));
+ execFileSync("git", ["clone", "--quiet", (await deploy.gitRepoPath(d.id))!, work]);
+ writeFileSync(join(work, "a.txt"), "0123456789!");
+ execFileSync("git", ["-c", "user.name=T", "-c", "user.email=t@t", "commit", "-aqm", "big"], { cwd: work });
+ execFileSync("git", ["push", "--quiet", (await deploy.gitRepoPath(d.id))!, "HEAD:current"], { cwd: work });
+ await deploy.pushGit(d.id, async () => ({ result: true, ok: true }));
+ assert.equal(applies(), 1);
+ const after = (await store.get(d.id))!;
+ assert.equal(after.currentVersion, 1);
+ assert.equal(after.versions.length, 1);
+});
diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.ts
index 5a516e88f..b1484cdec 100644
--- a/test/fly-deploy-provider.test.ts
+++ b/test/fly-deploy-provider.test.ts
@@ -387,16 +387,6 @@ test("apply: an app bundle over the machine-file cap is refused with its actual
assert.deepEqual(calls, [], "nothing is created on Fly for a bundle that could never be injected");
});
-test("apply: highly compressible source cannot bypass the unpacked-size cap", async () => {
- const { fetchImpl, calls } = fakeFly();
- const big = snapshot({ "zeros.bin": Buffer.alloc(20_000_001) });
- await assert.rejects(
- provider(fetchImpl).apply(deployment(ID), version(big)),
- /app source is too large.*20000001 bytes, maximum 20000000 bytes/,
- );
- assert.deepEqual(calls, []);
-});
-
test("apply: a machine that never starts reports its last state", async () => {
const { fetchImpl, machines } = fakeFly({ existingMachines: ["machine-old"], states: ["created"] });
await assert.rejects(
@@ -487,8 +477,11 @@ test("destroy: refuses a mismatched app and surfaces a failed deletion", async (
test("profile: shared Flycast manages suspension while existing standalone apps retain idle cleanup", () => {
const { fetchImpl } = fakeFly();
- assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false });
- assert.deepEqual(provider(fetchImpl, { sharedAppName: "company-app" }).profile, { managedScaleToZero: true });
+ assert.deepEqual(provider(fetchImpl).profile, { managedScaleToZero: false, maxAppBytes: 20_000_000 });
+ assert.deepEqual(provider(fetchImpl, { sharedAppName: "company-app" }).profile, {
+ managedScaleToZero: true,
+ maxAppBytes: 20_000_000,
+ });
});
test("missing fly configuration fails at the point of use with the env var that is missing", async () => {