Multiplayer agent harness for work.
Expire idle pending uploads so abandoned sessions free the upload quota (#2468)
Joshua Franceeba9ea9f4079
diff --git a/src/files/direct-file-upload.ts b/src/files/direct-file-upload.tsindex 638bf8b67..8c092068a 100644--- a/src/files/direct-file-upload.ts+++ b/src/files/direct-file-upload.ts@@ -25,6 +25,7 @@ import { createSweeper } from "../util/sweeper.ts";export const FILE_UPLOAD_PART_SIZE = 64 * 1024 * 1024;const MAX_DIRECT_FILE_BYTES = 100 * 1024 ** 3;const UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;+const UPLOAD_IDLE_MS = 60 * 60 * 1000;const URL_TTL_SECONDS = 15 * 60;const SHA256_BASE64 = /^[A-Za-z0-9+/]{43}=$/;@@ -157,7 +158,7 @@ export function createDirectFileUploads(options: {uploadId: "",state: "pending",createdAt: now(),- expiresAt: now() + UPLOAD_TTL_MS,+ expiresAt: now() + UPLOAD_IDLE_MS,};const matches = (existing: FileUpload) =>existing.actorId === row.actorId &&@@ -201,7 +202,9 @@ export function createDirectFileUploads(options: {if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > row.checksums.length)throw new FileUploadError("invalid part number");const length = Math.min(row.partSize, row.sizeBytes - (partNumber - 1) * row.partSize);- const expiresIn = Math.min(URL_TTL_SECONDS, Math.floor((row.expiresAt - now()) / 1000));+ const expiresAt = Math.min(row.createdAt + UPLOAD_TTL_MS, now() + UPLOAD_IDLE_MS);+ if (!(await store.extend(id, expiresAt))) throw new FileUploadError("upload is not pending or has expired", 409);+ const expiresIn = Math.min(URL_TTL_SECONDS, Math.floor((expiresAt - now()) / 1000));if (expiresIn < 1) throw new FileUploadError("upload expired", 409);const checksum = row.checksums[partNumber - 1]!;const url = await presign(diff --git a/src/files/file-upload-store.ts b/src/files/file-upload-store.tsindex 18530b0e9..58d260995 100644--- a/src/files/file-upload-store.ts+++ b/src/files/file-upload-store.ts@@ -22,6 +22,7 @@ export interface FileUploadStore {insert(upload: FileUpload): Promise<void>;get(id: string): Promise<FileUpload | null>;transition(id: string, from: FileUploadState[], to: FileUploadState): Promise<boolean>;+ extend(id: string, expiresAt: number): Promise<boolean>;expired(now: number): Promise<FileUpload[]>;}@@ -73,14 +74,18 @@ export function createPostgresFileUploadStore(connectionString: string): FileUplawait withPgTransaction(await db.pool(), async (client) => {await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`file-uploads:${upload.actorId}`]);const result = await client.query(- "SELECT count(*) AS count, coalesce(sum(size_bytes),0) AS bytes FROM file_uploads WHERE actor_id=$1 AND state NOT IN ('complete','aborted','failed')",- [upload.actorId],+ "SELECT count(*) AS count, coalesce(sum(size_bytes),0) AS bytes FROM file_uploads WHERE actor_id=$1 AND state NOT IN ('complete','aborted','failed') AND NOT (state='pending' AND expires_at <= $2)",+ [upload.actorId, upload.createdAt],);if (Number(result.rows[0].count) >= MAX_ACTIVE_UPLOADS ||Number(result.rows[0].bytes) + upload.sizeBytes > MAX_ACTIVE_UPLOAD_BYTES)- throw new FileUploadError("active upload quota exceeded", 429, "upload_quota");+ throw new FileUploadError(+ "active upload quota exceeded; finish or DELETE /v1/files/uploads/:id an unfinished upload, or wait an hour for idle ones to expire",+ 429,+ "upload_quota",+ );await client.query("INSERT INTO file_uploads(id,actor_id,state,size_bytes,expires_at,data) VALUES($1,$2,$3,$4,$5,$6)",[upload.id, upload.actorId, upload.state, upload.sizeBytes, upload.expiresAt, JSON.stringify(upload)],@@ -97,6 +102,16 @@ export function createPostgresFileUploadStore(connectionString: string): FileUpl.rowCount === 1);},+ async extend(id, expiresAt) {+ return (+ (+ await db.query(+ "UPDATE file_uploads SET expires_at=$2, data=jsonb_set(data,'{expiresAt}',to_jsonb($2::bigint)) WHERE id=$1 AND state='pending'",+ [id, expiresAt],+ )+ ).rowCount === 1+ );+ },async expired(now) {return (await db.q(diff --git a/src/files/upload-client.py b/src/files/upload-client.pyindex 8c29ad4d5..3f3c663a9 100644--- a/src/files/upload-client.py+++ b/src/files/upload-client.py@@ -24,6 +24,13 @@ def quote(value):return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"'+def error_code(content):+ try:+ return json.loads(content).get("error")+ except (ValueError, AttributeError):+ return None++def request(method, url, data=None, headers=None):config = [f"url = {quote(url)}", f"request = {quote(method)}", "silent", "show-error", "max-time = 180", 'write-out = "\\n%{http_code}"']config += [f"header = {quote(f'{name}: {value}')}" for name, value in (headers or {}).items()]@@ -44,7 +51,7 @@ def request(method, url, data=None, headers=None):status = int(status or b"0")if 200 <= status < 300:return content- if status < 500 and status != 429:+ if status < 500 and (status != 429 or error_code(content) == "upload_quota"):raise RequestError(status, f"request failed ({status}): {content[:4096].decode(errors='replace')}")failure = f"HTTP {status}: {content[:200].decode(errors='replace')}"else:@@ -110,14 +117,21 @@ def main():except (RequestError, RuntimeError) as error:print(f"could not cancel the previous unfinished upload: {error}", file=sys.stderr)state = None+ status = None+ if state:+ try:+ status = api("GET", f"/v1/files/uploads/{state['id']}")["upload"]+ except RequestError as error:+ if error.status != 404:+ raise+ expired = status and status["state"] == "pending" and status["expiresAt"] <= time.time() * 1000+ if status and "file" not in state and (expired or status["state"] in ("aborted", "failed")):+ print(f"starting a new upload: saved upload {state['id']} is {'expired' if expired else status['state']}", file=sys.stderr)+ state = status = Noneif not state:state = {"id": uuid.uuid4().hex, "manifest": manifest, "destination": destination, "next": 1}save(state_path, state)- try:- status = api("GET", f"/v1/files/uploads/{state['id']}")["upload"]- except RequestError as error:- if error.status != 404:- raise+ if status is None:status = api("POST", "/v1/files/uploads", {**manifest, "requestId": state["id"]})["upload"]if status["state"] == "pending":with args.file.open("rb") as source:diff --git a/test/direct-file-upload.test.ts b/test/direct-file-upload.test.tsindex 80453876f..423b7f398 100644--- a/test/direct-file-upload.test.ts+++ b/test/direct-file-upload.test.ts@@ -25,6 +25,12 @@ function fixture() {row.state = to;return true;},+ async extend(id, expiresAt) {+ const row = rows.get(id);+ if (!row || row.state !== "pending") return false;+ row.expiresAt = expiresAt;+ return true;+ },async expired(now) {return [...rows.values()].filter((r) => r.expiresAt <= now && r.state !== "complete" && r.state !== "aborted" && r.state !== "failed",@@ -110,12 +116,30 @@ function fixture() {loseResponse: () => {loseCompleteResponse = true;},- advance: () => {- time += 25 * 60 * 60 * 1000;+ advance: (ms = 25 * 60 * 60 * 1000) => {+ time += ms;},};}+test("an idle pending upload expires after an hour; signing parts keeps it alive up to a day", async () => {+ const hour = 60 * 60 * 1000;+ const f = fixture();+ const idle = await f.service.begin(f.input);+ f.advance(hour);+ await assert.rejects(f.service.sign(idle.id, 1), /expired/);+ await f.service.sweep();+ assert.equal((await f.store.get(idle.id))!.state, "aborted");++ const busy = await f.service.begin(f.input);+ for (let i = 0; i < 24; i++) {+ await f.service.sign(busy.id, 1);+ f.advance(hour - 1);+ }+ await assert.rejects(f.service.sign(busy.id, 1), /expired/);+ assert.equal((await f.store.get(busy.id))!.expiresAt, busy.createdAt + 24 * hour);+});+test("signed parts bind the exact size, checksum, upload and number; unpublished parts are invisible", async () => {const f = fixture();const row = await f.service.begin(f.input);diff --git a/test/file-upload-client.test.ts b/test/file-upload-client.test.tsindex acc9ba89c..25e038f95 100644--- a/test/file-upload-client.test.ts+++ b/test/file-upload-client.test.ts@@ -7,6 +7,7 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { fileURLToPath } from "node:url";import { deflateRawSync } from "node:zlib";+import { createHash } from "node:crypto";import { mintSignedPayload } from "../src/auth/signed-token.ts";test("publisher resumes an existing transfer after new uploads are disabled", async () => {@@ -28,7 +29,7 @@ test("publisher resumes an existing transfer after new uploads are disabled", asreturn;}initiated = true;- response.end(JSON.stringify({ upload: { state: "pending" } }));+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));} else if (request.method === "PUT") {puts++;response.end();@@ -42,7 +43,9 @@ test("publisher resumes an existing transfer after new uploads are disabled", asresponse.writeHead(404).end("{}");return;}- response.end(JSON.stringify({ upload: { state: complete ? "complete" : "pending" } }));+ response.end(+ JSON.stringify({ upload: { state: complete ? "complete" : "pending", expiresAt: Date.now() + 60_000 } }),+ );}});await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));@@ -124,7 +127,8 @@ test("publisher cancels an unfinished upload when the file changed", async () =>const url = request.url!;if (request.method === "DELETE") response.end(JSON.stringify({ ok: true }));else if (request.method === "PUT") response.end();- else if (url === "/v1/files/uploads") response.end(JSON.stringify({ upload: { state: "pending" } }));+ else if (url === "/v1/files/uploads")+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));else if (url.endsWith("/parts/1")) response.end(JSON.stringify({ url: origin + "/part", headers: {} }));else if (url.endsWith("/complete")) response.end(JSON.stringify({ file: { id: "published" } }));else response.writeHead(404).end("{}");@@ -167,3 +171,62 @@ test("publisher cancels an unfinished upload when the file changed", async () =>await rm(directory, { recursive: true, force: true });}});++test("publisher starts a fresh upload when the saved session was aborted or expired", async () => {+ for (const saved of [{ state: "aborted" }, { state: "pending", expiresAt: 1 }]) {+ const directory = await mkdtemp(join(tmpdir(), "qm-publisher-"));+ const requests: string[] = [];+ let origin = "";+ const stale = "b".repeat(32);+ const server = createServer(async (request, response) => {+ for await (const _ of request) void _;+ requests.push(`${request.method} ${request.url}`);+ response.setHeader("content-type", "application/json");+ const url = request.url!;+ if (request.method === "DELETE") response.end(JSON.stringify({ ok: true }));+ else if (request.method === "PUT") response.end();+ else if (url === `/v1/files/uploads/${stale}`) response.end(JSON.stringify({ upload: saved }));+ else if (url === "/v1/files/uploads")+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));+ else if (url.endsWith("/parts/1")) response.end(JSON.stringify({ url: origin + "/part", headers: {} }));+ else if (url.endsWith("/complete")) response.end(JSON.stringify({ file: { id: "published" } }));+ else response.writeHead(404).end("{}");+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const address = server.address();+ assert.ok(address && typeof address !== "string");+ origin = `http://127.0.0.1:${address.port}`;+ const source = join(directory, "note.txt");+ const token =+ Buffer.from(JSON.stringify({ actorId: "user", scopeId: "personal:user" })).toString("base64url") + ".signature";+ await writeFile(source, "abc");+ const checksum = createHash("sha256").update("abc").digest("base64");+ await writeFile(+ source + ".qm-upload.json",+ JSON.stringify({+ id: stale,+ manifest: { name: "note.txt", mimetype: "text/plain", sizeBytes: 3, checksums: [checksum] },+ destination: { origin, actor: "user", scope: "personal:user" },+ next: 1,+ }),+ );+ try {+ const code = await new Promise<number | null>((resolve, reject) => {+ const child = spawn(+ "python3",+ [fileURLToPath(new URL("../src/files/upload-client.py", import.meta.url)), source],+ { env: { ...process.env, AGENT_API_URL: origin, AGENT_API_TOKEN: token }, stdio: "ignore" },+ );+ child.on("error", reject);+ child.on("exit", resolve);+ });+ assert.equal(code, 0);+ assert.ok(requests.includes("POST /v1/files/uploads"));+ assert.ok(!requests.includes(`POST /v1/files/uploads/${stale}/complete`));+ assert.ok(!requests.some((r) => r.startsWith("DELETE")));+ } finally {+ await new Promise<void>((resolve) => server.close(() => resolve()));+ await rm(directory, { recursive: true, force: true });+ }+ }+});diff --git a/test/postgres-file-upload-store.test.ts b/test/postgres-file-upload-store.test.tsindex 86a3bac66..3a392f28c 100644--- a/test/postgres-file-upload-store.test.ts+++ b/test/postgres-file-upload-store.test.ts@@ -28,7 +28,7 @@ test(checksums: ["hash"],uploadId: "provider-upload",state: "pending",- expiresAt: 0,+ expiresAt: Date.now() + 60_000,createdAt: Date.now(),});const uploads = Array.from({ length: MAX_ACTIVE_UPLOADS + 3 }, row);@@ -47,7 +47,7 @@ test(second.transition(accepted.id, ["pending"], "aborting"),]);assert.equal(claims.filter(Boolean).length, 1);- assert.ok((await second.expired(Date.now())).some((r) => r.id === accepted.id));+ assert.ok((await second.expired(Date.now() + 120_000)).some((r) => r.id === accepted.id));for (const item of uploads) await first.transition(item.id, ["pending", "completing", "aborting"], "aborted");await second.insert(row());},@@ -112,3 +112,31 @@ test("terminal failed uploads release active quota", { skip: !url }, async () =>}assert.equal((await store.get(ids.at(-1)!))!.state, "failed");});++test("expired pending uploads stop holding quota; extend only moves pending rows", { skip: !url }, async () => {+ const store = createPostgresFileUploadStore(url!);+ const actorId = `idle-quota-${randomUUID()}`;+ const now = Date.now();+ const row = (expiresAt: number): FileUpload => ({+ id: randomUUID().replaceAll("-", ""),+ actorId,+ scopeId: scopeId("personal", actorId),+ name: "file",+ mimetype: "text/plain",+ sizeBytes: 1,+ partSize: 64 * 1024 * 1024,+ checksums: ["hash"],+ uploadId: "upload",+ state: "pending",+ createdAt: now,+ expiresAt,+ });+ const stale = Array.from({ length: MAX_ACTIVE_UPLOADS }, () => row(now - 1));+ for (const r of stale) await store.insert(r);+ const live = row(now + 60_000);+ await store.insert(live);+ assert.ok(await store.extend(live.id, now + 120_000));+ assert.equal((await store.get(live.id))!.expiresAt, now + 120_000);+ await store.transition(stale[0]!.id, ["pending"], "aborted");+ assert.equal(await store.extend(stale[0]!.id, now + 120_000), false);+});