Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Expire idle pending uploads so abandoned sessions free the upload quota (#2468)

Joshua Franceeba9ea9f4079
diff --git a/src/files/direct-file-upload.ts b/src/files/direct-file-upload.ts
index 638bf8b67..8c092068a 100644
--- a/src/files/direct-file-upload.ts
+++ b/src/files/direct-file-upload.ts
@@ -25,6 +25,7 @@ import { createSweeper } from "../util/sweeper.ts";
export const FILE_UPLOAD_PART_SIZE = 64 * 1024 * 1024;
const MAX_DIRECT_FILE_BYTES = 100 * 1024 ** 3;
const UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
+const UPLOAD_IDLE_MS = 60 * 60 * 1000;
const URL_TTL_SECONDS = 15 * 60;
const SHA256_BASE64 = /^[A-Za-z0-9+/]{43}=$/;
@@ -157,7 +158,7 @@ export function createDirectFileUploads(options: {
uploadId: "",
state: "pending",
createdAt: now(),
- expiresAt: now() + UPLOAD_TTL_MS,
+ expiresAt: now() + UPLOAD_IDLE_MS,
};
const matches = (existing: FileUpload) =>
existing.actorId === row.actorId &&
@@ -201,7 +202,9 @@ export function createDirectFileUploads(options: {
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > row.checksums.length)
throw new FileUploadError("invalid part number");
const length = Math.min(row.partSize, row.sizeBytes - (partNumber - 1) * row.partSize);
- const expiresIn = Math.min(URL_TTL_SECONDS, Math.floor((row.expiresAt - now()) / 1000));
+ const expiresAt = Math.min(row.createdAt + UPLOAD_TTL_MS, now() + UPLOAD_IDLE_MS);
+ if (!(await store.extend(id, expiresAt))) throw new FileUploadError("upload is not pending or has expired", 409);
+ const expiresIn = Math.min(URL_TTL_SECONDS, Math.floor((expiresAt - now()) / 1000));
if (expiresIn < 1) throw new FileUploadError("upload expired", 409);
const checksum = row.checksums[partNumber - 1]!;
const url = await presign(
diff --git a/src/files/file-upload-store.ts b/src/files/file-upload-store.ts
index 18530b0e9..58d260995 100644
--- a/src/files/file-upload-store.ts
+++ b/src/files/file-upload-store.ts
@@ -22,6 +22,7 @@ export interface FileUploadStore {
insert(upload: FileUpload): Promise<void>;
get(id: string): Promise<FileUpload | null>;
transition(id: string, from: FileUploadState[], to: FileUploadState): Promise<boolean>;
+ extend(id: string, expiresAt: number): Promise<boolean>;
expired(now: number): Promise<FileUpload[]>;
}
@@ -73,14 +74,18 @@ export function createPostgresFileUploadStore(connectionString: string): FileUpl
await withPgTransaction(await db.pool(), async (client) => {
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`file-uploads:${upload.actorId}`]);
const result = await client.query(
- "SELECT count(*) AS count, coalesce(sum(size_bytes),0) AS bytes FROM file_uploads WHERE actor_id=$1 AND state NOT IN ('complete','aborted','failed')",
- [upload.actorId],
+ "SELECT count(*) AS count, coalesce(sum(size_bytes),0) AS bytes FROM file_uploads WHERE actor_id=$1 AND state NOT IN ('complete','aborted','failed') AND NOT (state='pending' AND expires_at <= $2)",
+ [upload.actorId, upload.createdAt],
);
if (
Number(result.rows[0].count) >= MAX_ACTIVE_UPLOADS ||
Number(result.rows[0].bytes) + upload.sizeBytes > MAX_ACTIVE_UPLOAD_BYTES
)
- throw new FileUploadError("active upload quota exceeded", 429, "upload_quota");
+ throw new FileUploadError(
+ "active upload quota exceeded; finish or DELETE /v1/files/uploads/:id an unfinished upload, or wait an hour for idle ones to expire",
+ 429,
+ "upload_quota",
+ );
await client.query(
"INSERT INTO file_uploads(id,actor_id,state,size_bytes,expires_at,data) VALUES($1,$2,$3,$4,$5,$6)",
[upload.id, upload.actorId, upload.state, upload.sizeBytes, upload.expiresAt, JSON.stringify(upload)],
@@ -97,6 +102,16 @@ export function createPostgresFileUploadStore(connectionString: string): FileUpl
.rowCount === 1
);
},
+ async extend(id, expiresAt) {
+ return (
+ (
+ await db.query(
+ "UPDATE file_uploads SET expires_at=$2, data=jsonb_set(data,'{expiresAt}',to_jsonb($2::bigint)) WHERE id=$1 AND state='pending'",
+ [id, expiresAt],
+ )
+ ).rowCount === 1
+ );
+ },
async expired(now) {
return (
await db.q(
diff --git a/src/files/upload-client.py b/src/files/upload-client.py
index 8c29ad4d5..3f3c663a9 100644
--- a/src/files/upload-client.py
+++ b/src/files/upload-client.py
@@ -24,6 +24,13 @@ def quote(value):
return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"'
+def error_code(content):
+ try:
+ return json.loads(content).get("error")
+ except (ValueError, AttributeError):
+ return None
+
+
def request(method, url, data=None, headers=None):
config = [f"url = {quote(url)}", f"request = {quote(method)}", "silent", "show-error", "max-time = 180", 'write-out = "\\n%{http_code}"']
config += [f"header = {quote(f'{name}: {value}')}" for name, value in (headers or {}).items()]
@@ -44,7 +51,7 @@ def request(method, url, data=None, headers=None):
status = int(status or b"0")
if 200 <= status < 300:
return content
- if status < 500 and status != 429:
+ if status < 500 and (status != 429 or error_code(content) == "upload_quota"):
raise RequestError(status, f"request failed ({status}): {content[:4096].decode(errors='replace')}")
failure = f"HTTP {status}: {content[:200].decode(errors='replace')}"
else:
@@ -110,14 +117,21 @@ def main():
except (RequestError, RuntimeError) as error:
print(f"could not cancel the previous unfinished upload: {error}", file=sys.stderr)
state = None
+ status = None
+ if state:
+ try:
+ status = api("GET", f"/v1/files/uploads/{state['id']}")["upload"]
+ except RequestError as error:
+ if error.status != 404:
+ raise
+ expired = status and status["state"] == "pending" and status["expiresAt"] <= time.time() * 1000
+ if status and "file" not in state and (expired or status["state"] in ("aborted", "failed")):
+ print(f"starting a new upload: saved upload {state['id']} is {'expired' if expired else status['state']}", file=sys.stderr)
+ state = status = None
if not state:
state = {"id": uuid.uuid4().hex, "manifest": manifest, "destination": destination, "next": 1}
save(state_path, state)
- try:
- status = api("GET", f"/v1/files/uploads/{state['id']}")["upload"]
- except RequestError as error:
- if error.status != 404:
- raise
+ if status is None:
status = api("POST", "/v1/files/uploads", {**manifest, "requestId": state["id"]})["upload"]
if status["state"] == "pending":
with args.file.open("rb") as source:
diff --git a/test/direct-file-upload.test.ts b/test/direct-file-upload.test.ts
index 80453876f..423b7f398 100644
--- a/test/direct-file-upload.test.ts
+++ b/test/direct-file-upload.test.ts
@@ -25,6 +25,12 @@ function fixture() {
row.state = to;
return true;
},
+ async extend(id, expiresAt) {
+ const row = rows.get(id);
+ if (!row || row.state !== "pending") return false;
+ row.expiresAt = expiresAt;
+ return true;
+ },
async expired(now) {
return [...rows.values()].filter(
(r) => r.expiresAt <= now && r.state !== "complete" && r.state !== "aborted" && r.state !== "failed",
@@ -110,12 +116,30 @@ function fixture() {
loseResponse: () => {
loseCompleteResponse = true;
},
- advance: () => {
- time += 25 * 60 * 60 * 1000;
+ advance: (ms = 25 * 60 * 60 * 1000) => {
+ time += ms;
},
};
}
+test("an idle pending upload expires after an hour; signing parts keeps it alive up to a day", async () => {
+ const hour = 60 * 60 * 1000;
+ const f = fixture();
+ const idle = await f.service.begin(f.input);
+ f.advance(hour);
+ await assert.rejects(f.service.sign(idle.id, 1), /expired/);
+ await f.service.sweep();
+ assert.equal((await f.store.get(idle.id))!.state, "aborted");
+
+ const busy = await f.service.begin(f.input);
+ for (let i = 0; i < 24; i++) {
+ await f.service.sign(busy.id, 1);
+ f.advance(hour - 1);
+ }
+ await assert.rejects(f.service.sign(busy.id, 1), /expired/);
+ assert.equal((await f.store.get(busy.id))!.expiresAt, busy.createdAt + 24 * hour);
+});
+
test("signed parts bind the exact size, checksum, upload and number; unpublished parts are invisible", async () => {
const f = fixture();
const row = await f.service.begin(f.input);
diff --git a/test/file-upload-client.test.ts b/test/file-upload-client.test.ts
index acc9ba89c..25e038f95 100644
--- a/test/file-upload-client.test.ts
+++ b/test/file-upload-client.test.ts
@@ -7,6 +7,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { deflateRawSync } from "node:zlib";
+import { createHash } from "node:crypto";
import { mintSignedPayload } from "../src/auth/signed-token.ts";
test("publisher resumes an existing transfer after new uploads are disabled", async () => {
@@ -28,7 +29,7 @@ test("publisher resumes an existing transfer after new uploads are disabled", as
return;
}
initiated = true;
- response.end(JSON.stringify({ upload: { state: "pending" } }));
+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));
} else if (request.method === "PUT") {
puts++;
response.end();
@@ -42,7 +43,9 @@ test("publisher resumes an existing transfer after new uploads are disabled", as
response.writeHead(404).end("{}");
return;
}
- response.end(JSON.stringify({ upload: { state: complete ? "complete" : "pending" } }));
+ response.end(
+ JSON.stringify({ upload: { state: complete ? "complete" : "pending", expiresAt: Date.now() + 60_000 } }),
+ );
}
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
@@ -124,7 +127,8 @@ test("publisher cancels an unfinished upload when the file changed", async () =>
const url = request.url!;
if (request.method === "DELETE") response.end(JSON.stringify({ ok: true }));
else if (request.method === "PUT") response.end();
- else if (url === "/v1/files/uploads") response.end(JSON.stringify({ upload: { state: "pending" } }));
+ else if (url === "/v1/files/uploads")
+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));
else if (url.endsWith("/parts/1")) response.end(JSON.stringify({ url: origin + "/part", headers: {} }));
else if (url.endsWith("/complete")) response.end(JSON.stringify({ file: { id: "published" } }));
else response.writeHead(404).end("{}");
@@ -167,3 +171,62 @@ test("publisher cancels an unfinished upload when the file changed", async () =>
await rm(directory, { recursive: true, force: true });
}
});
+
+test("publisher starts a fresh upload when the saved session was aborted or expired", async () => {
+ for (const saved of [{ state: "aborted" }, { state: "pending", expiresAt: 1 }]) {
+ const directory = await mkdtemp(join(tmpdir(), "qm-publisher-"));
+ const requests: string[] = [];
+ let origin = "";
+ const stale = "b".repeat(32);
+ const server = createServer(async (request, response) => {
+ for await (const _ of request) void _;
+ requests.push(`${request.method} ${request.url}`);
+ response.setHeader("content-type", "application/json");
+ const url = request.url!;
+ if (request.method === "DELETE") response.end(JSON.stringify({ ok: true }));
+ else if (request.method === "PUT") response.end();
+ else if (url === `/v1/files/uploads/${stale}`) response.end(JSON.stringify({ upload: saved }));
+ else if (url === "/v1/files/uploads")
+ response.end(JSON.stringify({ upload: { state: "pending", expiresAt: Date.now() + 60_000 } }));
+ else if (url.endsWith("/parts/1")) response.end(JSON.stringify({ url: origin + "/part", headers: {} }));
+ else if (url.endsWith("/complete")) response.end(JSON.stringify({ file: { id: "published" } }));
+ else response.writeHead(404).end("{}");
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ assert.ok(address && typeof address !== "string");
+ origin = `http://127.0.0.1:${address.port}`;
+ const source = join(directory, "note.txt");
+ const token =
+ Buffer.from(JSON.stringify({ actorId: "user", scopeId: "personal:user" })).toString("base64url") + ".signature";
+ await writeFile(source, "abc");
+ const checksum = createHash("sha256").update("abc").digest("base64");
+ await writeFile(
+ source + ".qm-upload.json",
+ JSON.stringify({
+ id: stale,
+ manifest: { name: "note.txt", mimetype: "text/plain", sizeBytes: 3, checksums: [checksum] },
+ destination: { origin, actor: "user", scope: "personal:user" },
+ next: 1,
+ }),
+ );
+ try {
+ const code = await new Promise<number | null>((resolve, reject) => {
+ const child = spawn(
+ "python3",
+ [fileURLToPath(new URL("../src/files/upload-client.py", import.meta.url)), source],
+ { env: { ...process.env, AGENT_API_URL: origin, AGENT_API_TOKEN: token }, stdio: "ignore" },
+ );
+ child.on("error", reject);
+ child.on("exit", resolve);
+ });
+ assert.equal(code, 0);
+ assert.ok(requests.includes("POST /v1/files/uploads"));
+ assert.ok(!requests.includes(`POST /v1/files/uploads/${stale}/complete`));
+ assert.ok(!requests.some((r) => r.startsWith("DELETE")));
+ } finally {
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ await rm(directory, { recursive: true, force: true });
+ }
+ }
+});
diff --git a/test/postgres-file-upload-store.test.ts b/test/postgres-file-upload-store.test.ts
index 86a3bac66..3a392f28c 100644
--- a/test/postgres-file-upload-store.test.ts
+++ b/test/postgres-file-upload-store.test.ts
@@ -28,7 +28,7 @@ test(
checksums: ["hash"],
uploadId: "provider-upload",
state: "pending",
- expiresAt: 0,
+ expiresAt: Date.now() + 60_000,
createdAt: Date.now(),
});
const uploads = Array.from({ length: MAX_ACTIVE_UPLOADS + 3 }, row);
@@ -47,7 +47,7 @@ test(
second.transition(accepted.id, ["pending"], "aborting"),
]);
assert.equal(claims.filter(Boolean).length, 1);
- assert.ok((await second.expired(Date.now())).some((r) => r.id === accepted.id));
+ assert.ok((await second.expired(Date.now() + 120_000)).some((r) => r.id === accepted.id));
for (const item of uploads) await first.transition(item.id, ["pending", "completing", "aborting"], "aborted");
await second.insert(row());
},
@@ -112,3 +112,31 @@ test("terminal failed uploads release active quota", { skip: !url }, async () =>
}
assert.equal((await store.get(ids.at(-1)!))!.state, "failed");
});
+
+test("expired pending uploads stop holding quota; extend only moves pending rows", { skip: !url }, async () => {
+ const store = createPostgresFileUploadStore(url!);
+ const actorId = `idle-quota-${randomUUID()}`;
+ const now = Date.now();
+ const row = (expiresAt: number): FileUpload => ({
+ id: randomUUID().replaceAll("-", ""),
+ actorId,
+ scopeId: scopeId("personal", actorId),
+ name: "file",
+ mimetype: "text/plain",
+ sizeBytes: 1,
+ partSize: 64 * 1024 * 1024,
+ checksums: ["hash"],
+ uploadId: "upload",
+ state: "pending",
+ createdAt: now,
+ expiresAt,
+ });
+ const stale = Array.from({ length: MAX_ACTIVE_UPLOADS }, () => row(now - 1));
+ for (const r of stale) await store.insert(r);
+ const live = row(now + 60_000);
+ await store.insert(live);
+ assert.ok(await store.extend(live.id, now + 120_000));
+ assert.equal((await store.get(live.id))!.expiresAt, now + 120_000);
+ await store.transition(stale[0]!.id, ["pending"], "aborted");
+ assert.equal(await store.extend(stale[0]!.id, now + 120_000), false);
+});