Multiplayer agent harness for work.
Make the Files upload helper work through HTTPS proxies (#2246)
Regan Bell479038962187
diff --git a/docs/files-publication.md b/docs/files-publication.mdindex 8c0ab444b..38636744e 100644--- a/docs/files-publication.md+++ b/docs/files-publication.md@@ -12,7 +12,7 @@ curl -fsS "$AGENT_API_URL/v1/files/upload-client" \python3 /tmp/qm-upload.py path/to/report.pdf```-The helper prints the published file and its content URL only after completion. It uses Python's standard library, holds at most one 64 MiB part in memory, retries transfers, and obtains another signed URL if one expires. A local `<filename>.qm-upload.json` tracks progress without storing credentials or signed URLs. Rerunning the same command resumes the same upload and returns the same artifact. `--state` selects another progress file; `--name` changes the published name. Keep the source file unchanged during transfer. The helper binds its progress to the API origin, actor, conversation scope, and complete part manifest.+The helper prints the published file and its content URL only after completion. It needs Python and curl (curl carries every request, so it works through the sandbox's HTTPS egress proxy), holds at most one 64 MiB part in memory, retries transfers, and obtains another signed URL if one expires. A local `<filename>.qm-upload.json` tracks progress without storing credentials or signed URLs. Rerunning the same command resumes the same upload and returns the same artifact. If the file changed since an unfinished attempt, the helper cancels that attempt and starts a new upload, so it does not hold one of the four active-upload slots. `--state` selects another progress file; `--name` changes the published name. Keep the source file unchanged during transfer. The helper binds its progress to the API origin, actor, conversation scope, and complete part manifest.The file belongs to the conversation scope. An agent token cannot publish into another scope or inspect another scope's upload sessions, even if its actor belongs to both scopes. Retirement of a sandbox does not affect a completed file. An unfinished upload is not a file and does not appear in Files.diff --git a/src/files/upload-client.py b/src/files/upload-client.pyindex 4d238f9a3..8c29ad4d5 100644--- a/src/files/upload-client.py+++ b/src/files/upload-client.py@@ -4,10 +4,9 @@ import hashlibimport jsonimport mimetypesimport os+import subprocessimport sysimport time-import urllib.error-import urllib.requestimport uuidimport zlibfrom pathlib import Path@@ -21,19 +20,37 @@ class RequestError(RuntimeError):self.status = status+def quote(value):+ return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"'++def request(method, url, data=None, headers=None):+ config = [f"url = {quote(url)}", f"request = {quote(method)}", "silent", "show-error", "max-time = 180", 'write-out = "\\n%{http_code}"']+ config += [f"header = {quote(f'{name}: {value}')}" for name, value in (headers or {}).items()]+ if data is not None:+ config.append('data-binary = "@-"')+ failure = ""for attempt in range(5):- try:- req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)- with urllib.request.urlopen(req, timeout=180) as response:- return response.read()- except urllib.error.HTTPError as error:- if error.code < 500 and error.code != 429:- raise RequestError(error.code, f"request failed ({error.code}): {error.read(4096).decode(errors='replace')}") from None- except (urllib.error.URLError, TimeoutError):- pass+ read_end, write_end = os.pipe()+ with os.fdopen(write_end, "w") as pipe:+ pipe.write("\n".join(config) + "\n")+ with os.fdopen(read_end) as pipe:+ try:+ result = subprocess.run(["curl", "-K", f"/dev/fd/{pipe.fileno()}"], input=data or b"", capture_output=True, pass_fds=(pipe.fileno(),))+ except FileNotFoundError:+ raise RuntimeError("curl is required; install it and rerun") from None+ content, _, status = result.stdout.rpartition(b"\n")+ if result.returncode == 0:+ status = int(status or b"0")+ if 200 <= status < 300:+ return content+ if status < 500 and status != 429:+ raise RequestError(status, f"request failed ({status}): {content[:4096].decode(errors='replace')}")+ failure = f"HTTP {status}: {content[:200].decode(errors='replace')}"+ else:+ failure = result.stderr.decode(errors="replace").strip() or f"curl exited {result.returncode}"if attempt == 4:- raise RuntimeError("transfer failed after retries; rerun with the same state file")+ raise RuntimeError(f"transfer failed after retries ({failure}); rerun with the same state file")time.sleep(2 ** attempt)@@ -83,11 +100,17 @@ def main():manifest = {"name": args.name or args.file.name, "mimetype": mimetypes.guess_type(args.file.name)[0] or "application/octet-stream", "sizeBytes": size, "checksums": checksums}if args.scope:manifest["scopeId"] = args.scope- if state_path.exists():- state = json.loads(state_path.read_text())- if state["manifest"] != manifest or state.get("destination") != destination:- raise RuntimeError("file or destination changed; use another --state path for a new upload")- else:+ state = json.loads(state_path.read_text()) if state_path.exists() else None+ if state and state.get("destination") != destination:+ raise RuntimeError("destination changed; use another --state path for a new upload")+ if state and state["manifest"] != manifest:+ if "file" not in state:+ try:+ api("DELETE", f"/v1/files/uploads/{state['id']}")+ except (RequestError, RuntimeError) as error:+ print(f"could not cancel the previous unfinished upload: {error}", file=sys.stderr)+ state = None+ if not state:state = {"id": uuid.uuid4().hex, "manifest": manifest, "destination": destination, "next": 1}save(state_path, state)try:diff --git a/test/file-upload-client.test.ts b/test/file-upload-client.test.tsindex 6c5b7d298..acc9ba89c 100644--- a/test/file-upload-client.test.ts+++ b/test/file-upload-client.test.ts@@ -112,3 +112,58 @@ test("publisher resumes an existing transfer after new uploads are disabled", asawait rm(directory, { recursive: true, force: true });}});++test("publisher cancels an unfinished upload when the file changed", async () => {+ const directory = await mkdtemp(join(tmpdir(), "qm-publisher-"));+ const requests: string[] = [];+ let origin = "";+ const server = createServer(async (request, response) => {+ for await (const _ of request) void _;+ requests.push(`${request.method} ${request.url}`);+ response.setHeader("content-type", "application/json");+ const url = request.url!;+ if (request.method === "DELETE") response.end(JSON.stringify({ ok: true }));+ else if (request.method === "PUT") response.end();+ else if (url === "/v1/files/uploads") response.end(JSON.stringify({ upload: { state: "pending" } }));+ else if (url.endsWith("/parts/1")) response.end(JSON.stringify({ url: origin + "/part", headers: {} }));+ else if (url.endsWith("/complete")) response.end(JSON.stringify({ file: { id: "published" } }));+ else response.writeHead(404).end("{}");+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const address = server.address();+ assert.ok(address && typeof address !== "string");+ origin = `http://127.0.0.1:${address.port}`;+ const source = join(directory, "state.md");+ const token =+ Buffer.from(JSON.stringify({ actorId: "user", scopeId: "personal:user" })).toString("base64url") + ".signature";+ await writeFile(source, "new contents");+ await writeFile(+ source + ".qm-upload.json",+ JSON.stringify({+ id: "a".repeat(32),+ manifest: { name: "state.md", mimetype: "text/markdown", sizeBytes: 3, checksums: ["old"] },+ destination: { origin, actor: "user", scope: "personal:user" },+ next: 1,+ }),+ );+ try {+ const code = await new Promise<number | null>((resolve, reject) => {+ const child = spawn(+ "python3",+ [fileURLToPath(new URL("../src/files/upload-client.py", import.meta.url)), source],+ {+ env: { ...process.env, AGENT_API_URL: origin, AGENT_API_TOKEN: token },+ stdio: "ignore",+ },+ );+ child.on("error", reject);+ child.on("exit", resolve);+ });+ assert.equal(code, 0);+ assert.equal(requests[0], `DELETE /v1/files/uploads/${"a".repeat(32)}`);+ assert.ok(requests.includes("POST /v1/files/uploads"));+ } finally {+ await new Promise<void>((resolve) => server.close(() => resolve()));+ await rm(directory, { recursive: true, force: true });+ }+});