Agent Zero AI framework
Make workspace HTTP transfers atomic
Alessandrof08d63cb2e58
diff --git a/api/AGENTS.md b/api/AGENTS.mdindex 02344d435..35b3628ef 100644--- a/api/AGENTS.md+++ b/api/AGENTS.md@@ -19,6 +19,10 @@- Keep CSRF and authentication protections intact for browser-facing state-changing endpoints.- WebSocket handlers must derive from `helpers.ws.WsHandler` and validate event data before using it.- Do not return secrets, raw environment values, private files, or unfiltered exception details to clients.+- Upload bodies are streamed to same-directory `.partial-*` files, hashed and+ fsynced, then atomically replaced; retain the legacy filename response while+ adding ordered size/SHA-256 receipts. Download responses remain streamed and+ include Content-Length plus `X-Content-SHA256` for atomic client verification.- This directory is a file-documented DOX profile: every direct `*.py` endpoint or WebSocket module must have a same-directory `*.py.dox.md` file named by appending `.dox.md` to the full Python filename.- The `*.py.dox.md` file owns endpoint purpose, request/response concepts, auth/CSRF/API-key/loopback assumptions, side effects, important helper dependencies, and verification guidance.- When a Python endpoint is added, removed, renamed, or behaviorally changed, update its matching `*.py.dox.md` in the same change.diff --git a/api/download_work_dir_file.py b/api/download_work_dir_file.pyindex 034156389..49c5f25f8 100644--- a/api/download_work_dir_file.py+++ b/api/download_work_dir_file.py@@ -1,4 +1,5 @@import base64+import hashlibfrom io import BytesIOimport mimetypesimport os@@ -11,6 +12,29 @@ from api import file_infofrom urllib.parse import quote+_HASH_CHUNK_BYTES = 1024 * 1024+++def file_size_and_sha256(file_source: str | BytesIO) -> tuple[int, str]:+ digest = hashlib.sha256()+ if isinstance(file_source, str):+ size = os.path.getsize(file_source)+ with open(file_source, "rb") as handle:+ for chunk in iter(lambda: handle.read(_HASH_CHUNK_BYTES), b""):+ digest.update(chunk)+ return size, digest.hexdigest()+ if isinstance(file_source, BytesIO):+ current_pos = file_source.tell()+ file_source.seek(0)+ size = 0+ while chunk := file_source.read(_HASH_CHUNK_BYTES):+ size += len(chunk)+ digest.update(chunk)+ file_source.seek(current_pos)+ return size, digest.hexdigest()+ raise ValueError(f"Unsupported file source type: {type(file_source)}")++def stream_file_download(file_source, download_name, chunk_size=8192):"""@@ -24,18 +48,7 @@ def stream_file_download(file_source, download_name, chunk_size=8192):Returns:Flask Response object with streaming content"""- # Calculate file size for Content-Length header- if isinstance(file_source, str):- # File path - get size from filesystem- file_size = os.path.getsize(file_source)- elif isinstance(file_source, BytesIO):- # BytesIO object - get size from buffer- current_pos = file_source.tell()- file_source.seek(0, 2) # Seek to end- file_size = file_source.tell()- file_source.seek(current_pos) # Restore original position- else:- raise ValueError(f"Unsupported file source type: {type(file_source)}")+ file_size, sha256 = file_size_and_sha256(file_source)def generate():if isinstance(file_source, str):@@ -68,6 +81,7 @@ def stream_file_download(file_source, download_name, chunk_size=8192):headers={'Content-Disposition': make_disposition(download_name),'Content-Length': str(file_size), # Critical for browser progress bars+ 'X-Content-SHA256': sha256,'Cache-Control': 'no-cache','X-Accel-Buffering': 'no', # Disable nginx buffering'Accept-Ranges': 'bytes' # Allow browser to resume downloadsdiff --git a/api/download_work_dir_file.py.dox.md b/api/download_work_dir_file.py.dox.mdindex 87085765a..c7feb9cc6 100644--- a/api/download_work_dir_file.py.dox.md+++ b/api/download_work_dir_file.py.dox.md@@ -15,6 +15,9 @@- `get_methods(cls)`- `async process(self, input: Input, request: Request) -> Output`- Top-level functions:+- `file_size_and_sha256(file_source) -> tuple[int, str]`: Stream a disk file or+ `BytesIO` source to produce download integrity metadata without loading a disk+ file into memory.- `stream_file_download(file_source, download_name, chunk_size=...)`: Create a streaming response for file downloads that shows progress in browser.- `make_disposition(download_name: str) -> str`- `resolve_download_path(path: str) -> str`: Resolve a requested download path and keep it within the runtime base dir.@@ -27,6 +30,8 @@- `DownloadFile` is an `ApiHandler`.- `DownloadFile` defines `process(...)`.- `DownloadFile` defines `get_methods(...)`.+- Every streamed response carries `Content-Length` and `X-Content-SHA256`; the+ CLI verifies both before atomically exposing a host destination.- Observed side-effect areas: filesystem reads, network calls.- Imported dependency areas include: `api`, `base64`, `flask`, `helpers`, `helpers.api`, `io`, `mimetypes`, `os`, `pathlib`, `urllib.parse`.diff --git a/api/upload.py b/api/upload.pyindex f334fb0af..0d7660c9f 100644--- a/api/upload.py+++ b/api/upload.py@@ -1,8 +1,48 @@+import hashlib+import os+import tempfile+from typing import Any+from helpers.api import ApiHandler, Request, Responsefrom helpers import filesfrom helpers.security import safe_filename+_UPLOAD_CHUNK_BYTES = 1024 * 1024+++def save_upload_atomic(file_storage: Any, target_path: str) -> dict[str, Any]:+ directory = os.path.dirname(target_path) or "."+ os.makedirs(directory, exist_ok=True)+ fd, temp_path = tempfile.mkstemp(prefix=".partial-", dir=directory)+ digest = hashlib.sha256()+ size = 0+ try:+ with os.fdopen(fd, "wb") as handle:+ while chunk := file_storage.stream.read(_UPLOAD_CHUNK_BYTES):+ handle.write(chunk)+ digest.update(chunk)+ size += len(chunk)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temp_path, target_path)+ try:+ directory_fd = os.open(directory, os.O_RDONLY)+ except OSError:+ directory_fd = None+ if directory_fd is not None:+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)+ finally:+ try:+ os.unlink(temp_path)+ except FileNotFoundError:+ pass+ return {"size": size, "sha256": digest.hexdigest()}++class UploadFile(ApiHandler):async def process(self, input: dict, request: Request) -> dict | Response:if "file" not in request.files:@@ -10,6 +50,7 @@ class UploadFile(ApiHandler):file_list = request.files.getlist("file") # Handle multiple filessaved_filenames = []+ saved_files = []for file in file_list:if file and self.allowed_file(file.filename): # Check file type@@ -18,13 +59,17 @@ class UploadFile(ApiHandler):filename = safe_filename(file.filename)if not filename:continue- file.save(files.get_abs_path("usr/uploads", filename))+ metadata = save_upload_atomic(+ file,+ files.get_abs_path("usr/uploads", filename),+ )saved_filenames.append(filename)+ saved_files.append({"filename": filename, **metadata})- return {"filenames": saved_filenames} # Return saved filenames+ return {"filenames": saved_filenames, "files": saved_files}def allowed_file(self,filename):return True# ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "txt", "pdf", "csv", "html", "json", "md"}- # return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS\ No newline at end of file+ # return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONSdiff --git a/api/upload.py.dox.md b/api/upload.py.dox.mdindex 9fccb571f..246fef0ae 100644--- a/api/upload.py.dox.md+++ b/api/upload.py.dox.md@@ -14,6 +14,11 @@- `UploadFile` (`ApiHandler`)- `async process(self, input: dict, request: Request) -> dict | Response`- `allowed_file(self, filename)`+- Top-level functions:+- `save_upload_atomic(file_storage, target_path) -> dict`: Stream one upload to+ a same-directory `.partial-*` file, hash and fsync it, atomically replace the+ final path, fsync the directory where supported, and remove the partial on+ every failure.## Runtime Contracts@@ -21,6 +26,9 @@- Update this file whenever request payloads, authentication or CSRF requirements, response shapes, route side effects, or WebSocket event contracts change.- `UploadFile` is an `ApiHandler`.- `UploadFile` defines `process(...)`.+- The response retains legacy `filenames` and adds ordered `files` entries with+ `filename`, byte `size`, and `sha256` so streaming clients can verify what Core+ committed.- Observed side-effect areas: filesystem reads, settings/state persistence.- Imported dependency areas include: `helpers`, `helpers.api`, `helpers.security`.diff --git a/tests/test_download_work_dir_file.py b/tests/test_download_work_dir_file.pynew file mode 100644index 000000000..be5badccf--- /dev/null+++ b/tests/test_download_work_dir_file.py@@ -0,0 +1,34 @@+from __future__ import annotations++from io import BytesIO+from pathlib import Path++from api.download_work_dir_file import stream_file_download+++def test_stream_file_download_includes_integrity_headers(tmp_path: Path) -> None:+ payload = b"download-payload" * 10_000+ source = tmp_path / "payload.bin"+ source.write_bytes(payload)++ response = stream_file_download(str(source), "payload.bin", chunk_size=4096)++ assert response.headers["Content-Length"] == str(len(payload))+ assert response.headers["X-Content-SHA256"] == (+ "c27444ae06bb85e92d4b8379a5c3f8a46d2ed910aff0c64b87da33544e606cfa"+ )+ assert b"".join(response.response) == payload+++def test_stream_file_download_hashes_bytes_io_without_changing_position() -> None:+ payload = b"memory-payload"+ source = BytesIO(payload)+ source.seek(3)++ response = stream_file_download(source, "payload.bin")++ assert response.headers["Content-Length"] == str(len(payload))+ assert response.headers["X-Content-SHA256"] == (+ "125c607bfe36606154f28bcbe339bf05aad4412bb67726651369b22f89dc1102"+ )+ assert b"".join(response.response) == payloaddiff --git a/tests/test_upload_atomic.py b/tests/test_upload_atomic.pynew file mode 100644index 000000000..64ab8f59a--- /dev/null+++ b/tests/test_upload_atomic.py@@ -0,0 +1,72 @@+from __future__ import annotations++from io import BytesIO+from pathlib import Path+from types import SimpleNamespace++import pytest++from api import upload+++class _UploadFiles:+ def __init__(self, files: list[object]) -> None:+ self._files = files++ def __contains__(self, key: str) -> bool:+ return key == "file"++ def getlist(self, key: str) -> list[object]:+ return list(self._files) if key == "file" else []+++async def test_upload_streams_atomically_and_returns_integrity_metadata(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ payload = b"0123456789abcdef" * 131_072+ storage = SimpleNamespace(filename="payload.bin", stream=BytesIO(payload))+ monkeypatch.setattr(+ upload.files,+ "get_abs_path",+ lambda *_parts: str(tmp_path / "payload.bin"),+ )+ request = SimpleNamespace(files=_UploadFiles([storage]))++ result = await upload.UploadFile(None, None).process({}, request)++ assert result == {+ "filenames": ["payload.bin"],+ "files": [+ {+ "filename": "payload.bin",+ "size": len(payload),+ "sha256": "9e5c630590af086d064e518b3c1d1cf98e9f4bc9df4fa7cfa32879079a250f62",+ }+ ],+ }+ assert (tmp_path / "payload.bin").read_bytes() == payload+ assert list(tmp_path.glob(".partial-*")) == []+++def test_upload_failure_preserves_existing_file_and_removes_partial(tmp_path: Path) -> None:+ target = tmp_path / "payload.bin"+ target.write_bytes(b"original")++ class BrokenStream:+ calls = 0++ def read(self, _size: int) -> bytes:+ self.calls += 1+ if self.calls == 1:+ return b"replacement-prefix"+ raise OSError("simulated upload disconnect")++ storage = SimpleNamespace(stream=BrokenStream())++ with pytest.raises(OSError, match="simulated upload disconnect"):+ upload.save_upload_atomic(storage, str(target))++ assert target.read_bytes() == b"original"+ assert list(tmp_path.glob(".partial-*")) == []