Skip to content

Agent Zero AI framework

Make workspace HTTP transfers atomic

Alessandrof08d63cb2e58
diff --git a/api/AGENTS.md b/api/AGENTS.md
index 02344d435..35b3628ef 100644
--- a/api/AGENTS.md
+++ b/api/AGENTS.md
@@ -19,6 +19,10 @@
- Keep CSRF and authentication protections intact for browser-facing state-changing endpoints.
- WebSocket handlers must derive from `helpers.ws.WsHandler` and validate event data before using it.
- Do not return secrets, raw environment values, private files, or unfiltered exception details to clients.
+- Upload bodies are streamed to same-directory `.partial-*` files, hashed and
+ fsynced, then atomically replaced; retain the legacy filename response while
+ adding ordered size/SHA-256 receipts. Download responses remain streamed and
+ include Content-Length plus `X-Content-SHA256` for atomic client verification.
- This directory is a file-documented DOX profile: every direct `*.py` endpoint or WebSocket module must have a same-directory `*.py.dox.md` file named by appending `.dox.md` to the full Python filename.
- The `*.py.dox.md` file owns endpoint purpose, request/response concepts, auth/CSRF/API-key/loopback assumptions, side effects, important helper dependencies, and verification guidance.
- When a Python endpoint is added, removed, renamed, or behaviorally changed, update its matching `*.py.dox.md` in the same change.
diff --git a/api/download_work_dir_file.py b/api/download_work_dir_file.py
index 034156389..49c5f25f8 100644
--- a/api/download_work_dir_file.py
+++ b/api/download_work_dir_file.py
@@ -1,4 +1,5 @@
import base64
+import hashlib
from io import BytesIO
import mimetypes
import os
@@ -11,6 +12,29 @@ from api import file_info
from urllib.parse import quote
+_HASH_CHUNK_BYTES = 1024 * 1024
+
+
+def file_size_and_sha256(file_source: str | BytesIO) -> tuple[int, str]:
+ digest = hashlib.sha256()
+ if isinstance(file_source, str):
+ size = os.path.getsize(file_source)
+ with open(file_source, "rb") as handle:
+ for chunk in iter(lambda: handle.read(_HASH_CHUNK_BYTES), b""):
+ digest.update(chunk)
+ return size, digest.hexdigest()
+ if isinstance(file_source, BytesIO):
+ current_pos = file_source.tell()
+ file_source.seek(0)
+ size = 0
+ while chunk := file_source.read(_HASH_CHUNK_BYTES):
+ size += len(chunk)
+ digest.update(chunk)
+ file_source.seek(current_pos)
+ return size, digest.hexdigest()
+ raise ValueError(f"Unsupported file source type: {type(file_source)}")
+
+
def stream_file_download(file_source, download_name, chunk_size=8192):
"""
@@ -24,18 +48,7 @@ def stream_file_download(file_source, download_name, chunk_size=8192):
Returns:
Flask Response object with streaming content
"""
- # Calculate file size for Content-Length header
- if isinstance(file_source, str):
- # File path - get size from filesystem
- file_size = os.path.getsize(file_source)
- elif isinstance(file_source, BytesIO):
- # BytesIO object - get size from buffer
- current_pos = file_source.tell()
- file_source.seek(0, 2) # Seek to end
- file_size = file_source.tell()
- file_source.seek(current_pos) # Restore original position
- else:
- raise ValueError(f"Unsupported file source type: {type(file_source)}")
+ file_size, sha256 = file_size_and_sha256(file_source)
def generate():
if isinstance(file_source, str):
@@ -68,6 +81,7 @@ def stream_file_download(file_source, download_name, chunk_size=8192):
headers={
'Content-Disposition': make_disposition(download_name),
'Content-Length': str(file_size), # Critical for browser progress bars
+ 'X-Content-SHA256': sha256,
'Cache-Control': 'no-cache',
'X-Accel-Buffering': 'no', # Disable nginx buffering
'Accept-Ranges': 'bytes' # Allow browser to resume downloads
diff --git a/api/download_work_dir_file.py.dox.md b/api/download_work_dir_file.py.dox.md
index 87085765a..c7feb9cc6 100644
--- a/api/download_work_dir_file.py.dox.md
+++ b/api/download_work_dir_file.py.dox.md
@@ -15,6 +15,9 @@
- `get_methods(cls)`
- `async process(self, input: Input, request: Request) -> Output`
- Top-level functions:
+- `file_size_and_sha256(file_source) -> tuple[int, str]`: Stream a disk file or
+ `BytesIO` source to produce download integrity metadata without loading a disk
+ file into memory.
- `stream_file_download(file_source, download_name, chunk_size=...)`: Create a streaming response for file downloads that shows progress in browser.
- `make_disposition(download_name: str) -> str`
- `resolve_download_path(path: str) -> str`: Resolve a requested download path and keep it within the runtime base dir.
@@ -27,6 +30,8 @@
- `DownloadFile` is an `ApiHandler`.
- `DownloadFile` defines `process(...)`.
- `DownloadFile` defines `get_methods(...)`.
+- Every streamed response carries `Content-Length` and `X-Content-SHA256`; the
+ CLI verifies both before atomically exposing a host destination.
- Observed side-effect areas: filesystem reads, network calls.
- Imported dependency areas include: `api`, `base64`, `flask`, `helpers`, `helpers.api`, `io`, `mimetypes`, `os`, `pathlib`, `urllib.parse`.
diff --git a/api/upload.py b/api/upload.py
index f334fb0af..0d7660c9f 100644
--- a/api/upload.py
+++ b/api/upload.py
@@ -1,8 +1,48 @@
+import hashlib
+import os
+import tempfile
+from typing import Any
+
from helpers.api import ApiHandler, Request, Response
from helpers import files
from helpers.security import safe_filename
+_UPLOAD_CHUNK_BYTES = 1024 * 1024
+
+
+def save_upload_atomic(file_storage: Any, target_path: str) -> dict[str, Any]:
+ directory = os.path.dirname(target_path) or "."
+ os.makedirs(directory, exist_ok=True)
+ fd, temp_path = tempfile.mkstemp(prefix=".partial-", dir=directory)
+ digest = hashlib.sha256()
+ size = 0
+ try:
+ with os.fdopen(fd, "wb") as handle:
+ while chunk := file_storage.stream.read(_UPLOAD_CHUNK_BYTES):
+ handle.write(chunk)
+ digest.update(chunk)
+ size += len(chunk)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temp_path, target_path)
+ try:
+ directory_fd = os.open(directory, os.O_RDONLY)
+ except OSError:
+ directory_fd = None
+ if directory_fd is not None:
+ try:
+ os.fsync(directory_fd)
+ finally:
+ os.close(directory_fd)
+ finally:
+ try:
+ os.unlink(temp_path)
+ except FileNotFoundError:
+ pass
+ return {"size": size, "sha256": digest.hexdigest()}
+
+
class UploadFile(ApiHandler):
async def process(self, input: dict, request: Request) -> dict | Response:
if "file" not in request.files:
@@ -10,6 +50,7 @@ class UploadFile(ApiHandler):
file_list = request.files.getlist("file") # Handle multiple files
saved_filenames = []
+ saved_files = []
for file in file_list:
if file and self.allowed_file(file.filename): # Check file type
@@ -18,13 +59,17 @@ class UploadFile(ApiHandler):
filename = safe_filename(file.filename)
if not filename:
continue
- file.save(files.get_abs_path("usr/uploads", filename))
+ metadata = save_upload_atomic(
+ file,
+ files.get_abs_path("usr/uploads", filename),
+ )
saved_filenames.append(filename)
+ saved_files.append({"filename": filename, **metadata})
- return {"filenames": saved_filenames} # Return saved filenames
+ return {"filenames": saved_filenames, "files": saved_files}
def allowed_file(self,filename):
return True
# ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "txt", "pdf", "csv", "html", "json", "md"}
- # return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
\ No newline at end of file
+ # return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
diff --git a/api/upload.py.dox.md b/api/upload.py.dox.md
index 9fccb571f..246fef0ae 100644
--- a/api/upload.py.dox.md
+++ b/api/upload.py.dox.md
@@ -14,6 +14,11 @@
- `UploadFile` (`ApiHandler`)
- `async process(self, input: dict, request: Request) -> dict | Response`
- `allowed_file(self, filename)`
+- Top-level functions:
+- `save_upload_atomic(file_storage, target_path) -> dict`: Stream one upload to
+ a same-directory `.partial-*` file, hash and fsync it, atomically replace the
+ final path, fsync the directory where supported, and remove the partial on
+ every failure.
## Runtime Contracts
@@ -21,6 +26,9 @@
- Update this file whenever request payloads, authentication or CSRF requirements, response shapes, route side effects, or WebSocket event contracts change.
- `UploadFile` is an `ApiHandler`.
- `UploadFile` defines `process(...)`.
+- The response retains legacy `filenames` and adds ordered `files` entries with
+ `filename`, byte `size`, and `sha256` so streaming clients can verify what Core
+ committed.
- Observed side-effect areas: filesystem reads, settings/state persistence.
- Imported dependency areas include: `helpers`, `helpers.api`, `helpers.security`.
diff --git a/tests/test_download_work_dir_file.py b/tests/test_download_work_dir_file.py
new file mode 100644
index 000000000..be5badccf
--- /dev/null
+++ b/tests/test_download_work_dir_file.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from io import BytesIO
+from pathlib import Path
+
+from api.download_work_dir_file import stream_file_download
+
+
+def test_stream_file_download_includes_integrity_headers(tmp_path: Path) -> None:
+ payload = b"download-payload" * 10_000
+ source = tmp_path / "payload.bin"
+ source.write_bytes(payload)
+
+ response = stream_file_download(str(source), "payload.bin", chunk_size=4096)
+
+ assert response.headers["Content-Length"] == str(len(payload))
+ assert response.headers["X-Content-SHA256"] == (
+ "c27444ae06bb85e92d4b8379a5c3f8a46d2ed910aff0c64b87da33544e606cfa"
+ )
+ assert b"".join(response.response) == payload
+
+
+def test_stream_file_download_hashes_bytes_io_without_changing_position() -> None:
+ payload = b"memory-payload"
+ source = BytesIO(payload)
+ source.seek(3)
+
+ response = stream_file_download(source, "payload.bin")
+
+ assert response.headers["Content-Length"] == str(len(payload))
+ assert response.headers["X-Content-SHA256"] == (
+ "125c607bfe36606154f28bcbe339bf05aad4412bb67726651369b22f89dc1102"
+ )
+ assert b"".join(response.response) == payload
diff --git a/tests/test_upload_atomic.py b/tests/test_upload_atomic.py
new file mode 100644
index 000000000..64ab8f59a
--- /dev/null
+++ b/tests/test_upload_atomic.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from io import BytesIO
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from api import upload
+
+
+class _UploadFiles:
+ def __init__(self, files: list[object]) -> None:
+ self._files = files
+
+ def __contains__(self, key: str) -> bool:
+ return key == "file"
+
+ def getlist(self, key: str) -> list[object]:
+ return list(self._files) if key == "file" else []
+
+
+async def test_upload_streams_atomically_and_returns_integrity_metadata(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ payload = b"0123456789abcdef" * 131_072
+ storage = SimpleNamespace(filename="payload.bin", stream=BytesIO(payload))
+ monkeypatch.setattr(
+ upload.files,
+ "get_abs_path",
+ lambda *_parts: str(tmp_path / "payload.bin"),
+ )
+ request = SimpleNamespace(files=_UploadFiles([storage]))
+
+ result = await upload.UploadFile(None, None).process({}, request)
+
+ assert result == {
+ "filenames": ["payload.bin"],
+ "files": [
+ {
+ "filename": "payload.bin",
+ "size": len(payload),
+ "sha256": "9e5c630590af086d064e518b3c1d1cf98e9f4bc9df4fa7cfa32879079a250f62",
+ }
+ ],
+ }
+ assert (tmp_path / "payload.bin").read_bytes() == payload
+ assert list(tmp_path.glob(".partial-*")) == []
+
+
+def test_upload_failure_preserves_existing_file_and_removes_partial(tmp_path: Path) -> None:
+ target = tmp_path / "payload.bin"
+ target.write_bytes(b"original")
+
+ class BrokenStream:
+ calls = 0
+
+ def read(self, _size: int) -> bytes:
+ self.calls += 1
+ if self.calls == 1:
+ return b"replacement-prefix"
+ raise OSError("simulated upload disconnect")
+
+ storage = SimpleNamespace(stream=BrokenStream())
+
+ with pytest.raises(OSError, match="simulated upload disconnect"):
+ upload.save_upload_atomic(storage, str(target))
+
+ assert target.read_bytes() == b"original"
+ assert list(tmp_path.glob(".partial-*")) == []