A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat(files): stream uploads to disk instead of buffering (#9527)
Myles Scolnick865d65f1789e
diff --git a/marimo/_server/api/endpoints/file_explorer.py b/marimo/_server/api/endpoints/file_explorer.pyindex 391779c919..8b3c24557c 100644--- a/marimo/_server/api/endpoints/file_explorer.py+++ b/marimo/_server/api/endpoints/file_explorer.py@@ -4,11 +4,15 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom starlette.authentication import requires+from starlette.exceptions import HTTPExceptionfrom marimo import _loggersfrom marimo._server.api.deps import AppStatefrom marimo._server.api.utils import parse_multipart_request, parse_request-from marimo._server.files.os_file_system import OSFileSystem+from marimo._server.files.os_file_system import (+ OSFileSystem,+ UploadTooLargeError,+)from marimo._server.models.files import (FileCopyRequest,FileCopyResponse,@@ -120,16 +124,24 @@ async def create_file_or_directory($ref: "#/components/schemas/FileCreateResponse""""try:- parsed = await parse_multipart_request(+ async with parse_multipart_request(request, FileCreateMultipartRequest- )- info = file_system.create_file_or_directory(- parsed.body.path,- parsed.body.type,- parsed.body.name,- parsed.files.get("file"),- )+ ) as parsed:+ upload = parsed.files.get("file")+ # Directories and the default-template notebook take the+ # in-memory path; only real file content streams.+ if upload is not None and parsed.body.type in ("file", "notebook"):+ info = await file_system.stream_create_file(+ parsed.body.path, parsed.body.name, upload+ )+ else:+ info = file_system.create_file_or_directory(+ parsed.body.path, parsed.body.type, parsed.body.name, None+ )return FileCreateResponse(success=True, info=info)+ except UploadTooLargeError as e:+ LOGGER.warning(f"Rejected oversize upload: {e}")+ raise HTTPException(status_code=413, detail=str(e)) from eexcept Exception as e:LOGGER.error(f"Error creating file or directory: {e}")return FileCreateResponse(success=False, message=str(e))diff --git a/marimo/_server/api/utils.py b/marimo/_server/api/utils.pyindex 5751149718..08a10fdda9 100644--- a/marimo/_server/api/utils.py+++ b/marimo/_server/api/utils.py@@ -5,6 +5,7 @@ import osimport subprocessimport sysimport webbrowser+from contextlib import asynccontextmanagerfrom dataclasses import dataclassfrom pathlib import Pathfrom shutil import which@@ -25,6 +26,9 @@ from marimo._types.ids import ConsumerIdfrom marimo._utils.parse_dataclass import parse_rawif TYPE_CHECKING:+ from collections.abc import AsyncIterator++ from starlette.datastructures import UploadFilefrom starlette.requests import Requestfrom marimo._session.session import Session@@ -44,40 +48,39 @@ S = TypeVar("S", bound=msgspec.Struct)@dataclassclass MultipartRequest(Generic[S]):- """Result of parsing a multipart/form-data request body."""+ """Parsed multipart body. `files` holds un-read `UploadFile` handles+ so callers can stream large parts instead of buffering."""body: S- files: dict[str, bytes]+ files: dict[str, UploadFile]+@asynccontextmanagerasync def parse_multipart_request(request: Request, cls: type[S]-) -> MultipartRequest[S]:- """Parse a multipart/form-data body into a msgspec.Struct + file bytes.+) -> AsyncIterator[MultipartRequest[S]]:+ """Parse a multipart/form-data body into a msgspec.Struct + uploads.- String form fields are validated against `cls`. File upload parts are- read fully into memory and returned in `files`, keyed by form-field- name (callers look them up explicitly rather than via the struct).+ Must be used as an async context manager: `UploadFile` parts stay+ readable for the body of the `async with`, and their spooled temp+ files are closed on exit.Raises msgspec.ValidationError if required string fields are missingor invalid."""- # Imported lazily so this module stays import-safe in environments- # without starlette (e.g. pyodide).+ # Lazy import so this module stays import-safe under pyodide.from starlette.datastructures import UploadFile- # Use as an async context manager so any spooled temp files backing- # UploadFile parts are closed after parsing.async with request.form() as form:string_payload: dict[str, Any] = {}- files: dict[str, bytes] = {}+ files: dict[str, UploadFile] = {}for key, value in form.multi_items():if isinstance(value, UploadFile):- files[key] = await value.read()+ files[key] = valueelif isinstance(value, str):string_payload[key] = value- body = msgspec.convert(string_payload, cls, strict=False)- return MultipartRequest(body=body, files=files)+ body = msgspec.convert(string_payload, cls, strict=False)+ yield MultipartRequest(body=body, files=files)@runtime_checkablediff --git a/marimo/_server/files/os_file_system.py b/marimo/_server/files/os_file_system.pyindex e835d6e087..8c1c76d734 100644--- a/marimo/_server/files/os_file_system.py+++ b/marimo/_server/files/os_file_system.py@@ -8,9 +8,10 @@ import platformimport reimport shutilimport subprocess+import tempfilefrom collections import dequefrom pathlib import Path-from typing import Literal+from typing import Literal, Protocolfrom marimo import _loggersfrom marimo._server.files.file_system import FileSystem@@ -36,6 +37,29 @@ DISALLOWED_NAMES = ["..",]+# 1 MiB. Large enough to amortize syscall overhead, small enough to keep+# peak memory bounded when streaming.+_STREAM_CHUNK_SIZE = 1024 * 1024++# Hard cap on streamed uploads. Streaming removes the implicit OOM ceiling+# that buffered uploads had, so without a cap an authenticated client could+# exhaust disk. 1 GiB covers normal notebook-data use cases with margin.+MAX_UPLOAD_BYTES = 1024 * 1024 * 1024+++class UploadTooLargeError(ValueError):+ """Raised when a streamed upload exceeds `MAX_UPLOAD_BYTES`.++ Separate type (vs. a bare `ValueError`) so the HTTP layer can map it+ to a 413 response instead of the generic error path.+ """+++class AsyncByteSource(Protocol):+ """Structural type for things like Starlette's `UploadFile`."""++ async def read(self, size: int = -1, /) -> bytes: ...+class OSFileSystem(FileSystem):def get_root(self) -> str:@@ -133,33 +157,32 @@ class OSFileSystem(FileSystem):except UnicodeDecodeError:return file_path.read_bytes()- def create_file_or_directory(- self,- path: str,- file_type: Literal["file", "directory", "notebook"],- name: str,- contents: bytes | None,- ) -> FileInfo:+ @staticmethod+ def _validate_create_name(name: str) -> None:+ """Reject names that are empty, reserved, or traverse out of the+ parent. Centralized so HTTP, WASM, and streaming paths all share it.+ """if name in DISALLOWED_NAMES:raise ValueError(f"Cannot create file or directory with name {name}")if name.strip() == "":raise ValueError("Cannot create file or directory with empty name")- # Names that traverse out of `path` or escape via separators are- # rejected. Validation belongs here (not in the endpoint) so every- # caller of OSFileSystem — HTTP, WASM bridge, scripts — is covered.- if (- "/" in name- or "\\" in name- or "\x00" in name- or name in (".", "..")- ):+ if "/" in name or "\\" in name or "\x00" in name:raise ValueError(f"Invalid name {name!r}: must not contain path separators ""or refer to a parent directory")+ def create_file_or_directory(+ self,+ path: str,+ file_type: Literal["file", "directory", "notebook"],+ name: str,+ contents: bytes | None,+ ) -> FileInfo:+ self._validate_create_name(name)+full_path = Path(path) / namefull_path = _generate_unique_path(full_path)@@ -192,6 +215,63 @@ class OSFileSystem(FileSystem):),).file+ async def stream_create_file(+ self,+ path: str,+ name: str,+ source: AsyncByteSource,+ ) -> FileInfo:+ """Stream-write an uploaded file to disk, chunk by chunk.++ Avoids loading the full payload into memory (the HTTP multipart+ path can otherwise buffer 100 MB at once). Writes to a ``.part``+ temp file and atomically renames on success so a failed upload+ doesn't leave a half-written file at the final path.+ """+ self._validate_create_name(name)++ parent = Path(path)+ os.makedirs(parent, exist_ok=True)++ # Atomic O_CREAT|O_EXCL reservation closes the TOCTOU window between+ # picking a unique name and writing to it.+ full_path = _claim_unique_path(parent / name)++ tmp = tempfile.NamedTemporaryFile(+ dir=full_path.parent,+ prefix=full_path.name + ".",+ suffix=".part",+ delete=False,+ )+ tmp_path = tmp.name+ try:+ # Sync writes are bounded to one chunk between awaits, so event+ # loop blockage stays small without pulling in aiofiles.+ written = 0+ with tmp:+ while chunk := await source.read(_STREAM_CHUNK_SIZE):+ written += len(chunk)+ if written > MAX_UPLOAD_BYTES:+ raise UploadTooLargeError(+ f"Upload exceeds maximum size of "+ f"{MAX_UPLOAD_BYTES} bytes"+ )+ tmp.write(chunk)+ os.replace(tmp_path, full_path)+ except BaseException:+ # Both paths may or may not exist depending on where we failed;+ # FileNotFoundError on either is expected.+ for p in (tmp_path, str(full_path)):+ try:+ os.unlink(p)+ except FileNotFoundError:+ pass+ raise++ # `get_details` would re-read the file (and base64-encode binary),+ # defeating the streaming win.+ return self._get_file_info(str(full_path))+def delete_file_or_directory(self, path: str) -> bool:if os.path.isdir(path):safe_rmtree(path)@@ -485,6 +565,29 @@ def _generate_unique_path(new_path: str | Path) -> Path:i += 1+def _claim_unique_path(target: Path) -> Path:+ """Race-safe variant of `_generate_unique_path`: opens with O_EXCL and+ walks numbered suffixes on collision. Returns an empty file at the+ claimed path; the caller writes into or replaces it.+ """+ name_without_extension = target.stem+ extension = target.suffix+ candidate = target+ i = 0+ while True:+ try:+ fd = os.open(+ candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644+ )+ os.close(fd)+ return candidate+ except FileExistsError:+ i += 1+ candidate = target.parent / (+ f"{name_without_extension}_{i}{extension}"+ )++def _is_allowed_paths(path: str | Path, new_path: str | Path) -> bool:file_name = os.path.basename(new_path)if file_name in DISALLOWED_NAMES or not file_name.strip():diff --git a/marimo/_server/models/files.py b/marimo/_server/models/files.pyindex d4cd57b26a..248b713d77 100644--- a/marimo/_server/models/files.py+++ b/marimo/_server/models/files.py@@ -58,7 +58,13 @@ class FileCreateRequest(msgspec.Struct, rename="camel"):class FileCreateMultipartRequest(msgspec.Struct, rename="camel"):- """multipart/form-data body for POST /api/files/create."""+ """multipart/form-data body for POST /api/files/create.++ Schema-only: this struct exists to describe the multipart shape in+ OpenAPI. At runtime, the endpoint reads the string fields from+ ``MultipartRequest.body`` and the uploaded bytes from+ ``MultipartRequest.files["file"]`` — ``body.file`` is never populated.+ """# The path where to create the file or directorypath: str@@ -66,8 +72,8 @@ class FileCreateMultipartRequest(msgspec.Struct, rename="camel"):type: FileCreateType# The name of the file or directoryname: str- # The raw file bytes (optional). When omitted, an empty file is created- # (or, for 'notebook' type, a default notebook template).+ # OpenAPI-only (see class docstring). `format: binary` makes the+ # generated spec emit a file-upload schema rather than a base64 string.file: Annotated[str | None, msgspec.Meta(extra_json_schema={"format": "binary"})] = Nonediff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 66ba981d5c..556cc6366d 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -1690,7 +1690,11 @@ components:title: FileCopyResponsetype: objectFileCreateMultipartRequest:- description: multipart/form-data body for POST /api/files/create.+ description: "multipart/form-data body for POST /api/files/create.\n\n Schema-only:\+ \ this struct exists to describe the multipart shape in\n OpenAPI. At runtime,\+ \ the endpoint reads the string fields from\n ``MultipartRequest.body``\+ \ and the uploaded bytes from\n ``MultipartRequest.files[\"file\"]`` \u2014\+ \ ``body.file`` is never populated."properties:file:anyOf:diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex d7fbe124d9..8085de21a0 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -4378,6 +4378,11 @@ export interface components {/*** FileCreateMultipartRequest* @description multipart/form-data body for POST /api/files/create.+ *+ * Schema-only: this struct exists to describe the multipart shape in+ * OpenAPI. At runtime, the endpoint reads the string fields from+ * ``MultipartRequest.body`` and the uploaded bytes from+ * ``MultipartRequest.files["file"]`` — ``body.file`` is never populated.*/FileCreateMultipartRequest: {/**diff --git a/tests/_server/api/endpoints/test_file_explorer.py b/tests/_server/api/endpoints/test_file_explorer.pyindex e64b4b78ff..f54869bbca 100644--- a/tests/_server/api/endpoints/test_file_explorer.py+++ b/tests/_server/api/endpoints/test_file_explorer.py@@ -179,6 +179,27 @@ def test_create_rejects_path_traversal_name(client: TestClient) -> None:assert not os.path.exists(parent_path)+def test_create_returns_413_when_upload_exceeds_size_cap(+ client: TestClient, monkeypatch: pytest.MonkeyPatch+) -> None:+ # Shrink the cap so a small payload trips it.+ monkeypatch.setattr(+ "marimo._server.files.os_file_system.MAX_UPLOAD_BYTES", 4+ )+ response = client.post(+ "/api/files/create",+ headers=HEADERS,+ data={+ "path": test_dir,+ "type": "file",+ "name": "too_big.bin",+ },+ files={"file": ("too_big.bin", b"way too much data")},+ )+ assert response.status_code == 413, response.text+ assert not os.path.exists(os.path.join(test_dir, "too_big.bin"))++def test_update_file(client: TestClient) -> None:response = client.post("/api/files/update",diff --git a/tests/_server/api/test_api_utils.py b/tests/_server/api/test_api_utils.pyindex 5f4a09a4e6..56e009e543 100644--- a/tests/_server/api/test_api_utils.py+++ b/tests/_server/api/test_api_utils.py@@ -23,16 +23,19 @@ class _SampleForm(msgspec.Struct):def _build_app(captured: dict[str, object]) -> TestClient:async def endpoint(request: Request) -> JSONResponse:- parsed = await parse_multipart_request(request, _SampleForm)- captured["body"] = parsed.body- captured["files"] = parsed.files+ async with parse_multipart_request(request, _SampleForm) as parsed:+ captured["body"] = parsed.body+ captured["files"] = dict(parsed.files)+ upload = parsed.files.get("upload")+ if upload is not None:+ captured["upload_bytes"] = await upload.read()return JSONResponse({"ok": True})app = Starlette(routes=[Route("/test", endpoint, methods=["POST"])])return TestClient(app)-def test_parse_multipart_request_strings_and_file_bytes() -> None:+def test_parse_multipart_request_strings_and_file_upload() -> None:captured: dict[str, object] = {}client = _build_app(captured)response = client.post(@@ -45,7 +48,10 @@ def test_parse_multipart_request_strings_and_file_bytes() -> None:assert isinstance(body, _SampleForm)assert body.name == "marimo"assert body.count == 42- assert captured["files"] == {"upload": b"\x00\x01\x02\xff"}+ files = captured["files"]+ assert isinstance(files, dict)+ assert set(files.keys()) == {"upload"}+ assert captured["upload_bytes"] == b"\x00\x01\x02\xff"def test_parse_multipart_request_omitted_file_yields_empty_dict() -> None:diff --git a/tests/_server/files/test_os_file_system.py b/tests/_server/files/test_os_file_system.pyindex 9b7f0e508d..543fe90893 100644--- a/tests/_server/files/test_os_file_system.py+++ b/tests/_server/files/test_os_file_system.py@@ -8,6 +8,8 @@ import pytestfrom marimo._server.files.os_file_system import (OSFileSystem,+ UploadTooLargeError,+ _claim_unique_path,_generate_unique_path,_is_allowed_paths,)@@ -133,6 +135,156 @@ def test_create_rejects_path_traversal(assert not (test_dir.parent / "escaped.txt").exists()+class _ChunkedSource:+ """Async byte source emitting predetermined chunks; optionally raises+ after `fail_after` reads to simulate mid-stream failures."""++ def __init__(+ self, chunks: list[bytes], *, fail_after: int | None = None+ ) -> None:+ self._chunks = list(chunks)+ self._fail_after = fail_after+ self._reads = 0++ async def read(self, size: int = -1, /) -> bytes:+ if self._fail_after is not None and self._reads >= self._fail_after:+ raise RuntimeError("stream failed")+ if not self._chunks:+ return b""+ chunk = self._chunks[0]+ if size < 0 or len(chunk) <= size:+ out = self._chunks.pop(0)+ else:+ out, self._chunks[0] = chunk[:size], chunk[size:]+ self._reads += 1+ return out+++def _part_files(directory: Path) -> list[Path]:+ return list(directory.glob("*.part"))+++async def test_stream_create_file_writes_chunks(+ test_dir: Path, fs: OSFileSystem+) -> None:+ source = _ChunkedSource([b"hello ", b"streamed ", b"world"])+ info = await fs.stream_create_file(str(test_dir), "out.bin", source)+ out_path = test_dir / "out.bin"+ assert out_path.read_bytes() == b"hello streamed world"+ assert info.path == str(out_path)+ assert _part_files(test_dir) == []+++async def test_stream_create_file_writes_empty_file(+ test_dir: Path, fs: OSFileSystem+) -> None:+ info = await fs.stream_create_file(+ str(test_dir), "empty.bin", _ChunkedSource([])+ )+ out_path = test_dir / "empty.bin"+ assert out_path.exists()+ assert out_path.read_bytes() == b""+ assert info.path == str(out_path)+ assert _part_files(test_dir) == []+++async def test_stream_create_file_generates_unique_path(+ test_dir: Path, fs: OSFileSystem+) -> None:+ (test_dir / "dup.bin").write_bytes(b"existing")+ info = await fs.stream_create_file(+ str(test_dir), "dup.bin", _ChunkedSource([b"new"])+ )+ assert info.path == str(test_dir / "dup_1.bin")+ assert (test_dir / "dup.bin").read_bytes() == b"existing"+ assert (test_dir / "dup_1.bin").read_bytes() == b"new"+++ "name",+ [+ "../escape.bin",+ "../../escape.bin",+ "subdir/nested.bin",+ "..",+ ".",+ "with\\backslash.bin",+ "embed\x00null.bin",+ "",+ " ",+ ],+)+async def test_stream_create_file_rejects_traversal(+ test_dir: Path, fs: OSFileSystem, name: str+) -> None:+ with pytest.raises(ValueError):+ await fs.stream_create_file(+ str(test_dir), name, _ChunkedSource([b"x"])+ )+ assert not (test_dir.parent / "escape.bin").exists()+ assert _part_files(test_dir) == []+++async def test_stream_create_file_cleans_up_on_immediate_failure(+ test_dir: Path, fs: OSFileSystem+) -> None:+ source = _ChunkedSource([], fail_after=0)+ with pytest.raises(RuntimeError):+ await fs.stream_create_file(str(test_dir), "no_file.bin", source)+ assert not (test_dir / "no_file.bin").exists()+ assert _part_files(test_dir) == []+++async def test_stream_create_file_cleans_up_on_mid_stream_failure(+ test_dir: Path, fs: OSFileSystem+) -> None:+ # Source fails after writes have already landed in the .part file.+ source = _ChunkedSource([b"first ", b"second ", b"third"], fail_after=2)+ with pytest.raises(RuntimeError):+ await fs.stream_create_file(str(test_dir), "partial.bin", source)+ assert not (test_dir / "partial.bin").exists()+ assert _part_files(test_dir) == []+++async def test_stream_create_file_enforces_size_cap(+ test_dir: Path, fs: OSFileSystem, monkeypatch: pytest.MonkeyPatch+) -> None:+ # Shrink the cap so the test is fast; the production value is 1 GiB.+ monkeypatch.setattr(+ "marimo._server.files.os_file_system.MAX_UPLOAD_BYTES", 8+ )+ source = _ChunkedSource([b"aaaa", b"bbbb", b"too much"])+ with pytest.raises(UploadTooLargeError, match="exceeds maximum size"):+ await fs.stream_create_file(str(test_dir), "big.bin", source)+ assert not (test_dir / "big.bin").exists()+ assert _part_files(test_dir) == []+++async def test_stream_create_file_claims_path_atomically(+ test_dir: Path, fs: OSFileSystem+) -> None:+ # Concurrent file at the target name must not be clobbered by the+ # streaming write — the atomic claim should bump to a new suffix.+ (test_dir / "race.bin").write_bytes(b"existing")+ info = await fs.stream_create_file(+ str(test_dir), "race.bin", _ChunkedSource([b"new"])+ )+ assert info.path == str(test_dir / "race_1.bin")+ assert (test_dir / "race.bin").read_bytes() == b"existing"+ assert (test_dir / "race_1.bin").read_bytes() == b"new"+++def test_claim_unique_path_atomic_reservation(test_dir: Path) -> None:+ claimed_a = _claim_unique_path(test_dir / "claim.bin")+ assert claimed_a == test_dir / "claim.bin"+ assert claimed_a.exists()+ # An existing (even empty) reservation forces the next caller off the+ # base name — this is what closes the TOCTOU window.+ claimed_b = _claim_unique_path(test_dir / "claim.bin")+ assert claimed_b == test_dir / "claim_1.bin"+ assert claimed_b.exists()++def test_list_files(test_dir: Path, fs: OSFileSystem) -> None:test_create_file(test_dir, fs)test_create_directory(test_dir, fs)