Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(files): stream uploads to disk instead of buffering (#9527)

Myles Scolnick865d65f1789e
diff --git a/marimo/_server/api/endpoints/file_explorer.py b/marimo/_server/api/endpoints/file_explorer.py
index 391779c919..8b3c24557c 100644
--- a/marimo/_server/api/endpoints/file_explorer.py
+++ b/marimo/_server/api/endpoints/file_explorer.py
@@ -4,11 +4,15 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from starlette.authentication import requires
+from starlette.exceptions import HTTPException
from marimo import _loggers
from marimo._server.api.deps import AppState
from marimo._server.api.utils import parse_multipart_request, parse_request
-from marimo._server.files.os_file_system import OSFileSystem
+from marimo._server.files.os_file_system import (
+ OSFileSystem,
+ UploadTooLargeError,
+)
from marimo._server.models.files import (
FileCopyRequest,
FileCopyResponse,
@@ -120,16 +124,24 @@ async def create_file_or_directory(
$ref: "#/components/schemas/FileCreateResponse"
"""
try:
- parsed = await parse_multipart_request(
+ async with parse_multipart_request(
request, FileCreateMultipartRequest
- )
- info = file_system.create_file_or_directory(
- parsed.body.path,
- parsed.body.type,
- parsed.body.name,
- parsed.files.get("file"),
- )
+ ) as parsed:
+ upload = parsed.files.get("file")
+ # Directories and the default-template notebook take the
+ # in-memory path; only real file content streams.
+ if upload is not None and parsed.body.type in ("file", "notebook"):
+ info = await file_system.stream_create_file(
+ parsed.body.path, parsed.body.name, upload
+ )
+ else:
+ info = file_system.create_file_or_directory(
+ parsed.body.path, parsed.body.type, parsed.body.name, None
+ )
return FileCreateResponse(success=True, info=info)
+ except UploadTooLargeError as e:
+ LOGGER.warning(f"Rejected oversize upload: {e}")
+ raise HTTPException(status_code=413, detail=str(e)) from e
except Exception as e:
LOGGER.error(f"Error creating file or directory: {e}")
return FileCreateResponse(success=False, message=str(e))
diff --git a/marimo/_server/api/utils.py b/marimo/_server/api/utils.py
index 5751149718..08a10fdda9 100644
--- a/marimo/_server/api/utils.py
+++ b/marimo/_server/api/utils.py
@@ -5,6 +5,7 @@ import os
import subprocess
import sys
import webbrowser
+from contextlib import asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
from shutil import which
@@ -25,6 +26,9 @@ from marimo._types.ids import ConsumerId
from marimo._utils.parse_dataclass import parse_raw
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
+ from starlette.datastructures import UploadFile
from starlette.requests import Request
from marimo._session.session import Session
@@ -44,40 +48,39 @@ S = TypeVar("S", bound=msgspec.Struct)
@dataclass
class MultipartRequest(Generic[S]):
- """Result of parsing a multipart/form-data request body."""
+ """Parsed multipart body. `files` holds un-read `UploadFile` handles
+ so callers can stream large parts instead of buffering."""
body: S
- files: dict[str, bytes]
+ files: dict[str, UploadFile]
+@asynccontextmanager
async def parse_multipart_request(
request: Request, cls: type[S]
-) -> MultipartRequest[S]:
- """Parse a multipart/form-data body into a msgspec.Struct + file bytes.
+) -> AsyncIterator[MultipartRequest[S]]:
+ """Parse a multipart/form-data body into a msgspec.Struct + uploads.
- String form fields are validated against `cls`. File upload parts are
- read fully into memory and returned in `files`, keyed by form-field
- name (callers look them up explicitly rather than via the struct).
+ Must be used as an async context manager: `UploadFile` parts stay
+ readable for the body of the `async with`, and their spooled temp
+ files are closed on exit.
Raises msgspec.ValidationError if required string fields are missing
or invalid.
"""
- # Imported lazily so this module stays import-safe in environments
- # without starlette (e.g. pyodide).
+ # Lazy import so this module stays import-safe under pyodide.
from starlette.datastructures import UploadFile
- # Use as an async context manager so any spooled temp files backing
- # UploadFile parts are closed after parsing.
async with request.form() as form:
string_payload: dict[str, Any] = {}
- files: dict[str, bytes] = {}
+ files: dict[str, UploadFile] = {}
for key, value in form.multi_items():
if isinstance(value, UploadFile):
- files[key] = await value.read()
+ files[key] = value
elif isinstance(value, str):
string_payload[key] = value
- body = msgspec.convert(string_payload, cls, strict=False)
- return MultipartRequest(body=body, files=files)
+ body = msgspec.convert(string_payload, cls, strict=False)
+ yield MultipartRequest(body=body, files=files)
@runtime_checkable
diff --git a/marimo/_server/files/os_file_system.py b/marimo/_server/files/os_file_system.py
index e835d6e087..8c1c76d734 100644
--- a/marimo/_server/files/os_file_system.py
+++ b/marimo/_server/files/os_file_system.py
@@ -8,9 +8,10 @@ import platform
import re
import shutil
import subprocess
+import tempfile
from collections import deque
from pathlib import Path
-from typing import Literal
+from typing import Literal, Protocol
from marimo import _loggers
from marimo._server.files.file_system import FileSystem
@@ -36,6 +37,29 @@ DISALLOWED_NAMES = [
"..",
]
+# 1 MiB. Large enough to amortize syscall overhead, small enough to keep
+# peak memory bounded when streaming.
+_STREAM_CHUNK_SIZE = 1024 * 1024
+
+# Hard cap on streamed uploads. Streaming removes the implicit OOM ceiling
+# that buffered uploads had, so without a cap an authenticated client could
+# exhaust disk. 1 GiB covers normal notebook-data use cases with margin.
+MAX_UPLOAD_BYTES = 1024 * 1024 * 1024
+
+
+class UploadTooLargeError(ValueError):
+ """Raised when a streamed upload exceeds `MAX_UPLOAD_BYTES`.
+
+ Separate type (vs. a bare `ValueError`) so the HTTP layer can map it
+ to a 413 response instead of the generic error path.
+ """
+
+
+class AsyncByteSource(Protocol):
+ """Structural type for things like Starlette's `UploadFile`."""
+
+ async def read(self, size: int = -1, /) -> bytes: ...
+
class OSFileSystem(FileSystem):
def get_root(self) -> str:
@@ -133,33 +157,32 @@ class OSFileSystem(FileSystem):
except UnicodeDecodeError:
return file_path.read_bytes()
- def create_file_or_directory(
- self,
- path: str,
- file_type: Literal["file", "directory", "notebook"],
- name: str,
- contents: bytes | None,
- ) -> FileInfo:
+ @staticmethod
+ def _validate_create_name(name: str) -> None:
+ """Reject names that are empty, reserved, or traverse out of the
+ parent. Centralized so HTTP, WASM, and streaming paths all share it.
+ """
if name in DISALLOWED_NAMES:
raise ValueError(
f"Cannot create file or directory with name {name}"
)
if name.strip() == "":
raise ValueError("Cannot create file or directory with empty name")
- # Names that traverse out of `path` or escape via separators are
- # rejected. Validation belongs here (not in the endpoint) so every
- # caller of OSFileSystem — HTTP, WASM bridge, scripts — is covered.
- if (
- "/" in name
- or "\\" in name
- or "\x00" in name
- or name in (".", "..")
- ):
+ if "/" in name or "\\" in name or "\x00" in name:
raise ValueError(
f"Invalid name {name!r}: must not contain path separators "
"or refer to a parent directory"
)
+ def create_file_or_directory(
+ self,
+ path: str,
+ file_type: Literal["file", "directory", "notebook"],
+ name: str,
+ contents: bytes | None,
+ ) -> FileInfo:
+ self._validate_create_name(name)
+
full_path = Path(path) / name
full_path = _generate_unique_path(full_path)
@@ -192,6 +215,63 @@ class OSFileSystem(FileSystem):
),
).file
+ async def stream_create_file(
+ self,
+ path: str,
+ name: str,
+ source: AsyncByteSource,
+ ) -> FileInfo:
+ """Stream-write an uploaded file to disk, chunk by chunk.
+
+ Avoids loading the full payload into memory (the HTTP multipart
+ path can otherwise buffer 100 MB at once). Writes to a ``.part``
+ temp file and atomically renames on success so a failed upload
+ doesn't leave a half-written file at the final path.
+ """
+ self._validate_create_name(name)
+
+ parent = Path(path)
+ os.makedirs(parent, exist_ok=True)
+
+ # Atomic O_CREAT|O_EXCL reservation closes the TOCTOU window between
+ # picking a unique name and writing to it.
+ full_path = _claim_unique_path(parent / name)
+
+ tmp = tempfile.NamedTemporaryFile(
+ dir=full_path.parent,
+ prefix=full_path.name + ".",
+ suffix=".part",
+ delete=False,
+ )
+ tmp_path = tmp.name
+ try:
+ # Sync writes are bounded to one chunk between awaits, so event
+ # loop blockage stays small without pulling in aiofiles.
+ written = 0
+ with tmp:
+ while chunk := await source.read(_STREAM_CHUNK_SIZE):
+ written += len(chunk)
+ if written > MAX_UPLOAD_BYTES:
+ raise UploadTooLargeError(
+ f"Upload exceeds maximum size of "
+ f"{MAX_UPLOAD_BYTES} bytes"
+ )
+ tmp.write(chunk)
+ os.replace(tmp_path, full_path)
+ except BaseException:
+ # Both paths may or may not exist depending on where we failed;
+ # FileNotFoundError on either is expected.
+ for p in (tmp_path, str(full_path)):
+ try:
+ os.unlink(p)
+ except FileNotFoundError:
+ pass
+ raise
+
+ # `get_details` would re-read the file (and base64-encode binary),
+ # defeating the streaming win.
+ return self._get_file_info(str(full_path))
+
def delete_file_or_directory(self, path: str) -> bool:
if os.path.isdir(path):
safe_rmtree(path)
@@ -485,6 +565,29 @@ def _generate_unique_path(new_path: str | Path) -> Path:
i += 1
+def _claim_unique_path(target: Path) -> Path:
+ """Race-safe variant of `_generate_unique_path`: opens with O_EXCL and
+ walks numbered suffixes on collision. Returns an empty file at the
+ claimed path; the caller writes into or replaces it.
+ """
+ name_without_extension = target.stem
+ extension = target.suffix
+ candidate = target
+ i = 0
+ while True:
+ try:
+ fd = os.open(
+ candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644
+ )
+ os.close(fd)
+ return candidate
+ except FileExistsError:
+ i += 1
+ candidate = target.parent / (
+ f"{name_without_extension}_{i}{extension}"
+ )
+
+
def _is_allowed_paths(path: str | Path, new_path: str | Path) -> bool:
file_name = os.path.basename(new_path)
if file_name in DISALLOWED_NAMES or not file_name.strip():
diff --git a/marimo/_server/models/files.py b/marimo/_server/models/files.py
index d4cd57b26a..248b713d77 100644
--- a/marimo/_server/models/files.py
+++ b/marimo/_server/models/files.py
@@ -58,7 +58,13 @@ class FileCreateRequest(msgspec.Struct, rename="camel"):
class FileCreateMultipartRequest(msgspec.Struct, rename="camel"):
- """multipart/form-data body for POST /api/files/create."""
+ """multipart/form-data body for POST /api/files/create.
+
+ Schema-only: this struct exists to describe the multipart shape in
+ OpenAPI. At runtime, the endpoint reads the string fields from
+ ``MultipartRequest.body`` and the uploaded bytes from
+ ``MultipartRequest.files["file"]`` — ``body.file`` is never populated.
+ """
# The path where to create the file or directory
path: str
@@ -66,8 +72,8 @@ class FileCreateMultipartRequest(msgspec.Struct, rename="camel"):
type: FileCreateType
# The name of the file or directory
name: str
- # The raw file bytes (optional). When omitted, an empty file is created
- # (or, for 'notebook' type, a default notebook template).
+ # OpenAPI-only (see class docstring). `format: binary` makes the
+ # generated spec emit a file-upload schema rather than a base64 string.
file: Annotated[
str | None, msgspec.Meta(extra_json_schema={"format": "binary"})
] = None
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 66ba981d5c..556cc6366d 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -1690,7 +1690,11 @@ components:
title: FileCopyResponse
type: object
FileCreateMultipartRequest:
- description: multipart/form-data body for POST /api/files/create.
+ description: "multipart/form-data body for POST /api/files/create.\n\n Schema-only:\
+ \ this struct exists to describe the multipart shape in\n OpenAPI. At runtime,\
+ \ the endpoint reads the string fields from\n ``MultipartRequest.body``\
+ \ and the uploaded bytes from\n ``MultipartRequest.files[\"file\"]`` \u2014\
+ \ ``body.file`` is never populated."
properties:
file:
anyOf:
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index d7fbe124d9..8085de21a0 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4378,6 +4378,11 @@ export interface components {
/**
* FileCreateMultipartRequest
* @description multipart/form-data body for POST /api/files/create.
+ *
+ * Schema-only: this struct exists to describe the multipart shape in
+ * OpenAPI. At runtime, the endpoint reads the string fields from
+ * ``MultipartRequest.body`` and the uploaded bytes from
+ * ``MultipartRequest.files["file"]`` — ``body.file`` is never populated.
*/
FileCreateMultipartRequest: {
/**
diff --git a/tests/_server/api/endpoints/test_file_explorer.py b/tests/_server/api/endpoints/test_file_explorer.py
index e64b4b78ff..f54869bbca 100644
--- a/tests/_server/api/endpoints/test_file_explorer.py
+++ b/tests/_server/api/endpoints/test_file_explorer.py
@@ -179,6 +179,27 @@ def test_create_rejects_path_traversal_name(client: TestClient) -> None:
assert not os.path.exists(parent_path)
+def test_create_returns_413_when_upload_exceeds_size_cap(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # Shrink the cap so a small payload trips it.
+ monkeypatch.setattr(
+ "marimo._server.files.os_file_system.MAX_UPLOAD_BYTES", 4
+ )
+ response = client.post(
+ "/api/files/create",
+ headers=HEADERS,
+ data={
+ "path": test_dir,
+ "type": "file",
+ "name": "too_big.bin",
+ },
+ files={"file": ("too_big.bin", b"way too much data")},
+ )
+ assert response.status_code == 413, response.text
+ assert not os.path.exists(os.path.join(test_dir, "too_big.bin"))
+
+
def test_update_file(client: TestClient) -> None:
response = client.post(
"/api/files/update",
diff --git a/tests/_server/api/test_api_utils.py b/tests/_server/api/test_api_utils.py
index 5f4a09a4e6..56e009e543 100644
--- a/tests/_server/api/test_api_utils.py
+++ b/tests/_server/api/test_api_utils.py
@@ -23,16 +23,19 @@ class _SampleForm(msgspec.Struct):
def _build_app(captured: dict[str, object]) -> TestClient:
async def endpoint(request: Request) -> JSONResponse:
- parsed = await parse_multipart_request(request, _SampleForm)
- captured["body"] = parsed.body
- captured["files"] = parsed.files
+ async with parse_multipart_request(request, _SampleForm) as parsed:
+ captured["body"] = parsed.body
+ captured["files"] = dict(parsed.files)
+ upload = parsed.files.get("upload")
+ if upload is not None:
+ captured["upload_bytes"] = await upload.read()
return JSONResponse({"ok": True})
app = Starlette(routes=[Route("/test", endpoint, methods=["POST"])])
return TestClient(app)
-def test_parse_multipart_request_strings_and_file_bytes() -> None:
+def test_parse_multipart_request_strings_and_file_upload() -> None:
captured: dict[str, object] = {}
client = _build_app(captured)
response = client.post(
@@ -45,7 +48,10 @@ def test_parse_multipart_request_strings_and_file_bytes() -> None:
assert isinstance(body, _SampleForm)
assert body.name == "marimo"
assert body.count == 42
- assert captured["files"] == {"upload": b"\x00\x01\x02\xff"}
+ files = captured["files"]
+ assert isinstance(files, dict)
+ assert set(files.keys()) == {"upload"}
+ assert captured["upload_bytes"] == b"\x00\x01\x02\xff"
def test_parse_multipart_request_omitted_file_yields_empty_dict() -> None:
diff --git a/tests/_server/files/test_os_file_system.py b/tests/_server/files/test_os_file_system.py
index 9b7f0e508d..543fe90893 100644
--- a/tests/_server/files/test_os_file_system.py
+++ b/tests/_server/files/test_os_file_system.py
@@ -8,6 +8,8 @@ import pytest
from marimo._server.files.os_file_system import (
OSFileSystem,
+ UploadTooLargeError,
+ _claim_unique_path,
_generate_unique_path,
_is_allowed_paths,
)
@@ -133,6 +135,156 @@ def test_create_rejects_path_traversal(
assert not (test_dir.parent / "escaped.txt").exists()
+class _ChunkedSource:
+ """Async byte source emitting predetermined chunks; optionally raises
+ after `fail_after` reads to simulate mid-stream failures."""
+
+ def __init__(
+ self, chunks: list[bytes], *, fail_after: int | None = None
+ ) -> None:
+ self._chunks = list(chunks)
+ self._fail_after = fail_after
+ self._reads = 0
+
+ async def read(self, size: int = -1, /) -> bytes:
+ if self._fail_after is not None and self._reads >= self._fail_after:
+ raise RuntimeError("stream failed")
+ if not self._chunks:
+ return b""
+ chunk = self._chunks[0]
+ if size < 0 or len(chunk) <= size:
+ out = self._chunks.pop(0)
+ else:
+ out, self._chunks[0] = chunk[:size], chunk[size:]
+ self._reads += 1
+ return out
+
+
+def _part_files(directory: Path) -> list[Path]:
+ return list(directory.glob("*.part"))
+
+
+async def test_stream_create_file_writes_chunks(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ source = _ChunkedSource([b"hello ", b"streamed ", b"world"])
+ info = await fs.stream_create_file(str(test_dir), "out.bin", source)
+ out_path = test_dir / "out.bin"
+ assert out_path.read_bytes() == b"hello streamed world"
+ assert info.path == str(out_path)
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_writes_empty_file(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ info = await fs.stream_create_file(
+ str(test_dir), "empty.bin", _ChunkedSource([])
+ )
+ out_path = test_dir / "empty.bin"
+ assert out_path.exists()
+ assert out_path.read_bytes() == b""
+ assert info.path == str(out_path)
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_generates_unique_path(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ (test_dir / "dup.bin").write_bytes(b"existing")
+ info = await fs.stream_create_file(
+ str(test_dir), "dup.bin", _ChunkedSource([b"new"])
+ )
+ assert info.path == str(test_dir / "dup_1.bin")
+ assert (test_dir / "dup.bin").read_bytes() == b"existing"
+ assert (test_dir / "dup_1.bin").read_bytes() == b"new"
+
+
+ "name",
+ [
+ "../escape.bin",
+ "../../escape.bin",
+ "subdir/nested.bin",
+ "..",
+ ".",
+ "with\\backslash.bin",
+ "embed\x00null.bin",
+ "",
+ " ",
+ ],
+)
+async def test_stream_create_file_rejects_traversal(
+ test_dir: Path, fs: OSFileSystem, name: str
+) -> None:
+ with pytest.raises(ValueError):
+ await fs.stream_create_file(
+ str(test_dir), name, _ChunkedSource([b"x"])
+ )
+ assert not (test_dir.parent / "escape.bin").exists()
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_cleans_up_on_immediate_failure(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ source = _ChunkedSource([], fail_after=0)
+ with pytest.raises(RuntimeError):
+ await fs.stream_create_file(str(test_dir), "no_file.bin", source)
+ assert not (test_dir / "no_file.bin").exists()
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_cleans_up_on_mid_stream_failure(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ # Source fails after writes have already landed in the .part file.
+ source = _ChunkedSource([b"first ", b"second ", b"third"], fail_after=2)
+ with pytest.raises(RuntimeError):
+ await fs.stream_create_file(str(test_dir), "partial.bin", source)
+ assert not (test_dir / "partial.bin").exists()
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_enforces_size_cap(
+ test_dir: Path, fs: OSFileSystem, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # Shrink the cap so the test is fast; the production value is 1 GiB.
+ monkeypatch.setattr(
+ "marimo._server.files.os_file_system.MAX_UPLOAD_BYTES", 8
+ )
+ source = _ChunkedSource([b"aaaa", b"bbbb", b"too much"])
+ with pytest.raises(UploadTooLargeError, match="exceeds maximum size"):
+ await fs.stream_create_file(str(test_dir), "big.bin", source)
+ assert not (test_dir / "big.bin").exists()
+ assert _part_files(test_dir) == []
+
+
+async def test_stream_create_file_claims_path_atomically(
+ test_dir: Path, fs: OSFileSystem
+) -> None:
+ # Concurrent file at the target name must not be clobbered by the
+ # streaming write — the atomic claim should bump to a new suffix.
+ (test_dir / "race.bin").write_bytes(b"existing")
+ info = await fs.stream_create_file(
+ str(test_dir), "race.bin", _ChunkedSource([b"new"])
+ )
+ assert info.path == str(test_dir / "race_1.bin")
+ assert (test_dir / "race.bin").read_bytes() == b"existing"
+ assert (test_dir / "race_1.bin").read_bytes() == b"new"
+
+
+def test_claim_unique_path_atomic_reservation(test_dir: Path) -> None:
+ claimed_a = _claim_unique_path(test_dir / "claim.bin")
+ assert claimed_a == test_dir / "claim.bin"
+ assert claimed_a.exists()
+ # An existing (even empty) reservation forces the next caller off the
+ # base name — this is what closes the TOCTOU window.
+ claimed_b = _claim_unique_path(test_dir / "claim.bin")
+ assert claimed_b == test_dir / "claim_1.bin"
+ assert claimed_b.exists()
+
+
def test_list_files(test_dir: Path, fs: OSFileSystem) -> None:
test_create_file(test_dir, fs)
test_create_directory(test_dir, fs)