A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
enhancement: Stream virtual files instead of buffering entirely in memory (#8499)
Myles Scolnick1198d4a4ba72
diff --git a/marimo/_runtime/virtual_file/__init__.py b/marimo/_runtime/virtual_file/__init__.pyindex fcdf1a3d3c..4a9b3bfffc 100644--- a/marimo/_runtime/virtual_file/__init__.py+++ b/marimo/_runtime/virtual_file/__init__.py@@ -21,6 +21,7 @@ from marimo._runtime.virtual_file.virtual_file import (VirtualFileRegistryItem,random_filename,read_virtual_file,+ read_virtual_file_chunked,)__all__ = [@@ -37,4 +38,5 @@ __all__ = ["VirtualFileRegistry","random_filename","read_virtual_file",+ "read_virtual_file_chunked",]diff --git a/marimo/_runtime/virtual_file/storage.py b/marimo/_runtime/virtual_file/storage.pyindex d87d6a2dea..6fd7a653e9 100644--- a/marimo/_runtime/virtual_file/storage.py+++ b/marimo/_runtime/virtual_file/storage.py@@ -7,7 +7,9 @@ from typing import TYPE_CHECKING, Protocolfrom marimo._utils.platform import is_pyodideif TYPE_CHECKING:- from collections.abc import Iterable+ from collections.abc import Iterable, Iterator++DEFAULT_CHUNK_SIZE = 256 * 1024 # 256KBif not is_pyodide():# the shared_memory module is not supported in the Pyodide distribution@@ -29,6 +31,22 @@ class VirtualFileStorage(Protocol):"""...+ def read_chunked(+ self,+ key: str,+ byte_length: int,+ chunk_size: int = DEFAULT_CHUNK_SIZE,+ ) -> Iterator[bytes]:+ """Read buffer data by key in chunks.++ Yields chunks of bytes, avoiding allocating the full buffer at once.+ Useful for streaming large files over HTTP.++ Raises:+ KeyError: If key not found+ """+ ...+def remove(self, key: str) -> None:"""Remove stored data by key."""...@@ -110,7 +128,10 @@ class SharedMemoryStorage(VirtualFileStorage):shm = Nonetry:shm = shared_memory.SharedMemory(name=key)- buffer_contents = bytes(shm.buf)[:byte_length]+ # Slice the memoryview first, then copy — avoids allocating+ # a bytes object for the entire buffer when only a prefix+ # is needed.+ buffer_contents = bytes(shm.buf[:byte_length])except FileNotFoundError as err:raise KeyError(f"Virtual file not found: {key}") from errfinally:@@ -118,6 +139,33 @@ class SharedMemoryStorage(VirtualFileStorage):shm.close()return buffer_contents+ def read_chunked(+ self,+ key: str,+ byte_length: int,+ chunk_size: int = DEFAULT_CHUNK_SIZE,+ ) -> Iterator[bytes]:+ if is_pyodide():+ raise RuntimeError(+ "Shared memory is not supported on this platform"+ )+ shm = None+ view = None+ try:+ shm = shared_memory.SharedMemory(name=key)+ view = shm.buf[:byte_length]+ for i in range(0, byte_length, chunk_size):+ yield bytes(view[i : i + chunk_size])+ except FileNotFoundError as err:+ raise KeyError(f"Virtual file not found: {key}") from err+ finally:+ # Release the memoryview before closing the shared memory,+ # otherwise close() fails with "cannot close exported pointers".+ if view is not None:+ view.release()+ if shm is not None:+ shm.close()+def remove(self, key: str) -> None:if key in self._storage:if sys.platform == "win32":@@ -165,6 +213,19 @@ class InMemoryStorage(VirtualFileStorage):raise KeyError(f"Virtual file not found: {key}")return self._storage[key][:byte_length]+ def read_chunked(+ self,+ key: str,+ byte_length: int,+ chunk_size: int = DEFAULT_CHUNK_SIZE,+ ) -> Iterator[bytes]:+ if key not in self._storage:+ raise KeyError(f"Virtual file not found: {key}")+ buffer = self._storage[key]+ end = min(byte_length, len(buffer))+ for i in range(0, end, chunk_size):+ yield buffer[i : min(i + chunk_size, end)]+def remove(self, key: str) -> None:if key in self._storage:del self._storage[key]@@ -214,3 +275,26 @@ class VirtualFileStorageManager:# Use SharedMemoryStorage to read by name across processesreturn SharedMemoryStorage().read(filename, byte_length)return storage.read(filename, byte_length)++ def read_chunked(+ self,+ filename: str,+ byte_length: int,+ chunk_size: int = DEFAULT_CHUNK_SIZE,+ ) -> Iterator[bytes]:+ """Read from storage in chunks, with cross-process fallback.++ Yields chunks of bytes for streaming. Avoids holding the entire+ file in memory as a single bytes object.++ Raises:+ KeyError: If file not found+ RuntimeError: When ``SharedMemoryStorage`` is used on the Pyodide platform.+ """+ storage = self.storage+ if storage is None:+ yield from SharedMemoryStorage().read_chunked(+ filename, byte_length, chunk_size+ )+ else:+ yield from storage.read_chunked(filename, byte_length, chunk_size)diff --git a/marimo/_runtime/virtual_file/virtual_file.py b/marimo/_runtime/virtual_file/virtual_file.pyindex f2420b5e7f..5f989eb0b7 100644--- a/marimo/_runtime/virtual_file/virtual_file.py+++ b/marimo/_runtime/virtual_file/virtual_file.py@@ -9,6 +9,9 @@ import stringimport threadingfrom typing import TYPE_CHECKING, Optional, cast+if TYPE_CHECKING:+ from collections.abc import Iterator+from marimo import _loggersfrom marimo._messaging.mimetypes import KnownMimeTypefrom marimo._runtime.cell_lifecycle_item import CellLifecycleItem@@ -304,3 +307,22 @@ def read_virtual_file(filename: str, byte_length: int) -> bytes:HTTPStatus.NOT_FOUND,detail="File not found",) from err+++def read_virtual_file_chunked(+ filename: str, byte_length: int+) -> Iterator[bytes]:+ """Read a virtual file in chunks for streaming responses.++ Yields chunks of bytes, avoiding holding the entire file in memory+ as a single bytes object.+ """+ try:+ yield from VirtualFileStorageManager().read_chunked(+ filename, byte_length+ )+ except KeyError as err:+ raise HTTPException(+ HTTPStatus.NOT_FOUND,+ detail="File not found",+ ) from errdiff --git a/marimo/_server/api/endpoints/assets.py b/marimo/_server/api/endpoints/assets.pyindex 46b9e8096e..48e9ec0bf9 100644--- a/marimo/_server/api/endpoints/assets.py+++ b/marimo/_server/api/endpoints/assets.py@@ -13,6 +13,7 @@ from starlette.responses import (HTMLResponse,RedirectResponse,Response,+ StreamingResponse,)from starlette.staticfiles import StaticFiles@@ -20,7 +21,10 @@ from marimo import _loggersfrom marimo._cli.sandbox import SandboxModefrom marimo._config.manager import get_default_config_managerfrom marimo._output.utils import uri_decode_component, uri_encode_component-from marimo._runtime.virtual_file import EMPTY_VIRTUAL_FILE, read_virtual_file+from marimo._runtime.virtual_file import (+ EMPTY_VIRTUAL_FILE,+ read_virtual_file_chunked,+)from marimo._server.api.deps import AppStatefrom marimo._server.files.path_validator import PathValidatorfrom marimo._server.router import APIRouter@@ -390,12 +394,15 @@ def virtual_file(detail="Invalid byte length in virtual file request",)- buffer_contents = read_virtual_file(filename, int(byte_length))+ chunks = read_virtual_file_chunked(filename, int(byte_length))mimetype, _ = mimetypes.guess_type(filename)- return Response(- content=buffer_contents,+ return StreamingResponse(+ content=chunks,media_type=mimetype,- headers={"Cache-Control": "max-age=86400"},+ headers={+ "Cache-Control": "max-age=86400",+ "Content-Length": byte_length,+ },)diff --git a/tests/_runtime/test_storage.py b/tests/_runtime/test_storage.pyindex 792cb334bf..0fa1c784f4 100644--- a/tests/_runtime/test_storage.py+++ b/tests/_runtime/test_storage.py@@ -10,6 +10,49 @@ from marimo._runtime.virtual_file.storage import ()+class TestInMemoryStorageReadChunked:+ def test_read_chunked_basic(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"hello world")+ chunks = list(storage.read_chunked("test_key", 11))+ assert b"".join(chunks) == b"hello world"++ def test_read_chunked_with_byte_length(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"hello world")+ chunks = list(storage.read_chunked("test_key", 5))+ assert b"".join(chunks) == b"hello"++ def test_read_chunked_multiple_chunks(self) -> None:+ storage = InMemoryStorage()+ data = b"a" * 100+ storage.store("test_key", data)+ # Use a small chunk size to force multiple chunks+ chunks = list(storage.read_chunked("test_key", 100, chunk_size=30))+ assert b"".join(chunks) == data+ assert len(chunks) == 4 # 30 + 30 + 30 + 10++ def test_read_chunked_nonexistent_raises_keyerror(self) -> None:+ storage = InMemoryStorage()+ with pytest.raises(KeyError, match="Virtual file not found"):+ list(storage.read_chunked("nonexistent", 10))++ def test_read_chunked_chunk_sizes(self) -> None:+ """Verify each chunk is at most chunk_size bytes."""+ storage = InMemoryStorage()+ data = b"x" * 250+ storage.store("test_key", data)+ chunk_size = 64+ chunks = list(+ storage.read_chunked("test_key", 250, chunk_size=chunk_size)+ )+ for chunk in chunks[:-1]:+ assert len(chunk) == chunk_size+ # Last chunk may be smaller+ assert len(chunks[-1]) <= chunk_size+ assert b"".join(chunks) == data++class TestInMemoryStorage:def test_store_and_read(self) -> None:storage = InMemoryStorage()@@ -156,6 +199,77 @@ class TestSharedMemoryStorage:storage.shutdown()storage.shutdown() # Should not raise+ def test_read_chunked_basic(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_chunk_1", b"hello world")+ chunks = list(storage.read_chunked("marimo_chunk_1", 11))+ assert b"".join(chunks) == b"hello world"+ finally:+ storage.shutdown()++ def test_read_chunked_with_byte_length(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_chunk_2", b"hello world")+ chunks = list(storage.read_chunked("marimo_chunk_2", 5))+ assert b"".join(chunks) == b"hello"+ finally:+ storage.shutdown()++ def test_read_chunked_multiple_chunks(self) -> None:+ storage = SharedMemoryStorage()+ try:+ data = b"a" * 100+ storage.store("marimo_chunk_3", data)+ chunks = list(+ storage.read_chunked("marimo_chunk_3", 100, chunk_size=30)+ )+ assert b"".join(chunks) == data+ assert len(chunks) == 4 # 30 + 30 + 30 + 10+ finally:+ storage.shutdown()++ def test_read_chunked_nonexistent_raises_keyerror(self) -> None:+ storage = SharedMemoryStorage()+ try:+ with pytest.raises(KeyError, match="Virtual file not found"):+ list(storage.read_chunked("nonexistent_chunk", 10))+ finally:+ storage.shutdown()++ def test_read_chunked_cross_process(self) -> None:+ """Test chunked read works across fresh instances."""+ storage1 = SharedMemoryStorage()+ try:+ data = b"cross process chunked"+ storage1.store("marimo_chunk_cross", data)+ storage2 = SharedMemoryStorage()+ chunks = list(+ storage2.read_chunked(+ "marimo_chunk_cross", len(data), chunk_size=5+ )+ )+ assert b"".join(chunks) == data+ finally:+ storage1.shutdown()++ def test_read_chunked_data_integrity(self) -> None:+ """Test that chunked read produces identical data to regular read."""+ storage = SharedMemoryStorage()+ try:+ data = bytes(range(256)) * 4 # 1024 bytes of varied data+ storage.store("marimo_chunk_integ", data)+ regular = storage.read("marimo_chunk_integ", len(data))+ chunked = b"".join(+ storage.read_chunked(+ "marimo_chunk_integ", len(data), chunk_size=100+ )+ )+ assert regular == chunked == data+ finally:+ storage.shutdown()+class TestVirtualFileStorageManager:def test_singleton(self) -> None:@@ -199,3 +313,31 @@ class TestVirtualFileStorageManager:finally:manager.storage = original_storageshm_storage.shutdown()++ def test_read_chunked_with_storage(self) -> None:+ manager = VirtualFileStorageManager()+ original_storage = manager.storage+ try:+ storage = InMemoryStorage()+ storage.store("test_file", b"test data chunked")+ manager.storage = storage+ chunks = list(manager.read_chunked("test_file", 17))+ assert b"".join(chunks) == b"test data chunked"+ finally:+ manager.storage = original_storage++ def test_read_chunked_falls_back_to_shared_memory(self) -> None:+ manager = VirtualFileStorageManager()+ original_storage = manager.storage+ shm_storage = SharedMemoryStorage()+ try:+ shm_storage.store("marimo_fb_chunk", b"fallback chunked")+ manager.storage = None+ chunks = list(+ manager.read_chunked("marimo_fb_chunk", 16, chunk_size=4)+ )+ assert b"".join(chunks) == b"fallback chunked"+ assert len(chunks) == 4 # 16 / 4 = 4 chunks+ finally:+ manager.storage = original_storage+ shm_storage.shutdown()