A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: do not use shared memory in run case (#7637)
Dylan Madisetti797c77994bdb
diff --git a/marimo/_convert/markdown/__init__.py b/marimo/_convert/markdown/__init__.pyindex 5e078ad009..236d950d01 100644--- a/marimo/_convert/markdown/__init__.py+++ b/marimo/_convert/markdown/__init__.py@@ -1,3 +1,4 @@+# Copyright 2026 Marimo. All rights reserved.from __future__ import annotationsimport astdiff --git a/marimo/_convert/script.py b/marimo/_convert/script.pyindex 31c79d1b9f..841aac0e6a 100644--- a/marimo/_convert/script.py+++ b/marimo/_convert/script.py@@ -1,3 +1,4 @@+# Copyright 2026 Marimo. All rights reserved.from __future__ import annotationsfrom marimo._ast import codegendiff --git a/marimo/_runtime/context/kernel_context.py b/marimo/_runtime/context/kernel_context.pyindex f1a6a59558..63368cf919 100644--- a/marimo/_runtime/context/kernel_context.py+++ b/marimo/_runtime/context/kernel_context.py@@ -150,9 +150,21 @@ def create_kernel_context() -> KernelRuntimeContext:from marimo._plugins.ui._core.registry import UIElementRegistryfrom marimo._runtime.state import StateRegistry- from marimo._runtime.virtual_file import VirtualFileRegistry+ from marimo._runtime.virtual_file import (+ InMemoryStorage,+ SharedMemoryStorage,+ VirtualFileRegistry,+ )from marimo._save.stores import get_store+ # Use shared memory in edit mode,+ # in-memory storage in run mode (same process)+ storage = (+ SharedMemoryStorage()+ if mode == SessionMode.EDIT+ else InMemoryStorage()+ )+return KernelRuntimeContext(_kernel=kernel,_session_mode=mode,@@ -163,7 +175,7 @@ def create_kernel_context(cache_store=get_store(kernel.app_metadata.filename),cell_lifecycle_registry=CellLifecycleRegistry(),app_kernel_runner_registry=AppKernelRunnerRegistry(),- virtual_file_registry=VirtualFileRegistry(),+ virtual_file_registry=VirtualFileRegistry(storage=storage),virtual_files_supported=virtual_files_supported,stream=stream,stdout=stdout,diff --git a/marimo/_runtime/context/script_context.py b/marimo/_runtime/context/script_context.pyindex 5994ef6c49..12537c8d67 100644--- a/marimo/_runtime/context/script_context.py+++ b/marimo/_runtime/context/script_context.py@@ -146,7 +146,10 @@ def initialize_script_context(Must be called exactly once for each client thread."""- from marimo._runtime.virtual_file import VirtualFileRegistry+ from marimo._runtime.virtual_file import (+ InMemoryStorage,+ VirtualFileRegistry,+ )from marimo._save.stores import get_storeruntime_context = ScriptRuntimeContext(@@ -157,7 +160,7 @@ def initialize_script_context(cache_store=get_store(filename),cell_lifecycle_registry=CellLifecycleRegistry(),app_kernel_runner_registry=AppKernelRunnerRegistry(),- virtual_file_registry=VirtualFileRegistry(),+ virtual_file_registry=VirtualFileRegistry(storage=InMemoryStorage()),virtual_files_supported=False,stream=stream,stdout=None,diff --git a/marimo/_runtime/virtual_file/__init__.py b/marimo/_runtime/virtual_file/__init__.pynew file mode 100644index 0000000000..fcdf1a3d3c--- /dev/null+++ b/marimo/_runtime/virtual_file/__init__.py@@ -0,0 +1,40 @@+# Copyright 2026 Marimo. All rights reserved.+"""Virtual file storage and management for marimo.++This module provides storage backends and file management for virtual files+created during notebook execution.+"""++from __future__ import annotations++from marimo._runtime.virtual_file.storage import (+ InMemoryStorage,+ SharedMemoryStorage,+ VirtualFileStorage,+ VirtualFileStorageManager,+)+from marimo._runtime.virtual_file.virtual_file import (+ EMPTY_VIRTUAL_FILE,+ VirtualFile,+ VirtualFileLifecycleItem,+ VirtualFileRegistry,+ VirtualFileRegistryItem,+ random_filename,+ read_virtual_file,+)++__all__ = [+ # Storage+ "VirtualFileStorage",+ "SharedMemoryStorage",+ "InMemoryStorage",+ "VirtualFileStorageManager",+ # Virtual files+ "VirtualFile",+ "EMPTY_VIRTUAL_FILE",+ "VirtualFileLifecycleItem",+ "VirtualFileRegistryItem",+ "VirtualFileRegistry",+ "random_filename",+ "read_virtual_file",+]diff --git a/marimo/_runtime/virtual_file/storage.py b/marimo/_runtime/virtual_file/storage.pynew file mode 100644index 0000000000..95cc556d3c--- /dev/null+++ b/marimo/_runtime/virtual_file/storage.py@@ -0,0 +1,190 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import sys+from typing import Protocol++from marimo._utils.platform import is_pyodide++if not is_pyodide():+ # the shared_memory module is not supported in the Pyodide distribution+ from multiprocessing import shared_memory+++class VirtualFileStorage(Protocol):+ """Protocol for virtual file storage backends."""++ def store(self, key: str, buffer: bytes) -> None:+ """Store buffer data by key."""+ ...++ def read(self, key: str, byte_length: int) -> bytes:+ """Read buffer data by key.++ Raises:+ KeyError: If key not found+ """+ ...++ def remove(self, key: str) -> None:+ """Remove stored data by key."""+ ...++ def shutdown(self) -> None:+ """Clean up all storage resources."""+ ...++ def has(self, key: str) -> bool:+ """Check if key exists in storage."""+ ...+++class SharedMemoryStorage(VirtualFileStorage):+ """Storage backend using multiprocessing shared memory.++ Used in `edit` mode when kernel runs in a separate process.+ """++ def __init__(self) -> None:+ if is_pyodide():+ raise RuntimeError(+ "SharedMemoryStorage is not supported on Pyodide"+ )+ self._storage: dict[str, shared_memory.SharedMemory] = {}+ self._shutting_down = False++ def store(self, key: str, buffer: bytes) -> None:+ if key in self._storage:+ return # Already stored++ # Immediately writes the contents of the file to an in-memory+ # buffer; not lazy.+ #+ # To retrieve the buffer from another process, use:+ #+ # ```+ # try:+ # shm = shared_memory.SharedMemory(name=key)+ # buffer_contents = bytes(shm.buf)+ # except FileNotFoundError:+ # # virtual file was removed+ # ```+ shm = shared_memory.SharedMemory(+ name=key,+ create=True,+ size=len(buffer),+ )+ shm.buf[: len(buffer)] = buffer+ # we can safely close this shm, since we don't need to access its+ # buffer; we do need to keep it around so we can unlink it later+ if sys.platform != "win32":+ # don't call close() on Windows, due to a bug in the Windows+ # Python implementation. On Windows, close() actually unlinks+ # (destroys) the shared_memory:+ # https://stackoverflow.com/questions/63713241/segmentation-fault-using-python-shared-memory/63717188#63717188+ shm.close()+ # We have to keep a reference to the shared memory to prevent it from+ # being destroyed on Windows+ self._storage[key] = shm++ def read(self, key: str, byte_length: int) -> bytes:+ if is_pyodide():+ raise RuntimeError(+ "Shared memory is not supported on this platform"+ )+ # Read from shared memory by name (works cross-process)+ shm = None+ try:+ shm = shared_memory.SharedMemory(name=key)+ buffer_contents = bytes(shm.buf)[:byte_length]+ except FileNotFoundError as err:+ raise KeyError(f"Virtual file not found: {key}") from err+ finally:+ if shm is not None:+ shm.close()+ return buffer_contents++ def remove(self, key: str) -> None:+ if key in self._storage:+ if sys.platform == "win32":+ self._storage[key].close()+ self._storage[key].unlink()+ del self._storage[key]++ def shutdown(self) -> None:+ if self._shutting_down:+ return+ try:+ self._shutting_down = True+ for shm in self._storage.values():+ if sys.platform == "win32":+ shm.close()+ shm.unlink()+ self._storage.clear()+ finally:+ self._shutting_down = False++ def has(self, key: str) -> bool:+ return key in self._storage+++class InMemoryStorage(VirtualFileStorage):+ """Storage backend using simple in-memory dictionary.++ Used in `run` mode when kernel runs in the same process as the server.+ """++ def __init__(self) -> None:+ self._storage: dict[str, bytes] = {}++ def store(self, key: str, buffer: bytes) -> None:+ self._storage[key] = buffer++ def read(self, key: str, byte_length: int) -> bytes:+ if key not in self._storage:+ raise KeyError(f"Virtual file not found: {key}")+ return self._storage[key][:byte_length]++ def remove(self, key: str) -> None:+ if key in self._storage:+ del self._storage[key]++ def shutdown(self) -> None:+ self._storage.clear()++ def has(self, key: str) -> bool:+ return key in self._storage+++class VirtualFileStorageManager:+ """Singleton manager for virtual file storage access."""++ _instance: VirtualFileStorageManager | None = None+ _storage: VirtualFileStorage | None = None++ def __new__(cls) -> VirtualFileStorageManager:+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ return cls._instance++ @property+ def storage(self) -> VirtualFileStorage | None:+ return self._storage++ @storage.setter+ def storage(self, value: VirtualFileStorage | None) -> None:+ self._storage = value++ def read(self, filename: str, byte_length: int) -> bytes:+ """Read from storage, with cross-process fallback for EDIT mode server.++ Raises:+ KeyError: If file not found+ RuntimeError: When ``SharedMemoryStorage`` is used on the Pyodide platform.+ """+ storage = self.storage+ if storage is None:+ # Never initialized so in a separate thread from the kernel.+ # Use SharedMemoryStorage to read by name across processes+ return SharedMemoryStorage().read(filename, byte_length)+ return storage.read(filename, byte_length)diff --git a/marimo/_runtime/virtual_file.py b/marimo/_runtime/virtual_file/virtual_file.pysimilarity index 75%rename from marimo/_runtime/virtual_file.pyrename to marimo/_runtime/virtual_file/virtual_file.pyindex 26ae0f15f3..72e56645f4 100644--- a/marimo/_runtime/virtual_file.py+++ b/marimo/_runtime/virtual_file/virtual_file.py@@ -6,7 +6,6 @@ import dataclassesimport mimetypesimport randomimport string-import sysimport threadingfrom typing import TYPE_CHECKING, Optional, cast@@ -14,9 +13,12 @@ from marimo import _loggersfrom marimo._messaging.mimetypes import KnownMimeTypefrom marimo._runtime.cell_lifecycle_item import CellLifecycleItemfrom marimo._runtime.context import ContextNotInitializedError+from marimo._runtime.virtual_file.storage import (+ VirtualFileStorage,+ VirtualFileStorageManager,+)from marimo._utils.data_uri import build_data_urlfrom marimo._utils.http import HTTPException, HTTPStatus-from marimo._utils.platform import is_pyodideif TYPE_CHECKING:from collections.abc import Iterable@@ -25,11 +27,6 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()-if not is_pyodide():- # the shared_memory module is not supported in the Pyodide distribution- from multiprocessing import shared_memory--_ALPHABET = string.ascii_letters + string.digits@@ -161,8 +158,6 @@ class VirtualFileLifecycleItem(CellLifecycleItem):@dataclasses.dataclassclass VirtualFileRegistryItem:- # contents of the file- shm: shared_memory.SharedMemory# number of HTML objects that are referencing this virtual filerefcount: int@@ -179,10 +174,15 @@ class VirtualFileRegistry:exposes methods for incrementing, decrementing, and getting the counts."""+ storage: VirtualFileStorageregistry: dict[str, VirtualFileRegistryItem] = dataclasses.field(default_factory=dict)- shutting_down = False+ shutting_down: bool = False++ def __post_init__(self) -> None:+ # Set singleton reference for read_virtual_file()+ VirtualFileStorageManager().storage = self.storagedef __del__(self) -> None:self.shutdown()@@ -224,43 +224,14 @@ class VirtualFileRegistry:# Skip adding if buffer is emptyif len(buffer) == 0:return- # Immediately writes the contents of the file to an in-memory- # buffer; not lazy.- #- # To retrieve the buffer from another process, use:- #- # ```- # try:- # shm = shared_memory.SharedMemory(name=key)- # buffer_contents = bytes(shm.buf)- # except FileNotFoundError:- # # virtual file was removed- # ```- shm = shared_memory.SharedMemory(- name=key,- create=True,- size=len(buffer),- )- shm.buf[: len(buffer)] = buffer- # we can safely close this shm, since we don't need to access its- # buffer; we do need to keep it around so we can unlink it later- if sys.platform != "win32":- # don't call close() on Windows, due to a bug in the Windows- # Python implementation. On Windows, close() actually unlinks- # (destroys) the shared_memory:- # https://stackoverflow.com/questions/63713241/segmentation-fault-using-python-shared-memory/63717188#63717188- shm.close()- # We have to keep a reference to the shared memory to prevent it from- # being destroyed on Windows- self.registry[key] = VirtualFileRegistryItem(shm=shm, refcount=0)++ self.storage.store(key, buffer)+ self.registry[key] = VirtualFileRegistryItem(refcount=0)def remove(self, virtual_file: VirtualFile) -> None:key = virtual_file.filenameif key in self.registry:- if sys.platform == "win32":- self.registry[key].shm.close()- # destroy the shared memory- self.registry[key].shm.unlink()+ self.storage.remove(key)del self.registry[key]def shutdown(self) -> None:@@ -272,10 +243,7 @@ class VirtualFileRegistry:returntry:self.shutting_down = True- for item in self.registry.values():- if sys.platform == "win32":- item.shm.close()- item.shm.unlink()+ self.storage.shutdown()self.registry.clear()finally:self.shutting_down = False@@ -286,27 +254,10 @@ def _without_leading_dot(ext: str) -> str:def read_virtual_file(filename: str, byte_length: int) -> bytes:- if not shared_memory:- raise RuntimeError("Shared memory is not supported on this platform")-- key = filename- shm = Nonetry:- # NB: this can't be collapsed into a one-liner!- # doing it in one line yields a 'released memoryview ...'- # because shared_memory has built in ref-tracking + GC- shm = shared_memory.SharedMemory(name=key)- buffer_contents = bytes(shm.buf)[: int(byte_length)]- except FileNotFoundError as err:- LOGGER.debug(- "Error retrieving shared memory for virtual file: %s", err- )+ return VirtualFileStorageManager().read(filename, byte_length)+ except KeyError as err:raise HTTPException(HTTPStatus.NOT_FOUND,detail="File not found",) from err- finally:- if shm is not None:- shm.close()-- return buffer_contentsdiff --git a/marimo/_server/config.py b/marimo/_server/config.pyindex 5c4c347600..e16a326c87 100644--- a/marimo/_server/config.py+++ b/marimo/_server/config.py@@ -1,3 +1,4 @@+# Copyright 2026 Marimo. All rights reserved.from __future__ import annotationsfrom dataclasses import dataclassdiff --git a/marimo/_smoke_tests/pdf.py b/marimo/_smoke_tests/pdf.pyindex b185a6deaa..abb0a0d353 100644--- a/marimo/_smoke_tests/pdf.py+++ b/marimo/_smoke_tests/pdf.py@@ -2,7 +2,7 @@import marimo-__generated_with = "0.15.5"+__generated_with = "0.18.4"app = marimo.App(width="full")@@ -16,7 +16,9 @@ def _():@app.celldef _(mo):- mo.md("# PDFs")+ mo.md("""+ # PDFs+ """)returndiff --git a/marimo/_utils/print.py b/marimo/_utils/print.pyindex e7499944cd..f07c80ffc9 100644--- a/marimo/_utils/print.py+++ b/marimo/_utils/print.py@@ -1,3 +1,4 @@+# Copyright 2026 Marimo. All rights reserved.from __future__ import annotationsfrom typing import Anydiff --git a/tests/_runtime/test_storage.py b/tests/_runtime/test_storage.pynew file mode 100644index 0000000000..792cb334bf--- /dev/null+++ b/tests/_runtime/test_storage.py@@ -0,0 +1,201 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import pytest++from marimo._runtime.virtual_file.storage import (+ InMemoryStorage,+ SharedMemoryStorage,+ VirtualFileStorageManager,+)+++class TestInMemoryStorage:+ def test_store_and_read(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"hello world")+ result = storage.read("test_key", 11)+ assert result == b"hello world"++ def test_read_with_byte_length(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"hello world")+ result = storage.read("test_key", 5)+ assert result == b"hello"++ def test_read_nonexistent_raises_keyerror(self) -> None:+ storage = InMemoryStorage()+ with pytest.raises(KeyError, match="Virtual file not found"):+ storage.read("nonexistent", 10)++ def test_store_overwrites(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"original")+ storage.store("test_key", b"updated")+ result = storage.read("test_key", 7)+ assert result == b"updated"++ def test_remove(self) -> None:+ storage = InMemoryStorage()+ storage.store("test_key", b"hello")+ assert storage.has("test_key")+ storage.remove("test_key")+ assert not storage.has("test_key")++ def test_remove_nonexistent_no_error(self) -> None:+ storage = InMemoryStorage()+ storage.remove("nonexistent") # Should not raise++ def test_has(self) -> None:+ storage = InMemoryStorage()+ assert not storage.has("test_key")+ storage.store("test_key", b"hello")+ assert storage.has("test_key")++ def test_shutdown_clears_storage(self) -> None:+ storage = InMemoryStorage()+ storage.store("key1", b"data1")+ storage.store("key2", b"data2")+ assert storage.has("key1")+ assert storage.has("key2")+ storage.shutdown()+ assert not storage.has("key1")+ assert not storage.has("key2")+++class TestSharedMemoryStorage:+ def test_store_and_read(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_test_1", b"hello world")+ result = storage.read("marimo_test_1", 11)+ assert result == b"hello world"+ finally:+ storage.shutdown()++ def test_read_with_byte_length(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_test_2", b"hello world")+ result = storage.read("marimo_test_2", 5)+ assert result == b"hello"+ finally:+ storage.shutdown()++ def test_read_nonexistent_raises_keyerror(self) -> None:+ storage = SharedMemoryStorage()+ try:+ with pytest.raises(KeyError, match="Virtual file not found"):+ storage.read("nonexistent_key_xyz", 10)+ finally:+ storage.shutdown()++ def test_store_duplicate_skipped(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_test_3", b"original")+ # Second store should be a no-op (not overwrite)+ storage.store("marimo_test_3", b"updated_longer")+ result = storage.read("marimo_test_3", 8)+ assert result == b"original"+ finally:+ storage.shutdown()++ def test_remove(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.store("marimo_test_4", b"hello")+ assert storage.has("marimo_test_4")+ storage.remove("marimo_test_4")+ assert not storage.has("marimo_test_4")+ finally:+ storage.shutdown()++ def test_remove_nonexistent_no_error(self) -> None:+ storage = SharedMemoryStorage()+ try:+ storage.remove("nonexistent") # Should not raise+ finally:+ storage.shutdown()++ def test_has(self) -> None:+ storage = SharedMemoryStorage()+ try:+ assert not storage.has("marimo_test_5")+ storage.store("marimo_test_5", b"hello")+ assert storage.has("marimo_test_5")+ finally:+ storage.shutdown()++ def test_shutdown_clears_storage(self) -> None:+ storage = SharedMemoryStorage()+ storage.store("marimo_test_6", b"data1")+ storage.store("marimo_test_7", b"data2")+ assert storage.has("marimo_test_6")+ assert storage.has("marimo_test_7")+ storage.shutdown()+ assert not storage.has("marimo_test_6")+ assert not storage.has("marimo_test_7")++ def test_cross_process_read(self) -> None:+ """Test that shared memory can be read by name from a fresh instance."""+ storage1 = SharedMemoryStorage()+ try:+ storage1.store("marimo_test_cross", b"cross process data")+ # Create a new instance and read by name+ storage2 = SharedMemoryStorage()+ result = storage2.read("marimo_test_cross", 18)+ assert result == b"cross process data"+ finally:+ storage1.shutdown()++ def test_shutdown_is_reentrant(self) -> None:+ """Test that shutdown can be called multiple times safely."""+ storage = SharedMemoryStorage()+ storage.store("marimo_test_8", b"data")+ storage.shutdown()+ storage.shutdown() # Should not raise+++class TestVirtualFileStorageManager:+ def test_singleton(self) -> None:+ manager1 = VirtualFileStorageManager()+ manager2 = VirtualFileStorageManager()+ assert manager1 is manager2++ def test_storage_property(self) -> None:+ manager = VirtualFileStorageManager()+ original_storage = manager.storage+ try:+ storage = InMemoryStorage()+ manager.storage = storage+ assert manager.storage is storage+ finally:+ manager.storage = original_storage++ def test_read_with_storage(self) -> None:+ manager = VirtualFileStorageManager()+ original_storage = manager.storage+ try:+ storage = InMemoryStorage()+ storage.store("test_file", b"test data")+ manager.storage = storage+ result = manager.read("test_file", 9)+ assert result == b"test data"+ finally:+ manager.storage = original_storage++ def test_read_without_storage_falls_back_to_shared_memory(self) -> None:+ manager = VirtualFileStorageManager()+ original_storage = manager.storage+ # Store data in shared memory directly+ shm_storage = SharedMemoryStorage()+ try:+ shm_storage.store("marimo_fallback_test", b"fallback data")+ # Set manager storage to None to trigger fallback+ manager.storage = None+ result = manager.read("marimo_fallback_test", 13)+ assert result == b"fallback data"+ finally:+ manager.storage = original_storage+ shm_storage.shutdown()