Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: do not use shared memory in run case (#7637)

Dylan Madisetti797c77994bdb
diff --git a/marimo/_convert/markdown/__init__.py b/marimo/_convert/markdown/__init__.py
index 5e078ad009..236d950d01 100644
--- a/marimo/_convert/markdown/__init__.py
+++ b/marimo/_convert/markdown/__init__.py
@@ -1,3 +1,4 @@
+# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
import ast
diff --git a/marimo/_convert/script.py b/marimo/_convert/script.py
index 31c79d1b9f..841aac0e6a 100644
--- a/marimo/_convert/script.py
+++ b/marimo/_convert/script.py
@@ -1,3 +1,4 @@
+# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
from marimo._ast import codegen
diff --git a/marimo/_runtime/context/kernel_context.py b/marimo/_runtime/context/kernel_context.py
index f1a6a59558..63368cf919 100644
--- a/marimo/_runtime/context/kernel_context.py
+++ b/marimo/_runtime/context/kernel_context.py
@@ -150,9 +150,21 @@ def create_kernel_context(
) -> KernelRuntimeContext:
from marimo._plugins.ui._core.registry import UIElementRegistry
from marimo._runtime.state import StateRegistry
- from marimo._runtime.virtual_file import VirtualFileRegistry
+ from marimo._runtime.virtual_file import (
+ InMemoryStorage,
+ SharedMemoryStorage,
+ VirtualFileRegistry,
+ )
from marimo._save.stores import get_store
+ # Use shared memory in edit mode,
+ # in-memory storage in run mode (same process)
+ storage = (
+ SharedMemoryStorage()
+ if mode == SessionMode.EDIT
+ else InMemoryStorage()
+ )
+
return KernelRuntimeContext(
_kernel=kernel,
_session_mode=mode,
@@ -163,7 +175,7 @@ def create_kernel_context(
cache_store=get_store(kernel.app_metadata.filename),
cell_lifecycle_registry=CellLifecycleRegistry(),
app_kernel_runner_registry=AppKernelRunnerRegistry(),
- virtual_file_registry=VirtualFileRegistry(),
+ virtual_file_registry=VirtualFileRegistry(storage=storage),
virtual_files_supported=virtual_files_supported,
stream=stream,
stdout=stdout,
diff --git a/marimo/_runtime/context/script_context.py b/marimo/_runtime/context/script_context.py
index 5994ef6c49..12537c8d67 100644
--- a/marimo/_runtime/context/script_context.py
+++ b/marimo/_runtime/context/script_context.py
@@ -146,7 +146,10 @@ def initialize_script_context(
Must be called exactly once for each client thread.
"""
- from marimo._runtime.virtual_file import VirtualFileRegistry
+ from marimo._runtime.virtual_file import (
+ InMemoryStorage,
+ VirtualFileRegistry,
+ )
from marimo._save.stores import get_store
runtime_context = ScriptRuntimeContext(
@@ -157,7 +160,7 @@ def initialize_script_context(
cache_store=get_store(filename),
cell_lifecycle_registry=CellLifecycleRegistry(),
app_kernel_runner_registry=AppKernelRunnerRegistry(),
- virtual_file_registry=VirtualFileRegistry(),
+ virtual_file_registry=VirtualFileRegistry(storage=InMemoryStorage()),
virtual_files_supported=False,
stream=stream,
stdout=None,
diff --git a/marimo/_runtime/virtual_file/__init__.py b/marimo/_runtime/virtual_file/__init__.py
new file mode 100644
index 0000000000..fcdf1a3d3c
--- /dev/null
+++ b/marimo/_runtime/virtual_file/__init__.py
@@ -0,0 +1,40 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Virtual file storage and management for marimo.
+
+This module provides storage backends and file management for virtual files
+created during notebook execution.
+"""
+
+from __future__ import annotations
+
+from marimo._runtime.virtual_file.storage import (
+ InMemoryStorage,
+ SharedMemoryStorage,
+ VirtualFileStorage,
+ VirtualFileStorageManager,
+)
+from marimo._runtime.virtual_file.virtual_file import (
+ EMPTY_VIRTUAL_FILE,
+ VirtualFile,
+ VirtualFileLifecycleItem,
+ VirtualFileRegistry,
+ VirtualFileRegistryItem,
+ random_filename,
+ read_virtual_file,
+)
+
+__all__ = [
+ # Storage
+ "VirtualFileStorage",
+ "SharedMemoryStorage",
+ "InMemoryStorage",
+ "VirtualFileStorageManager",
+ # Virtual files
+ "VirtualFile",
+ "EMPTY_VIRTUAL_FILE",
+ "VirtualFileLifecycleItem",
+ "VirtualFileRegistryItem",
+ "VirtualFileRegistry",
+ "random_filename",
+ "read_virtual_file",
+]
diff --git a/marimo/_runtime/virtual_file/storage.py b/marimo/_runtime/virtual_file/storage.py
new file mode 100644
index 0000000000..95cc556d3c
--- /dev/null
+++ b/marimo/_runtime/virtual_file/storage.py
@@ -0,0 +1,190 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import sys
+from typing import Protocol
+
+from marimo._utils.platform import is_pyodide
+
+if not is_pyodide():
+ # the shared_memory module is not supported in the Pyodide distribution
+ from multiprocessing import shared_memory
+
+
+class VirtualFileStorage(Protocol):
+ """Protocol for virtual file storage backends."""
+
+ def store(self, key: str, buffer: bytes) -> None:
+ """Store buffer data by key."""
+ ...
+
+ def read(self, key: str, byte_length: int) -> bytes:
+ """Read buffer data by key.
+
+ Raises:
+ KeyError: If key not found
+ """
+ ...
+
+ def remove(self, key: str) -> None:
+ """Remove stored data by key."""
+ ...
+
+ def shutdown(self) -> None:
+ """Clean up all storage resources."""
+ ...
+
+ def has(self, key: str) -> bool:
+ """Check if key exists in storage."""
+ ...
+
+
+class SharedMemoryStorage(VirtualFileStorage):
+ """Storage backend using multiprocessing shared memory.
+
+ Used in `edit` mode when kernel runs in a separate process.
+ """
+
+ def __init__(self) -> None:
+ if is_pyodide():
+ raise RuntimeError(
+ "SharedMemoryStorage is not supported on Pyodide"
+ )
+ self._storage: dict[str, shared_memory.SharedMemory] = {}
+ self._shutting_down = False
+
+ def store(self, key: str, buffer: bytes) -> None:
+ if key in self._storage:
+ return # Already stored
+
+ # Immediately writes the contents of the file to an in-memory
+ # buffer; not lazy.
+ #
+ # To retrieve the buffer from another process, use:
+ #
+ # ```
+ # try:
+ # shm = shared_memory.SharedMemory(name=key)
+ # buffer_contents = bytes(shm.buf)
+ # except FileNotFoundError:
+ # # virtual file was removed
+ # ```
+ shm = shared_memory.SharedMemory(
+ name=key,
+ create=True,
+ size=len(buffer),
+ )
+ shm.buf[: len(buffer)] = buffer
+ # we can safely close this shm, since we don't need to access its
+ # buffer; we do need to keep it around so we can unlink it later
+ if sys.platform != "win32":
+ # don't call close() on Windows, due to a bug in the Windows
+ # Python implementation. On Windows, close() actually unlinks
+ # (destroys) the shared_memory:
+ # https://stackoverflow.com/questions/63713241/segmentation-fault-using-python-shared-memory/63717188#63717188
+ shm.close()
+ # We have to keep a reference to the shared memory to prevent it from
+ # being destroyed on Windows
+ self._storage[key] = shm
+
+ def read(self, key: str, byte_length: int) -> bytes:
+ if is_pyodide():
+ raise RuntimeError(
+ "Shared memory is not supported on this platform"
+ )
+ # Read from shared memory by name (works cross-process)
+ shm = None
+ try:
+ shm = shared_memory.SharedMemory(name=key)
+ buffer_contents = bytes(shm.buf)[:byte_length]
+ except FileNotFoundError as err:
+ raise KeyError(f"Virtual file not found: {key}") from err
+ finally:
+ if shm is not None:
+ shm.close()
+ return buffer_contents
+
+ def remove(self, key: str) -> None:
+ if key in self._storage:
+ if sys.platform == "win32":
+ self._storage[key].close()
+ self._storage[key].unlink()
+ del self._storage[key]
+
+ def shutdown(self) -> None:
+ if self._shutting_down:
+ return
+ try:
+ self._shutting_down = True
+ for shm in self._storage.values():
+ if sys.platform == "win32":
+ shm.close()
+ shm.unlink()
+ self._storage.clear()
+ finally:
+ self._shutting_down = False
+
+ def has(self, key: str) -> bool:
+ return key in self._storage
+
+
+class InMemoryStorage(VirtualFileStorage):
+ """Storage backend using simple in-memory dictionary.
+
+ Used in `run` mode when kernel runs in the same process as the server.
+ """
+
+ def __init__(self) -> None:
+ self._storage: dict[str, bytes] = {}
+
+ def store(self, key: str, buffer: bytes) -> None:
+ self._storage[key] = buffer
+
+ def read(self, key: str, byte_length: int) -> bytes:
+ if key not in self._storage:
+ raise KeyError(f"Virtual file not found: {key}")
+ return self._storage[key][:byte_length]
+
+ def remove(self, key: str) -> None:
+ if key in self._storage:
+ del self._storage[key]
+
+ def shutdown(self) -> None:
+ self._storage.clear()
+
+ def has(self, key: str) -> bool:
+ return key in self._storage
+
+
+class VirtualFileStorageManager:
+ """Singleton manager for virtual file storage access."""
+
+ _instance: VirtualFileStorageManager | None = None
+ _storage: VirtualFileStorage | None = None
+
+ def __new__(cls) -> VirtualFileStorageManager:
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ @property
+ def storage(self) -> VirtualFileStorage | None:
+ return self._storage
+
+ @storage.setter
+ def storage(self, value: VirtualFileStorage | None) -> None:
+ self._storage = value
+
+ def read(self, filename: str, byte_length: int) -> bytes:
+ """Read from storage, with cross-process fallback for EDIT mode server.
+
+ Raises:
+ KeyError: If file not found
+ RuntimeError: When ``SharedMemoryStorage`` is used on the Pyodide platform.
+ """
+ storage = self.storage
+ if storage is None:
+ # Never initialized so in a separate thread from the kernel.
+ # Use SharedMemoryStorage to read by name across processes
+ return SharedMemoryStorage().read(filename, byte_length)
+ return storage.read(filename, byte_length)
diff --git a/marimo/_runtime/virtual_file.py b/marimo/_runtime/virtual_file/virtual_file.py
similarity index 75%
rename from marimo/_runtime/virtual_file.py
rename to marimo/_runtime/virtual_file/virtual_file.py
index 26ae0f15f3..72e56645f4 100644
--- a/marimo/_runtime/virtual_file.py
+++ b/marimo/_runtime/virtual_file/virtual_file.py
@@ -6,7 +6,6 @@ import dataclasses
import mimetypes
import random
import string
-import sys
import threading
from typing import TYPE_CHECKING, Optional, cast
@@ -14,9 +13,12 @@ from marimo import _loggers
from marimo._messaging.mimetypes import KnownMimeType
from marimo._runtime.cell_lifecycle_item import CellLifecycleItem
from marimo._runtime.context import ContextNotInitializedError
+from marimo._runtime.virtual_file.storage import (
+ VirtualFileStorage,
+ VirtualFileStorageManager,
+)
from marimo._utils.data_uri import build_data_url
from marimo._utils.http import HTTPException, HTTPStatus
-from marimo._utils.platform import is_pyodide
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -25,11 +27,6 @@ if TYPE_CHECKING:
LOGGER = _loggers.marimo_logger()
-if not is_pyodide():
- # the shared_memory module is not supported in the Pyodide distribution
- from multiprocessing import shared_memory
-
-
_ALPHABET = string.ascii_letters + string.digits
@@ -161,8 +158,6 @@ class VirtualFileLifecycleItem(CellLifecycleItem):
@dataclasses.dataclass
class VirtualFileRegistryItem:
- # contents of the file
- shm: shared_memory.SharedMemory
# number of HTML objects that are referencing this virtual file
refcount: int
@@ -179,10 +174,15 @@ class VirtualFileRegistry:
exposes methods for incrementing, decrementing, and getting the counts.
"""
+ storage: VirtualFileStorage
registry: dict[str, VirtualFileRegistryItem] = dataclasses.field(
default_factory=dict
)
- shutting_down = False
+ shutting_down: bool = False
+
+ def __post_init__(self) -> None:
+ # Set singleton reference for read_virtual_file()
+ VirtualFileStorageManager().storage = self.storage
def __del__(self) -> None:
self.shutdown()
@@ -224,43 +224,14 @@ class VirtualFileRegistry:
# Skip adding if buffer is empty
if len(buffer) == 0:
return
- # Immediately writes the contents of the file to an in-memory
- # buffer; not lazy.
- #
- # To retrieve the buffer from another process, use:
- #
- # ```
- # try:
- # shm = shared_memory.SharedMemory(name=key)
- # buffer_contents = bytes(shm.buf)
- # except FileNotFoundError:
- # # virtual file was removed
- # ```
- shm = shared_memory.SharedMemory(
- name=key,
- create=True,
- size=len(buffer),
- )
- shm.buf[: len(buffer)] = buffer
- # we can safely close this shm, since we don't need to access its
- # buffer; we do need to keep it around so we can unlink it later
- if sys.platform != "win32":
- # don't call close() on Windows, due to a bug in the Windows
- # Python implementation. On Windows, close() actually unlinks
- # (destroys) the shared_memory:
- # https://stackoverflow.com/questions/63713241/segmentation-fault-using-python-shared-memory/63717188#63717188
- shm.close()
- # We have to keep a reference to the shared memory to prevent it from
- # being destroyed on Windows
- self.registry[key] = VirtualFileRegistryItem(shm=shm, refcount=0)
+
+ self.storage.store(key, buffer)
+ self.registry[key] = VirtualFileRegistryItem(refcount=0)
def remove(self, virtual_file: VirtualFile) -> None:
key = virtual_file.filename
if key in self.registry:
- if sys.platform == "win32":
- self.registry[key].shm.close()
- # destroy the shared memory
- self.registry[key].shm.unlink()
+ self.storage.remove(key)
del self.registry[key]
def shutdown(self) -> None:
@@ -272,10 +243,7 @@ class VirtualFileRegistry:
return
try:
self.shutting_down = True
- for item in self.registry.values():
- if sys.platform == "win32":
- item.shm.close()
- item.shm.unlink()
+ self.storage.shutdown()
self.registry.clear()
finally:
self.shutting_down = False
@@ -286,27 +254,10 @@ def _without_leading_dot(ext: str) -> str:
def read_virtual_file(filename: str, byte_length: int) -> bytes:
- if not shared_memory:
- raise RuntimeError("Shared memory is not supported on this platform")
-
- key = filename
- shm = None
try:
- # NB: this can't be collapsed into a one-liner!
- # doing it in one line yields a 'released memoryview ...'
- # because shared_memory has built in ref-tracking + GC
- shm = shared_memory.SharedMemory(name=key)
- buffer_contents = bytes(shm.buf)[: int(byte_length)]
- except FileNotFoundError as err:
- LOGGER.debug(
- "Error retrieving shared memory for virtual file: %s", err
- )
+ return VirtualFileStorageManager().read(filename, byte_length)
+ except KeyError as err:
raise HTTPException(
HTTPStatus.NOT_FOUND,
detail="File not found",
) from err
- finally:
- if shm is not None:
- shm.close()
-
- return buffer_contents
diff --git a/marimo/_server/config.py b/marimo/_server/config.py
index 5c4c347600..e16a326c87 100644
--- a/marimo/_server/config.py
+++ b/marimo/_server/config.py
@@ -1,3 +1,4 @@
+# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
from dataclasses import dataclass
diff --git a/marimo/_smoke_tests/pdf.py b/marimo/_smoke_tests/pdf.py
index b185a6deaa..abb0a0d353 100644
--- a/marimo/_smoke_tests/pdf.py
+++ b/marimo/_smoke_tests/pdf.py
@@ -2,7 +2,7 @@
import marimo
-__generated_with = "0.15.5"
+__generated_with = "0.18.4"
app = marimo.App(width="full")
@@ -16,7 +16,9 @@ def _():
@app.cell
def _(mo):
- mo.md("# PDFs")
+ mo.md("""
+ # PDFs
+ """)
return
diff --git a/marimo/_utils/print.py b/marimo/_utils/print.py
index e7499944cd..f07c80ffc9 100644
--- a/marimo/_utils/print.py
+++ b/marimo/_utils/print.py
@@ -1,3 +1,4 @@
+# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
from typing import Any
diff --git a/tests/_runtime/test_storage.py b/tests/_runtime/test_storage.py
new file mode 100644
index 0000000000..792cb334bf
--- /dev/null
+++ b/tests/_runtime/test_storage.py
@@ -0,0 +1,201 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import pytest
+
+from marimo._runtime.virtual_file.storage import (
+ InMemoryStorage,
+ SharedMemoryStorage,
+ VirtualFileStorageManager,
+)
+
+
+class TestInMemoryStorage:
+ def test_store_and_read(self) -> None:
+ storage = InMemoryStorage()
+ storage.store("test_key", b"hello world")
+ result = storage.read("test_key", 11)
+ assert result == b"hello world"
+
+ def test_read_with_byte_length(self) -> None:
+ storage = InMemoryStorage()
+ storage.store("test_key", b"hello world")
+ result = storage.read("test_key", 5)
+ assert result == b"hello"
+
+ def test_read_nonexistent_raises_keyerror(self) -> None:
+ storage = InMemoryStorage()
+ with pytest.raises(KeyError, match="Virtual file not found"):
+ storage.read("nonexistent", 10)
+
+ def test_store_overwrites(self) -> None:
+ storage = InMemoryStorage()
+ storage.store("test_key", b"original")
+ storage.store("test_key", b"updated")
+ result = storage.read("test_key", 7)
+ assert result == b"updated"
+
+ def test_remove(self) -> None:
+ storage = InMemoryStorage()
+ storage.store("test_key", b"hello")
+ assert storage.has("test_key")
+ storage.remove("test_key")
+ assert not storage.has("test_key")
+
+ def test_remove_nonexistent_no_error(self) -> None:
+ storage = InMemoryStorage()
+ storage.remove("nonexistent") # Should not raise
+
+ def test_has(self) -> None:
+ storage = InMemoryStorage()
+ assert not storage.has("test_key")
+ storage.store("test_key", b"hello")
+ assert storage.has("test_key")
+
+ def test_shutdown_clears_storage(self) -> None:
+ storage = InMemoryStorage()
+ storage.store("key1", b"data1")
+ storage.store("key2", b"data2")
+ assert storage.has("key1")
+ assert storage.has("key2")
+ storage.shutdown()
+ assert not storage.has("key1")
+ assert not storage.has("key2")
+
+
+class TestSharedMemoryStorage:
+ def test_store_and_read(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ storage.store("marimo_test_1", b"hello world")
+ result = storage.read("marimo_test_1", 11)
+ assert result == b"hello world"
+ finally:
+ storage.shutdown()
+
+ def test_read_with_byte_length(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ storage.store("marimo_test_2", b"hello world")
+ result = storage.read("marimo_test_2", 5)
+ assert result == b"hello"
+ finally:
+ storage.shutdown()
+
+ def test_read_nonexistent_raises_keyerror(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ with pytest.raises(KeyError, match="Virtual file not found"):
+ storage.read("nonexistent_key_xyz", 10)
+ finally:
+ storage.shutdown()
+
+ def test_store_duplicate_skipped(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ storage.store("marimo_test_3", b"original")
+ # Second store should be a no-op (not overwrite)
+ storage.store("marimo_test_3", b"updated_longer")
+ result = storage.read("marimo_test_3", 8)
+ assert result == b"original"
+ finally:
+ storage.shutdown()
+
+ def test_remove(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ storage.store("marimo_test_4", b"hello")
+ assert storage.has("marimo_test_4")
+ storage.remove("marimo_test_4")
+ assert not storage.has("marimo_test_4")
+ finally:
+ storage.shutdown()
+
+ def test_remove_nonexistent_no_error(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ storage.remove("nonexistent") # Should not raise
+ finally:
+ storage.shutdown()
+
+ def test_has(self) -> None:
+ storage = SharedMemoryStorage()
+ try:
+ assert not storage.has("marimo_test_5")
+ storage.store("marimo_test_5", b"hello")
+ assert storage.has("marimo_test_5")
+ finally:
+ storage.shutdown()
+
+ def test_shutdown_clears_storage(self) -> None:
+ storage = SharedMemoryStorage()
+ storage.store("marimo_test_6", b"data1")
+ storage.store("marimo_test_7", b"data2")
+ assert storage.has("marimo_test_6")
+ assert storage.has("marimo_test_7")
+ storage.shutdown()
+ assert not storage.has("marimo_test_6")
+ assert not storage.has("marimo_test_7")
+
+ def test_cross_process_read(self) -> None:
+ """Test that shared memory can be read by name from a fresh instance."""
+ storage1 = SharedMemoryStorage()
+ try:
+ storage1.store("marimo_test_cross", b"cross process data")
+ # Create a new instance and read by name
+ storage2 = SharedMemoryStorage()
+ result = storage2.read("marimo_test_cross", 18)
+ assert result == b"cross process data"
+ finally:
+ storage1.shutdown()
+
+ def test_shutdown_is_reentrant(self) -> None:
+ """Test that shutdown can be called multiple times safely."""
+ storage = SharedMemoryStorage()
+ storage.store("marimo_test_8", b"data")
+ storage.shutdown()
+ storage.shutdown() # Should not raise
+
+
+class TestVirtualFileStorageManager:
+ def test_singleton(self) -> None:
+ manager1 = VirtualFileStorageManager()
+ manager2 = VirtualFileStorageManager()
+ assert manager1 is manager2
+
+ def test_storage_property(self) -> None:
+ manager = VirtualFileStorageManager()
+ original_storage = manager.storage
+ try:
+ storage = InMemoryStorage()
+ manager.storage = storage
+ assert manager.storage is storage
+ finally:
+ manager.storage = original_storage
+
+ def test_read_with_storage(self) -> None:
+ manager = VirtualFileStorageManager()
+ original_storage = manager.storage
+ try:
+ storage = InMemoryStorage()
+ storage.store("test_file", b"test data")
+ manager.storage = storage
+ result = manager.read("test_file", 9)
+ assert result == b"test data"
+ finally:
+ manager.storage = original_storage
+
+ def test_read_without_storage_falls_back_to_shared_memory(self) -> None:
+ manager = VirtualFileStorageManager()
+ original_storage = manager.storage
+ # Store data in shared memory directly
+ shm_storage = SharedMemoryStorage()
+ try:
+ shm_storage.store("marimo_fallback_test", b"fallback data")
+ # Set manager storage to None to trigger fallback
+ manager.storage = None
+ result = manager.read("marimo_fallback_test", 13)
+ assert result == b"fallback data"
+ finally:
+ manager.storage = original_storage
+ shm_storage.shutdown()