Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: support multiple vfile registries with shared inmemorystorage (#8145)

Dylan Madisettia95efe3ab98e
diff --git a/marimo/_runtime/virtual_file/storage.py b/marimo/_runtime/virtual_file/storage.py
index 5964ddc9c7..d87d6a2dea 100644
--- a/marimo/_runtime/virtual_file/storage.py
+++ b/marimo/_runtime/virtual_file/storage.py
@@ -2,10 +2,13 @@
from __future__ import annotations
import sys
-from typing import Protocol
+from typing import TYPE_CHECKING, Protocol
from marimo._utils.platform import is_pyodide
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
if not is_pyodide():
# the shared_memory module is not supported in the Pyodide distribution
from multiprocessing import shared_memory
@@ -30,14 +33,24 @@ class VirtualFileStorage(Protocol):
"""Remove stored data by key."""
...
- def shutdown(self) -> None:
- """Clean up all storage resources."""
+ def shutdown(self, keys: Iterable[str] | None = None) -> None:
+ """Clean up storage resources.
+
+ Args:
+ keys: If provided, only remove these keys. If None, clear all.
+ Implementations may ignore this if storage is not shared.
+ """
...
def has(self, key: str) -> bool:
"""Check if key exists in storage."""
...
+ @property
+ def stale(self) -> bool:
+ """Whether storage has been fully shut down and is no longer usable."""
+ ...
+
class SharedMemoryStorage(VirtualFileStorage):
"""Storage backend using multiprocessing shared memory.
@@ -48,6 +61,11 @@ class SharedMemoryStorage(VirtualFileStorage):
def __init__(self) -> None:
self._storage: dict[str, shared_memory.SharedMemory] = {}
self._shutting_down = False
+ self._stale = False
+
+ @property
+ def stale(self) -> bool:
+ return self._stale
def store(self, key: str, buffer: bytes) -> None:
if key in self._storage:
@@ -107,7 +125,8 @@ class SharedMemoryStorage(VirtualFileStorage):
self._storage[key].unlink()
del self._storage[key]
- def shutdown(self) -> None:
+ def shutdown(self, keys: Iterable[str] | None = None) -> None:
+ del keys # Always clear all - not shared
if self._shutting_down:
return
try:
@@ -118,6 +137,7 @@ class SharedMemoryStorage(VirtualFileStorage):
shm.unlink()
self._storage.clear()
finally:
+ self._stale = True
self._shutting_down = False
def has(self, key: str) -> bool:
@@ -133,6 +153,10 @@ class InMemoryStorage(VirtualFileStorage):
def __init__(self) -> None:
self._storage: dict[str, bytes] = {}
+ @property
+ def stale(self) -> bool:
+ return False # Never stale - can be shared
+
def store(self, key: str, buffer: bytes) -> None:
self._storage[key] = buffer
@@ -145,8 +169,12 @@ class InMemoryStorage(VirtualFileStorage):
if key in self._storage:
del self._storage[key]
- def shutdown(self) -> None:
- self._storage.clear()
+ def shutdown(self, keys: Iterable[str] | None = None) -> None:
+ if keys is not None:
+ for key in keys:
+ self.remove(key)
+ else:
+ self._storage.clear()
def has(self, key: str) -> bool:
return key in self._storage
@@ -165,6 +193,8 @@ class VirtualFileStorageManager:
@property
def storage(self) -> VirtualFileStorage | None:
+ if self._storage is not None and self._storage.stale:
+ self._storage = None
return self._storage
@storage.setter
diff --git a/marimo/_runtime/virtual_file/virtual_file.py b/marimo/_runtime/virtual_file/virtual_file.py
index 72e56645f4..0f05663604 100644
--- a/marimo/_runtime/virtual_file/virtual_file.py
+++ b/marimo/_runtime/virtual_file/virtual_file.py
@@ -182,7 +182,16 @@ class VirtualFileRegistry:
def __post_init__(self) -> None:
# Set singleton reference for read_virtual_file()
- VirtualFileStorageManager().storage = self.storage
+ manager = VirtualFileStorageManager()
+ if manager.storage is None:
+ # Not set yet, _or_ was stale
+ manager.storage = self.storage
+ elif self.storage is not manager.storage:
+ LOGGER.warning(
+ "Expected shared global storage but VirtualFileRegistry was initialized "
+ "with new storage instance. Overriding with global storage.",
+ )
+ self.storage = manager.storage
def __del__(self) -> None:
self.shutdown()
@@ -243,7 +252,7 @@ class VirtualFileRegistry:
return
try:
self.shutting_down = True
- self.storage.shutdown()
+ self.storage.shutdown(keys=self.registry.keys())
self.registry.clear()
finally:
self.shutting_down = False
diff --git a/tests/_runtime/test_virtual_file.py b/tests/_runtime/test_virtual_file.py
index 6373f226b0..9de6fde346 100644
--- a/tests/_runtime/test_virtual_file.py
+++ b/tests/_runtime/test_virtual_file.py
@@ -4,7 +4,13 @@ from __future__ import annotations
from marimo._runtime.commands import DeleteCellCommand
from marimo._runtime.context import get_context
from marimo._runtime.runtime import Kernel
-from tests.conftest import ExecReqProvider
+from marimo._runtime.virtual_file.storage import InMemoryStorage
+from marimo._runtime.virtual_file.virtual_file import (
+ VirtualFileLifecycleItem,
+ VirtualFileRegistry,
+ read_virtual_file,
+)
+from tests.conftest import ExecReqProvider, MockedKernel
async def test_virtual_file_creation(
@@ -301,3 +307,22 @@ async def test_virtual_file_empty_buffer(
)
# Empty buffers should not be added to the registry
assert len(get_context().virtual_file_registry.registry) == 0
+
+
+def test_virtual_file_registry_shared_inmemory_storage(
+ run_mode_kernel: MockedKernel, # noqa: ARG001
+) -> None:
+ ctx = get_context()
+
+ # Create vfile in existing vfile registry
+ item = VirtualFileLifecycleItem(ext="pdf", buffer=b"abc")
+ item.create(context=ctx)
+ vf = item.virtual_file
+
+ assert read_virtual_file(vf.filename, 3) == b"abc"
+
+ # Simulate a second session initializing its own vfile registry
+ VirtualFileRegistry(storage=InMemoryStorage())
+
+ # Ensure old file should still readable
+ assert read_virtual_file(vf.filename, 3) == b"abc"