A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: support multiple vfile registries with shared inmemorystorage (#8145)
Dylan Madisettia95efe3ab98e
diff --git a/marimo/_runtime/virtual_file/storage.py b/marimo/_runtime/virtual_file/storage.pyindex 5964ddc9c7..d87d6a2dea 100644--- a/marimo/_runtime/virtual_file/storage.py+++ b/marimo/_runtime/virtual_file/storage.py@@ -2,10 +2,13 @@from __future__ import annotationsimport sys-from typing import Protocol+from typing import TYPE_CHECKING, Protocolfrom marimo._utils.platform import is_pyodide+if TYPE_CHECKING:+ from collections.abc import Iterable+if not is_pyodide():# the shared_memory module is not supported in the Pyodide distributionfrom multiprocessing import shared_memory@@ -30,14 +33,24 @@ class VirtualFileStorage(Protocol):"""Remove stored data by key."""...- def shutdown(self) -> None:- """Clean up all storage resources."""+ def shutdown(self, keys: Iterable[str] | None = None) -> None:+ """Clean up storage resources.++ Args:+ keys: If provided, only remove these keys. If None, clear all.+ Implementations may ignore this if storage is not shared.+ """...def has(self, key: str) -> bool:"""Check if key exists in storage."""...+ @property+ def stale(self) -> bool:+ """Whether storage has been fully shut down and is no longer usable."""+ ...+class SharedMemoryStorage(VirtualFileStorage):"""Storage backend using multiprocessing shared memory.@@ -48,6 +61,11 @@ class SharedMemoryStorage(VirtualFileStorage):def __init__(self) -> None:self._storage: dict[str, shared_memory.SharedMemory] = {}self._shutting_down = False+ self._stale = False++ @property+ def stale(self) -> bool:+ return self._staledef store(self, key: str, buffer: bytes) -> None:if key in self._storage:@@ -107,7 +125,8 @@ class SharedMemoryStorage(VirtualFileStorage):self._storage[key].unlink()del self._storage[key]- def shutdown(self) -> None:+ def shutdown(self, keys: Iterable[str] | None = None) -> None:+ del keys # Always clear all - not sharedif self._shutting_down:returntry:@@ -118,6 +137,7 @@ class SharedMemoryStorage(VirtualFileStorage):shm.unlink()self._storage.clear()finally:+ self._stale = Trueself._shutting_down = Falsedef has(self, key: str) -> bool:@@ -133,6 +153,10 @@ class InMemoryStorage(VirtualFileStorage):def __init__(self) -> None:self._storage: dict[str, bytes] = {}+ @property+ def stale(self) -> bool:+ return False # Never stale - can be shared+def store(self, key: str, buffer: bytes) -> None:self._storage[key] = buffer@@ -145,8 +169,12 @@ class InMemoryStorage(VirtualFileStorage):if key in self._storage:del self._storage[key]- def shutdown(self) -> None:- self._storage.clear()+ def shutdown(self, keys: Iterable[str] | None = None) -> None:+ if keys is not None:+ for key in keys:+ self.remove(key)+ else:+ self._storage.clear()def has(self, key: str) -> bool:return key in self._storage@@ -165,6 +193,8 @@ class VirtualFileStorageManager:@propertydef storage(self) -> VirtualFileStorage | None:+ if self._storage is not None and self._storage.stale:+ self._storage = Nonereturn self._storage@storage.setterdiff --git a/marimo/_runtime/virtual_file/virtual_file.py b/marimo/_runtime/virtual_file/virtual_file.pyindex 72e56645f4..0f05663604 100644--- a/marimo/_runtime/virtual_file/virtual_file.py+++ b/marimo/_runtime/virtual_file/virtual_file.py@@ -182,7 +182,16 @@ class VirtualFileRegistry:def __post_init__(self) -> None:# Set singleton reference for read_virtual_file()- VirtualFileStorageManager().storage = self.storage+ manager = VirtualFileStorageManager()+ if manager.storage is None:+ # Not set yet, _or_ was stale+ manager.storage = self.storage+ elif self.storage is not manager.storage:+ LOGGER.warning(+ "Expected shared global storage but VirtualFileRegistry was initialized "+ "with new storage instance. Overriding with global storage.",+ )+ self.storage = manager.storagedef __del__(self) -> None:self.shutdown()@@ -243,7 +252,7 @@ class VirtualFileRegistry:returntry:self.shutting_down = True- self.storage.shutdown()+ self.storage.shutdown(keys=self.registry.keys())self.registry.clear()finally:self.shutting_down = Falsediff --git a/tests/_runtime/test_virtual_file.py b/tests/_runtime/test_virtual_file.pyindex 6373f226b0..9de6fde346 100644--- a/tests/_runtime/test_virtual_file.py+++ b/tests/_runtime/test_virtual_file.py@@ -4,7 +4,13 @@ from __future__ import annotationsfrom marimo._runtime.commands import DeleteCellCommandfrom marimo._runtime.context import get_contextfrom marimo._runtime.runtime import Kernel-from tests.conftest import ExecReqProvider+from marimo._runtime.virtual_file.storage import InMemoryStorage+from marimo._runtime.virtual_file.virtual_file import (+ VirtualFileLifecycleItem,+ VirtualFileRegistry,+ read_virtual_file,+)+from tests.conftest import ExecReqProvider, MockedKernelasync def test_virtual_file_creation(@@ -301,3 +307,22 @@ async def test_virtual_file_empty_buffer()# Empty buffers should not be added to the registryassert len(get_context().virtual_file_registry.registry) == 0+++def test_virtual_file_registry_shared_inmemory_storage(+ run_mode_kernel: MockedKernel, # noqa: ARG001+) -> None:+ ctx = get_context()++ # Create vfile in existing vfile registry+ item = VirtualFileLifecycleItem(ext="pdf", buffer=b"abc")+ item.create(context=ctx)+ vf = item.virtual_file++ assert read_virtual_file(vf.filename, 3) == b"abc"++ # Simulate a second session initializing its own vfile registry+ VirtualFileRegistry(storage=InMemoryStorage())++ # Ensure old file should still readable+ assert read_virtual_file(vf.filename, 3) == b"abc"