Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: selectively prune modules on pytest (#10908)

Dylan Madisettie5686b919627
diff --git a/marimo/_runtime/pytest.py b/marimo/_runtime/pytest.py
index 78470f9576..d4dfa6d15d 100644
--- a/marimo/_runtime/pytest.py
+++ b/marimo/_runtime/pytest.py
@@ -15,6 +15,7 @@ from marimo._cli.print import bold, green
from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime.capture import capture_stdout
from marimo._runtime.context import safe_get_context
+from marimo._runtime.reload.autoreload import is_user_module
from marimo._runtime.runtime import notebook_location
MARIMO_TEST_BLOCK_REGEX = re.compile(rf"{MARIMO_TEST_STUB_NAME}_\d+[(?::)\.]+")
@@ -382,9 +383,16 @@ def run_pytest(
)
finally:
del os.environ["MARIMO_PYTEST_WASM"]
- # Note, in pytester, there are also exceptions for zope and readline.
- # However, those deps should already be in module_snapshot, since
- # dependencies are required before the given cell runs.
+ # Evict only project-local modules pytest imported during the run
+ # (conftest, helpers) so edits are picked up next run. Installed
+ # packages are kept: they do not change between runs, and
+ # re-executing them repeats one-shot import side effects such as
+ # native operator registration.
+ module_snapshot.update(
+ (name, module)
+ for name, module in sys.modules.items()
+ if name not in module_snapshot and not is_user_module(module)
+ )
sys.modules.clear()
sys.modules.update(module_snapshot)
sys.path[:] = path_snapshot
diff --git a/marimo/_runtime/reload/autoreload.py b/marimo/_runtime/reload/autoreload.py
index f7b4553d00..9be1168dc1 100644
--- a/marimo/_runtime/reload/autoreload.py
+++ b/marimo/_runtime/reload/autoreload.py
@@ -112,6 +112,19 @@ def _normalized_path(f: str) -> str:
return os.path.normcase(os.path.realpath(f))
+def is_user_module(module: types.ModuleType) -> bool:
+ """True for modules whose source lives outside stdlib/site-packages.
+
+ Editable installs (e.g. `pip install -e .`) point `__file__` at the
+ source tree, so they are correctly classified as user code. Modules
+ without a `__file__` (builtins, frozen, namespace packages) are not.
+ """
+ f = safe_getattr(module, "__file__", None)
+ if not f:
+ return False
+ return not _normalized_path(f).startswith(_non_user_module_roots())
+
+
def modules_imported_by_cell(
cell: CellImpl, sys_modules: dict[str, types.ModuleType]
) -> set[str]:
@@ -200,7 +213,7 @@ class ModuleReloader:
self.lock = threading.Lock()
self._module_dependency_finder = ModuleDependencyFinder()
# modname -> cached `__file__` for modules classified as non-user.
- # Populated by every `check()` call (memoizing `_is_user_module`);
+ # Populated by every `check()` call (memoizing `is_user_module`);
# consumed only when `skip_non_user_modules=True`. Stored value is
# used to invalidate the entry if `sys.modules[modname]` is later
# rebound to a module with a different `__file__` (e.g. a user
@@ -210,17 +223,6 @@ class ModuleReloader:
# Timestamp existing modules
self.check(modules=sys.modules, reload=False)
- def _is_user_module(self, module: types.ModuleType) -> bool:
- """True for modules whose source lives outside stdlib/site-packages.
-
- Editable installs (e.g. `pip install -e .`) point `__file__` at the
- source tree, so they are correctly classified as user code.
- """
- f = safe_getattr(module, "__file__", None)
- if not f:
- return False
- return not _normalized_path(f).startswith(_non_user_module_roots())
-
def filename_and_mtime(
self, module: types.ModuleType
) -> ModuleMTime | None:
@@ -316,7 +318,7 @@ class ModuleReloader:
is_non_user = False
else:
is_non_user = False
- if not is_non_user and not self._is_user_module(m):
+ if not is_non_user and not is_user_module(m):
self._skip[modname] = current_file
is_non_user = True
if is_non_user and skip_non_user_modules:
diff --git a/tests/_runtime/reload/test_autoreload.py b/tests/_runtime/reload/test_autoreload.py
index 9a8cbbcd25..ed8a6f7d3c 100644
--- a/tests/_runtime/reload/test_autoreload.py
+++ b/tests/_runtime/reload/test_autoreload.py
@@ -11,6 +11,7 @@ from unittest.mock import Mock
from reload_test_utils import update_file
+import marimo._runtime.reload.autoreload as autoreload_mod
from marimo._ast.visitor import ImportData
from marimo._runtime.reload.autoreload import (
ModuleDependencyFinder,
@@ -428,14 +429,12 @@ class TestModuleReloaderMethods:
class TestSkipCache:
def test_is_user_module_stdlib(self):
- reloader = ModuleReloader()
- assert reloader._is_user_module(sys.modules["os"]) is False
- assert reloader._is_user_module(sys.modules["pathlib"]) is False
+ assert autoreload_mod.is_user_module(sys.modules["os"]) is False
+ assert autoreload_mod.is_user_module(sys.modules["pathlib"]) is False
def test_is_user_module_builtin_has_no_file(self):
- reloader = ModuleReloader()
- assert reloader._is_user_module(sys.modules["sys"]) is False
- assert reloader._is_user_module(sys.modules["builtins"]) is False
+ assert autoreload_mod.is_user_module(sys.modules["sys"]) is False
+ assert autoreload_mod.is_user_module(sys.modules["builtins"]) is False
def test_is_user_module_user_code(
self, tmp_path: pathlib.Path, py_modname: str
@@ -444,8 +443,7 @@ class TestSkipCache:
py_file = tmp_path / pathlib.Path(py_modname + ".py")
py_file.write_text("x = 1")
mod = importlib.import_module(py_modname)
- reloader = ModuleReloader()
- assert reloader._is_user_module(mod) is True
+ assert autoreload_mod.is_user_module(mod) is True
def test_both_paths_populate_skip(self):
# The cache is shared memoization for the classification step;
@@ -494,8 +492,6 @@ class TestSkipCache:
# `skip_non_user_modules=False` call must still stat it and detect
# edits. Without this, `auto_reload` users editing files inside an
# installed package would silently stop getting hot reloads.
- import marimo._runtime.reload.autoreload as autoreload_mod
-
sys.path.append(str(tmp_path))
py_file = tmp_path / pathlib.Path(py_modname + ".py")
py_file.write_text("x = 1")
@@ -510,7 +506,7 @@ class TestSkipCache:
)
reloader = ModuleReloader()
- assert reloader._is_user_module(mod) is False
+ assert autoreload_mod.is_user_module(mod) is False
# Hot path classifies and caches.
reloader.check(sys.modules, reload=False, skip_non_user_modules=True)
diff --git a/tests/_runtime/test_pytest_runtime.py b/tests/_runtime/test_pytest_runtime.py
index 9debb87cbe..42d44591d9 100644
--- a/tests/_runtime/test_pytest_runtime.py
+++ b/tests/_runtime/test_pytest_runtime.py
@@ -441,3 +441,105 @@ def test_sync_hook_closes_event_loop(monkeypatch, failure: str | None) -> None:
assert loop.is_closed()
finally:
loop.close()
+
+
+def test_rerun_keeps_lazily_imported_modules(tmp_path, monkeypatch) -> None:
+ """Installed modules first imported during a run survive to the next run.
+
+ `run_pytest` restores `sys.modules` after each run so project-local
+ imports (conftest, helpers) reload with edits. Installed packages must be
+ kept: evicting one that a test imported lazily (e.g. `torch.manual_seed`
+ -> `torch._dynamo` -> `TORCH_LIBRARY` registration) re-executes its body
+ next run, while state outside the Python module (C++ dispatcher) persists,
+ raising "Only a single TORCH_LIBRARY can be used to register the
+ namespace".
+ """
+ import marimo
+ from marimo._runtime.pytest import run_pytest
+ from marimo._runtime.reload import autoreload
+
+ # Stand in for an installed package: extend the stdlib/site-packages
+ # roots so `fake_torch` classifies as non-user code.
+ roots = autoreload._non_user_module_roots()
+ installed = os.path.normcase(os.path.realpath(tmp_path / "installed"))
+ monkeypatch.setattr(
+ autoreload,
+ "_non_user_module_roots",
+ lambda: roots + (installed + os.sep,),
+ )
+ (tmp_path / "installed").mkdir()
+ pkg = tmp_path / "installed" / "fake_torch"
+ pkg.mkdir()
+ (pkg / "__init__.py").write_text(
+ "from . import _C\n\n"
+ "def seed():\n"
+ " # Lazy submodule import, as in torch._compile\n"
+ " import fake_torch._dynamo # noqa: F401\n"
+ )
+ # Registry state outliving the Python module, like the C++ dispatcher.
+ (pkg / "_C.py").write_text(
+ "REGISTERED: set[str] = set()\n\n"
+ "def dispatch_library(ns):\n"
+ " if ns in REGISTERED:\n"
+ " raise RuntimeError(f'Only a single TORCH_LIBRARY for {ns}')\n"
+ " REGISTERED.add(ns)\n"
+ )
+ (pkg / "_dynamo.py").write_text(
+ "from . import _C\n\n_C.dispatch_library('_inductor_test')\n"
+ )
+ # Project-local helper, imported lazily by the test.
+ (tmp_path / "local_helper.py").write_text("VALUE = 1\n")
+ monkeypatch.syspath_prepend(str(tmp_path / "installed"))
+ monkeypatch.syspath_prepend(str(tmp_path))
+
+ app = marimo.App()
+
+ @app.cell
+ def _():
+ import fake_torch # type: ignore[import-not-found]
+
+ return (fake_torch,)
+
+ @app.cell
+ def _(fake_torch):
+ def test_seed():
+ import local_helper # type: ignore[import-not-found]
+
+ fake_torch.seed()
+ assert local_helper.VALUE == 1
+
+ return
+
+ notebook = tmp_path / "notebook.py"
+ notebook.write_text("import marimo\napp = marimo.App()\n")
+
+ previous = os.environ.get("PYTEST_CURRENT_TEST", "")
+ os.environ.pop("PYTEST_CURRENT_TEST", None)
+ asyncio.run(asyncio.sleep(0.1))
+ try:
+ _, lcls = app.run()
+ lcls = dict(lcls)
+ results = [
+ run_pytest(defs={"test_seed"}, lcls=lcls, notebook_path=notebook)
+ for _ in range(2)
+ ]
+ dynamo_kept = "fake_torch._dynamo" in sys.modules
+ helper_kept = "local_helper" in sys.modules
+ finally:
+ if previous:
+ os.environ["PYTEST_CURRENT_TEST"] = previous
+ for name in [
+ n
+ for n in sys.modules
+ if n.startswith("fake_torch") or n == "local_helper"
+ ]:
+ del sys.modules[name]
+
+ for run, result in enumerate(results, start=1):
+ assert (result.passed, result.failed, result.errors) == (1, 0, 0), (
+ f"run {run}: {result.output}"
+ )
+ # Installed modules survive the run; project-local ones are evicted so
+ # edits are picked up on the next run.
+ assert dynamo_kept
+ assert not helper_kept