A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: selectively prune modules on pytest (#10908)
Dylan Madisettie5686b919627
diff --git a/marimo/_runtime/pytest.py b/marimo/_runtime/pytest.pyindex 78470f9576..d4dfa6d15d 100644--- a/marimo/_runtime/pytest.py+++ b/marimo/_runtime/pytest.py@@ -15,6 +15,7 @@ from marimo._cli.print import bold, greenfrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime.capture import capture_stdoutfrom marimo._runtime.context import safe_get_context+from marimo._runtime.reload.autoreload import is_user_modulefrom marimo._runtime.runtime import notebook_locationMARIMO_TEST_BLOCK_REGEX = re.compile(rf"{MARIMO_TEST_STUB_NAME}_\d+[(?::)\.]+")@@ -382,9 +383,16 @@ def run_pytest()finally:del os.environ["MARIMO_PYTEST_WASM"]- # Note, in pytester, there are also exceptions for zope and readline.- # However, those deps should already be in module_snapshot, since- # dependencies are required before the given cell runs.+ # Evict only project-local modules pytest imported during the run+ # (conftest, helpers) so edits are picked up next run. Installed+ # packages are kept: they do not change between runs, and+ # re-executing them repeats one-shot import side effects such as+ # native operator registration.+ module_snapshot.update(+ (name, module)+ for name, module in sys.modules.items()+ if name not in module_snapshot and not is_user_module(module)+ )sys.modules.clear()sys.modules.update(module_snapshot)sys.path[:] = path_snapshotdiff --git a/marimo/_runtime/reload/autoreload.py b/marimo/_runtime/reload/autoreload.pyindex f7b4553d00..9be1168dc1 100644--- a/marimo/_runtime/reload/autoreload.py+++ b/marimo/_runtime/reload/autoreload.py@@ -112,6 +112,19 @@ def _normalized_path(f: str) -> str:return os.path.normcase(os.path.realpath(f))+def is_user_module(module: types.ModuleType) -> bool:+ """True for modules whose source lives outside stdlib/site-packages.++ Editable installs (e.g. `pip install -e .`) point `__file__` at the+ source tree, so they are correctly classified as user code. Modules+ without a `__file__` (builtins, frozen, namespace packages) are not.+ """+ f = safe_getattr(module, "__file__", None)+ if not f:+ return False+ return not _normalized_path(f).startswith(_non_user_module_roots())++def modules_imported_by_cell(cell: CellImpl, sys_modules: dict[str, types.ModuleType]) -> set[str]:@@ -200,7 +213,7 @@ class ModuleReloader:self.lock = threading.Lock()self._module_dependency_finder = ModuleDependencyFinder()# modname -> cached `__file__` for modules classified as non-user.- # Populated by every `check()` call (memoizing `_is_user_module`);+ # Populated by every `check()` call (memoizing `is_user_module`);# consumed only when `skip_non_user_modules=True`. Stored value is# used to invalidate the entry if `sys.modules[modname]` is later# rebound to a module with a different `__file__` (e.g. a user@@ -210,17 +223,6 @@ class ModuleReloader:# Timestamp existing modulesself.check(modules=sys.modules, reload=False)- def _is_user_module(self, module: types.ModuleType) -> bool:- """True for modules whose source lives outside stdlib/site-packages.-- Editable installs (e.g. `pip install -e .`) point `__file__` at the- source tree, so they are correctly classified as user code.- """- f = safe_getattr(module, "__file__", None)- if not f:- return False- return not _normalized_path(f).startswith(_non_user_module_roots())-def filename_and_mtime(self, module: types.ModuleType) -> ModuleMTime | None:@@ -316,7 +318,7 @@ class ModuleReloader:is_non_user = Falseelse:is_non_user = False- if not is_non_user and not self._is_user_module(m):+ if not is_non_user and not is_user_module(m):self._skip[modname] = current_fileis_non_user = Trueif is_non_user and skip_non_user_modules:diff --git a/tests/_runtime/reload/test_autoreload.py b/tests/_runtime/reload/test_autoreload.pyindex 9a8cbbcd25..ed8a6f7d3c 100644--- a/tests/_runtime/reload/test_autoreload.py+++ b/tests/_runtime/reload/test_autoreload.py@@ -11,6 +11,7 @@ from unittest.mock import Mockfrom reload_test_utils import update_file+import marimo._runtime.reload.autoreload as autoreload_modfrom marimo._ast.visitor import ImportDatafrom marimo._runtime.reload.autoreload import (ModuleDependencyFinder,@@ -428,14 +429,12 @@ class TestModuleReloaderMethods:class TestSkipCache:def test_is_user_module_stdlib(self):- reloader = ModuleReloader()- assert reloader._is_user_module(sys.modules["os"]) is False- assert reloader._is_user_module(sys.modules["pathlib"]) is False+ assert autoreload_mod.is_user_module(sys.modules["os"]) is False+ assert autoreload_mod.is_user_module(sys.modules["pathlib"]) is Falsedef test_is_user_module_builtin_has_no_file(self):- reloader = ModuleReloader()- assert reloader._is_user_module(sys.modules["sys"]) is False- assert reloader._is_user_module(sys.modules["builtins"]) is False+ assert autoreload_mod.is_user_module(sys.modules["sys"]) is False+ assert autoreload_mod.is_user_module(sys.modules["builtins"]) is Falsedef test_is_user_module_user_code(self, tmp_path: pathlib.Path, py_modname: str@@ -444,8 +443,7 @@ class TestSkipCache:py_file = tmp_path / pathlib.Path(py_modname + ".py")py_file.write_text("x = 1")mod = importlib.import_module(py_modname)- reloader = ModuleReloader()- assert reloader._is_user_module(mod) is True+ assert autoreload_mod.is_user_module(mod) is Truedef test_both_paths_populate_skip(self):# The cache is shared memoization for the classification step;@@ -494,8 +492,6 @@ class TestSkipCache:# `skip_non_user_modules=False` call must still stat it and detect# edits. Without this, `auto_reload` users editing files inside an# installed package would silently stop getting hot reloads.- import marimo._runtime.reload.autoreload as autoreload_mod-sys.path.append(str(tmp_path))py_file = tmp_path / pathlib.Path(py_modname + ".py")py_file.write_text("x = 1")@@ -510,7 +506,7 @@ class TestSkipCache:)reloader = ModuleReloader()- assert reloader._is_user_module(mod) is False+ assert autoreload_mod.is_user_module(mod) is False# Hot path classifies and caches.reloader.check(sys.modules, reload=False, skip_non_user_modules=True)diff --git a/tests/_runtime/test_pytest_runtime.py b/tests/_runtime/test_pytest_runtime.pyindex 9debb87cbe..42d44591d9 100644--- a/tests/_runtime/test_pytest_runtime.py+++ b/tests/_runtime/test_pytest_runtime.py@@ -441,3 +441,105 @@ def test_sync_hook_closes_event_loop(monkeypatch, failure: str | None) -> None:assert loop.is_closed()finally:loop.close()+++def test_rerun_keeps_lazily_imported_modules(tmp_path, monkeypatch) -> None:+ """Installed modules first imported during a run survive to the next run.++ `run_pytest` restores `sys.modules` after each run so project-local+ imports (conftest, helpers) reload with edits. Installed packages must be+ kept: evicting one that a test imported lazily (e.g. `torch.manual_seed`+ -> `torch._dynamo` -> `TORCH_LIBRARY` registration) re-executes its body+ next run, while state outside the Python module (C++ dispatcher) persists,+ raising "Only a single TORCH_LIBRARY can be used to register the+ namespace".+ """+ import marimo+ from marimo._runtime.pytest import run_pytest+ from marimo._runtime.reload import autoreload++ # Stand in for an installed package: extend the stdlib/site-packages+ # roots so `fake_torch` classifies as non-user code.+ roots = autoreload._non_user_module_roots()+ installed = os.path.normcase(os.path.realpath(tmp_path / "installed"))+ monkeypatch.setattr(+ autoreload,+ "_non_user_module_roots",+ lambda: roots + (installed + os.sep,),+ )+ (tmp_path / "installed").mkdir()+ pkg = tmp_path / "installed" / "fake_torch"+ pkg.mkdir()+ (pkg / "__init__.py").write_text(+ "from . import _C\n\n"+ "def seed():\n"+ " # Lazy submodule import, as in torch._compile\n"+ " import fake_torch._dynamo # noqa: F401\n"+ )+ # Registry state outliving the Python module, like the C++ dispatcher.+ (pkg / "_C.py").write_text(+ "REGISTERED: set[str] = set()\n\n"+ "def dispatch_library(ns):\n"+ " if ns in REGISTERED:\n"+ " raise RuntimeError(f'Only a single TORCH_LIBRARY for {ns}')\n"+ " REGISTERED.add(ns)\n"+ )+ (pkg / "_dynamo.py").write_text(+ "from . import _C\n\n_C.dispatch_library('_inductor_test')\n"+ )+ # Project-local helper, imported lazily by the test.+ (tmp_path / "local_helper.py").write_text("VALUE = 1\n")+ monkeypatch.syspath_prepend(str(tmp_path / "installed"))+ monkeypatch.syspath_prepend(str(tmp_path))++ app = marimo.App()++ @app.cell+ def _():+ import fake_torch # type: ignore[import-not-found]++ return (fake_torch,)++ @app.cell+ def _(fake_torch):+ def test_seed():+ import local_helper # type: ignore[import-not-found]++ fake_torch.seed()+ assert local_helper.VALUE == 1++ return++ notebook = tmp_path / "notebook.py"+ notebook.write_text("import marimo\napp = marimo.App()\n")++ previous = os.environ.get("PYTEST_CURRENT_TEST", "")+ os.environ.pop("PYTEST_CURRENT_TEST", None)+ asyncio.run(asyncio.sleep(0.1))+ try:+ _, lcls = app.run()+ lcls = dict(lcls)+ results = [+ run_pytest(defs={"test_seed"}, lcls=lcls, notebook_path=notebook)+ for _ in range(2)+ ]+ dynamo_kept = "fake_torch._dynamo" in sys.modules+ helper_kept = "local_helper" in sys.modules+ finally:+ if previous:+ os.environ["PYTEST_CURRENT_TEST"] = previous+ for name in [+ n+ for n in sys.modules+ if n.startswith("fake_torch") or n == "local_helper"+ ]:+ del sys.modules[name]++ for run, result in enumerate(results, start=1):+ assert (result.passed, result.failed, result.errors) == (1, 0, 0), (+ f"run {run}: {result.output}"+ )+ # Installed modules survive the run; project-local ones are evicted so+ # edits are picked up on the next run.+ assert dynamo_kept+ assert not helper_kept