Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

perf: module watcher perf improvements (#7218)

Myles Scolnick65d6e9b7b3bc
diff --git a/marimo/_runtime/reload/autoreload.py b/marimo/_runtime/reload/autoreload.py
index 61a1e4ced3..2f09b060a5 100644
--- a/marimo/_runtime/reload/autoreload.py
+++ b/marimo/_runtime/reload/autoreload.py
@@ -96,7 +96,7 @@ class ModuleDependencyFinder:
self._failed_module_filenames: set[str] = set()
def find_dependencies(
- self, module: types.ModuleType, excludes: list[str]
+ self, module: types.ModuleType, excludes: set[str]
) -> dict[str, types.ModuleType]:
if not hasattr(module, "__file__") or module.__file__ is None:
return {}
@@ -231,15 +231,14 @@ class ModuleReloader:
continue
py_filename, pymtime = module_mtime.name, module_mtime.mtime
- try:
- if pymtime <= self.modules_mtimes[modname]:
- continue
- except KeyError:
+ existing_mtime = self.modules_mtimes.get(modname)
+ if existing_mtime is None:
self.modules_mtimes[modname] = pymtime
continue
- else:
- if self.failed.get(py_filename, None) == pymtime:
- continue
+ if pymtime <= existing_mtime:
+ continue
+ if self.failed.get(py_filename, None) == pymtime:
+ continue
self.modules_mtimes[modname] = pymtime
modified_modules.add(m)
@@ -249,12 +248,12 @@ class ModuleReloader:
if not reload:
return modified_modules
- for modname in self.stale_modules:
+ # Pre-filter stale modules to only those present in modules dict
+ relevant_stale_modules = self.stale_modules & modules.keys()
+ for modname in relevant_stale_modules:
# Reload after the check loop: if there are any
# previously discovered stale modules, reload those as well
- m = modules.get(modname, None)
- if m is None:
- continue
+ m = modules[modname]
module_mtime = self.filename_and_mtime(m)
if module_mtime is None:
@@ -280,7 +279,7 @@ class ModuleReloader:
return modified_modules
def get_module_dependencies(
- self, module: types.ModuleType, excludes: list[str]
+ self, module: types.ModuleType, excludes: set[str]
) -> dict[str, types.ModuleType]:
return self._module_dependency_finder.find_dependencies(
module, excludes
@@ -340,8 +339,9 @@ def update_class(old: object, new: object) -> None:
except (AttributeError, TypeError):
pass # skip non-writable attributes
- for key in list(new.__dict__.keys()):
- if key not in list(old.__dict__.keys()):
+ old_dict_keys = set(old.__dict__.keys())
+ for key in new.__dict__.keys():
+ if key not in old_dict_keys:
try:
setattr(old, key, getattr(new, key))
except (AttributeError, TypeError):
@@ -432,14 +432,8 @@ def superreload(
old_objects = {}
# collect old objects in the module
- for name, obj in list(module.__dict__.items()):
- if not append_obj(module, old_objects, name, obj):
- continue
- key = (module.__name__, name)
- try:
- old_objects.setdefault(key, []).append(weakref.ref(obj))
- except TypeError:
- pass
+ for name, obj in module.__dict__.items():
+ append_obj(module, old_objects, name, obj)
# reload module
old_dict: dict[str, Any] | None = None
@@ -472,7 +466,7 @@ def superreload(
raise
# iterate over all objects and update functions & classes
- for name, new_obj in list(module.__dict__.items()):
+ for name, new_obj in module.__dict__.items():
key = (module.__name__, name)
if key not in old_objects:
continue
diff --git a/marimo/_runtime/reload/module_watcher.py b/marimo/_runtime/reload/module_watcher.py
index 2457c78e35..a8fd412722 100644
--- a/marimo/_runtime/reload/module_watcher.py
+++ b/marimo/_runtime/reload/module_watcher.py
@@ -30,17 +30,21 @@ def is_submodule(src_name: str, target_name: str) -> bool:
eg: "marimo.plugins" is a parent of "marimo.plugins.ui", returns True
"""
- src_parts = src_name.split(".")
- target_parts = target_name.split(".")
- if len(src_parts) > len(target_parts):
+ # Fast path: check if target starts with src
+ if not target_name.startswith(src_name):
return False
- return all(src_parts[i] == target_parts[i] for i in range(len(src_parts)))
+ # Exact match means it's the same module, which counts as submodule
+ if target_name == src_name:
+ return True
+ # Must be followed by a dot to be a true submodule (prevents matching "foo" with "foobar")
+ return target_name[len(src_name)] == "."
def _depends_on(
src_module: types.ModuleType,
target_modules: set[types.ModuleType],
- excludes: list[str],
+ target_filenames: set[str | None],
+ excludes: set[str],
reloader: ModuleReloader,
) -> bool:
"""Returns whether src_module depends on any of target_filenames"""
@@ -50,10 +54,6 @@ def _depends_on(
module_dependencies = reloader.get_module_dependencies(
src_module, excludes=excludes
)
-
- target_filenames = set(
- t.__file__ for t in target_modules if hasattr(t, "__file__")
- )
for found_module in itertools.chain(
[src_module], module_dependencies.values()
):
@@ -83,13 +83,29 @@ def _is_third_party_module(module: types.ModuleType) -> bool:
return "site-packages" in pathlib.Path(filepath).parts
-def _get_excluded_modules(modules: dict[str, types.ModuleType]) -> list[str]:
- return [
+# Cache for excluded modules to avoid recomputing on every check
+# Only caches most recent result to prevent memory bloat
+_excluded_modules_cache: tuple[frozenset[str], set[str]] | None = None
+
+
+def _get_excluded_modules(modules: dict[str, types.ModuleType]) -> set[str]:
+ global _excluded_modules_cache
+ # Use module names as cache key - if same modules, use cached result
+ cache_key = frozenset(modules.keys())
+ if (
+ _excluded_modules_cache is not None
+ and _excluded_modules_cache[0] == cache_key
+ ):
+ return _excluded_modules_cache[1]
+
+ result = set(
modname
for modname in modules
if (m := modules.get(modname)) is not None
and _is_third_party_module(m)
- ]
+ )
+ _excluded_modules_cache = (cache_key, result)
+ return result
def _check_modules(
@@ -103,10 +119,17 @@ def _check_modules(
# TODO(akshayka): could also exclude modules part of the standard library;
# haven't found a reliable way to do this, however.
excludes = _get_excluded_modules(sys_modules)
+
+ target_modules = set(m for m in modified_modules if m is not None)
+ target_filenames = set(
+ t.__file__ for t in target_modules if hasattr(t, "__file__")
+ )
+
for modname, module in modules.items():
if _depends_on(
src_module=module,
- target_modules=set(m for m in modified_modules if m is not None),
+ target_modules=target_modules,
+ target_filenames=target_filenames,
excludes=excludes,
reloader=reloader,
):
@@ -164,7 +187,8 @@ def watch_modules(
LOGGER.debug("Acquired graph lock.")
for modname in stale_modules.keys():
# prune definitions that are derived from stale modules
- cell = graph.cells[modname_to_cell_id[modname]]
+ cell_id = modname_to_cell_id[modname]
+ cell = graph.cells[cell_id]
defs_to_prune = [
import_data.definition
for import_data in cell.imports
diff --git a/tests/_runtime/reload/test_autoreload.py b/tests/_runtime/reload/test_autoreload.py
index 12412289bb..f862c21c89 100644
--- a/tests/_runtime/reload/test_autoreload.py
+++ b/tests/_runtime/reload/test_autoreload.py
@@ -1,15 +1,31 @@
from __future__ import annotations
+import gc
import importlib
import pathlib
import sys
import textwrap
+import types
+from unittest.mock import Mock
from reload_test_utils import update_file
+from marimo._ast.visitor import ImportData
from marimo._runtime.reload.autoreload import (
ModuleDependencyFinder,
ModuleReloader,
+ StrongRef,
+ append_obj,
+ isinstance2,
+ modules_imported_by_cell,
+ safe_getattr,
+ safe_hasattr,
+ superreload,
+ update_class,
+ update_function,
+ update_generic,
+ update_instances,
+ update_property,
)
@@ -108,10 +124,10 @@ class TestModuleDependencyFinder:
from tests._runtime.reload.reload_data import a, b, c, d
finder = ModuleDependencyFinder()
- a_deps = set(list(finder.find_dependencies(a, excludes=[]).keys()))
- b_deps = set(list(finder.find_dependencies(b, excludes=[]).keys()))
- c_deps = set(list(finder.find_dependencies(c, excludes=[]).keys()))
- d_deps = set(list(finder.find_dependencies(d, excludes=[]).keys()))
+ a_deps = set(list(finder.find_dependencies(a, excludes=set()).keys()))
+ b_deps = set(list(finder.find_dependencies(b, excludes=set()).keys()))
+ c_deps = set(list(finder.find_dependencies(c, excludes=set()).keys()))
+ d_deps = set(list(finder.find_dependencies(d, excludes=set()).keys()))
assert a_deps == set(["__main__", "reload_data", "reload_data.c"])
assert b_deps == set(["__main__", "reload_data", "reload_data.d"])
@@ -124,8 +140,650 @@ class TestModuleDependencyFinder:
finder = ModuleDependencyFinder()
assert not finder.cached(a)
- finder.find_dependencies(a, excludes=[])
+ finder.find_dependencies(a, excludes=set())
assert finder.cached(a)
finder.evict_from_cache(a)
assert not finder.cached(a)
+
+ def test_dependencies_module_without_file(self):
+ """Test handling modules without __file__ attribute"""
+ finder = ModuleDependencyFinder()
+ mod = types.ModuleType("test_no_file")
+ deps = finder.find_dependencies(mod, excludes=set())
+ assert deps == {}
+ assert not finder.cached(mod)
+
+ def test_dependencies_module_with_syntax_error(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test handling modules that have syntax errors"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("x = 1")
+ mod = importlib.import_module(py_modname)
+
+ finder = ModuleDependencyFinder()
+ # First call should work
+ deps = finder.find_dependencies(mod, excludes=set())
+ assert finder.cached(mod)
+
+ # Introduce syntax error
+ update_file(py_file, "invalid python +++")
+ finder.evict_from_cache(mod)
+
+ # Should return empty dict on syntax error
+ deps = finder.find_dependencies(mod, excludes=set())
+ assert deps == {}
+
+ def test_dependencies_failed_module_cached(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test that failed modules are added to failed cache"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ # Create a module that will fail when analyzed
+ py_file.write_text("x = 1")
+ mod = importlib.import_module(py_modname)
+
+ finder = ModuleDependencyFinder()
+ # Mock run_script to raise an exception other than SyntaxError
+ import modulefinder
+
+ original_run_script = modulefinder.ModuleFinder.run_script
+
+ def failing_run_script(self, pathname):
+ del self, pathname
+ raise RuntimeError("Simulated failure")
+
+ modulefinder.ModuleFinder.run_script = failing_run_script
+ try:
+ deps = finder.find_dependencies(mod, excludes=set())
+ assert deps == {}
+ assert mod.__file__ in finder._failed_module_filenames
+
+ # Second call should use failed cache
+ deps2 = finder.find_dependencies(mod, excludes=set())
+ assert deps2 == {}
+ finally:
+ modulefinder.ModuleFinder.run_script = original_run_script
+
+
+class TestSafeGetAttr:
+ """Tests for safe_getattr and safe_hasattr functions"""
+
+ def test_safe_getattr_normal(self):
+ """Test safe_getattr with normal attribute access"""
+ mod = types.ModuleType("test")
+ mod.attr = "value"
+ assert safe_getattr(mod, "attr") == "value"
+
+ def test_safe_getattr_missing_with_default(self):
+ """Test safe_getattr with missing attribute and default"""
+ mod = types.ModuleType("test")
+ assert safe_getattr(mod, "missing", "default") == "default"
+
+ def test_safe_getattr_missing_no_default(self):
+ """Test safe_getattr with missing attribute and no default"""
+ mod = types.ModuleType("test")
+ assert safe_getattr(mod, "missing") is None
+
+ def test_safe_getattr_module_not_found(self):
+ """Test safe_getattr handling ModuleNotFoundError"""
+
+ # Create a mock object that raises ModuleNotFoundError on getattr
+ class MockModule:
+ def __getattribute__(self, name):
+ if name != "__class__":
+ raise ModuleNotFoundError("test error")
+ return super().__getattribute__(name)
+
+ mock = MockModule()
+ result = safe_getattr(mock, "any_attr", "fallback")
+ assert result == "fallback"
+
+ def test_safe_hasattr_normal(self):
+ """Test safe_hasattr with normal attribute"""
+ mod = types.ModuleType("test")
+ mod.attr = "value"
+ assert safe_hasattr(mod, "attr") is True
+ assert safe_hasattr(mod, "missing") is False
+
+ def test_safe_hasattr_module_not_found(self):
+ """Test safe_hasattr handling ModuleNotFoundError"""
+
+ class MockModule:
+ def __getattribute__(self, name):
+ if name != "__class__":
+ raise ModuleNotFoundError("test error")
+ return super().__getattribute__(name)
+
+ mock = MockModule()
+ assert safe_hasattr(mock, "any_attr") is False
+
+
+class TestModulesImportedByCell:
+ """Tests for modules_imported_by_cell function"""
+
+ def test_simple_import(self):
+ """Test cell with simple module import"""
+ cell = Mock()
+ cell.imports = [
+ ImportData(module="os", definition="os", imported_symbol=None)
+ ]
+ sys_modules = {"os": sys.modules["os"]}
+
+ result = modules_imported_by_cell(cell, sys_modules)
+ assert result == {"os"}
+
+ def test_from_import(self):
+ """Test cell with from...import statement"""
+ cell = Mock()
+ cell.imports = [
+ ImportData(
+ module="os", definition="path", imported_symbol="os.path"
+ )
+ ]
+ sys_modules = {
+ "os": sys.modules["os"],
+ "os.path": sys.modules["os.path"],
+ }
+
+ result = modules_imported_by_cell(cell, sys_modules)
+ assert "os" in result
+ assert "os.path" in result
+
+ def test_import_not_in_sys_modules(self):
+ """Test that imports not in sys.modules are ignored"""
+ cell = Mock()
+ cell.imports = [
+ ImportData(
+ module="nonexistent_module",
+ definition="x",
+ imported_symbol=None,
+ )
+ ]
+ sys_modules = {}
+
+ result = modules_imported_by_cell(cell, sys_modules)
+ assert result == set()
+
+ def test_multiple_imports(self):
+ """Test cell with multiple imports"""
+ cell = Mock()
+ cell.imports = [
+ ImportData(module="sys", definition="sys", imported_symbol=None),
+ ImportData(module="os", definition="os", imported_symbol=None),
+ ImportData(
+ module="pathlib",
+ definition="Path",
+ imported_symbol="pathlib.Path",
+ ),
+ ]
+ sys_modules = {
+ "sys": sys.modules["sys"],
+ "os": sys.modules["os"],
+ "pathlib": sys.modules["pathlib"],
+ "pathlib.Path": types.ModuleType(
+ "pathlib.Path"
+ ), # pretend it's a module
+ }
+
+ result = modules_imported_by_cell(cell, sys_modules)
+ assert "sys" in result
+ assert "os" in result
+ assert "pathlib" in result
+
+
+class TestModuleReloaderMethods:
+ """Tests for ModuleReloader methods"""
+
+ def test_filename_and_mtime_normal_module(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test filename_and_mtime with normal module"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("x = 1")
+ mod = importlib.import_module(py_modname)
+
+ reloader = ModuleReloader()
+ result = reloader.filename_and_mtime(mod)
+
+ assert result is not None
+ assert result.name == str(py_file)
+ assert result.mtime > 0
+
+ def test_filename_and_mtime_no_file(self):
+ """Test filename_and_mtime with module without __file__"""
+ mod = types.ModuleType("test")
+ reloader = ModuleReloader()
+ result = reloader.filename_and_mtime(mod)
+ assert result is None
+
+ def test_filename_and_mtime_builtin(self):
+ """Test filename_and_mtime with builtin module"""
+ reloader = ModuleReloader()
+ result = reloader.filename_and_mtime(sys.modules["sys"])
+ assert result is None
+
+ def test_filename_and_mtime_main_module(self):
+ """Test filename_and_mtime with __main__ module"""
+ mod = types.ModuleType("__main__")
+ mod.__file__ = "/tmp/test.py"
+ reloader = ModuleReloader()
+ result = reloader.filename_and_mtime(mod)
+ assert result is None
+
+ def test_cell_uses_stale_modules_true(self):
+ """Test cell_uses_stale_modules returns True for stale modules"""
+ reloader = ModuleReloader()
+ reloader.stale_modules = {"os", "sys"}
+
+ cell = Mock()
+ cell.imports = [
+ ImportData(module="os", definition="os", imported_symbol=None)
+ ]
+
+ result = reloader.cell_uses_stale_modules(cell)
+ assert result is True
+
+ def test_cell_uses_stale_modules_false(self):
+ """Test cell_uses_stale_modules returns False for fresh modules"""
+ reloader = ModuleReloader()
+ reloader.stale_modules = {"some_other_module"}
+
+ cell = Mock()
+ cell.imports = [
+ ImportData(module="os", definition="os", imported_symbol=None)
+ ]
+
+ result = reloader.cell_uses_stale_modules(cell)
+ assert result is False
+
+ def test_check_reload_clears_stale_modules(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test that check with reload=True clears stale_modules"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("x = 1")
+ mod = importlib.import_module(py_modname)
+
+ reloader = ModuleReloader()
+ reloader.check(sys.modules, reload=False)
+
+ # Modify the file
+ update_file(py_file, "x = 2")
+
+ # Check should mark as stale
+ reloader.check(sys.modules, reload=False)
+ assert py_modname in reloader.stale_modules
+
+ # Reload should clear stale modules
+ reloader.check(sys.modules, reload=True)
+ assert len(reloader.stale_modules) == 0
+
+
+class TestUpdateFunctions:
+ """Tests for update_* functions"""
+
+ def test_update_function_code(self):
+ """Test that update_function updates function code"""
+
+ def old_func():
+ return 1
+
+ def new_func():
+ return 2
+
+ update_function(old_func, new_func)
+ # After update, old_func should have new_func's code
+ assert old_func() == 2
+
+ def test_update_function_defaults(self):
+ """Test that update_function updates default arguments"""
+
+ def old_func(x=1):
+ return x
+
+ def new_func(x=2):
+ return x
+
+ update_function(old_func, new_func)
+ assert old_func() == 2
+
+ def test_update_class_methods(self):
+ """Test that update_class updates class methods"""
+
+ class OldClass:
+ def method(self):
+ return 1
+
+ class NewClass:
+ def method(self):
+ return 2
+
+ update_class(OldClass, NewClass)
+ instance = OldClass()
+ assert instance.method() == 2
+
+ def test_update_class_removes_obsolete_attributes(self):
+ """Test that update_class removes attributes not in new class"""
+
+ class OldClass:
+ old_attr = 1
+ kept_attr = 2
+
+ class NewClass:
+ kept_attr = 3
+ new_attr = 4
+
+ update_class(OldClass, NewClass)
+ assert not hasattr(OldClass, "old_attr")
+ assert OldClass.kept_attr == 3
+ assert OldClass.new_attr == 4
+
+ def test_update_instances(self):
+ """Test that update_instances updates existing instances"""
+
+ class OldClass:
+ def method(self):
+ return 1
+
+ instance = OldClass()
+ assert instance.method() == 1
+
+ class NewClass:
+ def method(self):
+ return 2
+
+ # Update instances to use new class
+ update_instances(OldClass, NewClass)
+
+ # Existing instance should now use NewClass
+ assert instance.__class__ is NewClass
+ assert instance.method() == 2
+
+ def test_update_property(self):
+ """Test that update_property updates property get/set/del"""
+ get_value = [1]
+
+ class OldClass:
+ @property
+ def prop(self):
+ return get_value[0]
+
+ class NewClass:
+ @property
+ def prop(self):
+ return get_value[0] * 2
+
+ old_prop = OldClass.prop
+ new_prop = NewClass.prop
+
+ update_property(old_prop, new_prop)
+
+ # After update, old property should behave like new
+ instance = OldClass()
+ assert instance.prop == 2
+
+ def test_update_generic_function(self):
+ """Test update_generic delegates to update_function"""
+
+ def old_func():
+ return 1
+
+ def new_func():
+ return 2
+
+ result = update_generic(old_func, new_func)
+ assert result is True
+ assert old_func() == 2
+
+ def test_update_generic_class(self):
+ """Test update_generic delegates to update_class"""
+
+ class OldClass:
+ value = 1
+
+ class NewClass:
+ value = 2
+
+ result = update_generic(OldClass, NewClass)
+ assert result is True
+ assert OldClass.value == 2
+
+ def test_update_generic_unsupported_type(self):
+ """Test update_generic returns False for unsupported types"""
+ result = update_generic("string", "another")
+ assert result is False
+
+ result = update_generic(123, 456)
+ assert result is False
+
+
+class TestStrongRef:
+ """Tests for StrongRef class"""
+
+ def test_strongref_holds_reference(self):
+ """Test that StrongRef keeps object alive"""
+ obj = {"test": "value"}
+ ref = StrongRef(obj)
+
+ # Clear original reference
+ del obj
+ gc.collect()
+
+ # StrongRef should still hold the object
+ assert ref() == {"test": "value"}
+
+ def test_strongref_callable(self):
+ """Test that StrongRef can be called to get object"""
+ obj = [1, 2, 3]
+ ref = StrongRef(obj)
+ assert ref() is obj
+
+
+class TestAppendObj:
+ """Tests for append_obj function"""
+
+ def test_append_obj_in_module(self):
+ """Test append_obj adds object from same module"""
+ mod = types.ModuleType("test_mod")
+
+ class TestClass:
+ pass
+
+ TestClass.__module__ = "test_mod"
+ mod.__name__ = "test_mod"
+
+ d = {}
+ result = append_obj(mod, d, "TestClass", TestClass)
+
+ assert result is True
+ assert ("test_mod", "TestClass") in d
+ assert len(d[("test_mod", "TestClass")]) == 1
+
+ def test_append_obj_not_in_module(self):
+ """Test append_obj ignores objects from different module"""
+ mod = types.ModuleType("test_mod")
+ mod.__name__ = "test_mod"
+
+ class TestClass:
+ pass
+
+ TestClass.__module__ = "other_mod"
+
+ d = {}
+ result = append_obj(mod, d, "TestClass", TestClass)
+
+ assert result is False
+ assert ("test_mod", "TestClass") not in d
+
+ def test_append_obj_no_module_attr(self):
+ """Test append_obj with object without __module__"""
+ mod = types.ModuleType("test_mod")
+ mod.__name__ = "test_mod"
+
+ obj = "string" # strings don't have __module__
+
+ d = {}
+ result = append_obj(mod, d, "obj", obj)
+
+ assert result is False
+
+ def test_append_obj_multiple_objects(self):
+ """Test append_obj with multiple objects"""
+ mod = types.ModuleType("test_mod")
+ mod.__name__ = "test_mod"
+
+ class TestClass1:
+ pass
+
+ class TestClass2:
+ pass
+
+ TestClass1.__module__ = "test_mod"
+ TestClass2.__module__ = "test_mod"
+
+ d = {}
+ append_obj(mod, d, "TestClass1", TestClass1)
+ append_obj(mod, d, "TestClass2", TestClass2)
+
+ assert ("test_mod", "TestClass1") in d
+ assert ("test_mod", "TestClass2") in d
+
+
+class TestIsinstance2:
+ """Tests for isinstance2 helper function"""
+
+ def test_isinstance2_both_match(self):
+ """Test isinstance2 when both are of the type"""
+ assert isinstance2(int, int, type) is True
+ assert isinstance2(str, str, type) is True
+
+ def test_isinstance2_first_not_match(self):
+ """Test isinstance2 when first is not of the type"""
+ assert isinstance2("string", int, type) is False
+
+ def test_isinstance2_second_not_match(self):
+ """Test isinstance2 when second is not of the type"""
+ assert isinstance2(int, "string", type) is False
+
+ def test_isinstance2_neither_match(self):
+ """Test isinstance2 when neither is of the type"""
+ assert isinstance2("string", 123, type) is False
+
+ def test_isinstance2_with_functions(self):
+ """Test isinstance2 with function type"""
+
+ def func1():
+ pass
+
+ def func2():
+ pass
+
+ assert isinstance2(func1, func2, types.FunctionType) is True
+ assert isinstance2(func1, "not_func", types.FunctionType) is False
+
+
+class TestSuperreload:
+ """Tests for superreload function"""
+
+ def test_superreload_basic(self, tmp_path: pathlib.Path, py_modname: str):
+ """Test basic superreload functionality"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("value = 1")
+
+ mod = importlib.import_module(py_modname)
+ assert mod.value == 1
+
+ # Update module
+ update_file(py_file, "value = 2")
+
+ old_objects = {}
+ superreload(mod, old_objects)
+
+ assert mod.value == 2
+
+ def test_superreload_tracks_old_objects(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test that superreload tracks old objects"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text(
+ textwrap.dedent("""
+ class MyClass:
+ value = 1
+
+ def my_func():
+ return 1
+ """)
+ )
+
+ mod = importlib.import_module(py_modname)
+ MyClass = mod.MyClass
+ my_func = mod.my_func
+
+ old_objects = {}
+ superreload(mod, old_objects)
+
+ # Should have tracked the class and function
+ assert (py_modname, "MyClass") in old_objects
+ assert (py_modname, "my_func") in old_objects
+
+ def test_superreload_with_error_raises(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test that superreload raises on error"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("value = 1")
+
+ mod = importlib.import_module(py_modname)
+
+ # Introduce syntax error
+ update_file(py_file, "invalid python +++")
+
+ import pytest
+
+ with pytest.raises(SyntaxError):
+ superreload(mod, {})
+
+ def test_superreload_updates_existing_objects(
+ self, tmp_path: pathlib.Path, py_modname: str
+ ):
+ """Test that superreload updates existing class instances"""
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text(
+ textwrap.dedent("""
+ class MyClass:
+ def method(self):
+ return 1
+ """)
+ )
+
+ mod = importlib.import_module(py_modname)
+ instance = mod.MyClass()
+ assert instance.method() == 1
+
+ # Update the module
+ update_file(
+ py_file,
+ """
+ class MyClass:
+ def method(self):
+ return 2
+ """,
+ )
+
+ old_objects = {}
+ superreload(mod, old_objects)
+ superreload(mod, old_objects) # second reload uses old_objects
+
+ # Instance should now use updated method
+ assert instance.method() == 2
diff --git a/tests/_runtime/reload/test_module_watcher.py b/tests/_runtime/reload/test_module_watcher.py
index ea3fb2425c..68623908a4 100644
--- a/tests/_runtime/reload/test_module_watcher.py
+++ b/tests/_runtime/reload/test_module_watcher.py
@@ -5,6 +5,7 @@ import copy
import pathlib
import sys
import textwrap
+import types
from queue import Queue
import pytest
@@ -12,6 +13,12 @@ from reload_test_utils import random_modname, update_file
from marimo._config.config import DEFAULT_CONFIG
from marimo._dependencies.dependencies import DependencyManager
+from marimo._runtime.reload.autoreload import ModuleReloader
+from marimo._runtime.reload.module_watcher import (
+ _check_modules,
+ _depends_on,
+ _get_excluded_modules,
+)
from marimo._runtime.requests import SetUserConfigRequest
from marimo._runtime.runtime import Kernel
from tests.conftest import ExecReqProvider
@@ -301,7 +308,13 @@ async def test_reload_nested_module_import_module_autorun(
update_file(nested_module, "func = lambda: 2")
# wait for the watcher to pick up the change
- queue.get(timeout=3)
+ retries = 0
+ while retries < 15:
+ await asyncio.sleep(INTERVAL)
+ retries += 1
+ if k.graph.cells[er_1.cell_id].stale:
+ break
+
assert k.graph.cells[er_1.cell_id].stale
assert k.graph.cells[er_2.cell_id].stale
assert not k.graph.cells[er_3.cell_id].stale
@@ -401,7 +414,13 @@ async def test_reload_third_party(
)
# wait for the watcher to pick up the change
- await asyncio.sleep(INTERVAL * 3)
+ retries = 0
+ while retries < 10:
+ await asyncio.sleep(INTERVAL)
+ retries += 1
+ if k.graph.cells[er_1.cell_id].stale:
+ break
+
assert k.graph.cells[er_1.cell_id].stale
assert k.graph.cells[er_2.cell_id].stale
assert not k.graph.cells[er_3.cell_id].stale
@@ -536,3 +555,451 @@ async def test_reload_function_in_import_block(
assert not k.graph.cells[er_3.cell_id].stale
assert k.globals["x"] == 2
assert k.globals["y"] == y
+
+
+class TestIsSubmodule:
+ """Unit tests for the is_submodule utility function"""
+
+ def test_parent_child_relationship(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Parent is a submodule of child
+ assert is_submodule("marimo", "marimo.plugins")
+ assert is_submodule("marimo.plugins", "marimo.plugins.ui")
+ assert is_submodule("a.b", "a.b.c.d")
+
+ def test_exact_match(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Module is a submodule of itself
+ assert is_submodule("marimo", "marimo")
+ assert is_submodule("marimo.plugins", "marimo.plugins")
+ assert is_submodule("a.b.c", "a.b.c")
+
+ def test_not_submodule(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Different modules
+ assert not is_submodule("marimo", "numpy")
+ assert not is_submodule("marimo.plugins", "marimo.runtime")
+ assert not is_submodule("a.b.c", "a.b.d")
+
+ def test_longer_src_than_target(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Src is longer than target - can't be parent
+ assert not is_submodule("marimo.plugins.ui", "marimo.plugins")
+ assert not is_submodule("marimo.plugins", "marimo")
+ assert not is_submodule("a.b.c", "a.b")
+
+ def test_similar_names_not_submodule(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Make sure "foo" doesn't match "foobar"
+ assert not is_submodule("foo", "foobar")
+ assert not is_submodule("marimo", "marimo_extra")
+ assert not is_submodule("a.b", "a.bc")
+
+ def test_empty_strings(self):
+ from marimo._runtime.reload.module_watcher import is_submodule
+
+ # Edge cases with empty strings
+ assert is_submodule("", "")
+ # Empty string is not a valid parent module
+ assert not is_submodule("", "marimo")
+ assert not is_submodule("marimo", "")
+
+
+class TestIsThirdPartyModule:
+ """Unit tests for the _is_third_party_module utility function"""
+
+ def test_module_without_file(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Module without __file__ attribute
+ mod = types.ModuleType("test_module")
+ assert not _is_third_party_module(mod)
+
+ def test_module_with_none_file(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Module with __file__ = None
+ mod = types.ModuleType("test_module")
+ mod.__file__ = None
+ assert not _is_third_party_module(mod)
+
+ def test_site_packages_module(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Module in site-packages
+ mod = types.ModuleType("test_module")
+ mod.__file__ = "/usr/lib/python3.10/site-packages/numpy/__init__.py"
+ assert _is_third_party_module(mod)
+
+ def test_local_module(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Local module not in site-packages
+ mod = types.ModuleType("test_module")
+ mod.__file__ = "/home/user/project/mymodule.py"
+ assert not _is_third_party_module(mod)
+
+ def test_site_packages_in_path_but_not_directory(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Edge case: "site-packages" appears in filename but not as directory
+ mod = types.ModuleType("test_module")
+ mod.__file__ = "/home/user/site-packages.py"
+ assert not _is_third_party_module(mod)
+
+ def test_actual_third_party_module(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # pytest should be in site-packages
+ assert _is_third_party_module(pytest)
+
+ def test_builtin_module(self):
+ from marimo._runtime.reload.module_watcher import (
+ _is_third_party_module,
+ )
+
+ # Builtin modules typically don't have __file__ or have None
+ assert not _is_third_party_module(sys)
+
+
+class TestDependsOn:
+ """Unit tests for the _depends_on function"""
+
+ def test_depends_on_direct_match(self):
+ """Test _depends_on returns True when src is in target_modules"""
+ src_module = types.ModuleType("src")
+ target_module = types.ModuleType("target")
+ target_modules = {src_module, target_module}
+ target_filenames = set()
+ reloader = ModuleReloader()
+
+ result = _depends_on(
+ src_module, target_modules, target_filenames, set(), reloader
+ )
+ assert result is True
+
+ def test_depends_on_by_filename(self, tmp_path: pathlib.Path):
+ """Test _depends_on detects dependency by filename"""
+ # Create a module with a file
+ py_file = tmp_path / "test_mod.py"
+ py_file.write_text("x = 1")
+
+ src_module = types.ModuleType("src")
+ src_module.__file__ = str(py_file)
+
+ target_modules = set()
+ target_filenames = {str(py_file)}
+ reloader = ModuleReloader()
+
+ result = _depends_on(
+ src_module, target_modules, target_filenames, set(), reloader
+ )
+ assert result is True
+
+ def test_depends_on_package_submodule(self):
+ """Test _depends_on detects package dependencies"""
+ # Create a package module (ends with __init__.py)
+ src_module = types.ModuleType("mypackage")
+ src_module.__file__ = "/path/to/mypackage/__init__.py"
+ src_module.__name__ = "mypackage"
+
+ target_module = types.ModuleType("mypackage.submodule")
+ target_module.__name__ = "mypackage.submodule"
+ target_module.__file__ = "/path/to/mypackage/submodule.py"
+
+ target_modules = {target_module}
+ target_filenames = {"/path/to/mypackage/submodule.py"}
+ reloader = ModuleReloader()
+
+ result = _depends_on(
+ src_module, target_modules, target_filenames, set(), reloader
+ )
+ assert result is True
+
+ def test_depends_on_no_dependency(self):
+ """Test _depends_on returns False when no dependency"""
+ src_module = types.ModuleType("src")
+ src_module.__file__ = "/path/to/src.py"
+
+ target_module = types.ModuleType("target")
+ target_module.__file__ = "/path/to/target.py"
+
+ target_modules = {target_module}
+ target_filenames = {"/path/to/target.py"}
+ reloader = ModuleReloader()
+
+ result = _depends_on(
+ src_module, target_modules, target_filenames, set(), reloader
+ )
+ assert result is False
+
+ def test_depends_on_module_without_file(self):
+ """Test _depends_on with module without __file__"""
+ src_module = types.ModuleType("src")
+ # No __file__ attribute
+
+ target_modules = set()
+ target_filenames = set()
+ reloader = ModuleReloader()
+
+ result = _depends_on(
+ src_module, target_modules, target_filenames, set(), reloader
+ )
+ assert result is False
+
+
+class TestGetExcludedModules:
+ """Unit tests for the _get_excluded_modules function"""
+
+ def test_get_excluded_modules_filters_site_packages(self):
+ """Test _get_excluded_modules returns site-packages modules"""
+ mod1 = types.ModuleType("local_mod")
+ mod1.__file__ = "/home/user/project/local_mod.py"
+
+ mod2 = types.ModuleType("third_party")
+ mod2.__file__ = "/usr/lib/python3.10/site-packages/third_party.py"
+
+ modules = {"local_mod": mod1, "third_party": mod2}
+
+ result = _get_excluded_modules(modules)
+ assert "third_party" in result
+ assert "local_mod" not in result
+
+ def test_get_excluded_modules_caching(self):
+ """Test _get_excluded_modules uses cache"""
+ mod1 = types.ModuleType("mod1")
+ mod1.__file__ = "/usr/lib/python3.10/site-packages/mod1.py"
+
+ modules = {"mod1": mod1}
+
+ # First call should compute
+ result1 = _get_excluded_modules(modules)
+ assert "mod1" in result1
+
+ # Second call with same modules should use cache
+ result2 = _get_excluded_modules(modules)
+ assert result2 is result1 # Same object reference
+
+ def test_get_excluded_modules_cache_invalidation(self):
+ """Test _get_excluded_modules cache updates with different modules"""
+ mod1 = types.ModuleType("mod1")
+ mod1.__file__ = "/usr/lib/python3.10/site-packages/mod1.py"
+
+ modules1 = {"mod1": mod1}
+ result1 = _get_excluded_modules(modules1)
+
+ # Different set of modules should recompute
+ mod2 = types.ModuleType("mod2")
+ mod2.__file__ = "/usr/lib/python3.10/site-packages/mod2.py"
+
+ modules2 = {"mod2": mod2}
+ result2 = _get_excluded_modules(modules2)
+
+ assert result1 != result2
+ assert "mod1" in result1
+ assert "mod2" in result2
+
+ def test_get_excluded_modules_empty(self):
+ """Test _get_excluded_modules with no third-party modules"""
+ mod1 = types.ModuleType("local_mod")
+ mod1.__file__ = "/home/user/project/local_mod.py"
+
+ modules = {"local_mod": mod1}
+
+ result = _get_excluded_modules(modules)
+ assert result == set()
+
+
+class TestCheckModules:
+ """Unit tests for the _check_modules function"""
+
+ def test_check_modules_detects_stale(self, tmp_path: pathlib.Path):
+ """Test _check_modules detects stale modules"""
+ import importlib
+
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / "test_check_mod.py"
+ py_file.write_text("x = 1")
+
+ mod = importlib.import_module("test_check_mod")
+ reloader = ModuleReloader()
+ reloader.check(sys.modules, reload=False)
+
+ # Modify the file
+ update_file(py_file, "x = 2")
+
+ # _check_modules should detect it's stale
+ modules = {"test_check_mod": mod}
+ stale = _check_modules(modules, reloader, sys.modules)
+
+ assert "test_check_mod" in stale
+
+ def test_check_modules_no_stale(self):
+ """Test _check_modules with no stale modules"""
+ reloader = ModuleReloader()
+ modules = {"os": sys.modules["os"]}
+
+ stale = _check_modules(modules, reloader, sys.modules)
+ assert len(stale) == 0
+
+ def test_check_modules_empty_input(self):
+ """Test _check_modules with empty modules dict"""
+ reloader = ModuleReloader()
+ modules = {}
+
+ stale = _check_modules(modules, reloader, sys.modules)
+ assert len(stale) == 0
+
+
+class TestModuleWatcherStop:
+ """Tests for ModuleWatcher.stop method"""
+
+ async def test_module_watcher_stop(
+ self, execution_kernel: Kernel, exec_req: ExecReqProvider
+ ):
+ """Test that ModuleWatcher.stop sets the exit event"""
+ del exec_req
+ k = execution_kernel
+ config = copy.deepcopy(DEFAULT_CONFIG)
+ config["runtime"]["auto_reload"] = "lazy"
+ k.set_user_config(SetUserConfigRequest(config=config))
+
+ # Give watcher time to start
+ await asyncio.sleep(INTERVAL)
+
+ # Stop the watcher
+ assert k.module_watcher is not None
+ assert not k.module_watcher.should_exit.is_set()
+
+ k.module_watcher.stop()
+
+ # should_exit should be set
+ assert k.module_watcher.should_exit.is_set()
+
+ async def test_module_watcher_processes_flag(
+ self, execution_kernel: Kernel, exec_req: ExecReqProvider
+ ):
+ """Test ModuleWatcher run_is_processed flag behavior"""
+ del exec_req
+ k = execution_kernel
+ config = copy.deepcopy(DEFAULT_CONFIG)
+ config["runtime"]["auto_reload"] = "lazy"
+ k.set_user_config(SetUserConfigRequest(config=config))
+
+ # Give watcher time to start
+ await asyncio.sleep(INTERVAL)
+
+ assert k.module_watcher is not None
+ # Initially should be set (no run in flight)
+ assert k.module_watcher.run_is_processed.is_set()
+
+
+class TestModuleWatcherEdgeCases:
+ """Tests for edge cases in module watching"""
+
+ async def test_module_watcher_handles_deleted_cell(
+ self,
+ tmp_path: pathlib.Path,
+ py_modname: str,
+ execution_kernel: Kernel,
+ exec_req: ExecReqProvider,
+ ):
+ """Test watcher handles cells being deleted from graph"""
+ k = execution_kernel
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text(
+ textwrap.dedent(
+ """
+ def foo():
+ return 1
+ """
+ )
+ )
+
+ config = copy.deepcopy(DEFAULT_CONFIG)
+ config["runtime"]["auto_reload"] = "lazy"
+ k.set_user_config(SetUserConfigRequest(config=config))
+
+ await k.run(
+ [
+ er_1 := exec_req.get(f"from {py_modname} import foo"),
+ er_2 := exec_req.get("x = foo()"),
+ ]
+ )
+ assert k.globals["x"] == 1
+
+ # Delete a cell from the graph
+ await k.delete_cell(er_2)
+
+ # Modify the file
+ update_file(
+ py_file,
+ """
+ def foo():
+ return 2
+ """,
+ )
+
+ # Wait for watcher to pick up change
+ await asyncio.sleep(INTERVAL * 3)
+
+ # Only er_1 should be stale (er_2 was deleted)
+ assert k.graph.cells[er_1.cell_id].stale
+ assert er_2.cell_id not in k.graph.cells
+
+ async def test_module_watcher_cache_invalidation(
+ self,
+ tmp_path: pathlib.Path,
+ py_modname: str,
+ execution_kernel: Kernel,
+ exec_req: ExecReqProvider,
+ ):
+ """Test that cell modules cache is properly invalidated"""
+ k = execution_kernel
+ sys.path.append(str(tmp_path))
+ py_file = tmp_path / pathlib.Path(py_modname + ".py")
+ py_file.write_text("def foo(): return 1")
+
+ config = copy.deepcopy(DEFAULT_CONFIG)
+ config["runtime"]["auto_reload"] = "lazy"
+ k.set_user_config(SetUserConfigRequest(config=config))
+
+ # Run a cell with import
+ await k.run([er_1 := exec_req.get(f"from {py_modname} import foo")])
+
+ # Modify the module
+ update_file(py_file, "def foo(): return 2")
+
+ # Wait for watcher
+ await asyncio.sleep(INTERVAL * 3)
+ assert k.graph.cells[er_1.cell_id].stale
+
+ # Run stale cells
+ await k.run_stale_cells()
+
+ # Now modify the cell's imports (simulate changing the import)
+ # This should invalidate the cache
+ await k.run(
+ [exec_req.get(f"from {py_modname} import foo; import sys")]
+ )
+
+ # The cache should handle the modified imports correctly
+ assert not k.graph.cells[er_1.cell_id].stale