A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: handle comprehension shadowing when parsing required references for variables (#7446)
Dylan Madisettidabed5d4d3ba
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.pyindex 4ec0d39221..eb5a083bee 100644--- a/marimo/_ast/visitor.py+++ b/marimo/_ast/visitor.py@@ -1006,12 +1006,20 @@ class ScopedVisitor(ast.NodeVisitor):# Handle refs on the block scope level, or capture cell level# references.+ # Only add to ref_stack if the variable is not defined in any+ # non-module ancestor block. This prevents function parameters from+ # being incorrectly marked as refs when used in nested scopes like+ # list comprehensions, while still capturing module-level references.if (isinstance(node.ctx, ast.Load)and self._is_defined(node.id)and node.id not in self.ref_stack[-1]and (- node.id not in self.block_stack[-1].defs+ # Check blocks[1:] - skip module block so module-level vars+ # are still tracked as refs, but function params aren't+ not any(+ node.id in block.defs for block in self.block_stack[1:]+ )or len(self.block_stack) == 1)):diff --git a/marimo/_save/save.py b/marimo/_save/save.pyindex a7fd31ffc7..c067305b12 100644--- a/marimo/_save/save.py+++ b/marimo/_save/save.py@@ -424,7 +424,7 @@ class _cache_call(CacheContext):try:if attempt.hit:attempt.restore(scope)- return attempt.meta["return"]+ return attempt.meta.get("return")start_time = time.time()response = self.__wrapped__(*args, **kwargs)diff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.pyindex 89510a8e7f..b72cd818d9 100644--- a/tests/_ast/test_visitor.py+++ b/tests/_ast/test_visitor.py@@ -305,6 +305,96 @@ def test_nested_comprehension_generator_with_named_expr() -> None:assert v.variable_data == {"x": [VariableData(kind="variable")]}+def test_function_param_in_comprehension_not_required_ref() -> None:+ """Function parameters used in list comprehensions should not be required_refs.++ Regression test: The parameter `extension` was incorrectly added to required_refs+ when used as the iterator in a list comprehension inside the function.+ See: test_shadowed_ui_variable_threadpool in tests/_save/test_cache.py+ """+ code = cleandoc(+ """+ def helper(extension):+ return [e for e in extension or []]+ """+ )+ v = visitor.ScopedVisitor()+ mod = ast.parse(code)+ v.visit(mod)++ assert v.defs == {"helper"}+ assert v.refs == set() # No external refs!+ # extension is a PARAMETER, not an external dependency+ # Compare to test_globals_in_functions: foo(a...) where a is not in required_refs+ assert v.variable_data == {+ "helper": [VariableData(kind="function", required_refs=set())]+ }+++def test_nested_function_param_in_comprehension_not_required_ref() -> None:+ """Ensure that additional nesting works."""+ code = cleandoc(+ """+ def helper():+ extension = []+ def foo():+ def bar():+ return [e for e in extension or []]+ return bar+ return foo+ """+ )+ v = visitor.ScopedVisitor()+ mod = ast.parse(code)+ v.visit(mod)++ assert v.defs == {"helper"}+ assert v.refs == set() # No external refs!+ assert v.variable_data == {+ "helper": [VariableData(kind="function", required_refs=set())]+ }+++def test_param_in_comprehension_has_required_ref() -> None:+ """Sanity check ref still is picked up"""+ code = cleandoc(+ """+ def helper():+ return [e for e in extension or []]+ """+ )+ v = visitor.ScopedVisitor()+ mod = ast.parse(code)+ v.visit(mod)++ assert v.defs == {"helper"}+ assert v.refs == {"extension"}+ assert v.variable_data == {+ "helper": [VariableData(kind="function", required_refs={"extension"})]+ }+++def test_shadowed_param_in_comprehension_not_required_ref() -> None:+ """Check that a shadowed variable doesn't capture ref in module scope."""+ code = cleandoc(+ """+ extension = []+ def helper(extension):+ return [e for e in extension or []]+ """+ )+ v = visitor.ScopedVisitor()+ mod = ast.parse(code)+ v.visit(mod)++ assert v.defs == {"helper", "extension"}+ assert v.refs == set()+ assert v.variable_data == {+ "helper": [VariableData(kind="function", required_refs=set())],+ "extension": [VariableData(kind="variable", required_refs=set())],+ }++def test_walrus_leaks_to_global_in_comprehension() -> None:code = "\n".join([diff --git a/tests/_save/test_cache.py b/tests/_save/test_cache.pyindex 23062fcc1b..25f93d91f4 100644--- a/tests/_save/test_cache.py+++ b/tests/_save/test_cache.py@@ -2500,6 +2500,56 @@ class TestCacheDecorator:assert g() == 2return (g, arr)+ @staticmethod+ def test_shadowed_ui_variable_threadpool(app) -> None:+ """Test shadow error with UI-derived variable and ThreadPoolExecutor.++ Bug requires:+ 1. UI element providing a cell-scoped variable (e.g. `extension`)+ 2. Helper function with same-named parameter using nested scope (list comp)+ 3. Cached function called via ThreadPoolExecutor.submit()++ Causes KeyError at hash.py because scope has '*extension' (ARG_PREFIX)+ but lookup uses 'extension'.+ """+ with app.setup:+ from concurrent.futures import ThreadPoolExecutor++ import marimo as mo++ @app.cell+ def _():+ ui_input = mo.ui.text(value="hello")+ return (ui_input,)++ @app.cell+ def _():+ def helper(extension: list[str] | None) -> int:+ # Nested scope using extension triggers the bug+ # for e in extension: ... works fine.+ return len([e for e in extension or []])++ @mo.cache+ def inner(extension: list[str] | None) -> int:+ assert len([e for e in extension or []]) == 5+ return helper(extension)++ return (inner,)++ @app.cell+ def _(inner, ui_input):+ extension = ui_input.value++ results = []+ # has to be in a thread submission+ # the following works fine+ assert inner(extension) == 5++ with ThreadPoolExecutor(max_workers=2) as executor:+ future = executor.submit(inner, extension)+ assert future.result() == 5+ return+class TestPersistentCache:async def test_pickle_context(