A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: conversion scope rename fails (#10748)
Dylan Madisetti1ece33af1d1c
diff --git a/marimo/_ast/transformers.py b/marimo/_ast/transformers.pyindex d61085ee0f..2cdc5c7748 100644--- a/marimo/_ast/transformers.py+++ b/marimo/_ast/transformers.py@@ -131,89 +131,6 @@ def get_hashable_ast(return module-class NameTransformer(ast.NodeTransformer):- def __init__(self, name_substitutions: dict[str, str]) -> None:- """Remaps names in an AST.-- Naively remaps all occurrences of names in an AST, given a substitution- dict mapping old names to new names. In particular does not take- scoping into account.- """- self._name_substitutions = name_substitutions- self.made_changes = False- super().__init__()-- def visit_Name(self, node: ast.Name) -> ast.Name:- self.generic_visit(node)- if node.id in self._name_substitutions:- self.made_changes = True- return ast.Name(- **{**node.__dict__, "id": self._name_substitutions[node.id]}- )- return node-- def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef:- self.generic_visit(node)- if node.name in self._name_substitutions:- self.made_changes = True- return ast.FunctionDef(- **{- **node.__dict__,- "name": self._name_substitutions[node.name],- }- )- return node-- def visit_AsyncFunctionDef(- self, node: ast.AsyncFunctionDef- ) -> ast.AsyncFunctionDef:- self.generic_visit(node)- if node.name in self._name_substitutions:- self.made_changes = True- return ast.AsyncFunctionDef(- **{- **node.__dict__,- "name": self._name_substitutions[node.name],- }- )- return node-- def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef:- self.generic_visit(node)- if node.name in self._name_substitutions:- self.made_changes = True- return ast.ClassDef(- **{- **node.__dict__,- "name": self._name_substitutions[node.name],- }- )- return node-- def visit_Assign(self, node: ast.Assign) -> ast.Assign:- self.generic_visit(node)- new_targets: list[Any] = []- for target in node.targets:- if (- isinstance(target, ast.Name)- and target.id in self._name_substitutions- ):- self.made_changes = True- new_targets.append(- ast.Name(- id=self._name_substitutions[target.id], ctx=ast.Store()- )- )- else:- new_targets.append(target)- return ast.Assign(- **{- **node.__dict__,- "targets": new_targets,- }- )--class RemoveImportTransformer(ast.NodeTransformer):"""Removes import that matches the given name.e.g. given import_name = "bar":diff --git a/marimo/_convert/ipynb/to_ir.py b/marimo/_convert/ipynb/to_ir.pyindex f25771bbd3..ae155233db 100644--- a/marimo/_convert/ipynb/to_ir.py+++ b/marimo/_convert/ipynb/to_ir.py@@ -16,7 +16,7 @@ from pymdownx.superfences import RE_NESTED_FENCE_START # type: ignorefrom marimo._ast.cell import CellConfigfrom marimo._ast.compiler import compile_cellfrom marimo._ast.names import DEFAULT_CELL_NAME-from marimo._ast.transformers import NameTransformer, RemoveImportTransformer+from marimo._ast.transformers import RemoveImportTransformerfrom marimo._ast.variables import is_localfrom marimo._ast.visitor import Block, NamedNode, ScopedVisitorfrom marimo._convert.common.format import (@@ -172,20 +172,10 @@ def transform_fixup_multiple_definitions(sources: list[str]) -> list[str]:"_" + name if not name.startswith("_") else name)- def transform(source: str) -> str:- try:- tree = ast.parse(source)- visitor = NameTransformer(name_transformations)- transformed_tree = visitor.visit(tree)- # Don't unparse if no changes were made- # otherwise we lose comments and formatting- if visitor.made_changes:- return ast.unparse(transformed_tree)- return source- except SyntaxError:- return source-- return [transform(source) for source in sources]+ return [+ _rename_in_scope(source, i, PrivateRenamer(name_transformations))+ for i, source in enumerate(sources)+ ]def transform_add_marimo_import(sources: list[CodeCell]) -> list[CodeCell]:@@ -996,6 +986,14 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:class Renamer:+ """Rename nodes in place from per-cell name mappings.++ `cell_remappings` maps each cell index to the names redefined there.+ A definition uses the latest mapping up to and including its own cell.+ A reference uses the latest mapping from earlier cells only, because+ the notebook is assumed to run top-to-bottom.+ """+def __init__(self, cell_remappings: dict[int, dict[str, str]]) -> None:self.cell_remappings = cell_remappingsself.made_changes = False@@ -1058,6 +1056,56 @@ class Renamer:self.made_changes = name != new_name+class PrivateRenamer(Renamer):+ """Apply one flat name mapping to definitions and references alike."""++ def __init__(self, mapping: dict[str, str]) -> None:+ super().__init__({})+ self._mapping = mapping++ def _maybe_rename(self, cell: int, name: str, is_reference: bool) -> str:+ del cell, is_reference+ return self._mapping.get(name, name)+++def _rename_in_scope(source: str, cell_idx: int, renamer: Renamer) -> str:+ """Rename top-level names in `source`. Inner scopes that rebind a name+ are left alone.++ Returns `source` unchanged when nothing was renamed, so comments and+ formatting survive.+ """+ try:+ tree = ast.parse(source)+ except SyntaxError:+ return source++ def on_def(node: NamedNode, name: str, block_stack: list[Block]) -> None:+ # NB. a load that resolves to a definition also arrives here. Rename+ # it only when the top-level block defines the name and no inner+ # block, such as a parameter or comprehension target, rebinds it.+ if name in block_stack[-1].global_names or (+ block_stack[0].is_defined(name)+ and not any(block.is_defined(name) for block in block_stack[1:])+ ):+ renamer.rename_named_node(cell_idx, node, is_reference=False)++ def on_ref(node: NamedNode) -> None:+ renamer.rename_named_node(cell_idx, node, is_reference=True)++ visitor = ScopedVisitor(ignore_local=True, on_def=on_def, on_ref=on_ref)+ try:+ new_tree = visitor.visit(tree)+ except SyntaxError:+ # NB. the visitor raises on constructs marimo rejects, such as+ # `import *`. Keep the cell as-is.+ return source++ if not renamer.made_changes:+ return source+ return ast.unparse(new_tree)++def _transform_aug_assign(sources: list[str]) -> list[str]:new_sources = sources.copy()for i, source in enumerate(sources):@@ -1093,12 +1141,10 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:Rename variables with duplicate definitions across multiple cells,even when the variables are declared in one cell and used in another.- We assume the notebook was meant to be run top-to-bottom,- so references to the name will be renamed to the last definition.-- If a new definition is derived from a previous definition,- then at the top of the cell, we add a new line that assigns- the new definition to the previous definition.+ We assume top-to-bottom execution. Each redefinition gets a numbered+ name. A reference uses the latest definition from an earlier cell. An+ augmented assignment is expanded first, so `a += 2` reads the previous+ cell's `a`.```# Cell 1@@ -1108,7 +1154,7 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:print(a)# Cell 3- a = 2+ a += 2# Cell 4a = 3@@ -1125,8 +1171,7 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:print(a)# Cell 3- a_1 = a- a_1 = a_1 + 2+ a_1 = a + 2# Cell 4a_2 = 3@@ -1192,68 +1237,11 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:sources = _transform_aug_assign(sources)- new_sources: list[str] = sources.copy()name_mappings = create_name_mappings(duplicates, set(definitions.keys()))-- for cell_idx, source in enumerate(sources):- renamer = Renamer(name_mappings)- try:- tree = ast.parse(source)- except SyntaxError:- continue-- def on_def(- node: NamedNode,- name: str,- block_stack: list[Block],- cell_idx: int = cell_idx,- renamer: Renamer = renamer,- ) -> None:- block_idx = 0 if name in block_stack[-1].global_names else -1- if block_idx == 0:- # all top-level definitions are renamed- renamer.rename_named_node(cell_idx, node, is_reference=False)- elif block_stack[0].is_defined(name) and not any(- block.is_defined(name) for block in block_stack[1:]- ):- # all ast.LOADs of top-level definitions are defined- renamer.rename_named_node(cell_idx, node, is_reference=False)-- def on_ref(- node: NamedNode,- cell_idx: int = cell_idx,- renamer: Renamer = renamer,- ) -> None:- renamer.rename_named_node(cell_idx, node, is_reference=True)-- visitor = ScopedVisitor(- ignore_local=True, on_def=on_def, on_ref=on_ref- )- try:- new_tree = visitor.visit(tree)- except SyntaxError:- # Cell contains constructs like `import *` that marimo- # doesn't support — skip renaming but keep the cell as-is- new_sources[cell_idx] = source- continue-- # Don't unparse if no changes were made- if not renamer.made_changes:- new_sources[cell_idx] = source- continue-- new_source_lines: list[str] = []-- # TODO- # Add assignments for dependencies- # for definition, dep in visitor.dependencies.items():- # new_source_lines.append(f"{definition} = {dep}")-- # Add the modified source- new_source_lines.append(ast.unparse(new_tree))- new_sources[cell_idx] = "\n".join(new_source_lines)-- return new_sources+ return [+ _rename_in_scope(source, cell_idx, Renamer(name_mappings))+ for cell_idx, source in enumerate(sources)+ ]def bind_cell_metadata(diff --git a/tests/_ast/test_transformers.py b/tests/_ast/test_transformers.pyindex 41159b5019..dca9b0c422 100644--- a/tests/_ast/test_transformers.py+++ b/tests/_ast/test_transformers.py@@ -13,7 +13,6 @@ from marimo._ast.transformers import (DeprivateVisitor,ExtractWithBlock,MangleArguments,- NameTransformer,RemoveImportTransformer,RemoveReturns,clean_to_modules,@@ -22,75 +21,6 @@ from marimo._ast.transformers import ()-def test_name_transformer() -> None:- # Name transformer should naively remap all occurrences of names in an AST,- # without taking scoping into account.- #- # It does not transform attributes.- code = """-def old_function():- old_variable = 42- return old_variable--class OldClass:- def __init__(self):- self.old_attribute = "hello"--old_global = "world"- """-- # Create an AST from the code- tree = ast.parse(code)-- # Define name substitutions- name_substitutions = {- "old_function": "new_function",- "old_variable": "new_variable",- "OldClass": "NewClass",- "old_attribute": "new_attribute",- "old_global": "new_global",- }-- # Apply the NameTransformer- transformer = NameTransformer(name_substitutions)- new_tree = transformer.visit(tree)-- # Convert the new AST back to code- new_code = ast.unparse(new_tree)-- # Expected transformed code- expected_code = """-def new_function():- new_variable = 42- return new_variable--class NewClass:-- def __init__(self):- self.old_attribute = 'hello'-new_global = 'world'-"""-- # Remove leading/trailing whitespace and normalize line endings- new_code = new_code.strip()- expected_code = expected_code.strip()-- # Assert that the transformation was successful- assert new_code == expected_code- assert transformer.made_changes---def test_name_transformer_no_changes() -> None:- code = "x = 1"- tree = ast.parse(code)- transformer = NameTransformer({"y": "z"})- new_tree = transformer.visit(tree)- new_code = ast.unparse(new_tree)-- assert new_code.strip() == code.strip()- assert not transformer.made_changes--def test_import_transformer_strip() -> None:code = """import thing.marimo # Only line that's reasonable.@@ -630,81 +560,3 @@ with cache: x = 1 # All on one linepre_module, with_module = extractor.generic_visit(tree.body)assert isinstance(pre_module, ast.Module)assert isinstance(with_module, ast.Module)---def test_name_transformer_assign_targets() -> None:- """Test NameTransformer with assignment targets."""- code = """-old_var = 42-old_var, other_var = 1, 2-"""- tree = ast.parse(code)-- transformer = NameTransformer({"old_var": "new_var"})- result = transformer.visit(tree)-- code_result = ast.unparse(result)- assert "new_var = 42" in code_result- assert (- "new_var, other_var" in code_result- ) # AST unparses tuples with parentheses- assert "old_var" not in code_result- assert transformer.made_changes---def test_name_transformer_class_def() -> None:- """Test NameTransformer with class definitions."""- code = """-class OldClass:- pass-"""- tree = ast.parse(code)-- transformer = NameTransformer({"OldClass": "NewClass"})- result = transformer.visit(tree)-- code_result = ast.unparse(result)- assert "class NewClass:" in code_result- assert "class OldClass:" not in code_result- assert transformer.made_changes---def test_name_transformer_no_substitutions() -> None:- """Test NameTransformer when no substitutions are made."""- code = """-def func():- x = 1- return x-"""- tree = ast.parse(code)-- transformer = NameTransformer({})- result = transformer.visit(tree)-- code_result = ast.unparse(result)- assert code_result.strip() == code.strip()- assert not transformer.made_changes---def test_name_transformer_function_def() -> None:- """Test NameTransformer with function definitions."""- code = """-def old_func():- pass--async def old_async_func():- pass-"""- tree = ast.parse(code)-- transformer = NameTransformer(- {"old_func": "new_func", "old_async_func": "new_async_func"}- )- result = transformer.visit(tree)-- code_result = ast.unparse(result)- assert "def new_func():" in code_result- assert "async def new_async_func():" in code_result- assert "def old_func():" not in code_result- assert "async def old_async_func():" not in code_result- assert transformer.made_changesdiff --git a/tests/_convert/ipynb/test_ipynb_to_ir.py b/tests/_convert/ipynb/test_ipynb_to_ir.pyindex 238ef05246..aff574ce76 100644--- a/tests/_convert/ipynb/test_ipynb_to_ir.py+++ b/tests/_convert/ipynb/test_ipynb_to_ir.py@@ -9,7 +9,9 @@ import pytestfrom marimo._convert.ipynb.to_ir import (CellsTransform,CodeCell,+ PrivateRenamer,Transform,+ _rename_in_scope,convert_from_ipynb_to_notebook_ir,transform_add_marimo_import,transform_add_subprocess_import,@@ -97,14 +99,13 @@ for i_1 in range(X_1.shape[0]):def test_transform_fixup_multiple_definitions_scope():- # Makes everything private to avoid conflicts.- # Comments are removed, unfortunately.+ # NB. `K` inside `foo` is a local, so only the cell-level `K` is renamed.sources = dd(["""K = 2\ndef foo():- KK = 1+ return K""","K = 1",]@@ -113,8 +114,8 @@ def foo():expected = ["""_K = 2\ndef foo():- _K- _K = 1+ K = 1+ return K""","_K = 1",]@@ -1531,3 +1532,229 @@ def test_remove_input_alongside_other_tags_only_consumes_remove_input():assert cell.options.get("hide_code") is Trueassert "# Cell tags: extra-tag" in cell.codeassert "remove-input" not in cell.code+++ ("source", "expected"),+ [+ pytest.param(+ """+ def double(n):+ return n * 2+ n = 5+ print(double(3))+ """,+ """+ def double(n):+ return n * 2+ _n = 5+ print(double(3))+ """,+ id="function-parameter",+ ),+ pytest.param(+ """+ n = 5+ f = lambda n: n + 1+ print(f(n))+ """,+ """+ _n = 5+ f = lambda n: n + 1+ print(f(_n))+ """,+ id="lambda-parameter",+ ),+ pytest.param(+ """+ n = 5+ xs = [n for n in range(3)]+ print(n, xs)+ """,+ """+ _n = 5+ xs = [n for n in range(3)]+ print(_n, xs)+ """,+ id="comprehension-target",+ ),+ pytest.param(+ """+ n = 5+ def g(path):+ for n in range(2):+ pass+ with open(path) as n:+ return n+ """,+ """+ _n = 5+ def g(path):+ for n in range(2):+ pass+ with open(path) as n:+ return n+ """,+ id="for-and-with-targets-in-function",+ ),+ pytest.param(+ """+ n = 5+ def h(m):+ return n + m+ """,+ """+ _n = 5+ def h(m):+ return _n + m+ """,+ id="closure-read-is-still-renamed",+ ),+ pytest.param(+ """+ n = 5+ def f(n=n):+ return n+ """,+ """+ _n = 5+ def f(n=_n):+ return n+ """,+ id="kwarg-default",+ ),+ ],+)+def test_transform_fixup_multiple_definitions_respects_inner_scopes(+ source: str, expected: str+):+ # NB. an inner-scope binding shadows the cell-level name. The rename must+ # not enter that scope.+ sources = dd([source, "n = 999"])+ result = transform_fixup_multiple_definitions(sources)+ assert_sources_equal(result, [expected, "_n = 999"])+++def test_convert_privatization_keeps_parameter_and_body_consistent():+ # NB. the converted function must read its argument, not the privatized+ # cell-level value.+ notebook = {+ "cells": [+ {+ "cell_type": "code",+ "metadata": {},+ "source": "def double(n):\n return n * 2\n\nn = 5\nprint(double(3))",+ "outputs": [],+ "execution_count": None,+ },+ {+ "cell_type": "code",+ "metadata": {},+ "source": "n = 999\nprint(n)",+ "outputs": [],+ "execution_count": None,+ },+ ],+ "metadata": {},+ "nbformat": 4,+ "nbformat_minor": 5,+ }+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))+ first = result.cells[0].code+ assert "def double(n):" in first+ assert "return n * 2" in first+ assert "_n = 5" in first++ namespace: dict[str, object] = {}+ exec(first, namespace) # noqa: S102+ assert namespace["double"](3) == 6 # type: ignore[operator]+++ ("source", "mapping", "expected"),+ [+ pytest.param(+ """+ old_global = 'world'+ old_global, other = 1, 2+ print(old_global.real)+ """,+ {"old_global": "new_global"},+ """+ new_global = 'world'+ new_global, other = 1, 2+ print(new_global.real)+ """,+ id="variable-and-tuple-targets",+ ),+ pytest.param(+ """+ def old_func():+ pass++ async def old_async_func():+ pass+ old_func()+ """,+ {"old_func": "new_func", "old_async_func": "new_async_func"},+ """+ def new_func():+ pass++ async def new_async_func():+ pass+ new_func()+ """,+ id="function-defs",+ ),+ pytest.param(+ """+ class OldClass:+ def __init__(self):+ self.old_attribute = 'hello'+ OldClass()+ """,+ {"OldClass": "NewClass", "old_attribute": "new_attribute"},+ """+ class NewClass:+ def __init__(self):+ self.old_attribute = 'hello'+ NewClass()+ """,+ id="class-def-and-attribute-untouched",+ ),+ pytest.param(+ """+ def old_function():+ old_variable = 42+ return old_variable+ """,+ {"old_function": "new_function", "old_variable": "new_variable"},+ """+ def new_function():+ old_variable = 42+ return old_variable+ """,+ id="function-local-untouched",+ ),+ ],+)+def test_rename_in_scope(+ source: str, mapping: dict[str, str], expected: str+) -> None:+ renamer = PrivateRenamer(mapping)+ result = _rename_in_scope(dedent(source), 0, renamer)+ assert renamer.made_changes+ assert_sources_equal([result], [expected])+++ "mapping", [{}, {"y": "z"}], ids=["empty-mapping", "no-match"]+)+def test_rename_in_scope_returns_source_verbatim_when_unchanged(+ mapping: dict[str, str],+) -> None:+ source = "x = 1 # keep this comment\n\n\ndef func():\n return x\n"+ renamer = PrivateRenamer(mapping)+ assert _rename_in_scope(source, 0, renamer) is source+ assert not renamer.made_changes