Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: conversion scope rename fails (#10748)

Dylan Madisetti1ece33af1d1c
diff --git a/marimo/_ast/transformers.py b/marimo/_ast/transformers.py
index d61085ee0f..2cdc5c7748 100644
--- a/marimo/_ast/transformers.py
+++ b/marimo/_ast/transformers.py
@@ -131,89 +131,6 @@ def get_hashable_ast(
return module
-class NameTransformer(ast.NodeTransformer):
- def __init__(self, name_substitutions: dict[str, str]) -> None:
- """Remaps names in an AST.
-
- Naively remaps all occurrences of names in an AST, given a substitution
- dict mapping old names to new names. In particular does not take
- scoping into account.
- """
- self._name_substitutions = name_substitutions
- self.made_changes = False
- super().__init__()
-
- def visit_Name(self, node: ast.Name) -> ast.Name:
- self.generic_visit(node)
- if node.id in self._name_substitutions:
- self.made_changes = True
- return ast.Name(
- **{**node.__dict__, "id": self._name_substitutions[node.id]}
- )
- return node
-
- def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef:
- self.generic_visit(node)
- if node.name in self._name_substitutions:
- self.made_changes = True
- return ast.FunctionDef(
- **{
- **node.__dict__,
- "name": self._name_substitutions[node.name],
- }
- )
- return node
-
- def visit_AsyncFunctionDef(
- self, node: ast.AsyncFunctionDef
- ) -> ast.AsyncFunctionDef:
- self.generic_visit(node)
- if node.name in self._name_substitutions:
- self.made_changes = True
- return ast.AsyncFunctionDef(
- **{
- **node.__dict__,
- "name": self._name_substitutions[node.name],
- }
- )
- return node
-
- def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef:
- self.generic_visit(node)
- if node.name in self._name_substitutions:
- self.made_changes = True
- return ast.ClassDef(
- **{
- **node.__dict__,
- "name": self._name_substitutions[node.name],
- }
- )
- return node
-
- def visit_Assign(self, node: ast.Assign) -> ast.Assign:
- self.generic_visit(node)
- new_targets: list[Any] = []
- for target in node.targets:
- if (
- isinstance(target, ast.Name)
- and target.id in self._name_substitutions
- ):
- self.made_changes = True
- new_targets.append(
- ast.Name(
- id=self._name_substitutions[target.id], ctx=ast.Store()
- )
- )
- else:
- new_targets.append(target)
- return ast.Assign(
- **{
- **node.__dict__,
- "targets": new_targets,
- }
- )
-
-
class RemoveImportTransformer(ast.NodeTransformer):
"""Removes import that matches the given name.
e.g. given import_name = "bar":
diff --git a/marimo/_convert/ipynb/to_ir.py b/marimo/_convert/ipynb/to_ir.py
index f25771bbd3..ae155233db 100644
--- a/marimo/_convert/ipynb/to_ir.py
+++ b/marimo/_convert/ipynb/to_ir.py
@@ -16,7 +16,7 @@ from pymdownx.superfences import RE_NESTED_FENCE_START # type: ignore
from marimo._ast.cell import CellConfig
from marimo._ast.compiler import compile_cell
from marimo._ast.names import DEFAULT_CELL_NAME
-from marimo._ast.transformers import NameTransformer, RemoveImportTransformer
+from marimo._ast.transformers import RemoveImportTransformer
from marimo._ast.variables import is_local
from marimo._ast.visitor import Block, NamedNode, ScopedVisitor
from marimo._convert.common.format import (
@@ -172,20 +172,10 @@ def transform_fixup_multiple_definitions(sources: list[str]) -> list[str]:
"_" + name if not name.startswith("_") else name
)
- def transform(source: str) -> str:
- try:
- tree = ast.parse(source)
- visitor = NameTransformer(name_transformations)
- transformed_tree = visitor.visit(tree)
- # Don't unparse if no changes were made
- # otherwise we lose comments and formatting
- if visitor.made_changes:
- return ast.unparse(transformed_tree)
- return source
- except SyntaxError:
- return source
-
- return [transform(source) for source in sources]
+ return [
+ _rename_in_scope(source, i, PrivateRenamer(name_transformations))
+ for i, source in enumerate(sources)
+ ]
def transform_add_marimo_import(sources: list[CodeCell]) -> list[CodeCell]:
@@ -996,6 +986,14 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:
class Renamer:
+ """Rename nodes in place from per-cell name mappings.
+
+ `cell_remappings` maps each cell index to the names redefined there.
+ A definition uses the latest mapping up to and including its own cell.
+ A reference uses the latest mapping from earlier cells only, because
+ the notebook is assumed to run top-to-bottom.
+ """
+
def __init__(self, cell_remappings: dict[int, dict[str, str]]) -> None:
self.cell_remappings = cell_remappings
self.made_changes = False
@@ -1058,6 +1056,56 @@ class Renamer:
self.made_changes = name != new_name
+class PrivateRenamer(Renamer):
+ """Apply one flat name mapping to definitions and references alike."""
+
+ def __init__(self, mapping: dict[str, str]) -> None:
+ super().__init__({})
+ self._mapping = mapping
+
+ def _maybe_rename(self, cell: int, name: str, is_reference: bool) -> str:
+ del cell, is_reference
+ return self._mapping.get(name, name)
+
+
+def _rename_in_scope(source: str, cell_idx: int, renamer: Renamer) -> str:
+ """Rename top-level names in `source`. Inner scopes that rebind a name
+ are left alone.
+
+ Returns `source` unchanged when nothing was renamed, so comments and
+ formatting survive.
+ """
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return source
+
+ def on_def(node: NamedNode, name: str, block_stack: list[Block]) -> None:
+ # NB. a load that resolves to a definition also arrives here. Rename
+ # it only when the top-level block defines the name and no inner
+ # block, such as a parameter or comprehension target, rebinds it.
+ if name in block_stack[-1].global_names or (
+ block_stack[0].is_defined(name)
+ and not any(block.is_defined(name) for block in block_stack[1:])
+ ):
+ renamer.rename_named_node(cell_idx, node, is_reference=False)
+
+ def on_ref(node: NamedNode) -> None:
+ renamer.rename_named_node(cell_idx, node, is_reference=True)
+
+ visitor = ScopedVisitor(ignore_local=True, on_def=on_def, on_ref=on_ref)
+ try:
+ new_tree = visitor.visit(tree)
+ except SyntaxError:
+ # NB. the visitor raises on constructs marimo rejects, such as
+ # `import *`. Keep the cell as-is.
+ return source
+
+ if not renamer.made_changes:
+ return source
+ return ast.unparse(new_tree)
+
+
def _transform_aug_assign(sources: list[str]) -> list[str]:
new_sources = sources.copy()
for i, source in enumerate(sources):
@@ -1093,12 +1141,10 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:
Rename variables with duplicate definitions across multiple cells,
even when the variables are declared in one cell and used in another.
- We assume the notebook was meant to be run top-to-bottom,
- so references to the name will be renamed to the last definition.
-
- If a new definition is derived from a previous definition,
- then at the top of the cell, we add a new line that assigns
- the new definition to the previous definition.
+ We assume top-to-bottom execution. Each redefinition gets a numbered
+ name. A reference uses the latest definition from an earlier cell. An
+ augmented assignment is expanded first, so `a += 2` reads the previous
+ cell's `a`.
```
# Cell 1
@@ -1108,7 +1154,7 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:
print(a)
# Cell 3
- a = 2
+ a += 2
# Cell 4
a = 3
@@ -1125,8 +1171,7 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:
print(a)
# Cell 3
- a_1 = a
- a_1 = a_1 + 2
+ a_1 = a + 2
# Cell 4
a_2 = 3
@@ -1192,68 +1237,11 @@ def transform_duplicate_definitions(sources: list[str]) -> list[str]:
sources = _transform_aug_assign(sources)
- new_sources: list[str] = sources.copy()
name_mappings = create_name_mappings(duplicates, set(definitions.keys()))
-
- for cell_idx, source in enumerate(sources):
- renamer = Renamer(name_mappings)
- try:
- tree = ast.parse(source)
- except SyntaxError:
- continue
-
- def on_def(
- node: NamedNode,
- name: str,
- block_stack: list[Block],
- cell_idx: int = cell_idx,
- renamer: Renamer = renamer,
- ) -> None:
- block_idx = 0 if name in block_stack[-1].global_names else -1
- if block_idx == 0:
- # all top-level definitions are renamed
- renamer.rename_named_node(cell_idx, node, is_reference=False)
- elif block_stack[0].is_defined(name) and not any(
- block.is_defined(name) for block in block_stack[1:]
- ):
- # all ast.LOADs of top-level definitions are defined
- renamer.rename_named_node(cell_idx, node, is_reference=False)
-
- def on_ref(
- node: NamedNode,
- cell_idx: int = cell_idx,
- renamer: Renamer = renamer,
- ) -> None:
- renamer.rename_named_node(cell_idx, node, is_reference=True)
-
- visitor = ScopedVisitor(
- ignore_local=True, on_def=on_def, on_ref=on_ref
- )
- try:
- new_tree = visitor.visit(tree)
- except SyntaxError:
- # Cell contains constructs like `import *` that marimo
- # doesn't support — skip renaming but keep the cell as-is
- new_sources[cell_idx] = source
- continue
-
- # Don't unparse if no changes were made
- if not renamer.made_changes:
- new_sources[cell_idx] = source
- continue
-
- new_source_lines: list[str] = []
-
- # TODO
- # Add assignments for dependencies
- # for definition, dep in visitor.dependencies.items():
- # new_source_lines.append(f"{definition} = {dep}")
-
- # Add the modified source
- new_source_lines.append(ast.unparse(new_tree))
- new_sources[cell_idx] = "\n".join(new_source_lines)
-
- return new_sources
+ return [
+ _rename_in_scope(source, cell_idx, Renamer(name_mappings))
+ for cell_idx, source in enumerate(sources)
+ ]
def bind_cell_metadata(
diff --git a/tests/_ast/test_transformers.py b/tests/_ast/test_transformers.py
index 41159b5019..dca9b0c422 100644
--- a/tests/_ast/test_transformers.py
+++ b/tests/_ast/test_transformers.py
@@ -13,7 +13,6 @@ from marimo._ast.transformers import (
DeprivateVisitor,
ExtractWithBlock,
MangleArguments,
- NameTransformer,
RemoveImportTransformer,
RemoveReturns,
clean_to_modules,
@@ -22,75 +21,6 @@ from marimo._ast.transformers import (
)
-def test_name_transformer() -> None:
- # Name transformer should naively remap all occurrences of names in an AST,
- # without taking scoping into account.
- #
- # It does not transform attributes.
- code = """
-def old_function():
- old_variable = 42
- return old_variable
-
-class OldClass:
- def __init__(self):
- self.old_attribute = "hello"
-
-old_global = "world"
- """
-
- # Create an AST from the code
- tree = ast.parse(code)
-
- # Define name substitutions
- name_substitutions = {
- "old_function": "new_function",
- "old_variable": "new_variable",
- "OldClass": "NewClass",
- "old_attribute": "new_attribute",
- "old_global": "new_global",
- }
-
- # Apply the NameTransformer
- transformer = NameTransformer(name_substitutions)
- new_tree = transformer.visit(tree)
-
- # Convert the new AST back to code
- new_code = ast.unparse(new_tree)
-
- # Expected transformed code
- expected_code = """
-def new_function():
- new_variable = 42
- return new_variable
-
-class NewClass:
-
- def __init__(self):
- self.old_attribute = 'hello'
-new_global = 'world'
-"""
-
- # Remove leading/trailing whitespace and normalize line endings
- new_code = new_code.strip()
- expected_code = expected_code.strip()
-
- # Assert that the transformation was successful
- assert new_code == expected_code
- assert transformer.made_changes
-
-
-def test_name_transformer_no_changes() -> None:
- code = "x = 1"
- tree = ast.parse(code)
- transformer = NameTransformer({"y": "z"})
- new_tree = transformer.visit(tree)
- new_code = ast.unparse(new_tree)
-
- assert new_code.strip() == code.strip()
- assert not transformer.made_changes
-
-
def test_import_transformer_strip() -> None:
code = """
import thing.marimo # Only line that's reasonable.
@@ -630,81 +560,3 @@ with cache: x = 1 # All on one line
pre_module, with_module = extractor.generic_visit(tree.body)
assert isinstance(pre_module, ast.Module)
assert isinstance(with_module, ast.Module)
-
-
-def test_name_transformer_assign_targets() -> None:
- """Test NameTransformer with assignment targets."""
- code = """
-old_var = 42
-old_var, other_var = 1, 2
-"""
- tree = ast.parse(code)
-
- transformer = NameTransformer({"old_var": "new_var"})
- result = transformer.visit(tree)
-
- code_result = ast.unparse(result)
- assert "new_var = 42" in code_result
- assert (
- "new_var, other_var" in code_result
- ) # AST unparses tuples with parentheses
- assert "old_var" not in code_result
- assert transformer.made_changes
-
-
-def test_name_transformer_class_def() -> None:
- """Test NameTransformer with class definitions."""
- code = """
-class OldClass:
- pass
-"""
- tree = ast.parse(code)
-
- transformer = NameTransformer({"OldClass": "NewClass"})
- result = transformer.visit(tree)
-
- code_result = ast.unparse(result)
- assert "class NewClass:" in code_result
- assert "class OldClass:" not in code_result
- assert transformer.made_changes
-
-
-def test_name_transformer_no_substitutions() -> None:
- """Test NameTransformer when no substitutions are made."""
- code = """
-def func():
- x = 1
- return x
-"""
- tree = ast.parse(code)
-
- transformer = NameTransformer({})
- result = transformer.visit(tree)
-
- code_result = ast.unparse(result)
- assert code_result.strip() == code.strip()
- assert not transformer.made_changes
-
-
-def test_name_transformer_function_def() -> None:
- """Test NameTransformer with function definitions."""
- code = """
-def old_func():
- pass
-
-async def old_async_func():
- pass
-"""
- tree = ast.parse(code)
-
- transformer = NameTransformer(
- {"old_func": "new_func", "old_async_func": "new_async_func"}
- )
- result = transformer.visit(tree)
-
- code_result = ast.unparse(result)
- assert "def new_func():" in code_result
- assert "async def new_async_func():" in code_result
- assert "def old_func():" not in code_result
- assert "async def old_async_func():" not in code_result
- assert transformer.made_changes
diff --git a/tests/_convert/ipynb/test_ipynb_to_ir.py b/tests/_convert/ipynb/test_ipynb_to_ir.py
index 238ef05246..aff574ce76 100644
--- a/tests/_convert/ipynb/test_ipynb_to_ir.py
+++ b/tests/_convert/ipynb/test_ipynb_to_ir.py
@@ -9,7 +9,9 @@ import pytest
from marimo._convert.ipynb.to_ir import (
CellsTransform,
CodeCell,
+ PrivateRenamer,
Transform,
+ _rename_in_scope,
convert_from_ipynb_to_notebook_ir,
transform_add_marimo_import,
transform_add_subprocess_import,
@@ -97,14 +99,13 @@ for i_1 in range(X_1.shape[0]):
def test_transform_fixup_multiple_definitions_scope():
- # Makes everything private to avoid conflicts.
- # Comments are removed, unfortunately.
+ # NB. `K` inside `foo` is a local, so only the cell-level `K` is renamed.
sources = dd(
[
"""K = 2\n
def foo():
- K
K = 1
+ return K
""",
"K = 1",
]
@@ -113,8 +114,8 @@ def foo():
expected = [
"""_K = 2\n
def foo():
- _K
- _K = 1
+ K = 1
+ return K
""",
"_K = 1",
]
@@ -1531,3 +1532,229 @@ def test_remove_input_alongside_other_tags_only_consumes_remove_input():
assert cell.options.get("hide_code") is True
assert "# Cell tags: extra-tag" in cell.code
assert "remove-input" not in cell.code
+
+
+ ("source", "expected"),
+ [
+ pytest.param(
+ """
+ def double(n):
+ return n * 2
+ n = 5
+ print(double(3))
+ """,
+ """
+ def double(n):
+ return n * 2
+ _n = 5
+ print(double(3))
+ """,
+ id="function-parameter",
+ ),
+ pytest.param(
+ """
+ n = 5
+ f = lambda n: n + 1
+ print(f(n))
+ """,
+ """
+ _n = 5
+ f = lambda n: n + 1
+ print(f(_n))
+ """,
+ id="lambda-parameter",
+ ),
+ pytest.param(
+ """
+ n = 5
+ xs = [n for n in range(3)]
+ print(n, xs)
+ """,
+ """
+ _n = 5
+ xs = [n for n in range(3)]
+ print(_n, xs)
+ """,
+ id="comprehension-target",
+ ),
+ pytest.param(
+ """
+ n = 5
+ def g(path):
+ for n in range(2):
+ pass
+ with open(path) as n:
+ return n
+ """,
+ """
+ _n = 5
+ def g(path):
+ for n in range(2):
+ pass
+ with open(path) as n:
+ return n
+ """,
+ id="for-and-with-targets-in-function",
+ ),
+ pytest.param(
+ """
+ n = 5
+ def h(m):
+ return n + m
+ """,
+ """
+ _n = 5
+ def h(m):
+ return _n + m
+ """,
+ id="closure-read-is-still-renamed",
+ ),
+ pytest.param(
+ """
+ n = 5
+ def f(n=n):
+ return n
+ """,
+ """
+ _n = 5
+ def f(n=_n):
+ return n
+ """,
+ id="kwarg-default",
+ ),
+ ],
+)
+def test_transform_fixup_multiple_definitions_respects_inner_scopes(
+ source: str, expected: str
+):
+ # NB. an inner-scope binding shadows the cell-level name. The rename must
+ # not enter that scope.
+ sources = dd([source, "n = 999"])
+ result = transform_fixup_multiple_definitions(sources)
+ assert_sources_equal(result, [expected, "_n = 999"])
+
+
+def test_convert_privatization_keeps_parameter_and_body_consistent():
+ # NB. the converted function must read its argument, not the privatized
+ # cell-level value.
+ notebook = {
+ "cells": [
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": "def double(n):\n return n * 2\n\nn = 5\nprint(double(3))",
+ "outputs": [],
+ "execution_count": None,
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": "n = 999\nprint(n)",
+ "outputs": [],
+ "execution_count": None,
+ },
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ }
+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))
+ first = result.cells[0].code
+ assert "def double(n):" in first
+ assert "return n * 2" in first
+ assert "_n = 5" in first
+
+ namespace: dict[str, object] = {}
+ exec(first, namespace) # noqa: S102
+ assert namespace["double"](3) == 6 # type: ignore[operator]
+
+
+ ("source", "mapping", "expected"),
+ [
+ pytest.param(
+ """
+ old_global = 'world'
+ old_global, other = 1, 2
+ print(old_global.real)
+ """,
+ {"old_global": "new_global"},
+ """
+ new_global = 'world'
+ new_global, other = 1, 2
+ print(new_global.real)
+ """,
+ id="variable-and-tuple-targets",
+ ),
+ pytest.param(
+ """
+ def old_func():
+ pass
+
+ async def old_async_func():
+ pass
+ old_func()
+ """,
+ {"old_func": "new_func", "old_async_func": "new_async_func"},
+ """
+ def new_func():
+ pass
+
+ async def new_async_func():
+ pass
+ new_func()
+ """,
+ id="function-defs",
+ ),
+ pytest.param(
+ """
+ class OldClass:
+ def __init__(self):
+ self.old_attribute = 'hello'
+ OldClass()
+ """,
+ {"OldClass": "NewClass", "old_attribute": "new_attribute"},
+ """
+ class NewClass:
+ def __init__(self):
+ self.old_attribute = 'hello'
+ NewClass()
+ """,
+ id="class-def-and-attribute-untouched",
+ ),
+ pytest.param(
+ """
+ def old_function():
+ old_variable = 42
+ return old_variable
+ """,
+ {"old_function": "new_function", "old_variable": "new_variable"},
+ """
+ def new_function():
+ old_variable = 42
+ return old_variable
+ """,
+ id="function-local-untouched",
+ ),
+ ],
+)
+def test_rename_in_scope(
+ source: str, mapping: dict[str, str], expected: str
+) -> None:
+ renamer = PrivateRenamer(mapping)
+ result = _rename_in_scope(dedent(source), 0, renamer)
+ assert renamer.made_changes
+ assert_sources_equal([result], [expected])
+
+
+ "mapping", [{}, {"y": "z"}], ids=["empty-mapping", "no-match"]
+)
+def test_rename_in_scope_returns_source_verbatim_when_unchanged(
+ mapping: dict[str, str],
+) -> None:
+ source = "x = 1 # keep this comment\n\n\ndef func():\n return x\n"
+ renamer = PrivateRenamer(mapping)
+ assert _rename_in_scope(source, 0, renamer) is source
+ assert not renamer.made_changes