A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: line parsing for debugpy (#7415)
Dylan Madisetti5ad469aaaa0e
diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.pyindex 3f3a54dcd3..3933d698be 100644--- a/marimo/_ast/parse.py+++ b/marimo/_ast/parse.py@@ -78,6 +78,20 @@ def fixed_dedent(text: str) -> str:return dedent("\n".join(map(refill, lines)))+def extract_lineno(node: Node) -> int:+ if not isinstance(+ node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef)+ ):+ return node.lineno++ # In the case of a decorated function etc., we should not point to the body,+ # but instead the first line.+ decorator = get_valid_decorator(node)+ if not decorator or not decorator.end_lineno:+ return node.lineno+ return decorator.end_lineno++class MarimoFileError(Exception):passdiff --git a/marimo/_schemas/serialization.py b/marimo/_schemas/serialization.pyindex eb117747e6..40b66cefda 100644--- a/marimo/_schemas/serialization.py+++ b/marimo/_schemas/serialization.py@@ -47,8 +47,12 @@ class CellDef(Node):_ast: Optional[NodeRef] = Nonedef __post_init__(self) -> None:+ from marimo._ast.parse import extract_lineno+if self._ast:- self.lineno = self._ast.lineno if self.lineno == 0 else self.lineno+ self.lineno = (+ extract_lineno(self._ast) if self.lineno == 0 else self.lineno+ )self.col_offset = (self._ast.col_offsetif self.col_offset == 0diff --git a/tests/_ast/codegen_data/test_stacked_decorators_toplevel.py b/tests/_ast/codegen_data/test_stacked_decorators_toplevel.pynew file mode 100644index 0000000000..bec16b08e1--- /dev/null+++ b/tests/_ast/codegen_data/test_stacked_decorators_toplevel.py@@ -0,0 +1,38 @@+import marimo++__generated_with = "0.18.2"+app = marimo.App()++with app.setup:+ import marimo as mo+ wrap = lambda x: x+++# comment after app.function+# comment after cache+def cached_func():+ return 42+++def normal_cell():+ a = 1+ return (a,)++[email protected]_definition+@wrap+class Foo: ...++[email protected](hide_code=True)+def _():+ with mo.cache("example"):+ print("Hello")+ return+++if __name__ == "__main__":+ app.run()diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.pyindex ba3cb88d44..01a655de15 100644--- a/tests/_ast/test_compiler.py+++ b/tests/_ast/test_compiler.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport osfrom functools import partial+from typing import Anyfrom unittest.mock import patchimport pytest@@ -602,3 +603,326 @@ class TestIrCellFactoryDebugpy:assert cell._name == "_"assert cell._cell.code == "x = 1"assert cell._cell.defs == {"x"}+++class TestSolveSourcePositionToplevelCells:+ """Test solve_source_position returns correct lineno for all cell types.++ The key invariant: the first AST-meaningful line of cell.code should be+ contained in the source file at the reported lineno.+ """++ DATA_FILE = os.path.join(+ os.path.dirname(__file__),+ "codegen_data",+ "test_stacked_decorators_toplevel.py",+ )++ def _assert_lineno_matches_source(+ self, cell_code: str, source: str, lineno: int+ ) -> None:+ """Assert cell AST maps correctly to source after applying lineno offset.++ lineno is a base offset: cell AST line N maps to raw source line (lineno + N).+ """+ import ast++ cell_ast = ast.parse(cell_code)+ source_lines = source.split("\n")++ # Get the first AST line number (accounting for decorators)+ first_stmt = cell_ast.body[0]+ if isinstance(+ first_stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)+ ):+ if first_stmt.decorator_list:+ first_ast_lineno = min(+ d.lineno for d in first_stmt.decorator_list+ )+ else:+ first_ast_lineno = first_stmt.lineno+ else:+ first_ast_lineno = first_stmt.lineno++ # After fix_source_position: cell line N becomes lineno + N+ raw_lineno = lineno + first_ast_lineno++ # Find the first non-comment line in cell.code (what the AST represents)+ cell_lines = cell_code.split("\n")+ first_ast_content = next(+ (+ ln.strip()+ for ln in cell_lines+ if ln.strip() and not ln.strip().startswith("#")+ ),+ cell_lines[0].strip(),+ )++ # Check the source line at the computed position+ raw_line = (+ source_lines[raw_lineno - 1]+ if raw_lineno <= len(source_lines)+ else ""+ )++ assert first_ast_content in raw_line, (+ f"Cell AST does not map correctly to source.\n"+ f"lineno={lineno}, first_ast_lineno={first_ast_lineno}, "+ f"raw_lineno={raw_lineno}\n"+ f"first AST content: {first_ast_content!r}\n"+ f"source line: {raw_line!r}"+ )++ def test_function_cell(self) -> None:+ """@app.function with stacked decorators."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if "def cached_func" in c.code)+ assert "@app.function" not in cell.code+ assert "@mo.cache" in cell.code++ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None+ self._assert_lineno_matches_source(cell.code, source, pos.lineno)++ def test_setup_cell(self) -> None:+ """with app.setup: block."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(+ c for c in notebook.cells if "import marimo as mo" in c.code+ )++ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None+ self._assert_lineno_matches_source(cell.code, source, pos.lineno)++ def test_class_definition_cell(self) -> None:+ """@app.class_definition with stacked decorator."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if "class Foo" in c.code)+ assert "@app.class_definition" not in cell.code+ assert "@wrap" in cell.code++ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None+ self._assert_lineno_matches_source(cell.code, source, pos.lineno)++ def test_normal_cell(self) -> None:+ """@app.cell basic."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if c.code.strip() == "a = 1")+ assert "@app.cell" not in cell.code++ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None+ self._assert_lineno_matches_source(cell.code, source, pos.lineno)++ def test_normal_cell_with_options(self) -> None:+ """@app.cell(hide_code=True)."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(+ c for c in notebook.cells if 'mo.cache("example")' in c.code+ )+ assert "@app.cell" not in cell.code++ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None+ self._assert_lineno_matches_source(cell.code, source, pos.lineno)+++class TestCellAstMatchesRawAst:+ """Test that fix_source_position correctly maps cell AST to raw file positions.++ After fix_source_position, the first AST node's lineno should point to the+ containing line in the raw source file.+ """++ DATA_FILE = os.path.join(+ os.path.dirname(__file__),+ "codegen_data",+ "test_stacked_decorators_toplevel.py",+ )++ def _get_first_ast_lineno(self, node: Any) -> int:+ """Get the first line number of an AST node, including decorators."""+ import ast++ # For decorated nodes, check decorator line numbers+ if isinstance(+ node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)+ ):+ if node.decorator_list:+ return min(d.lineno for d in node.decorator_list)+ return node.lineno++ def _assert_fixed_ast_contains_line(+ self,+ cell_code: str,+ raw_source: str,+ source_position: compiler.SourcePosition,+ ) -> None:+ """Assert cell AST after fix_source_position points to containing line."""+ import ast++ cell_ast = ast.parse(cell_code)+ source_lines = raw_source.split("\n")++ # Apply fix_source_position to adjust cell AST line numbers+ compiler.fix_source_position(cell_ast, source_position)++ # Get the first statement's line number after fix (including decorators)+ first_stmt = cell_ast.body[0]+ fixed_lineno = self._get_first_ast_lineno(first_stmt)++ # The line in raw source at fixed_lineno should contain the first AST line of cell.code+ # Find the first non-comment line in cell.code+ cell_lines = cell_code.split("\n")+ first_ast_line = next(+ (+ ln.strip()+ for ln in cell_lines+ if ln.strip() and not ln.strip().startswith("#")+ ),+ cell_lines[0].strip(),+ )+ # lineno is 1-indexed+ raw_line = (+ source_lines[fixed_lineno - 1]+ if fixed_lineno <= len(source_lines)+ else ""+ )++ assert first_ast_line in raw_line, (+ f"Fixed AST lineno={fixed_lineno} does not contain cell code.\n"+ f"first AST line: {first_ast_line!r}\n"+ f"raw source line: {raw_line!r}\n"+ f"source_position.lineno={source_position.lineno}"+ )++ def test_function_cell_ast(self) -> None:+ """@app.function cell AST after fix_source_position points to containing line."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if "def cached_func" in c.code)+ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None++ self._assert_fixed_ast_contains_line(cell.code, source, pos)++ def test_setup_cell_ast(self) -> None:+ """Setup cell AST after fix_source_position points to containing line."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(+ c for c in notebook.cells if "import marimo as mo" in c.code+ )+ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None++ self._assert_fixed_ast_contains_line(cell.code, source, pos)++ def test_class_definition_cell_ast(self) -> None:+ """@app.class_definition cell AST after fix_source_position points to containing line."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if "class Foo" in c.code)+ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None++ self._assert_fixed_ast_contains_line(cell.code, source, pos)++ def test_normal_cell_ast(self) -> None:+ """@app.cell AST after fix_source_position points to containing line."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(c for c in notebook.cells if c.code.strip() == "a = 1")+ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None++ self._assert_fixed_ast_contains_line(cell.code, source, pos)++ def test_normal_cell_with_options_ast(self) -> None:+ """@app.cell(hide_code=True) AST after fix_source_position points to containing line."""+ from marimo._ast.parse import parse_notebook++ with open(self.DATA_FILE) as f:+ source = f.read()++ notebook = parse_notebook(source)+ assert notebook is not None+ assert notebook.valid++ cell = next(+ c for c in notebook.cells if 'mo.cache("example")' in c.code+ )+ pos = compiler.solve_source_position(cell.code, self.DATA_FILE)+ assert pos is not None++ self._assert_fixed_ast_contains_line(cell.code, source, pos)