A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: basic Debugpy integration (#6311)
Dylan Madisetti149a1e0aa917
diff --git a/marimo/_ast/app.py b/marimo/_ast/app.pyindex a0e0cb9646..e16c73d923 100644--- a/marimo/_ast/app.py+++ b/marimo/_ast/app.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport astimport base64import inspect+import osimport sysimport threadingfrom collections.abc import Iterable, Iterator, Mapping@@ -594,6 +595,19 @@ class App:def run(self,) -> tuple[Sequence[Any], Mapping[str, Any]]:+ # Enabled specifically for debugging purposes.+ # see docs.marimo.io/guides/debugging+ if os.environ.get("MARIMO_SCRIPT_EDIT"):+ from marimo._cli.cli import edit++ if self._filename is None:+ raise RuntimeError(+ "MARIMO_SCRIPT_EDIT is set, but filename cannot be determined."+ )+ ctx = edit.make_context("edit", ["--watch", self._filename])+ edit.invoke(ctx)+ return ((), {})+self._maybe_initialize()glbls: dict[str, Any] = {}if self._setup is not None:diff --git a/marimo/_ast/cell_manager.py b/marimo/_ast/cell_manager.pyindex e54f5d36e1..0911d71c8b 100644--- a/marimo/_ast/cell_manager.py+++ b/marimo/_ast/cell_manager.py@@ -227,7 +227,8 @@ class CellManager:cell_id = CellId_t(SETUP_CELL_NAME)else:cell_id = self.create_cell_id()- cell = ir_cell_factory(cell_def, cell_id=cell_id)+ filename = app.filename if app is not None else None+ cell = ir_cell_factory(cell_def, cell_id=cell_id, filename=filename)cell_config = CellConfig.from_dict(cell_def.options,)diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.pyindex ca11ed78e3..83b3043ea3 100644--- a/marimo/_ast/compiler.py+++ b/marimo/_ast/compiler.py@@ -152,7 +152,16 @@ def compile_cell(source_position: Optional[SourcePosition] = None,carried_imports: list[ImportData] | None = None,test_rewrite: bool = False,+ filename: Optional[str] = None,) -> CellImpl:+ if filename is not None and source_position is None:+ source_position = solve_source_position(+ code,+ filename,+ )+ elif filename is not None and source_position is not None:+ source_position.filename = filename+# Replace non-breaking spaces with regular spaces -- some frontends# send nbsp in place of space, which is a syntax error.#@@ -213,7 +222,6 @@ def compile_cell(expr.col_offset = final_expr.end_col_offset # type: ignore[attr-defined]expr.end_col_offset = final_expr.end_col_offset # type: ignore[attr-defined]- filename: strif source_position:# Modify the "source" position for meaningful stacktracesfix_source_position(module, source_position)@@ -294,6 +302,36 @@ def compile_cell()+def solve_source_position(+ code: str, filename: str+) -> Optional[SourcePosition]:+ from marimo._ast.load import _maybe_contents+ from marimo._ast.parse import parse_notebook+ from marimo._utils.cell_matching import match_cell_ids_by_similarity++ contents = _maybe_contents(filename)+ if not contents:+ return None++ notebook = parse_notebook(contents)+ if notebook is None or not notebook.valid:+ return None+ on_disk = {+ CellId_t(str(i)): cell.code for i, cell in enumerate(notebook.cells)+ }+ matches = match_cell_ids_by_similarity(on_disk, {CellId_t("new"): code})+ if not matches or len(matches) != 1:+ return None+ (cell_index,) = matches.keys()+ index = int(cell_index)++ return SourcePosition(+ filename=filename,+ lineno=notebook.cells[index].lineno,+ col_offset=notebook.cells[index].col_offset,+ )++def get_source_position(f: Cls | Callable[..., Any], lineno: int, col_offset: int) -> Optional[SourcePosition]:@@ -427,9 +465,21 @@ def toplevel_cell_factory()-def ir_cell_factory(cell_def: CellDef, cell_id: CellId_t) -> Cell:+def ir_cell_factory(+ cell_def: CellDef, cell_id: CellId_t, filename: Optional[str] = None+) -> Cell:# NB. no need for test rewrite, anonymous file, etc.# Because this is never invoked in script mode.+ source_position = None+ # EXCEPT in the case of debugpy, where we need to preserve source position.+ if os.environ.get("DEBUGPY_RUNNING"):+ if filename and cell_def.lineno:+ source_position = SourcePosition(+ filename=filename,+ lineno=cell_def.lineno,+ col_offset=cell_def.col_offset,+ )+prefix = ""if isinstance(cell_def, (FunctionCell, ClassCell)):prefix = TOPLEVEL_CELL_PREFIX@@ -438,6 +488,7 @@ def ir_cell_factory(cell_def: CellDef, cell_id: CellId_t) -> Cell:_cell=compile_cell(cell_def.code,cell_id=cell_id,+ source_position=source_position,),)diff --git a/marimo/_runtime/dataflow.py b/marimo/_runtime/dataflow.pyindex c4abd044fd..f60925f95f 100644--- a/marimo/_runtime/dataflow.py+++ b/marimo/_runtime/dataflow.py@@ -594,6 +594,32 @@ class DirectedGraph:return processed | refsreturn processed - refs+ def copy(self, filename: None | str = None) -> DirectedGraph:+ """Return a deep copy of the graph by recompiling all cells.++ This is mainly useful in the case where recompilation must be done+ due to a dynamically changing notebook, where the line cache must be+ consistent with the cell code, e.g. for debugging.+ """+ from marimo._ast.compiler import compile_cell++ graph = DirectedGraph()+ with self.lock:+ for cid, old_cell in self.cells.items():+ cell = compile_cell(+ old_cell.code,+ cell_id=cid,+ filename=filename,+ )+ # Carry over import data manually+ imported_defs = old_cell.import_workspace.imported_defs+ is_import_block = old_cell.import_workspace.is_import_block+ cell.import_workspace.imported_defs = imported_defs+ cell.import_workspace.is_import_block = is_import_block+ # Reregister+ graph.register_cell(cid, cell)+ return graph+def transitive_closure(graph: DirectedGraph,diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex b49bd26b60..86ad568d0e 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -845,7 +845,9 @@ class Kernel:error: Optional[Error] = Nonetry:cell = compile_cell(- code, cell_id=cell_id, carried_imports=carried_imports+ code,+ cell_id=cell_id,+ carried_imports=carried_imports,)except Exception as e:cell = None@@ -1395,9 +1397,15 @@ class Kernel:if isinstance(run_result.exception, MarimoInterrupt):self.last_interrupt_timestamp = time.time()+ # Rebuild graph with sourceful positions+ # Note, this is relatively expensive, but a reasonable tradeoff+ graph = self.graph+ if os.getenv("DEBUGPY_RUNNING"):+ graph = self.graph.copy(self.app_metadata.filename)+runner = cell_runner.Runner(roots=roots,- graph=self.graph,+ graph=graph,glbls=self.globals,excluded_cells=set(self.errors.keys()),debugger=self.debugger,@@ -2747,7 +2755,7 @@ def launch_kernel(stdin = ThreadSafeStdin(stream) if is_edit_mode else Nonedebugger = (marimo_pdb.MarimoPdb(stdout=stdout, stdin=stdin)- if is_edit_mode+ if is_edit_mode and not bool(os.getenv("DEBUGPY_RUNNING"))else None)diff --git a/marimo/_smoke_tests/pdb_test.py b/marimo/_smoke_tests/pdb_test.pyindex c653240c1a..6716c73813 100644--- a/marimo/_smoke_tests/pdb_test.py+++ b/marimo/_smoke_tests/pdb_test.py@@ -1,20 +1,58 @@# Copyright 2024 Marimo. All rights reserved.+import marimo-__generated_with = "0.1.77"+__generated_with = "0.15.2"app = marimo.App()@app.cell-def __():+def _():+ def percentile(xs, p):+ """+ Return the p-th percentile of xs.+ """+ xs = sorted(xs)+ idx = round(p / 100 * len(xs)) # here there be bugs+ return xs[idx]+++ percentile([1, 2, 3], 100)+ return+++def _():+ # Compute triangle numbers+ triangle = 0+ triangle_count = 20+ for i in range(1, triangle_count):+ triangle += i # T_i = sum of 1..i+ # Debug at the 10th iteration+ # as a sanity check. Should be 55.+ if i == 10:+ breakpoint()+ return+++def _():import marimo as mo- return mo,+ return@app.cell-def __(mo):- # import pdb; pdb.set_trace()- mo.pdb.set_trace()+def _():+ import pdb++ pdb.set_trace()+ return+++def _():++returndiff --git a/tests/_ast/test_app.py b/tests/_ast/test_app.pyindex a1e9c0be87..b1068bc638 100644--- a/tests/_ast/test_app.py+++ b/tests/_ast/test_app.py@@ -1,12 +1,15 @@# Copyright 2024 Marimo. All rights reserved.from __future__ import annotations+import osimport pathlibimport subprocessimport sysimport textwrapfrom typing import TYPE_CHECKING, Any+from unittest.mock import patch+import clickimport pytestfrom marimo._ast.app import (diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.pyindex 38279df5fe..ba3cb88d44 100644--- a/tests/_ast/test_compiler.py+++ b/tests/_ast/test_compiler.py@@ -1,7 +1,9 @@# Copyright 2024 Marimo. All rights reserved.from __future__ import annotations+import osfrom functools import partial+from unittest.mock import patchimport pytest@@ -439,3 +441,164 @@ class TestSemicolon:cell = compiler.cell_factory(f, cell_id="0")assert eval(cell._cell.last_expr) is None+++class TestCompileCellFilename:+ """Test compile_cell function with filename parameter."""++ @staticmethod+ def test_compile_cell_with_filename_no_source_position() -> None:+ """Test compile_cell with filename when no source_position provided."""+ code = "x = 1"+ filename = "test_notebook.py"++ # Mock solve_source_position to return a source position+ with patch("marimo._ast.compiler.solve_source_position") as mock_solve:+ mock_solve.return_value = compiler.SourcePosition(+ filename=filename, lineno=1, col_offset=0+ )++ cell = compiler.compile_cell(+ code, cell_id="test", filename=filename+ )++ # Verify solve_source_position was called+ mock_solve.assert_called_once_with(code, filename)++ # Verify the cell was compiled successfully+ assert cell.code == code+ assert cell.defs == {"x"}++ @staticmethod+ def test_compile_cell_with_filename_and_source_position() -> None:+ """Test compile_cell with filename when source_position is provided."""+ code = "x = 1"+ filename = "test_notebook.py"+ source_position = compiler.SourcePosition(+ filename="original.py", lineno=5, col_offset=10+ )++ cell = compiler.compile_cell(+ code,+ cell_id="test",+ filename=filename,+ source_position=source_position,+ )++ # Verify the cell was compiled successfully+ assert cell.code == code+ assert cell.defs == {"x"}++ # Note: We can't easily test that source_position.filename was updated+ # without accessing internal state, but the function should work++ @staticmethod+ def test_compile_cell_without_filename() -> None:+ """Test compile_cell without filename parameter."""+ code = "x = 1"++ cell = compiler.compile_cell(code, cell_id="test")++ # Verify the cell was compiled successfully+ assert cell.code == code+ assert cell.defs == {"x"}++ @staticmethod+ def test_solve_source_position_function() -> None:+ """Test solve_source_position function."""+ # Create a temporary notebook file+ import os+ import tempfile++ notebook_content = """import marimo++__generated_with = "0.1.0"+app = marimo.App()++def __():+ x = 1+ return x,++def __(x):+ y = x + 1+ return y,+"""++ with tempfile.NamedTemporaryFile(+ mode="w", suffix=".py", delete=False+ ) as f:+ f.write(notebook_content)+ temp_filename = f.name++ try:+ # Test with code that matches a cell+ result = compiler.solve_source_position("x = 1", temp_filename)+ # The function might find a match, so just verify it returns a valid result or None+ if result is not None:+ assert result.filename == temp_filename+ assert result.lineno > 0+ assert result.col_offset >= 0++ # Test with code that doesn't match any cell+ result = compiler.solve_source_position("z = 999", temp_filename)+ # This might still find a match due to similarity, so just verify it's valid or None+ if result is not None:+ assert result.filename == temp_filename++ finally:+ os.unlink(temp_filename)++ @staticmethod+ def test_solve_source_position_invalid_file() -> None:+ """Test solve_source_position with invalid file."""+ # Mock the _maybe_contents function to return None for invalid files+ with patch("marimo._ast.load._maybe_contents") as mock_contents:+ mock_contents.return_value = None+ result = compiler.solve_source_position(+ "x = 1", "nonexistent_file.py"+ )+ assert result is None+++class TestIrCellFactoryDebugpy:+ """Test ir_cell_factory function with DEBUGPY_RUNNING environment variable."""++ @staticmethod+ def test_ir_cell_factory_with_debugpy_running() -> None:+ """Test ir_cell_factory with DEBUGPY_RUNNING environment variable."""+ from marimo._schemas.serialization import CellDef++ cell_def = CellDef(code="x = 1", options={}, lineno=5, col_offset=10)+ cell_id = "test_cell"+ filename = "test_notebook.py"++ with patch.dict(os.environ, {"DEBUGPY_RUNNING": "1"}):+ cell = compiler.ir_cell_factory(+ cell_def, cell_id, filename=filename+ )++ # Verify the cell was created successfully+ assert cell._name == "_"+ assert cell._cell.code == "x = 1"+ assert cell._cell.defs == {"x"}++ @staticmethod+ def test_ir_cell_factory_without_debugpy_running() -> None:+ """Test ir_cell_factory without DEBUGPY_RUNNING environment variable."""+ from marimo._schemas.serialization import CellDef++ cell_def = CellDef(code="x = 1", options={}, lineno=5, col_offset=10)+ cell_id = "test_cell"+ filename = "test_notebook.py"++ with patch.dict(os.environ, {}, clear=True):+ cell = compiler.ir_cell_factory(+ cell_def, cell_id, filename=filename+ )++ # Verify the cell was created successfully+ assert cell._name == "_"+ assert cell._cell.code == "x = 1"+ assert cell._cell.defs == {"x"}diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.pyindex 4426b89001..cf9e8730b1 100644--- a/tests/_runtime/test_dataflow.py+++ b/tests/_runtime/test_dataflow.py@@ -2,6 +2,7 @@from __future__ import annotationsfrom functools import partial+from unittest.mock import patchimport pytest@@ -1337,3 +1338,70 @@ def public_func():# Private variable shouldn't appear directlyassert "_private_var" not in refs+++def test_directed_graph_copy() -> None:+ """Test DirectedGraph.copy() method for recompiling cells with filename."""+ graph = dataflow.DirectedGraph()++ # Create some cells+ code = "import marimo as mo"+ cell1 = parse_cell(code)++ graph.register_cell("0", cell1)++ # Mock compile_cell to verify it's called with filename+ with patch("marimo._ast.compiler.compile_cell") as mock_compile:+ mock_compile.return_value = parse_cell(code)++ filename = "test_notebook.py"+ copied_graph = graph.copy(filename)++ # Verify compile_cell was called for each cell with filename+ assert mock_compile.call_count == 1+ for call in mock_compile.call_args_list:+ assert call[1]["filename"] == filename++ # Verify the copied graph has the same structure+ assert len(copied_graph.cells) == 1+ assert set(copied_graph.cells.keys()) == {"0"}++ # Verify import workspace data is preserved+ for cell_id in copied_graph.cells:+ original_cell = graph.cells[cell_id]+ copied_cell = copied_graph.cells[cell_id]+ assert (+ copied_cell.import_workspace.imported_defs+ == original_cell.import_workspace.imported_defs+ )+ assert (+ copied_cell.import_workspace.is_import_block+ == original_cell.import_workspace.is_import_block+ )+++def test_directed_graph_copy_no_filename() -> None:+ """Test DirectedGraph.copy() method without filename."""+ graph = dataflow.DirectedGraph()++ # Create a cell+ code = "y = x + 1"+ cell1 = parse_cell(code)+ graph.register_cell("0", cell1)++ # Mock compile_cell to verify it's called without filename+ with patch("marimo._ast.compiler.compile_cell") as mock_compile:+ mock_compile.return_value = parse_cell(code)++ copied_graph = graph.copy(None)++ # Verify compile_cell was called without filename+ assert mock_compile.call_count == 1+ call_args = mock_compile.call_args+ assert (+ "filename" not in call_args[1] or call_args[1]["filename"] is None+ )++ # Verify the copied graph has the same structure+ assert len(copied_graph.cells) == 1+ assert "0" in copied_graph.cells