Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: basic Debugpy integration (#6311)

Dylan Madisetti149a1e0aa917
diff --git a/marimo/_ast/app.py b/marimo/_ast/app.py
index a0e0cb9646..e16c73d923 100644
--- a/marimo/_ast/app.py
+++ b/marimo/_ast/app.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import ast
import base64
import inspect
+import os
import sys
import threading
from collections.abc import Iterable, Iterator, Mapping
@@ -594,6 +595,19 @@ class App:
def run(
self,
) -> tuple[Sequence[Any], Mapping[str, Any]]:
+ # Enabled specifically for debugging purposes.
+ # see docs.marimo.io/guides/debugging
+ if os.environ.get("MARIMO_SCRIPT_EDIT"):
+ from marimo._cli.cli import edit
+
+ if self._filename is None:
+ raise RuntimeError(
+ "MARIMO_SCRIPT_EDIT is set, but filename cannot be determined."
+ )
+ ctx = edit.make_context("edit", ["--watch", self._filename])
+ edit.invoke(ctx)
+ return ((), {})
+
self._maybe_initialize()
glbls: dict[str, Any] = {}
if self._setup is not None:
diff --git a/marimo/_ast/cell_manager.py b/marimo/_ast/cell_manager.py
index e54f5d36e1..0911d71c8b 100644
--- a/marimo/_ast/cell_manager.py
+++ b/marimo/_ast/cell_manager.py
@@ -227,7 +227,8 @@ class CellManager:
cell_id = CellId_t(SETUP_CELL_NAME)
else:
cell_id = self.create_cell_id()
- cell = ir_cell_factory(cell_def, cell_id=cell_id)
+ filename = app.filename if app is not None else None
+ cell = ir_cell_factory(cell_def, cell_id=cell_id, filename=filename)
cell_config = CellConfig.from_dict(
cell_def.options,
)
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index ca11ed78e3..83b3043ea3 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -152,7 +152,16 @@ def compile_cell(
source_position: Optional[SourcePosition] = None,
carried_imports: list[ImportData] | None = None,
test_rewrite: bool = False,
+ filename: Optional[str] = None,
) -> CellImpl:
+ if filename is not None and source_position is None:
+ source_position = solve_source_position(
+ code,
+ filename,
+ )
+ elif filename is not None and source_position is not None:
+ source_position.filename = filename
+
# Replace non-breaking spaces with regular spaces -- some frontends
# send nbsp in place of space, which is a syntax error.
#
@@ -213,7 +222,6 @@ def compile_cell(
expr.col_offset = final_expr.end_col_offset # type: ignore[attr-defined]
expr.end_col_offset = final_expr.end_col_offset # type: ignore[attr-defined]
- filename: str
if source_position:
# Modify the "source" position for meaningful stacktraces
fix_source_position(module, source_position)
@@ -294,6 +302,36 @@ def compile_cell(
)
+def solve_source_position(
+ code: str, filename: str
+) -> Optional[SourcePosition]:
+ from marimo._ast.load import _maybe_contents
+ from marimo._ast.parse import parse_notebook
+ from marimo._utils.cell_matching import match_cell_ids_by_similarity
+
+ contents = _maybe_contents(filename)
+ if not contents:
+ return None
+
+ notebook = parse_notebook(contents)
+ if notebook is None or not notebook.valid:
+ return None
+ on_disk = {
+ CellId_t(str(i)): cell.code for i, cell in enumerate(notebook.cells)
+ }
+ matches = match_cell_ids_by_similarity(on_disk, {CellId_t("new"): code})
+ if not matches or len(matches) != 1:
+ return None
+ (cell_index,) = matches.keys()
+ index = int(cell_index)
+
+ return SourcePosition(
+ filename=filename,
+ lineno=notebook.cells[index].lineno,
+ col_offset=notebook.cells[index].col_offset,
+ )
+
+
def get_source_position(
f: Cls | Callable[..., Any], lineno: int, col_offset: int
) -> Optional[SourcePosition]:
@@ -427,9 +465,21 @@ def toplevel_cell_factory(
)
-def ir_cell_factory(cell_def: CellDef, cell_id: CellId_t) -> Cell:
+def ir_cell_factory(
+ cell_def: CellDef, cell_id: CellId_t, filename: Optional[str] = None
+) -> Cell:
# NB. no need for test rewrite, anonymous file, etc.
# Because this is never invoked in script mode.
+ source_position = None
+ # EXCEPT in the case of debugpy, where we need to preserve source position.
+ if os.environ.get("DEBUGPY_RUNNING"):
+ if filename and cell_def.lineno:
+ source_position = SourcePosition(
+ filename=filename,
+ lineno=cell_def.lineno,
+ col_offset=cell_def.col_offset,
+ )
+
prefix = ""
if isinstance(cell_def, (FunctionCell, ClassCell)):
prefix = TOPLEVEL_CELL_PREFIX
@@ -438,6 +488,7 @@ def ir_cell_factory(cell_def: CellDef, cell_id: CellId_t) -> Cell:
_cell=compile_cell(
cell_def.code,
cell_id=cell_id,
+ source_position=source_position,
),
)
diff --git a/marimo/_runtime/dataflow.py b/marimo/_runtime/dataflow.py
index c4abd044fd..f60925f95f 100644
--- a/marimo/_runtime/dataflow.py
+++ b/marimo/_runtime/dataflow.py
@@ -594,6 +594,32 @@ class DirectedGraph:
return processed | refs
return processed - refs
+ def copy(self, filename: None | str = None) -> DirectedGraph:
+ """Return a deep copy of the graph by recompiling all cells.
+
+ This is mainly useful in the case where recompilation must be done
+ due to a dynamically changing notebook, where the line cache must be
+ consistent with the cell code, e.g. for debugging.
+ """
+ from marimo._ast.compiler import compile_cell
+
+ graph = DirectedGraph()
+ with self.lock:
+ for cid, old_cell in self.cells.items():
+ cell = compile_cell(
+ old_cell.code,
+ cell_id=cid,
+ filename=filename,
+ )
+ # Carry over import data manually
+ imported_defs = old_cell.import_workspace.imported_defs
+ is_import_block = old_cell.import_workspace.is_import_block
+ cell.import_workspace.imported_defs = imported_defs
+ cell.import_workspace.is_import_block = is_import_block
+ # Reregister
+ graph.register_cell(cid, cell)
+ return graph
+
def transitive_closure(
graph: DirectedGraph,
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index b49bd26b60..86ad568d0e 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -845,7 +845,9 @@ class Kernel:
error: Optional[Error] = None
try:
cell = compile_cell(
- code, cell_id=cell_id, carried_imports=carried_imports
+ code,
+ cell_id=cell_id,
+ carried_imports=carried_imports,
)
except Exception as e:
cell = None
@@ -1395,9 +1397,15 @@ class Kernel:
if isinstance(run_result.exception, MarimoInterrupt):
self.last_interrupt_timestamp = time.time()
+ # Rebuild graph with sourceful positions
+ # Note, this is relatively expensive, but a reasonable tradeoff
+ graph = self.graph
+ if os.getenv("DEBUGPY_RUNNING"):
+ graph = self.graph.copy(self.app_metadata.filename)
+
runner = cell_runner.Runner(
roots=roots,
- graph=self.graph,
+ graph=graph,
glbls=self.globals,
excluded_cells=set(self.errors.keys()),
debugger=self.debugger,
@@ -2747,7 +2755,7 @@ def launch_kernel(
stdin = ThreadSafeStdin(stream) if is_edit_mode else None
debugger = (
marimo_pdb.MarimoPdb(stdout=stdout, stdin=stdin)
- if is_edit_mode
+ if is_edit_mode and not bool(os.getenv("DEBUGPY_RUNNING"))
else None
)
diff --git a/marimo/_smoke_tests/pdb_test.py b/marimo/_smoke_tests/pdb_test.py
index c653240c1a..6716c73813 100644
--- a/marimo/_smoke_tests/pdb_test.py
+++ b/marimo/_smoke_tests/pdb_test.py
@@ -1,20 +1,58 @@
# Copyright 2024 Marimo. All rights reserved.
+
import marimo
-__generated_with = "0.1.77"
+__generated_with = "0.15.2"
app = marimo.App()
@app.cell
-def __():
+def _():
+ def percentile(xs, p):
+ """
+ Return the p-th percentile of xs.
+ """
+ xs = sorted(xs)
+ idx = round(p / 100 * len(xs)) # here there be bugs
+ return xs[idx]
+
+
+ percentile([1, 2, 3], 100)
+ return
+
+
+def _():
+ # Compute triangle numbers
+ triangle = 0
+ triangle_count = 20
+ for i in range(1, triangle_count):
+ triangle += i # T_i = sum of 1..i
+ # Debug at the 10th iteration
+ # as a sanity check. Should be 55.
+ if i == 10:
+ breakpoint()
+ return
+
+
+def _():
import marimo as mo
- return mo,
+ return
@app.cell
-def __(mo):
- # import pdb; pdb.set_trace()
- mo.pdb.set_trace()
+def _():
+ import pdb
+
+ pdb.set_trace()
+ return
+
+
+def _():
+
+
return
diff --git a/tests/_ast/test_app.py b/tests/_ast/test_app.py
index a1e9c0be87..b1068bc638 100644
--- a/tests/_ast/test_app.py
+++ b/tests/_ast/test_app.py
@@ -1,12 +1,15 @@
# Copyright 2024 Marimo. All rights reserved.
from __future__ import annotations
+import os
import pathlib
import subprocess
import sys
import textwrap
from typing import TYPE_CHECKING, Any
+from unittest.mock import patch
+import click
import pytest
from marimo._ast.app import (
diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.py
index 38279df5fe..ba3cb88d44 100644
--- a/tests/_ast/test_compiler.py
+++ b/tests/_ast/test_compiler.py
@@ -1,7 +1,9 @@
# Copyright 2024 Marimo. All rights reserved.
from __future__ import annotations
+import os
from functools import partial
+from unittest.mock import patch
import pytest
@@ -439,3 +441,164 @@ class TestSemicolon:
cell = compiler.cell_factory(f, cell_id="0")
assert eval(cell._cell.last_expr) is None
+
+
+class TestCompileCellFilename:
+ """Test compile_cell function with filename parameter."""
+
+ @staticmethod
+ def test_compile_cell_with_filename_no_source_position() -> None:
+ """Test compile_cell with filename when no source_position provided."""
+ code = "x = 1"
+ filename = "test_notebook.py"
+
+ # Mock solve_source_position to return a source position
+ with patch("marimo._ast.compiler.solve_source_position") as mock_solve:
+ mock_solve.return_value = compiler.SourcePosition(
+ filename=filename, lineno=1, col_offset=0
+ )
+
+ cell = compiler.compile_cell(
+ code, cell_id="test", filename=filename
+ )
+
+ # Verify solve_source_position was called
+ mock_solve.assert_called_once_with(code, filename)
+
+ # Verify the cell was compiled successfully
+ assert cell.code == code
+ assert cell.defs == {"x"}
+
+ @staticmethod
+ def test_compile_cell_with_filename_and_source_position() -> None:
+ """Test compile_cell with filename when source_position is provided."""
+ code = "x = 1"
+ filename = "test_notebook.py"
+ source_position = compiler.SourcePosition(
+ filename="original.py", lineno=5, col_offset=10
+ )
+
+ cell = compiler.compile_cell(
+ code,
+ cell_id="test",
+ filename=filename,
+ source_position=source_position,
+ )
+
+ # Verify the cell was compiled successfully
+ assert cell.code == code
+ assert cell.defs == {"x"}
+
+ # Note: We can't easily test that source_position.filename was updated
+ # without accessing internal state, but the function should work
+
+ @staticmethod
+ def test_compile_cell_without_filename() -> None:
+ """Test compile_cell without filename parameter."""
+ code = "x = 1"
+
+ cell = compiler.compile_cell(code, cell_id="test")
+
+ # Verify the cell was compiled successfully
+ assert cell.code == code
+ assert cell.defs == {"x"}
+
+ @staticmethod
+ def test_solve_source_position_function() -> None:
+ """Test solve_source_position function."""
+ # Create a temporary notebook file
+ import os
+ import tempfile
+
+ notebook_content = """import marimo
+
+__generated_with = "0.1.0"
+app = marimo.App()
+
+def __():
+ x = 1
+ return x,
+
+def __(x):
+ y = x + 1
+ return y,
+"""
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".py", delete=False
+ ) as f:
+ f.write(notebook_content)
+ temp_filename = f.name
+
+ try:
+ # Test with code that matches a cell
+ result = compiler.solve_source_position("x = 1", temp_filename)
+ # The function might find a match, so just verify it returns a valid result or None
+ if result is not None:
+ assert result.filename == temp_filename
+ assert result.lineno > 0
+ assert result.col_offset >= 0
+
+ # Test with code that doesn't match any cell
+ result = compiler.solve_source_position("z = 999", temp_filename)
+ # This might still find a match due to similarity, so just verify it's valid or None
+ if result is not None:
+ assert result.filename == temp_filename
+
+ finally:
+ os.unlink(temp_filename)
+
+ @staticmethod
+ def test_solve_source_position_invalid_file() -> None:
+ """Test solve_source_position with invalid file."""
+ # Mock the _maybe_contents function to return None for invalid files
+ with patch("marimo._ast.load._maybe_contents") as mock_contents:
+ mock_contents.return_value = None
+ result = compiler.solve_source_position(
+ "x = 1", "nonexistent_file.py"
+ )
+ assert result is None
+
+
+class TestIrCellFactoryDebugpy:
+ """Test ir_cell_factory function with DEBUGPY_RUNNING environment variable."""
+
+ @staticmethod
+ def test_ir_cell_factory_with_debugpy_running() -> None:
+ """Test ir_cell_factory with DEBUGPY_RUNNING environment variable."""
+ from marimo._schemas.serialization import CellDef
+
+ cell_def = CellDef(code="x = 1", options={}, lineno=5, col_offset=10)
+ cell_id = "test_cell"
+ filename = "test_notebook.py"
+
+ with patch.dict(os.environ, {"DEBUGPY_RUNNING": "1"}):
+ cell = compiler.ir_cell_factory(
+ cell_def, cell_id, filename=filename
+ )
+
+ # Verify the cell was created successfully
+ assert cell._name == "_"
+ assert cell._cell.code == "x = 1"
+ assert cell._cell.defs == {"x"}
+
+ @staticmethod
+ def test_ir_cell_factory_without_debugpy_running() -> None:
+ """Test ir_cell_factory without DEBUGPY_RUNNING environment variable."""
+ from marimo._schemas.serialization import CellDef
+
+ cell_def = CellDef(code="x = 1", options={}, lineno=5, col_offset=10)
+ cell_id = "test_cell"
+ filename = "test_notebook.py"
+
+ with patch.dict(os.environ, {}, clear=True):
+ cell = compiler.ir_cell_factory(
+ cell_def, cell_id, filename=filename
+ )
+
+ # Verify the cell was created successfully
+ assert cell._name == "_"
+ assert cell._cell.code == "x = 1"
+ assert cell._cell.defs == {"x"}
diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.py
index 4426b89001..cf9e8730b1 100644
--- a/tests/_runtime/test_dataflow.py
+++ b/tests/_runtime/test_dataflow.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from functools import partial
+from unittest.mock import patch
import pytest
@@ -1337,3 +1338,70 @@ def public_func():
# Private variable shouldn't appear directly
assert "_private_var" not in refs
+
+
+def test_directed_graph_copy() -> None:
+ """Test DirectedGraph.copy() method for recompiling cells with filename."""
+ graph = dataflow.DirectedGraph()
+
+ # Create some cells
+ code = "import marimo as mo"
+ cell1 = parse_cell(code)
+
+ graph.register_cell("0", cell1)
+
+ # Mock compile_cell to verify it's called with filename
+ with patch("marimo._ast.compiler.compile_cell") as mock_compile:
+ mock_compile.return_value = parse_cell(code)
+
+ filename = "test_notebook.py"
+ copied_graph = graph.copy(filename)
+
+ # Verify compile_cell was called for each cell with filename
+ assert mock_compile.call_count == 1
+ for call in mock_compile.call_args_list:
+ assert call[1]["filename"] == filename
+
+ # Verify the copied graph has the same structure
+ assert len(copied_graph.cells) == 1
+ assert set(copied_graph.cells.keys()) == {"0"}
+
+ # Verify import workspace data is preserved
+ for cell_id in copied_graph.cells:
+ original_cell = graph.cells[cell_id]
+ copied_cell = copied_graph.cells[cell_id]
+ assert (
+ copied_cell.import_workspace.imported_defs
+ == original_cell.import_workspace.imported_defs
+ )
+ assert (
+ copied_cell.import_workspace.is_import_block
+ == original_cell.import_workspace.is_import_block
+ )
+
+
+def test_directed_graph_copy_no_filename() -> None:
+ """Test DirectedGraph.copy() method without filename."""
+ graph = dataflow.DirectedGraph()
+
+ # Create a cell
+ code = "y = x + 1"
+ cell1 = parse_cell(code)
+ graph.register_cell("0", cell1)
+
+ # Mock compile_cell to verify it's called without filename
+ with patch("marimo._ast.compiler.compile_cell") as mock_compile:
+ mock_compile.return_value = parse_cell(code)
+
+ copied_graph = graph.copy(None)
+
+ # Verify compile_cell was called without filename
+ assert mock_compile.call_count == 1
+ call_args = mock_compile.call_args
+ assert (
+ "filename" not in call_args[1] or call_args[1]["filename"] is None
+ )
+
+ # Verify the copied graph has the same structure
+ assert len(copied_graph.cells) == 1
+ assert "0" in copied_graph.cells