Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: change tokenization to allow trailing comments (#10981)

Dylan Madisettif360c3dc0c21
diff --git a/marimo/_ast/codegen.py b/marimo/_ast/codegen.py
index 1000c571b5..e70cc8355a 100644
--- a/marimo/_ast/codegen.py
+++ b/marimo/_ast/codegen.py
@@ -181,12 +181,21 @@ def format_markdown(cell: CellImpl) -> str:
# tokenize.
tokens = tokenize.tokenize(io.BytesIO(cell.code.encode("utf-8")).readline)
tag = ""
- # Comment capture
- comments = {
- "prefix": "",
- "suffix": "",
- }
- key: str | None = "prefix"
+ # Anything outside the `mo.md(...)` call (comments, and the whitespace
+ # around them) is kept verbatim, so track where the call starts and ends
+ # in the source rather than rebuilding it from tokens, which drop
+ # whitespace.
+ line_starts = [0]
+ for match in re.finditer(r"\r\n|\r|\n", cell.code):
+ line_starts.append(match.end())
+
+ def offset(position: tuple[int, int]) -> int:
+ row, col = position
+ return line_starts[row - 1] + col
+
+ call_start = 0
+ call_end = len(cell.code)
+ depth = 0
tokenizes_fstring = sys.version_info >= (3, 12)
start_tokens = (
(tokenize.STRING, tokenize.FSTRING_START)
@@ -206,11 +215,14 @@ def format_markdown(cell: CellImpl) -> str:
start = start[1:]
fstring = "f" in tag.lower()
elif tok.string == "mo":
- key = None
+ call_start = offset(tok.start)
+ depth = 0
+ elif tok.string == "(":
+ depth += 1
elif tok.string == ")":
- key = "suffix"
- elif key in comments and tok.type != tokenize.ENCODING:
- comments[key] += tok.string
+ depth -= 1
+ if depth == 0:
+ call_end = offset(tok.end)
if fstring:
# We can blanket replace, because cell.markdown is not set
@@ -220,7 +232,7 @@ def format_markdown(cell: CellImpl) -> str:
# We always use """ as per front end.
body = construct_markdown_call(markdown, '"""', tag)
- return "".join([comments["prefix"], body, comments["suffix"]])
+ return "".join([cell.code[:call_start], body, cell.code[call_end:]])
def construct_markdown_call(markdown: str, quote: str, tag: str) -> str:
diff --git a/tests/_ast/test_codegen.py b/tests/_ast/test_codegen.py
index 167cc578f4..e44e939aa1 100644
--- a/tests/_ast/test_codegen.py
+++ b/tests/_ast/test_codegen.py
@@ -2265,3 +2265,39 @@ def _strip_header_footer(source: str) -> str:
code_start = source.index(header)
code_end = source.index('if __name__ == "__main__":')
return source[code_start + len(header) : code_end].strip()
+
+
+ ("code", "expected"),
+ [
+ # Already formatted: identity
+ ('mo.md(r"""\nA\n""")', 'mo.md(r"""\nA\n""")'),
+ # Inline comment after the call keeps its leading whitespace (#10976)
+ (
+ 'mo.md(r"""\nA\n""") # noqa: E501',
+ 'mo.md(r"""\nA\n""") # noqa: E501',
+ ),
+ # Comments on their own lines, before and after
+ (
+ '# lead\n\nmo.md(r"""\nA\n""")\n# trail',
+ '# lead\n\nmo.md(r"""\nA\n""")\n# trail',
+ ),
+ # Reformatting the call preserves the inline comment
+ (
+ 'mo.md(\n r"""\n A\n """\n) # noqa: E501',
+ 'mo.md(r"""\nA\n""") # noqa: E501',
+ ),
+ # Non-ASCII content does not shift the comment position
+ (
+ 'mo.md(r"""\nÄé\n""") # noqa: É',
+ 'mo.md(r"""\nÄé\n""") # noqa: É',
+ ),
+ # Parentheses around the call are kept
+ ('(mo.md("x"))', '(mo.md("""\nx\n"""))'),
+ ('mo.md(("x"))', 'mo.md("""\nx\n""")'),
+ ],
+)
+def test_format_markdown_preserves_surrounding_source(
+ code: str, expected: str
+) -> None:
+ assert codegen.format_markdown(compile_cell(code)) == expected
diff --git a/tests/_lint/test_files/markdown_dedent.py b/tests/_lint/test_files/markdown_dedent.py
index f695a45561..f42b15d0c9 100644
--- a/tests/_lint/test_files/markdown_dedent.py
+++ b/tests/_lint/test_files/markdown_dedent.py
@@ -77,5 +77,12 @@ def _(mo):
mo.md(fr"siiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiingle line")
return
+def _(mo):
+ mo.md(r"""
+ Inline comment after the call. No issue
+ """) # noqa: E501
+ return
+
if __name__ == "__main__":
app.run()
diff --git a/tests/_lint/test_markdown_dedent.py b/tests/_lint/test_markdown_dedent.py
index fae078a3c7..27b7857e84 100644
--- a/tests/_lint/test_markdown_dedent.py
+++ b/tests/_lint/test_markdown_dedent.py
@@ -93,3 +93,71 @@ if __name__ == "__main__":
assert len(mf007_errors) == 0, (
"Should not flag correctly dedented markdown"
)
+
+
+def test_markdown_dedent_inline_comment_no_false_positive():
+ """An inline comment after the call must not trigger the rule (#10976)."""
+ code = '''
+import marimo
+
+__generated_with = "0.17.0"
+app = marimo.App()
+
+
+def _():
+ import marimo as mo
+ return (mo,)
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md(r"""
+ A
+ """) # noqa: E501
+ return
+
+
+if __name__ == "__main__":
+ app.run()
+'''
+ notebook = parse_notebook(code, filepath="notebook.py")
+ errors = lint_notebook(notebook, code)
+
+ mf007_errors = [e for e in errors if e.code == "MF007"]
+ assert len(mf007_errors) == 0
+
+
+def test_markdown_dedent_inline_comment_still_detects_indent():
+ """An inline comment must not mask markdown that really needs dedenting."""
+ code = '''
+import marimo
+
+__generated_with = "0.17.0"
+app = marimo.App()
+
+
+def _():
+ import marimo as mo
+ return (mo,)
+
+
+def _(mo):
+ mo.md(
+ r"""
+ A
+ """
+ ) # noqa: E501
+ return
+
+
+if __name__ == "__main__":
+ app.run()
+'''
+ notebook = parse_notebook(code, filepath="notebook.py")
+ errors = lint_notebook(notebook, code)
+
+ mf007_errors = [e for e in errors if e.code == "MF007"]
+ assert len(mf007_errors) == 1