A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: change tokenization to allow trailing comments (#10981)
Dylan Madisettif360c3dc0c21
diff --git a/marimo/_ast/codegen.py b/marimo/_ast/codegen.pyindex 1000c571b5..e70cc8355a 100644--- a/marimo/_ast/codegen.py+++ b/marimo/_ast/codegen.py@@ -181,12 +181,21 @@ def format_markdown(cell: CellImpl) -> str:# tokenize.tokens = tokenize.tokenize(io.BytesIO(cell.code.encode("utf-8")).readline)tag = ""- # Comment capture- comments = {- "prefix": "",- "suffix": "",- }- key: str | None = "prefix"+ # Anything outside the `mo.md(...)` call (comments, and the whitespace+ # around them) is kept verbatim, so track where the call starts and ends+ # in the source rather than rebuilding it from tokens, which drop+ # whitespace.+ line_starts = [0]+ for match in re.finditer(r"\r\n|\r|\n", cell.code):+ line_starts.append(match.end())++ def offset(position: tuple[int, int]) -> int:+ row, col = position+ return line_starts[row - 1] + col++ call_start = 0+ call_end = len(cell.code)+ depth = 0tokenizes_fstring = sys.version_info >= (3, 12)start_tokens = ((tokenize.STRING, tokenize.FSTRING_START)@@ -206,11 +215,14 @@ def format_markdown(cell: CellImpl) -> str:start = start[1:]fstring = "f" in tag.lower()elif tok.string == "mo":- key = None+ call_start = offset(tok.start)+ depth = 0+ elif tok.string == "(":+ depth += 1elif tok.string == ")":- key = "suffix"- elif key in comments and tok.type != tokenize.ENCODING:- comments[key] += tok.string+ depth -= 1+ if depth == 0:+ call_end = offset(tok.end)if fstring:# We can blanket replace, because cell.markdown is not set@@ -220,7 +232,7 @@ def format_markdown(cell: CellImpl) -> str:# We always use """ as per front end.body = construct_markdown_call(markdown, '"""', tag)- return "".join([comments["prefix"], body, comments["suffix"]])+ return "".join([cell.code[:call_start], body, cell.code[call_end:]])def construct_markdown_call(markdown: str, quote: str, tag: str) -> str:diff --git a/tests/_ast/test_codegen.py b/tests/_ast/test_codegen.pyindex 167cc578f4..e44e939aa1 100644--- a/tests/_ast/test_codegen.py+++ b/tests/_ast/test_codegen.py@@ -2265,3 +2265,39 @@ def _strip_header_footer(source: str) -> str:code_start = source.index(header)code_end = source.index('if __name__ == "__main__":')return source[code_start + len(header) : code_end].strip()+++ ("code", "expected"),+ [+ # Already formatted: identity+ ('mo.md(r"""\nA\n""")', 'mo.md(r"""\nA\n""")'),+ # Inline comment after the call keeps its leading whitespace (#10976)+ (+ 'mo.md(r"""\nA\n""") # noqa: E501',+ 'mo.md(r"""\nA\n""") # noqa: E501',+ ),+ # Comments on their own lines, before and after+ (+ '# lead\n\nmo.md(r"""\nA\n""")\n# trail',+ '# lead\n\nmo.md(r"""\nA\n""")\n# trail',+ ),+ # Reformatting the call preserves the inline comment+ (+ 'mo.md(\n r"""\n A\n """\n) # noqa: E501',+ 'mo.md(r"""\nA\n""") # noqa: E501',+ ),+ # Non-ASCII content does not shift the comment position+ (+ 'mo.md(r"""\nÄé\n""") # noqa: É',+ 'mo.md(r"""\nÄé\n""") # noqa: É',+ ),+ # Parentheses around the call are kept+ ('(mo.md("x"))', '(mo.md("""\nx\n"""))'),+ ('mo.md(("x"))', 'mo.md("""\nx\n""")'),+ ],+)+def test_format_markdown_preserves_surrounding_source(+ code: str, expected: str+) -> None:+ assert codegen.format_markdown(compile_cell(code)) == expecteddiff --git a/tests/_lint/test_files/markdown_dedent.py b/tests/_lint/test_files/markdown_dedent.pyindex f695a45561..f42b15d0c9 100644--- a/tests/_lint/test_files/markdown_dedent.py+++ b/tests/_lint/test_files/markdown_dedent.py@@ -77,5 +77,12 @@ def _(mo):mo.md(fr"siiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiingle line")return+def _(mo):+ mo.md(r"""+ Inline comment after the call. No issue+ """) # noqa: E501+ return+if __name__ == "__main__":app.run()diff --git a/tests/_lint/test_markdown_dedent.py b/tests/_lint/test_markdown_dedent.pyindex fae078a3c7..27b7857e84 100644--- a/tests/_lint/test_markdown_dedent.py+++ b/tests/_lint/test_markdown_dedent.py@@ -93,3 +93,71 @@ if __name__ == "__main__":assert len(mf007_errors) == 0, ("Should not flag correctly dedented markdown")+++def test_markdown_dedent_inline_comment_no_false_positive():+ """An inline comment after the call must not trigger the rule (#10976)."""+ code = '''+import marimo++__generated_with = "0.17.0"+app = marimo.App()+++def _():+ import marimo as mo+ return (mo,)++[email protected](hide_code=True)+def _(mo):+ mo.md(r"""+ A+ """) # noqa: E501+ return+++if __name__ == "__main__":+ app.run()+'''+ notebook = parse_notebook(code, filepath="notebook.py")+ errors = lint_notebook(notebook, code)++ mf007_errors = [e for e in errors if e.code == "MF007"]+ assert len(mf007_errors) == 0+++def test_markdown_dedent_inline_comment_still_detects_indent():+ """An inline comment must not mask markdown that really needs dedenting."""+ code = '''+import marimo++__generated_with = "0.17.0"+app = marimo.App()+++def _():+ import marimo as mo+ return (mo,)+++def _(mo):+ mo.md(+ r"""+ A+ """+ ) # noqa: E501+ return+++if __name__ == "__main__":+ app.run()+'''+ notebook = parse_notebook(code, filepath="notebook.py")+ errors = lint_notebook(notebook, code)++ mf007_errors = [e for e in errors if e.code == "MF007"]+ assert len(mf007_errors) == 1