A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(parse): count lines like ast/tokenize, not str.splitlines() (#9957)
Dylan Madisetti8b813edbb529
diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.pyindex b33869e805..df5c68e1f3 100644--- a/marimo/_ast/parse.py+++ b/marimo/_ast/parse.py@@ -54,6 +54,16 @@ def ast_parse(return cast(ast.Module, ast.parse(contents, **kwargs))+def split_source_lines(text: str) -> list[str]:+ """Split source into lines the way `ast`/`tokenize` count them.++ Unlike `str.splitlines()`, this only treats `\\n`, `\\r`, and `\\r\\n` as+ line breaks. `str.splitlines()` additionally splits on `\\f`, `\\v`, the+ `\\x1c`-`\\x1e` separators, and Unicode line separators.+ """+ return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")++def fixed_dedent(text: str) -> str:"""Manually edited code, can dedent"""# Added robustness for AI generated code@@ -105,7 +115,7 @@ class Extractor:def __init__(self, contents: str):self.contents = contents.strip()- self.lines = self.contents.splitlines() if self.contents else []+ self.lines = split_source_lines(self.contents) if self.contents else []def extract_from_offsets(self,diff --git a/tests/_ast/test_parse_control_chars.py b/tests/_ast/test_parse_control_chars.pynew file mode 100644index 0000000000..f2182d8213--- /dev/null+++ b/tests/_ast/test_parse_control_chars.py@@ -0,0 +1,65 @@+# Copyright 2026 Marimo. All rights reserved.+"""Regression tests: control characters in cell source must not crash parsing.++These reproduce real notebooks that marimo *itself generated* (note the+`__generated_with` header) but could no longer parse back — a round-trip bug.+"""++from __future__ import annotations++import tokenize++import pytest++from marimo._ast.parse import parse_notebook++# A form-feed (\x0c) inside an `mo.md` string. The tabs (\t, shown as `\times`+# in the literal) come from an unescaped LaTeX `\times`. marimo generated this+# file but, today, asserts out when parsing it back.+NOTEBOOK_FORMFEED_ASSERT = 'import marimo\n__generated_with = "0.17.7"\napp = marimo.App(\n)\[email protected]\ndef _(mo):\n mo.md("""\n P=\x0c rac{TP}{TP+FP}\n R=\x0c rac{TP}{TP+FN}\n F1 = \x0c rac{2\times P\times R}{P+R}\n """)\n return\n\[email protected]\ndef _(mo):\n mo.md("""\n """)\n return\n\[email protected]\ndef _():\n pass\n return\nif __name__ == "__main__":\n app.run()\n'++# Same root cause; parametrized `@app.cell(hide_code=True)` decorators make the+# incorrectly extracted fragment untokenizable -> tokenize.TokenError instead.+NOTEBOOK_FORMFEED_TOKEN = 'import marimo\n__generated_with = "0.17.4"\napp = marimo.App(width="medium", auto_download=["html"])\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""\n P=\x0c rac{TP}{TP+FP}\n R=\x0c rac{TP}{TP+FN}\n F1 = \x0c rac{2\times P\times R}{P+R}\n """)\n return\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""\n\n **Interpret your results**\n """)\n return\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""My answer:""")\n return\nif __name__ == "__main__":\n app.run()\n'+++ "source",+ [NOTEBOOK_FORMFEED_ASSERT, NOTEBOOK_FORMFEED_TOKEN],+ ids=["assert_case", "token_case"],+)+def test_parse_notebook_never_crashes_on_control_chars(source: str) -> None:+ # The contract: malformed/odd input yields a NotebookSerialization (which+ # may carry `.violations`), never an unhandled AssertionError/TokenError.+ try:+ notebook = parse_notebook(source)+ except (AssertionError, tokenize.TokenError) as e:+ pytest.fail(+ f"parse_notebook crashed on control chars: {type(e).__name__}: {e}"+ )+ assert notebook is not None+++ "source",+ [NOTEBOOK_FORMFEED_ASSERT, NOTEBOOK_FORMFEED_TOKEN],+ ids=["assert_case", "token_case"],+)+def test_marimo_check_does_not_crash_on_control_chars(+ source: str, tmp_path+) -> None:+ # `marimo check` must report a diagnostic / file error, not die with a+ # traceback. run_check should return a Linter even for broken notebooks.+ from marimo._lint import run_check++ nb = tmp_path / "notebook.py"+ nb.write_text(source)++ try:+ result = run_check((str(nb),))+ except (AssertionError, tokenize.TokenError) as e:+ pytest.fail(+ f"`marimo check` crashed on control chars: {type(e).__name__}: {e}"+ )+ assert result is not None+ assert len(result.files) == 1