Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(parse): count lines like ast/tokenize, not str.splitlines() (#9957)

Dylan Madisetti8b813edbb529
diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.py
index b33869e805..df5c68e1f3 100644
--- a/marimo/_ast/parse.py
+++ b/marimo/_ast/parse.py
@@ -54,6 +54,16 @@ def ast_parse(
return cast(ast.Module, ast.parse(contents, **kwargs))
+def split_source_lines(text: str) -> list[str]:
+ """Split source into lines the way `ast`/`tokenize` count them.
+
+ Unlike `str.splitlines()`, this only treats `\\n`, `\\r`, and `\\r\\n` as
+ line breaks. `str.splitlines()` additionally splits on `\\f`, `\\v`, the
+ `\\x1c`-`\\x1e` separators, and Unicode line separators.
+ """
+ return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+
+
def fixed_dedent(text: str) -> str:
"""Manually edited code, can dedent"""
# Added robustness for AI generated code
@@ -105,7 +115,7 @@ class Extractor:
def __init__(self, contents: str):
self.contents = contents.strip()
- self.lines = self.contents.splitlines() if self.contents else []
+ self.lines = split_source_lines(self.contents) if self.contents else []
def extract_from_offsets(
self,
diff --git a/tests/_ast/test_parse_control_chars.py b/tests/_ast/test_parse_control_chars.py
new file mode 100644
index 0000000000..f2182d8213
--- /dev/null
+++ b/tests/_ast/test_parse_control_chars.py
@@ -0,0 +1,65 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Regression tests: control characters in cell source must not crash parsing.
+
+These reproduce real notebooks that marimo *itself generated* (note the
+`__generated_with` header) but could no longer parse back — a round-trip bug.
+"""
+
+from __future__ import annotations
+
+import tokenize
+
+import pytest
+
+from marimo._ast.parse import parse_notebook
+
+# A form-feed (\x0c) inside an `mo.md` string. The tabs (\t, shown as `\times`
+# in the literal) come from an unescaped LaTeX `\times`. marimo generated this
+# file but, today, asserts out when parsing it back.
+NOTEBOOK_FORMFEED_ASSERT = 'import marimo\n__generated_with = "0.17.7"\napp = marimo.App(\n)\[email protected]\ndef _(mo):\n mo.md("""\n P=\x0c rac{TP}{TP+FP}\n R=\x0c rac{TP}{TP+FN}\n F1 = \x0c rac{2\times P\times R}{P+R}\n """)\n return\n\[email protected]\ndef _(mo):\n mo.md("""\n """)\n return\n\[email protected]\ndef _():\n pass\n return\nif __name__ == "__main__":\n app.run()\n'
+
+# Same root cause; parametrized `@app.cell(hide_code=True)` decorators make the
+# incorrectly extracted fragment untokenizable -> tokenize.TokenError instead.
+NOTEBOOK_FORMFEED_TOKEN = 'import marimo\n__generated_with = "0.17.4"\napp = marimo.App(width="medium", auto_download=["html"])\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""\n P=\x0c rac{TP}{TP+FP}\n R=\x0c rac{TP}{TP+FN}\n F1 = \x0c rac{2\times P\times R}{P+R}\n """)\n return\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""\n\n **Interpret your results**\n """)\n return\[email protected](hide_code=True)\ndef _(mo):\n mo.md("""My answer:""")\n return\nif __name__ == "__main__":\n app.run()\n'
+
+
+ "source",
+ [NOTEBOOK_FORMFEED_ASSERT, NOTEBOOK_FORMFEED_TOKEN],
+ ids=["assert_case", "token_case"],
+)
+def test_parse_notebook_never_crashes_on_control_chars(source: str) -> None:
+ # The contract: malformed/odd input yields a NotebookSerialization (which
+ # may carry `.violations`), never an unhandled AssertionError/TokenError.
+ try:
+ notebook = parse_notebook(source)
+ except (AssertionError, tokenize.TokenError) as e:
+ pytest.fail(
+ f"parse_notebook crashed on control chars: {type(e).__name__}: {e}"
+ )
+ assert notebook is not None
+
+
+ "source",
+ [NOTEBOOK_FORMFEED_ASSERT, NOTEBOOK_FORMFEED_TOKEN],
+ ids=["assert_case", "token_case"],
+)
+def test_marimo_check_does_not_crash_on_control_chars(
+ source: str, tmp_path
+) -> None:
+ # `marimo check` must report a diagnostic / file error, not die with a
+ # traceback. run_check should return a Linter even for broken notebooks.
+ from marimo._lint import run_check
+
+ nb = tmp_path / "notebook.py"
+ nb.write_text(source)
+
+ try:
+ result = run_check((str(nb),))
+ except (AssertionError, tokenize.TokenError) as e:
+ pytest.fail(
+ f"`marimo check` crashed on control chars: {type(e).__name__}: {e}"
+ )
+ assert result is not None
+ assert len(result.files) == 1