A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: broaden catches from TokenError to TokenError + SyntaxError (#10446)
Dylan Madisettid29c775ac491
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.pyindex 93061a9c9a..8dbb357ebc 100644--- a/marimo/_ast/compiler.py+++ b/marimo/_ast/compiler.py@@ -99,7 +99,7 @@ def ends_with_semicolon(code: str) -> bool:tokens = list(tokenize(io.BytesIO(code.strip().encode("utf-8")).readline))- except TokenError:+ except (TokenError, SyntaxError):# Fallback for code with syntax errors (e.g., unterminated strings)return code.rstrip().endswith(";")for token in reversed(tokens):diff --git a/marimo/_ast/dedent.py b/marimo/_ast/dedent.pyindex cb55ac7475..d902af1a98 100644--- a/marimo/_ast/dedent.py+++ b/marimo/_ast/dedent.py@@ -132,7 +132,7 @@ def smart_dedent(code: str) -> str:"""try:tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))- except TokenError:+ except (TokenError, SyntaxError):return textwrap.dedent(code)lines = split_source_lines(code, keepends=True)@@ -168,7 +168,7 @@ def fixed_dedent(text: str) -> str:try:tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))- except TokenError:+ except (TokenError, SyntaxError):tokens = []protected = _get_protected_lines(text, tokens)diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.pyindex 03a6ee1186..1ae5d51965 100644--- a/tests/_ast/test_compiler.py+++ b/tests/_ast/test_compiler.py@@ -520,6 +520,18 @@ class TestEndsWithSemicolon:def test_token_error_unterminated_triple_quote_with_semicolon() -> None:assert compiler.ends_with_semicolon('x = """unterminated;') is True+ @staticmethod+ def test_syntax_error_bad_coding_cookie() -> None:+ # ast.parse on a str ignores a bad coding cookie, but the bytes-based+ # tokenize() rejects it with a plain SyntaxError, not a TokenError.+ code = "# -*- coding: bogus -*-\nx = 1"+ assert compiler.ends_with_semicolon(code) is False++ @staticmethod+ def test_syntax_error_bad_coding_cookie_with_semicolon() -> None:+ code = "# -*- coding: bogus -*-\nx = 1;"+ assert compiler.ends_with_semicolon(code) is True+class TestCompileCellFilename:"""Test compile_cell function with filename parameter."""diff --git a/tests/_ast/test_dedent.py b/tests/_ast/test_dedent.pyindex 429213852e..1339121be2 100644--- a/tests/_ast/test_dedent.py+++ b/tests/_ast/test_dedent.py@@ -129,6 +129,12 @@ class TestSmartDedent:code = ' """hello\n\n world\n """\n x = 1\n'assert smart_dedent(code) == '"""hello\n\nworld\n"""\nx = 1\n'+ def test_unindent_mismatch_falls_back(self):+ # tokenize raises IndentationError (a SyntaxError, not a TokenError)+ # for a dedent to an unestablished level. Fall back, don't raise.+ code = " def f():\n x = 1\n f():\n"+ assert smart_dedent(code) == "def f():\n x = 1\n f():\n"+class TestFixedDedent:def test_basic(self):@@ -155,3 +161,16 @@ class TestFixedDedent:# spurious blank line before the auto-generated `return` on save).assert fixed_dedent(" import x\n") == "import x"assert fixed_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2"++ def test_unindent_mismatch_falls_back(self):+ # tokenize raises IndentationError for a dedent to an unestablished+ # level (issue #10415). The mismatched line keeps its offset from the+ # base indent.+ code = (+ "\n def apply_adjustments():\n value = 1\n"+ " return value\n\n apply_adjustments():\n "+ )+ assert fixed_dedent(code) == (+ "\ndef apply_adjustments():\n value = 1\n"+ " return value\n\n apply_adjustments():\n"+ )diff --git a/tests/_ast/test_parse.py b/tests/_ast/test_parse.pyindex ca0d429cc6..eb194120fc 100644--- a/tests/_ast/test_parse.py+++ b/tests/_ast/test_parse.py@@ -235,6 +235,36 @@ if __name__ == "__main__":assert len(notebook.violations) == 1assert "Expected string constant" in notebook.violations[0].description+ @staticmethod+ def test_unparsable_cell_with_unindent_mismatch() -> None:+ """Unparsable cell content that fails to tokenize still parses."""+ code = '''+import marimo+__generated_with = "0.0.0"+app = marimo.App()++app._unparsable_cell(+ r"""+ def apply_adjustments():+ value = 1+ return value++ apply_adjustments():+ """,+ name="_"+)++if __name__ == "__main__":+ app.run()+'''+ notebook = parse_notebook(code)+ assert notebook is not None+ assert len(notebook.cells) == 1+ assert notebook.cells[0].code == (+ "def apply_adjustments():\n value = 1\n"+ " return value\n\n apply_adjustments():"+ )+class TestTrailingComments:"""Tests for trailing comment preservation in @app.function and @app.class_definition."""