Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: broaden catches from TokenError to TokenError + SyntaxError (#10446)

Dylan Madisettid29c775ac491
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index 93061a9c9a..8dbb357ebc 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -99,7 +99,7 @@ def ends_with_semicolon(code: str) -> bool:
tokens = list(
tokenize(io.BytesIO(code.strip().encode("utf-8")).readline)
)
- except TokenError:
+ except (TokenError, SyntaxError):
# Fallback for code with syntax errors (e.g., unterminated strings)
return code.rstrip().endswith(";")
for token in reversed(tokens):
diff --git a/marimo/_ast/dedent.py b/marimo/_ast/dedent.py
index cb55ac7475..d902af1a98 100644
--- a/marimo/_ast/dedent.py
+++ b/marimo/_ast/dedent.py
@@ -132,7 +132,7 @@ def smart_dedent(code: str) -> str:
"""
try:
tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))
- except TokenError:
+ except (TokenError, SyntaxError):
return textwrap.dedent(code)
lines = split_source_lines(code, keepends=True)
@@ -168,7 +168,7 @@ def fixed_dedent(text: str) -> str:
try:
tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))
- except TokenError:
+ except (TokenError, SyntaxError):
tokens = []
protected = _get_protected_lines(text, tokens)
diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.py
index 03a6ee1186..1ae5d51965 100644
--- a/tests/_ast/test_compiler.py
+++ b/tests/_ast/test_compiler.py
@@ -520,6 +520,18 @@ class TestEndsWithSemicolon:
def test_token_error_unterminated_triple_quote_with_semicolon() -> None:
assert compiler.ends_with_semicolon('x = """unterminated;') is True
+ @staticmethod
+ def test_syntax_error_bad_coding_cookie() -> None:
+ # ast.parse on a str ignores a bad coding cookie, but the bytes-based
+ # tokenize() rejects it with a plain SyntaxError, not a TokenError.
+ code = "# -*- coding: bogus -*-\nx = 1"
+ assert compiler.ends_with_semicolon(code) is False
+
+ @staticmethod
+ def test_syntax_error_bad_coding_cookie_with_semicolon() -> None:
+ code = "# -*- coding: bogus -*-\nx = 1;"
+ assert compiler.ends_with_semicolon(code) is True
+
class TestCompileCellFilename:
"""Test compile_cell function with filename parameter."""
diff --git a/tests/_ast/test_dedent.py b/tests/_ast/test_dedent.py
index 429213852e..1339121be2 100644
--- a/tests/_ast/test_dedent.py
+++ b/tests/_ast/test_dedent.py
@@ -129,6 +129,12 @@ class TestSmartDedent:
code = ' """hello\n\n world\n """\n x = 1\n'
assert smart_dedent(code) == '"""hello\n\nworld\n"""\nx = 1\n'
+ def test_unindent_mismatch_falls_back(self):
+ # tokenize raises IndentationError (a SyntaxError, not a TokenError)
+ # for a dedent to an unestablished level. Fall back, don't raise.
+ code = " def f():\n x = 1\n f():\n"
+ assert smart_dedent(code) == "def f():\n x = 1\n f():\n"
+
class TestFixedDedent:
def test_basic(self):
@@ -155,3 +161,16 @@ class TestFixedDedent:
# spurious blank line before the auto-generated `return` on save).
assert fixed_dedent(" import x\n") == "import x"
assert fixed_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2"
+
+ def test_unindent_mismatch_falls_back(self):
+ # tokenize raises IndentationError for a dedent to an unestablished
+ # level (issue #10415). The mismatched line keeps its offset from the
+ # base indent.
+ code = (
+ "\n def apply_adjustments():\n value = 1\n"
+ " return value\n\n apply_adjustments():\n "
+ )
+ assert fixed_dedent(code) == (
+ "\ndef apply_adjustments():\n value = 1\n"
+ " return value\n\n apply_adjustments():\n"
+ )
diff --git a/tests/_ast/test_parse.py b/tests/_ast/test_parse.py
index ca0d429cc6..eb194120fc 100644
--- a/tests/_ast/test_parse.py
+++ b/tests/_ast/test_parse.py
@@ -235,6 +235,36 @@ if __name__ == "__main__":
assert len(notebook.violations) == 1
assert "Expected string constant" in notebook.violations[0].description
+ @staticmethod
+ def test_unparsable_cell_with_unindent_mismatch() -> None:
+ """Unparsable cell content that fails to tokenize still parses."""
+ code = '''
+import marimo
+__generated_with = "0.0.0"
+app = marimo.App()
+
+app._unparsable_cell(
+ r"""
+ def apply_adjustments():
+ value = 1
+ return value
+
+ apply_adjustments():
+ """,
+ name="_"
+)
+
+if __name__ == "__main__":
+ app.run()
+'''
+ notebook = parse_notebook(code)
+ assert notebook is not None
+ assert len(notebook.cells) == 1
+ assert notebook.cells[0].code == (
+ "def apply_adjustments():\n value = 1\n"
+ " return value\n\n apply_adjustments():"
+ )
+
class TestTrailingComments:
"""Tests for trailing comment preservation in @app.function and @app.class_definition."""