A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: safer ends_with_semicolon with token errors (#8548)
Myles Scolnick6b8ec98dbe06
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.pyindex 0325b7ea79..de9c73474c 100644--- a/marimo/_ast/compiler.py+++ b/marimo/_ast/compiler.py@@ -11,7 +11,7 @@ import sysimport textwrapimport token as token_typesimport warnings-from tokenize import tokenize+from tokenize import TokenError, tokenizefrom types import CodeType, FrameTypefrom typing import Any, Callable, Optional, TypeAlias, cast@@ -85,8 +85,14 @@ def ends_with_semicolon(code: str) -> bool:bool: True if the last non-comment line ends with a semicolon"""# Tokenize to check for semicolon- tokens = tokenize(io.BytesIO(code.strip().encode("utf-8")).readline)- for token in reversed(list(tokens)):+ try:+ tokens = list(+ tokenize(io.BytesIO(code.strip().encode("utf-8")).readline)+ )+ except TokenError:+ # Fallback for code with syntax errors (e.g., unterminated strings)+ return code.rstrip().endswith(";")+ for token in reversed(tokens):if token.type in (token_types.ENDMARKER,token_types.NEWLINE,diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.pyindex cfc0f93762..3ae12b2bf1 100644--- a/tests/_ast/test_compiler.py+++ b/tests/_ast/test_compiler.py@@ -444,6 +444,57 @@ class TestSemicolon:assert eval(cell._cell.last_expr) is None+class TestEndsWithSemicolon:+ @staticmethod+ def test_simple_semicolon() -> None:+ assert compiler.ends_with_semicolon("x = 1;") is True++ @staticmethod+ def test_no_semicolon() -> None:+ assert compiler.ends_with_semicolon("x = 1") is False++ @staticmethod+ def test_semicolon_with_trailing_whitespace() -> None:+ assert compiler.ends_with_semicolon("x = 1; ") is True++ @staticmethod+ def test_semicolon_before_comment() -> None:+ assert compiler.ends_with_semicolon("x = 1; # comment") is True++ @staticmethod+ def test_no_semicolon_with_comment() -> None:+ assert compiler.ends_with_semicolon("x = 1 # comment") is False++ @staticmethod+ def test_semicolon_in_string_not_counted() -> None:+ assert compiler.ends_with_semicolon('"hello;"') is False++ @staticmethod+ def test_empty_code() -> None:+ assert compiler.ends_with_semicolon("") is False++ @staticmethod+ def test_whitespace_only() -> None:+ assert compiler.ends_with_semicolon(" ") is False++ @staticmethod+ def test_token_error_unterminated_string() -> None:+ # Unterminated string causes TokenError; fallback should handle it+ assert compiler.ends_with_semicolon('x = "unterminated') is False++ @staticmethod+ def test_token_error_unterminated_string_with_semicolon() -> None:+ assert compiler.ends_with_semicolon('x = "unterminated;') is True++ @staticmethod+ def test_token_error_unterminated_triple_quote() -> None:+ assert compiler.ends_with_semicolon('x = """unterminated') is False++ @staticmethod+ def test_token_error_unterminated_triple_quote_with_semicolon() -> None:+ assert compiler.ends_with_semicolon('x = """unterminated;') is True++class TestCompileCellFilename:"""Test compile_cell function with filename parameter."""