Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: safer ends_with_semicolon with token errors (#8548)

Myles Scolnick6b8ec98dbe06
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index 0325b7ea79..de9c73474c 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -11,7 +11,7 @@ import sys
import textwrap
import token as token_types
import warnings
-from tokenize import tokenize
+from tokenize import TokenError, tokenize
from types import CodeType, FrameType
from typing import Any, Callable, Optional, TypeAlias, cast
@@ -85,8 +85,14 @@ def ends_with_semicolon(code: str) -> bool:
bool: True if the last non-comment line ends with a semicolon
"""
# Tokenize to check for semicolon
- tokens = tokenize(io.BytesIO(code.strip().encode("utf-8")).readline)
- for token in reversed(list(tokens)):
+ try:
+ tokens = list(
+ tokenize(io.BytesIO(code.strip().encode("utf-8")).readline)
+ )
+ except TokenError:
+ # Fallback for code with syntax errors (e.g., unterminated strings)
+ return code.rstrip().endswith(";")
+ for token in reversed(tokens):
if token.type in (
token_types.ENDMARKER,
token_types.NEWLINE,
diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.py
index cfc0f93762..3ae12b2bf1 100644
--- a/tests/_ast/test_compiler.py
+++ b/tests/_ast/test_compiler.py
@@ -444,6 +444,57 @@ class TestSemicolon:
assert eval(cell._cell.last_expr) is None
+class TestEndsWithSemicolon:
+ @staticmethod
+ def test_simple_semicolon() -> None:
+ assert compiler.ends_with_semicolon("x = 1;") is True
+
+ @staticmethod
+ def test_no_semicolon() -> None:
+ assert compiler.ends_with_semicolon("x = 1") is False
+
+ @staticmethod
+ def test_semicolon_with_trailing_whitespace() -> None:
+ assert compiler.ends_with_semicolon("x = 1; ") is True
+
+ @staticmethod
+ def test_semicolon_before_comment() -> None:
+ assert compiler.ends_with_semicolon("x = 1; # comment") is True
+
+ @staticmethod
+ def test_no_semicolon_with_comment() -> None:
+ assert compiler.ends_with_semicolon("x = 1 # comment") is False
+
+ @staticmethod
+ def test_semicolon_in_string_not_counted() -> None:
+ assert compiler.ends_with_semicolon('"hello;"') is False
+
+ @staticmethod
+ def test_empty_code() -> None:
+ assert compiler.ends_with_semicolon("") is False
+
+ @staticmethod
+ def test_whitespace_only() -> None:
+ assert compiler.ends_with_semicolon(" ") is False
+
+ @staticmethod
+ def test_token_error_unterminated_string() -> None:
+ # Unterminated string causes TokenError; fallback should handle it
+ assert compiler.ends_with_semicolon('x = "unterminated') is False
+
+ @staticmethod
+ def test_token_error_unterminated_string_with_semicolon() -> None:
+ assert compiler.ends_with_semicolon('x = "unterminated;') is True
+
+ @staticmethod
+ def test_token_error_unterminated_triple_quote() -> None:
+ assert compiler.ends_with_semicolon('x = """unterminated') is False
+
+ @staticmethod
+ def test_token_error_unterminated_triple_quote_with_semicolon() -> None:
+ assert compiler.ends_with_semicolon('x = """unterminated;') is True
+
+
class TestCompileCellFilename:
"""Test compile_cell function with filename parameter."""