Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: handle UTF-8 byte offsets in SQL definition extraction (#11187)

Myles Scolnick9c37a417b95b
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.py
index 4e0decbd23..e84a19e40f 100644
--- a/marimo/_ast/sql_visitor.py
+++ b/marimo/_ast/sql_visitor.py
@@ -106,7 +106,8 @@ def normalize_sql_f_string(node: ast.JoinedStr) -> str:
class _TokenExtractor:
def __init__(self, sql_statement: str, tokens: list[Any]) -> None:
- self.sql_statement = sql_statement
+ # DuckDB token positions are UTF-8 byte offsets.
+ self.sql_statement = sql_statement.encode("utf-8")
self.tokens = tokens
def token_str(self, i: int) -> str:
@@ -115,16 +116,16 @@ class _TokenExtractor:
start = token[0]
# If it starts with a quote, find the matching end quote
- if sql_statement[start] == '"':
- end = sql_statement.find('"', start + 1) + 1
- elif sql_statement[start] == "'":
- end = sql_statement.find("'", start + 1) + 1
- elif sql_statement[start:].startswith("e'"):
+ if sql_statement.startswith(b'"', start):
+ end = sql_statement.find(b'"', start + 1) + 1
+ elif sql_statement.startswith(b"'", start):
+ end = sql_statement.find(b"'", start + 1) + 1
+ elif sql_statement.startswith(b"e'", start):
start += 1
- end = sql_statement.find("'", start + 1) + 1
+ end = sql_statement.find(b"'", start + 1) + 1
else:
# For non-quoted tokens, find until space or comment
- maybe_end = re.search(r"[\s\-/]", sql_statement[start:])
+ maybe_end = re.search(rb"[\s\-/]", sql_statement[start:])
end = (
start + maybe_end.start() if maybe_end else len(sql_statement)
)
@@ -133,7 +134,7 @@ class _TokenExtractor:
# in (select * from x);;
end = min(end, tokens[i + 1][0])
- return sql_statement[start:end]
+ return sql_statement[start:end].decode("utf-8")
def is_keyword(self, i: int, match: str) -> bool:
import duckdb
diff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.py
index c9cf691b39..48a465ece1 100644
--- a/tests/_ast/test_sql_visitor.py
+++ b/tests/_ast/test_sql_visitor.py
@@ -173,6 +173,57 @@ class TestFindSQLDefs:
sql = "CREATE OR REPLACE VIEW test_view (id INT);"
assert find_sql_defs(sql) == SQLDefs(views=[SQLRef(table="test_view")])
+ @staticmethod
+ def test_find_sql_defs_unicode_comment_select() -> None:
+ assert find_sql_defs("-- комментарий\nSELECT 1 AS x") == SQLDefs()
+
+ @staticmethod
+ @pytest.mark.parametrize("text", ["тест", "测试", "🚀"])
+ @pytest.mark.parametrize(
+ "prefix", ["-- {text}\n", "/* {text} */", "SELECT '{text}';"]
+ )
+ def test_find_sql_defs_after_unicode(prefix: str, text: str) -> None:
+ prefix = prefix.format(text=text)
+ assert (
+ find_sql_defs(prefix + "SELECT a FROM t WHERE x = 1") == SQLDefs()
+ )
+ assert find_sql_defs(
+ prefix + "CREATE TABLE t(id INT); CREATE VIEW v AS SELECT * FROM t"
+ ) == SQLDefs(tables=[SQLRef(table="t")], views=[SQLRef(table="v")])
+
+ @staticmethod
+ @pytest.mark.parametrize("quote", ["", '"'])
+ def test_find_sql_defs_unicode_identifiers(quote: str) -> None:
+ catalog = f"{quote}каталог{quote}"
+ schema = f"{quote}схема{quote}"
+ table = f"{quote}表{quote}"
+ view = f"{quote}представление{quote}"
+ sql = (
+ f"CREATE SCHEMA {catalog}.{schema};"
+ f"CREATE TABLE {catalog}.{schema}.{table}(id INT);"
+ f"CREATE VIEW {catalog}.{schema}.{view} AS SELECT '🚀';"
+ )
+ assert find_sql_defs(sql) == SQLDefs(
+ tables=[SQLRef(table="表", catalog="каталог", schema="схема")],
+ views=[
+ SQLRef(
+ table="представление", catalog="каталог", schema="схема"
+ )
+ ],
+ schemas=["схема"],
+ reffed_catalogs=["каталог", "каталог", "каталог"],
+ reffed_schemas=["схема", "схема"],
+ )
+
+ @staticmethod
+ @pytest.mark.parametrize("quote", ["'", '"', "e'"])
+ @pytest.mark.parametrize("alias", ["", ' AS "каталог🚀"'])
+ def test_find_sql_defs_unicode_attach(quote: str, alias: str) -> None:
+ sql = f"ATTACH {quote}данные数据库.db{quote[-1]}{alias}"
+ assert find_sql_defs(sql) == SQLDefs(
+ catalogs=["каталог🚀" if alias else "данные数据库"]
+ )
+
@staticmethod
def test_find_sql_defs_temporary() -> None:
sql = "CREATE TEMPORARY TABLE temp_table (id INT);"