A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: handle UTF-8 byte offsets in SQL definition extraction (#11187)
Myles Scolnick9c37a417b95b
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.pyindex 4e0decbd23..e84a19e40f 100644--- a/marimo/_ast/sql_visitor.py+++ b/marimo/_ast/sql_visitor.py@@ -106,7 +106,8 @@ def normalize_sql_f_string(node: ast.JoinedStr) -> str:class _TokenExtractor:def __init__(self, sql_statement: str, tokens: list[Any]) -> None:- self.sql_statement = sql_statement+ # DuckDB token positions are UTF-8 byte offsets.+ self.sql_statement = sql_statement.encode("utf-8")self.tokens = tokensdef token_str(self, i: int) -> str:@@ -115,16 +116,16 @@ class _TokenExtractor:start = token[0]# If it starts with a quote, find the matching end quote- if sql_statement[start] == '"':- end = sql_statement.find('"', start + 1) + 1- elif sql_statement[start] == "'":- end = sql_statement.find("'", start + 1) + 1- elif sql_statement[start:].startswith("e'"):+ if sql_statement.startswith(b'"', start):+ end = sql_statement.find(b'"', start + 1) + 1+ elif sql_statement.startswith(b"'", start):+ end = sql_statement.find(b"'", start + 1) + 1+ elif sql_statement.startswith(b"e'", start):start += 1- end = sql_statement.find("'", start + 1) + 1+ end = sql_statement.find(b"'", start + 1) + 1else:# For non-quoted tokens, find until space or comment- maybe_end = re.search(r"[\s\-/]", sql_statement[start:])+ maybe_end = re.search(rb"[\s\-/]", sql_statement[start:])end = (start + maybe_end.start() if maybe_end else len(sql_statement))@@ -133,7 +134,7 @@ class _TokenExtractor:# in (select * from x);;end = min(end, tokens[i + 1][0])- return sql_statement[start:end]+ return sql_statement[start:end].decode("utf-8")def is_keyword(self, i: int, match: str) -> bool:import duckdbdiff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.pyindex c9cf691b39..48a465ece1 100644--- a/tests/_ast/test_sql_visitor.py+++ b/tests/_ast/test_sql_visitor.py@@ -173,6 +173,57 @@ class TestFindSQLDefs:sql = "CREATE OR REPLACE VIEW test_view (id INT);"assert find_sql_defs(sql) == SQLDefs(views=[SQLRef(table="test_view")])+ @staticmethod+ def test_find_sql_defs_unicode_comment_select() -> None:+ assert find_sql_defs("-- комментарий\nSELECT 1 AS x") == SQLDefs()++ @staticmethod+ @pytest.mark.parametrize("text", ["тест", "测试", "🚀"])+ @pytest.mark.parametrize(+ "prefix", ["-- {text}\n", "/* {text} */", "SELECT '{text}';"]+ )+ def test_find_sql_defs_after_unicode(prefix: str, text: str) -> None:+ prefix = prefix.format(text=text)+ assert (+ find_sql_defs(prefix + "SELECT a FROM t WHERE x = 1") == SQLDefs()+ )+ assert find_sql_defs(+ prefix + "CREATE TABLE t(id INT); CREATE VIEW v AS SELECT * FROM t"+ ) == SQLDefs(tables=[SQLRef(table="t")], views=[SQLRef(table="v")])++ @staticmethod+ @pytest.mark.parametrize("quote", ["", '"'])+ def test_find_sql_defs_unicode_identifiers(quote: str) -> None:+ catalog = f"{quote}каталог{quote}"+ schema = f"{quote}схема{quote}"+ table = f"{quote}表{quote}"+ view = f"{quote}представление{quote}"+ sql = (+ f"CREATE SCHEMA {catalog}.{schema};"+ f"CREATE TABLE {catalog}.{schema}.{table}(id INT);"+ f"CREATE VIEW {catalog}.{schema}.{view} AS SELECT '🚀';"+ )+ assert find_sql_defs(sql) == SQLDefs(+ tables=[SQLRef(table="表", catalog="каталог", schema="схема")],+ views=[+ SQLRef(+ table="представление", catalog="каталог", schema="схема"+ )+ ],+ schemas=["схема"],+ reffed_catalogs=["каталог", "каталог", "каталог"],+ reffed_schemas=["схема", "схема"],+ )++ @staticmethod+ @pytest.mark.parametrize("quote", ["'", '"', "e'"])+ @pytest.mark.parametrize("alias", ["", ' AS "каталог🚀"'])+ def test_find_sql_defs_unicode_attach(quote: str, alias: str) -> None:+ sql = f"ATTACH {quote}данные数据库.db{quote[-1]}{alias}"+ assert find_sql_defs(sql) == SQLDefs(+ catalogs=["каталог🚀" if alias else "данные数据库"]+ )+@staticmethoddef test_find_sql_defs_temporary() -> None:sql = "CREATE TEMPORARY TABLE temp_table (id INT);"