Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: normalize Unicode DuckDB SQL error locations (#11151)

Shahmir Varqhae5b92f38fff8
diff --git a/marimo/_sql/parse.py b/marimo/_sql/parse.py
index 96b153d2ad..04d7db1be0 100644
--- a/marimo/_sql/parse.py
+++ b/marimo/_sql/parse.py
@@ -8,6 +8,7 @@ import msgspec
from marimo import _loggers
from marimo._dependencies.dependencies import DependencyManager
+from marimo._sql.utils import is_duckdb_v2
if TYPE_CHECKING:
from collections.abc import Callable
@@ -137,11 +138,18 @@ def _parse_sql_duckdb(
return SqlParseResult(success=True, errors=[]), None
position = int(parsed_error.position or 0)
- subquery = query[:position]
+ # DuckDB 2 reports UTF-8 byte offsets; DuckDB 1 uses code points.
+ if is_duckdb_v2():
+ subquery = query.encode("utf-8")[:position].decode("utf-8")
+ position = len(subquery)
+ else:
+ subquery = query[:position]
line_number = subquery.count("\n") + 1
last_newline_idx = subquery.rfind("\n")
- column_number = position - last_newline_idx - 1
+ # CodeMirror positions use UTF-16 code units, including surrogate pairs.
+ line_prefix = subquery[last_newline_idx + 1 :]
+ column_number = len(line_prefix.encode("utf-16-le")) // 2
# Adjust column_number to account for any added quotes from bracket replacements
# SELECT {id} FRO users
diff --git a/marimo/_sql/utils.py b/marimo/_sql/utils.py
index ef3cf6e891..8a95359a7f 100644
--- a/marimo/_sql/utils.py
+++ b/marimo/_sql/utils.py
@@ -38,6 +38,14 @@ CHEAP_DISCOVERY_DATABASES = {
_NO_WASM_DUCKDB_RESULT = object()
+def is_duckdb_v2() -> bool:
+ """Whether the installed DuckDB uses version 2 or later, including previews."""
+ import duckdb
+ from packaging.version import Version
+
+ return Version(duckdb.__version__).major >= 2
+
+
def is_cheap_dialect(dialect: str) -> bool:
"""Whether schema/table discovery is cheap for the given SQL dialect.
diff --git a/tests/_sql/test_sql_parse.py b/tests/_sql/test_sql_parse.py
index c35d6fe838..2e94a06be6 100644
--- a/tests/_sql/test_sql_parse.py
+++ b/tests/_sql/test_sql_parse.py
@@ -14,10 +14,25 @@ from marimo._sql.parse import (
parse_sql,
replace_brackets_with_quotes,
)
+from marimo._sql.utils import is_duckdb_v2
HAS_DUCKDB = DependencyManager.duckdb.has()
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+ ("version", "expected"),
+ [("1.5.6", False), ("2.0.0.dev2610011535", True), ("2.0.0", True)],
+)
+def test_is_duckdb_v2(
+ monkeypatch: pytest.MonkeyPatch, version: str, expected: bool
+):
+ import duckdb
+
+ monkeypatch.setattr(duckdb, "__version__", version)
+ assert is_duckdb_v2() is expected
+
+
def test_sql_parse_result_with_errors():
"""Test SqlParseResult with errors."""
error = SqlParseError(
@@ -310,6 +325,26 @@ FRM table"""
assert error.line == 3
assert error.column >= 0
+ @pytest.mark.parametrize(
+ "query",
+ [
+ "SELECT 'é'; SELECT * FRM users",
+ "SELECT '😀'; SELECT * FRM users",
+ "SELECT 'é😀';\nSELECT * FRM users",
+ "SELECT 'é😀', {value};\nSELECT * FRM users",
+ "SELECT 'é😀', {value} FRM users",
+ "SELECT 'é😀', {value} FRM users WHERE id = {other}",
+ "SELECT 'é😀'\nFRM users",
+ ],
+ )
+ def test_unicode_error_position(self, query: str):
+ result, error = parse_sql(query, "duckdb")
+ prefix = query[: query.index("users")]
+ expected_line = prefix.count("\n") + 1
+ line_prefix = prefix.rsplit("\n", 1)[-1]
+ expected_column = len(line_prefix.encode("utf-16-le")) // 2
+ self.assert_line_column(result, error, expected_line, expected_column)
+
def test_error_position_after_offset(self):
"""Test position calculation with offset."""
query = """SELECT id FRM users"""