A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(sql): avoid pyarrow requirement for polars output in DuckDB engine (#9643)
Myles Scolnick202917c69567
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.pyindex 5469edb3db..68da662a57 100644--- a/marimo/_sql/engines/duckdb.py+++ b/marimo/_sql/engines/duckdb.py@@ -22,6 +22,7 @@ if TYPE_CHECKING:from collections.abc import Iteratorimport duckdb+ import polars as pl# Internal engine namesINTERNAL_DUCKDB_ENGINE = cast(VariableName, "__marimo_duckdb")@@ -83,9 +84,16 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):sql_output_format = self.sql_output_format()+ def to_polars() -> pl.DataFrame:+ import polars as pl++ # Use the Arrow PyCapsule interface (pl.DataFrame(relation))+ # instead of relation.pl() so that pyarrow is not required.+ return pl.DataFrame(relation)+return convert_to_output(sql_output_format=sql_output_format,- to_polars=lambda: relation.pl(),+ to_polars=to_polars,to_pandas=lambda: relation.df(),to_native=lambda: relation,)diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.pyindex b7bc8d0363..ab0998ca05 100644--- a/tests/_sql/test_duckdb.py+++ b/tests/_sql/test_duckdb.py@@ -2,8 +2,10 @@from __future__ import annotations+import sysfrom copy import deepcopyfrom typing import TYPE_CHECKING+from unittest import mockimport pytest@@ -228,8 +230,6 @@ def test_duckdb_engine_sql_output_formats(duckdb_connection: duckdb.DuckDBPyConnection,) -> None:"""Test DuckDBEngine execute with different SQL output formats."""- from unittest import mock-import pandas as pdimport polars as pl@@ -290,3 +290,58 @@ def test_duckdb_engine_sql_output_formats(result = engine.execute("SELECT * FROM test ORDER BY id")assert isinstance(result, (pd.DataFrame, pl.DataFrame))assert len(result) == 4+++ not HAS_DUCKDB or not HAS_POLARS,+ reason="DuckDB and Polars not installed",+)+ ("sql_output_format", "expected_type_name"),+ [+ ("polars", "DataFrame"),+ ("lazy-polars", "LazyFrame"),+ ("auto", "DataFrame"),+ ],+)+def test_duckdb_engine_polars_no_pyarrow(+ duckdb_connection: duckdb.DuckDBPyConnection,+ sql_output_format: str,+ expected_type_name: str,+) -> None:+ """Polars conversion should not require pyarrow.++ Uses the Arrow PyCapsule interface (`pl.DataFrame(relation)`) rather than+ `relation.pl()` which historically required pyarrow. Covers every output+ format that routes through `to_polars()` (polars, lazy-polars, and auto+ when polars is installed).+ """+ import polars as pl++ # Block `pyarrow` and any already-imported `pyarrow.*` submodules so that+ # fresh imports raise ModuleNotFoundError.+ blocked_pyarrow = {+ name: None+ for name in list(sys.modules)+ if name == "pyarrow" or name.startswith("pyarrow.")+ }+ blocked_pyarrow["pyarrow"] = None++ with (+ mock.patch.dict(sys.modules, blocked_pyarrow),+ mock.patch.object(+ DuckDBEngine, "sql_output_format", return_value=sql_output_format+ ),+ ):+ engine = DuckDBEngine(+ duckdb_connection,+ engine_name=VariableName("test_duckdb"),+ )+ result = engine.execute("SELECT * FROM test ORDER BY id")+ expected_type = getattr(pl, expected_type_name)+ assert isinstance(result, expected_type)+ # Collect lazy frames so we exercise the full polars conversion path.+ materialized = (+ result.collect() if expected_type_name == "LazyFrame" else result+ )+ assert len(materialized) == 4