Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(sql): avoid pyarrow requirement for polars output in DuckDB engine (#9643)

Myles Scolnick202917c69567
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.py
index 5469edb3db..68da662a57 100644
--- a/marimo/_sql/engines/duckdb.py
+++ b/marimo/_sql/engines/duckdb.py
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
from collections.abc import Iterator
import duckdb
+ import polars as pl
# Internal engine names
INTERNAL_DUCKDB_ENGINE = cast(VariableName, "__marimo_duckdb")
@@ -83,9 +84,16 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
sql_output_format = self.sql_output_format()
+ def to_polars() -> pl.DataFrame:
+ import polars as pl
+
+ # Use the Arrow PyCapsule interface (pl.DataFrame(relation))
+ # instead of relation.pl() so that pyarrow is not required.
+ return pl.DataFrame(relation)
+
return convert_to_output(
sql_output_format=sql_output_format,
- to_polars=lambda: relation.pl(),
+ to_polars=to_polars,
to_pandas=lambda: relation.df(),
to_native=lambda: relation,
)
diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.py
index b7bc8d0363..ab0998ca05 100644
--- a/tests/_sql/test_duckdb.py
+++ b/tests/_sql/test_duckdb.py
@@ -2,8 +2,10 @@
from __future__ import annotations
+import sys
from copy import deepcopy
from typing import TYPE_CHECKING
+from unittest import mock
import pytest
@@ -228,8 +230,6 @@ def test_duckdb_engine_sql_output_formats(
duckdb_connection: duckdb.DuckDBPyConnection,
) -> None:
"""Test DuckDBEngine execute with different SQL output formats."""
- from unittest import mock
-
import pandas as pd
import polars as pl
@@ -290,3 +290,58 @@ def test_duckdb_engine_sql_output_formats(
result = engine.execute("SELECT * FROM test ORDER BY id")
assert isinstance(result, (pd.DataFrame, pl.DataFrame))
assert len(result) == 4
+
+
+ not HAS_DUCKDB or not HAS_POLARS,
+ reason="DuckDB and Polars not installed",
+)
+ ("sql_output_format", "expected_type_name"),
+ [
+ ("polars", "DataFrame"),
+ ("lazy-polars", "LazyFrame"),
+ ("auto", "DataFrame"),
+ ],
+)
+def test_duckdb_engine_polars_no_pyarrow(
+ duckdb_connection: duckdb.DuckDBPyConnection,
+ sql_output_format: str,
+ expected_type_name: str,
+) -> None:
+ """Polars conversion should not require pyarrow.
+
+ Uses the Arrow PyCapsule interface (`pl.DataFrame(relation)`) rather than
+ `relation.pl()` which historically required pyarrow. Covers every output
+ format that routes through `to_polars()` (polars, lazy-polars, and auto
+ when polars is installed).
+ """
+ import polars as pl
+
+ # Block `pyarrow` and any already-imported `pyarrow.*` submodules so that
+ # fresh imports raise ModuleNotFoundError.
+ blocked_pyarrow = {
+ name: None
+ for name in list(sys.modules)
+ if name == "pyarrow" or name.startswith("pyarrow.")
+ }
+ blocked_pyarrow["pyarrow"] = None
+
+ with (
+ mock.patch.dict(sys.modules, blocked_pyarrow),
+ mock.patch.object(
+ DuckDBEngine, "sql_output_format", return_value=sql_output_format
+ ),
+ ):
+ engine = DuckDBEngine(
+ duckdb_connection,
+ engine_name=VariableName("test_duckdb"),
+ )
+ result = engine.execute("SELECT * FROM test ORDER BY id")
+ expected_type = getattr(pl, expected_type_name)
+ assert isinstance(result, expected_type)
+ # Collect lazy frames so we exercise the full polars conversion path.
+ materialized = (
+ result.collect() if expected_type_name == "LazyFrame" else result
+ )
+ assert len(materialized) == 4