A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: retry SQLAlchemy schema inference on Polars conversion failure (#11282)
Myles Scolnickcb71321349de
diff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.pyindex 1bfe6abf81..c7d267e77f 100644--- a/marimo/_sql/engines/sqlalchemy.py+++ b/marimo/_sql/engines/sqlalchemy.py@@ -210,7 +210,18 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):def convert_to_polars() -> pl.DataFrame:import polars as pl- return pl.DataFrame(rows)+ try:+ return pl.DataFrame(rows)+ except pl.exceptions.ComputeError as e:+ if "infer_schema_length" not in str(e):+ raise+ # Only scan all rows when sampled inference fails. Reuse+ # the fetched rows so SQL with side effects runs just once.+ try:+ return pl.DataFrame(rows, infer_schema_length=None)+ except pl.exceptions.SchemaError as retry_error:+ # Preserve auto's pandas fallback for incompatible types.+ raise e from retry_errordef convert_to_pandas() -> pd.DataFrame:import pandas as pddiff --git a/tests/_sql/test_sqlalchemy.py b/tests/_sql/test_sqlalchemy.pyindex 5617a10db3..875e264ee0 100644--- a/tests/_sql/test_sqlalchemy.py+++ b/tests/_sql/test_sqlalchemy.py@@ -902,6 +902,209 @@ def test_sqlalchemy_engine_get_cursor_metadata(assert metadata["sql_statement_type"] == "Query / Unknown"[email protected]("sqlalchemy", "polars")[email protected]("output_format", ["polars", "lazy-polars", "auto"])+ "values",+ [+ [None] * 100 + [0.5],+ [None] * 1000 + ["late", None],+ [1] * 100 + ["late"],+ [1] * 100 + [1e100],+ ],+ ids=[+ "late-float",+ "late-string-after-nulls",+ "late-string-after-ints",+ "integer-overflow",+ ],+)+def test_sqlalchemy_schema_inference_fallback(+ empty_sqlite_engine: sa.Engine,+ output_format: str,+ values: list[float | str | None],+) -> None:+ import polars as pl+ import sqlalchemy as sa+ from polars.testing import assert_frame_equal++ with empty_sqlite_engine.begin() as connection:+ connection.execute(sa.text("CREATE TABLE source (value)"))+ connection.execute(sa.text("CREATE TABLE destination (value)"))+ connection.execute(+ sa.text("INSERT INTO source VALUES (:value)"),+ [{"value": value} for value in values],+ )+ rows = connection.execute(+ sa.text("SELECT value FROM source ORDER BY rowid")+ ).fetchall()++ with pytest.raises(+ pl.exceptions.ComputeError, match="infer_schema_length"+ ):+ pl.DataFrame(rows)+ expected = pl.DataFrame(rows, infer_schema_length=None)+ engine = SQLAlchemyEngine(empty_sqlite_engine)+ executed = mock.Mock()+ sa.event.listen(empty_sqlite_engine, "before_cursor_execute", executed)+ try:+ with mock.patch.object(+ engine, "sql_output_format", return_value=output_format+ ):+ result = engine.execute(+ "INSERT INTO destination SELECT value FROM source ORDER BY rowid "+ "RETURNING value"+ )+ executed.assert_called_once()+ finally:+ sa.event.remove(empty_sqlite_engine, "before_cursor_execute", executed)++ if output_format == "lazy-polars":+ assert isinstance(result, pl.LazyFrame)+ result = result.collect()+ assert_frame_equal(result, expected)+ with empty_sqlite_engine.connect() as connection:+ assert connection.execute(+ sa.text("SELECT COUNT(*) FROM destination")+ ).scalar_one() == len(values)++[email protected]("sqlalchemy", "polars")+ "query",+ [+ "SELECT 1 AS value",+ "SELECT NULL AS value",+ "SELECT 1 AS value WHERE 0",+ ],+ ids=["valid", "all-null", "empty"],+)+def test_sqlalchemy_schema_inference_fast_path(+ empty_sqlite_engine: sa.Engine, query: str+) -> None:+ import polars as pl+ import sqlalchemy as sa+ from polars.testing import assert_frame_equal++ with empty_sqlite_engine.connect() as connection:+ expected = pl.DataFrame(connection.execute(sa.text(query)).fetchall())++ engine = SQLAlchemyEngine(empty_sqlite_engine)+ with (+ mock.patch.object(engine, "sql_output_format", return_value="polars"),+ mock.patch.object(pl, "DataFrame", wraps=pl.DataFrame) as constructor,+ ):+ result = engine.execute(query)+ constructor.assert_called_once()+ assert constructor.call_args.kwargs == {}+ assert_frame_equal(result, expected)++[email protected]("sqlalchemy", "polars")+ "error_type", ["ComputeError", "SchemaError", "ValueError"]+)+def test_sqlalchemy_schema_inference_unrelated_error(+ empty_sqlite_engine: sa.Engine, error_type: str+) -> None:+ import polars as pl++ error = (+ ValueError("unrelated error")+ if error_type == "ValueError"+ else getattr(pl.exceptions, error_type)("unrelated error")+ )+ engine = SQLAlchemyEngine(empty_sqlite_engine)+ with (+ mock.patch.object(engine, "sql_output_format", return_value="polars"),+ mock.patch.object(pl, "DataFrame", side_effect=error) as constructor,+ pytest.raises(type(error)) as exc_info,+ ):+ engine.execute("SELECT 1 AS value")+ assert exc_info.value is error+ constructor.assert_called_once()++[email protected]("sqlalchemy", "polars", "pandas")[email protected]("output_format", ["polars", "lazy-polars", "auto"])+ "message",+ ["cannot construct a common type", "increase infer_schema_length"],+)+def test_sqlalchemy_schema_inference_retry_failure(+ empty_sqlite_engine: sa.Engine, output_format: str, message: str+) -> None:+ import pandas as pd+ import polars as pl+ from pandas.testing import assert_frame_equal++ error = pl.exceptions.ComputeError(message)+ engine = SQLAlchemyEngine(empty_sqlite_engine)+ with (+ mock.patch.object(+ engine, "sql_output_format", return_value=output_format+ ),+ mock.patch.object(+ pl,+ "DataFrame",+ side_effect=[+ pl.exceptions.ComputeError(+ "consider increasing `infer_schema_length`"+ ),+ error,+ ],+ ) as constructor,+ ):+ if output_format == "auto":+ result = engine.execute("SELECT 1 AS value")+ assert_frame_equal(result, pd.DataFrame({"value": [1]}))+ else:+ with pytest.raises(pl.exceptions.ComputeError) as exc_info:+ engine.execute("SELECT 1 AS value")+ assert exc_info.value is error++ assert constructor.call_count == 2+ assert constructor.call_args.kwargs == {"infer_schema_length": None}+ assert (+ constructor.call_args_list[0].args[0] is constructor.call_args.args[0]+ )++[email protected]("sqlalchemy", "polars", "pandas")[email protected]("output_format", ["polars", "lazy-polars", "auto"])+def test_sqlalchemy_schema_inference_incompatible_types(+ empty_sqlite_engine: sa.Engine, output_format: str+) -> None:+ import pandas as pd+ import polars as pl+ from pandas.testing import assert_frame_equal++ query = """+ WITH RECURSIVE numbers(n) AS (+ SELECT 0 UNION ALL SELECT n + 1 FROM numbers WHERE n < 100+ )+ SELECT CASE WHEN n < 100 THEN 1 ELSE X'FF' END AS value+ FROM numbers ORDER BY n+ """+ engine = SQLAlchemyEngine(empty_sqlite_engine)+ with mock.patch.object(+ engine, "sql_output_format", return_value=output_format+ ):+ if output_format == "auto":+ result = engine.execute(query)+ assert_frame_equal(+ result, pd.DataFrame({"value": [1] * 100 + [b"\xff"]})+ )+ else:+ with pytest.raises(+ pl.exceptions.ComputeError, match="infer_schema_length"+ ) as exc_info:+ engine.execute(query)+ assert isinstance(+ exc_info.value.__cause__, pl.exceptions.SchemaError+ )++# ------------------------# Decoracotors# ------------------------