A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: add DataFusionFormatter (#9338)
Michael Chavinda81d7ec297281
diff --git a/marimo/_output/formatters/df_formatters.py b/marimo/_output/formatters/df_formatters.pyindex 52e0451867..dda837bef3 100644--- a/marimo/_output/formatters/df_formatters.py+++ b/marimo/_output/formatters/df_formatters.py@@ -160,6 +160,38 @@ class PyArrowFormatter(FormatterFactory):return table(df, selection=None, pagination=None)._mime_()+class DataFusionFormatter(FormatterFactory):+ @staticmethod+ def package_name() -> str:+ return "datafusion"++ def register(self) -> None:+ import datafusion # type: ignore[import-not-found]++ from marimo._output import formatting++ if not include_opinionated():+ return++ @formatting.opinionated_formatter(datafusion.DataFrame)+ def _show_marimo_datafusion_dataframe(+ df: datafusion.DataFrame,+ ) -> tuple[KnownMimeType, str]:+ try:+ # Avoid materializing the entire DataFusion DataFrame during+ # formatting; only load the first page of data.+ return table(+ df.limit(get_default_table_page_size()).to_arrow_table(),+ selection=None,+ pagination=False,+ _internal_lazy=True,+ _internal_preload=True,+ )._mime_()+ except BaseException as e:+ LOGGER.warning("Failed to format DataFusion DataFrame: %s", e)+ return ("text/html", df._repr_html_())++class PySparkFormatter(FormatterFactory):@staticmethoddef package_name() -> str:diff --git a/marimo/_output/formatters/formatters.py b/marimo/_output/formatters/formatters.pyindex fcea99ecaf..5a1ae706e5 100644--- a/marimo/_output/formatters/formatters.py+++ b/marimo/_output/formatters/formatters.py@@ -17,6 +17,7 @@ from marimo._output.formatters.arviz_formatters import ArviZFormatterfrom marimo._output.formatters.bokeh_formatters import BokehFormatterfrom marimo._output.formatters.cell import CellFormatterfrom marimo._output.formatters.df_formatters import (+ DataFusionFormatter,IbisFormatter,PolarsFormatter,PyArrowFormatter,@@ -52,6 +53,7 @@ if TYPE_CHECKING:THIRD_PARTY_FACTORIES: dict[str, FormatterFactory] = {AltairFormatter.package_name(): AltairFormatter(),MatplotlibFormatter.package_name(): MatplotlibFormatter(),+ DataFusionFormatter.package_name(): DataFusionFormatter(),IbisFormatter.package_name(): IbisFormatter(),PandasFormatter.package_name(): PandasFormatter(),PolarsFormatter.package_name(): PolarsFormatter(),diff --git a/tests/_output/formatters/test_formatters.py b/tests/_output/formatters/test_formatters.pyindex 462de8dbe4..603e9d7b99 100644--- a/tests/_output/formatters/test_formatters.py+++ b/tests/_output/formatters/test_formatters.py@@ -1,6 +1,7 @@from __future__ import annotationsimport importlib+import importlib.utilimport os.pathimport sysfrom typing import Any@@ -63,6 +64,7 @@ def test_path_finder_find_spec_non_recursive() -> None:HAS_DEPS = DependencyManager.pandas.has() and DependencyManager.polars.has()+HAS_DATAFUSION = importlib.util.find_spec("datafusion") is not None@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")@@ -145,6 +147,35 @@ def test_as_html_opinionated_formatter():assert "<marimo-table" not in html.text[email protected](not HAS_DATAFUSION, reason="datafusion is not installed")+def test_datafusion_formatter() -> None:+ register_formatters()++ from datafusion import SessionContext++ ctx = SessionContext()+ df = ctx.from_pydict({"A": [1, 2, 3], "B": ["a", "a", "a"]})++ # With DataFusion DataFrame+ formatter = get_formatter(df)+ assert formatter is not None+ mime, content = formatter(df)+ assert mime == "text/html"+ assert "<marimo-table" in content++ # With Plain — opinionated formatter should be bypassed+ obj = Plain(df)+ formatter = get_formatter(obj)+ assert formatter is not None+ mime, content = formatter(obj)+ assert mime == "text/html"+ assert "<marimo-table" not in content++ # as_html should produce the same table output for the bare DataFrame+ assert "<marimo-table" in as_html(df).text+ assert "<marimo-table" not in as_html(Plain(df)).text++def test_broken_formatter():class _ClsForBrokenFormatter: ...