Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: add DataFusionFormatter (#9338)

Michael Chavinda81d7ec297281
diff --git a/marimo/_output/formatters/df_formatters.py b/marimo/_output/formatters/df_formatters.py
index 52e0451867..dda837bef3 100644
--- a/marimo/_output/formatters/df_formatters.py
+++ b/marimo/_output/formatters/df_formatters.py
@@ -160,6 +160,38 @@ class PyArrowFormatter(FormatterFactory):
return table(df, selection=None, pagination=None)._mime_()
+class DataFusionFormatter(FormatterFactory):
+ @staticmethod
+ def package_name() -> str:
+ return "datafusion"
+
+ def register(self) -> None:
+ import datafusion # type: ignore[import-not-found]
+
+ from marimo._output import formatting
+
+ if not include_opinionated():
+ return
+
+ @formatting.opinionated_formatter(datafusion.DataFrame)
+ def _show_marimo_datafusion_dataframe(
+ df: datafusion.DataFrame,
+ ) -> tuple[KnownMimeType, str]:
+ try:
+ # Avoid materializing the entire DataFusion DataFrame during
+ # formatting; only load the first page of data.
+ return table(
+ df.limit(get_default_table_page_size()).to_arrow_table(),
+ selection=None,
+ pagination=False,
+ _internal_lazy=True,
+ _internal_preload=True,
+ )._mime_()
+ except BaseException as e:
+ LOGGER.warning("Failed to format DataFusion DataFrame: %s", e)
+ return ("text/html", df._repr_html_())
+
+
class PySparkFormatter(FormatterFactory):
@staticmethod
def package_name() -> str:
diff --git a/marimo/_output/formatters/formatters.py b/marimo/_output/formatters/formatters.py
index fcea99ecaf..5a1ae706e5 100644
--- a/marimo/_output/formatters/formatters.py
+++ b/marimo/_output/formatters/formatters.py
@@ -17,6 +17,7 @@ from marimo._output.formatters.arviz_formatters import ArviZFormatter
from marimo._output.formatters.bokeh_formatters import BokehFormatter
from marimo._output.formatters.cell import CellFormatter
from marimo._output.formatters.df_formatters import (
+ DataFusionFormatter,
IbisFormatter,
PolarsFormatter,
PyArrowFormatter,
@@ -52,6 +53,7 @@ if TYPE_CHECKING:
THIRD_PARTY_FACTORIES: dict[str, FormatterFactory] = {
AltairFormatter.package_name(): AltairFormatter(),
MatplotlibFormatter.package_name(): MatplotlibFormatter(),
+ DataFusionFormatter.package_name(): DataFusionFormatter(),
IbisFormatter.package_name(): IbisFormatter(),
PandasFormatter.package_name(): PandasFormatter(),
PolarsFormatter.package_name(): PolarsFormatter(),
diff --git a/tests/_output/formatters/test_formatters.py b/tests/_output/formatters/test_formatters.py
index 462de8dbe4..603e9d7b99 100644
--- a/tests/_output/formatters/test_formatters.py
+++ b/tests/_output/formatters/test_formatters.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import importlib
+import importlib.util
import os.path
import sys
from typing import Any
@@ -63,6 +64,7 @@ def test_path_finder_find_spec_non_recursive() -> None:
HAS_DEPS = DependencyManager.pandas.has() and DependencyManager.polars.has()
+HAS_DATAFUSION = importlib.util.find_spec("datafusion") is not None
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
@@ -145,6 +147,35 @@ def test_as_html_opinionated_formatter():
assert "<marimo-table" not in html.text
[email protected](not HAS_DATAFUSION, reason="datafusion is not installed")
+def test_datafusion_formatter() -> None:
+ register_formatters()
+
+ from datafusion import SessionContext
+
+ ctx = SessionContext()
+ df = ctx.from_pydict({"A": [1, 2, 3], "B": ["a", "a", "a"]})
+
+ # With DataFusion DataFrame
+ formatter = get_formatter(df)
+ assert formatter is not None
+ mime, content = formatter(df)
+ assert mime == "text/html"
+ assert "<marimo-table" in content
+
+ # With Plain — opinionated formatter should be bypassed
+ obj = Plain(df)
+ formatter = get_formatter(obj)
+ assert formatter is not None
+ mime, content = formatter(obj)
+ assert mime == "text/html"
+ assert "<marimo-table" not in content
+
+ # as_html should produce the same table output for the bare DataFrame
+ assert "<marimo-table" in as_html(df).text
+ assert "<marimo-table" not in as_html(Plain(df)).text
+
+
def test_broken_formatter():
class _ClsForBrokenFormatter: ...