Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: dataframe download encoding (#7385)

vavavavavavavavavacc71a4fda41b
diff --git a/marimo/_output/data/data.py b/marimo/_output/data/data.py
index 0bfbd634d7..ac37370f66 100644
--- a/marimo/_output/data/data.py
+++ b/marimo/_output/data/data.py
@@ -198,6 +198,7 @@ def is_bigint(value: int | float) -> bool:
def sanitize_json_bigint(
data: Union[str, dict[str, Any], list[dict[str, Any]]],
+ ensure_ascii: bool = True,
) -> str:
"""Sanitize JSON big numbers to a string.
@@ -239,4 +240,5 @@ def sanitize_json_bigint(
indent=None,
separators=(",", ":"),
default=enc_hook,
+ ensure_ascii=ensure_ascii,
)
diff --git a/marimo/_plugins/ui/_impl/charts/altair_transformer.py b/marimo/_plugins/ui/_impl/charts/altair_transformer.py
index efc7130613..d32be13865 100644
--- a/marimo/_plugins/ui/_impl/charts/altair_transformer.py
+++ b/marimo/_plugins/ui/_impl/charts/altair_transformer.py
@@ -105,7 +105,7 @@ def _data_to_json_string(data: _DataType) -> str:
tm = get_table_manager_or_none(data)
if tm:
- return tm.to_json().decode("utf-8")
+ return tm.to_json(ensure_ascii=True).decode("utf-8")
raise NotImplementedError(
"to_marimo_json only works with data expressed as a DataFrame "
diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
index b51412fb0f..7f34376275 100644
--- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py
+++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
@@ -108,6 +108,10 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
dataframes via Ibis.
show_download (bool, optional): Whether to show the download button.
Defaults to True.
+ download_csv_encoding (str, optional): Encoding used when downloading CSV.
+ Defaults to "utf-8". Set to "utf-8-sig" to include BOM for Excel.
+ download_json_ensure_ascii (bool, optional): Whether to escape non-ASCII characters
+ in JSON downloads. Defaults to True.
on_change (Optional[Callable[[DataFrameType], None]], optional): Optional callback
to run when this element's value changes.
lazy (Optional[bool], optional): When lazy is True, an 'Apply' button will be shown to apply transformations.
@@ -123,6 +127,9 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
page_size: Optional[int] = 5,
limit: Optional[int] = None,
show_download: bool = True,
+ *,
+ download_csv_encoding: str = "utf-8",
+ download_json_ensure_ascii: bool = True,
lazy: Optional[bool] = None,
) -> None:
validate_no_integer_columns(df)
@@ -150,6 +157,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
self._limit = limit
self._dataframe_name = dataframe_name
self._data = df
+ self._download_csv_encoding = download_csv_encoding
+ self._download_json_ensure_ascii = download_json_ensure_ascii
self._handler = handler
self._manager = self._get_cached_table_manager(df, self._limit)
self._transform_container = TransformsContainer(nw_df, handler)
@@ -178,6 +187,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
"total": rows,
"page-size": page_size,
"show-download": show_download,
+ "download-csv-encoding": download_csv_encoding,
+ "download-json-ensure-ascii": download_json_ensure_ascii,
"lazy": self._lazy,
},
functions=(
@@ -315,7 +326,12 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
# Get the table manager for the transformed data
manager = self._get_cached_table_manager(df, self._limit)
- return download_as(manager, args.format)
+ return download_as(
+ manager,
+ args.format,
+ csv_encoding=self._download_csv_encoding,
+ json_ensure_ascii=self._download_json_ensure_ascii,
+ )
def _apply_filters_query_sort(
self,
diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.py
index a13ae4455d..b24b234efd 100644
--- a/marimo/_plugins/ui/_impl/table.py
+++ b/marimo/_plugins/ui/_impl/table.py
@@ -1118,7 +1118,9 @@ class table(
# Fallback to JSON
try:
- data_url = mo_data.json(table_manager.to_json({})).url
+ data_url = mo_data.json(
+ table_manager.to_json({}, ensure_ascii=True)
+ ).url
return data_url, "json"
except Exception as e:
LOGGER.error(
diff --git a/marimo/_plugins/ui/_impl/tables/default_table.py b/marimo/_plugins/ui/_impl/tables/default_table.py
index d09c01f050..46ff8a31b0 100644
--- a/marimo/_plugins/ui/_impl/tables/default_table.py
+++ b/marimo/_plugins/ui/_impl/tables/default_table.py
@@ -104,8 +104,10 @@ class DefaultTableManager(TableManager[JsonTableData]):
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False,
+ ensure_ascii: bool = True,
) -> str:
del strict_json
+ del ensure_ascii
normalized = self._normalize_data(
self.apply_formatting(format_mapping).data
)
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
index 4c142a8f50..3f96b33193 100644
--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py
+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
@@ -96,10 +96,13 @@ class NarwhalsTableManager(
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False,
+ ensure_ascii: bool = True,
) -> str:
del strict_json
frame = self.apply_formatting(format_mapping).as_frame()
- return sanitize_json_bigint(frame.rows(named=True))
+ return sanitize_json_bigint(
+ frame.rows(named=True), ensure_ascii=ensure_ascii
+ )
def to_parquet(self) -> bytes:
stream = io.BytesIO()
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index 8b348f9e95..166397fb61 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -85,6 +85,7 @@ class PandasTableManagerFactory(TableManagerFactory):
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False,
+ ensure_ascii: bool = True,
) -> str:
def to_json(
result: pd.DataFrame,
@@ -145,7 +146,9 @@ class PandasTableManagerFactory(TableManagerFactory):
"Error handling complex or timedelta64 dtype",
exc_info=e,
)
- return sanitize_json_bigint(to_json(result))
+ return sanitize_json_bigint(
+ to_json(result), ensure_ascii=ensure_ascii
+ )
# Flatten row multi-index
if isinstance(result.index, pd.MultiIndex) or (
@@ -198,7 +201,9 @@ class PandasTableManagerFactory(TableManagerFactory):
"Indexes with more than one level are not well supported, call reset_index() or use mo.plain(df)"
)
- return sanitize_json_bigint(to_json(result))
+ return sanitize_json_bigint(
+ to_json(result), ensure_ascii=ensure_ascii
+ )
def _infer_dtype(self, column: ColumnName) -> str:
# Typically, pandas dtypes returns a generic dtype
diff --git a/marimo/_plugins/ui/_impl/tables/polars_table.py b/marimo/_plugins/ui/_impl/tables/polars_table.py
index 3bcf82e157..5f26b2e726 100644
--- a/marimo/_plugins/ui/_impl/tables/polars_table.py
+++ b/marimo/_plugins/ui/_impl/tables/polars_table.py
@@ -111,6 +111,7 @@ class PolarsTableManagerFactory(TableManagerFactory):
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False,
+ ensure_ascii: bool = True,
) -> str:
def to_json(
result: pl.DataFrame,
@@ -146,7 +147,9 @@ class PolarsTableManagerFactory(TableManagerFactory):
result = result.with_columns(
pl.col(column.name).cast(pl.List(pl.String))
)
- return sanitize_json_bigint(to_json(result))
+ return sanitize_json_bigint(
+ to_json(result), ensure_ascii=ensure_ascii
+ )
except (
BaseException
): # Sometimes, polars throws a generic exception
@@ -188,7 +191,9 @@ class PolarsTableManagerFactory(TableManagerFactory):
", ".join(f"'{col}'" for col in converted_columns),
)
- return sanitize_json_bigint(to_json(result))
+ return sanitize_json_bigint(
+ to_json(result), ensure_ascii=ensure_ascii
+ )
def _convert_time_to_string(
self, result: pl.DataFrame, column: pl.Series
diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.py
index 4828d75d45..d80791fc43 100644
--- a/marimo/_plugins/ui/_impl/tables/table_manager.py
+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py
@@ -103,8 +103,10 @@ class TableManager(abc.ABC, Generic[T]):
def to_csv(
self,
format_mapping: Optional[FormatMapping] = None,
+ encoding: str | None = "utf-8",
) -> bytes:
- return self.to_csv_str(format_mapping).encode("utf-8")
+ resolved_encoding = encoding or "utf-8"
+ return self.to_csv_str(format_mapping).encode(resolved_encoding)
def to_arrow_ipc(self) -> bytes:
raise NotImplementedError("Arrow format not supported")
@@ -114,6 +116,7 @@ class TableManager(abc.ABC, Generic[T]):
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False,
+ ensure_ascii: bool = True,
) -> str:
pass
@@ -121,8 +124,15 @@ class TableManager(abc.ABC, Generic[T]):
self,
format_mapping: Optional[FormatMapping] = None,
strict_json: bool = False, # Whether the result should be strictly JSON compliant (eg. nan -> null)
+ encoding: str | None = "utf-8",
+ ensure_ascii: bool = True,
) -> bytes:
- return self.to_json_str(format_mapping, strict_json).encode("utf-8")
+ resolved_encoding = encoding or "utf-8"
+ return self.to_json_str(
+ format_mapping=format_mapping,
+ strict_json=strict_json,
+ ensure_ascii=ensure_ascii,
+ ).encode(resolved_encoding)
@abc.abstractmethod
def to_parquet(self) -> bytes:
diff --git a/marimo/_plugins/ui/_impl/utils/dataframe.py b/marimo/_plugins/ui/_impl/utils/dataframe.py
index eaf20a473f..36466fea6c 100644
--- a/marimo/_plugins/ui/_impl/utils/dataframe.py
+++ b/marimo/_plugins/ui/_impl/utils/dataframe.py
@@ -29,7 +29,11 @@ TableData = Union[
def download_as(
- manager: TableManager[Any], ext: str, drop_marimo_index: bool = False
+ manager: TableManager[Any],
+ ext: str,
+ drop_marimo_index: bool = False,
+ csv_encoding: str | None = "utf-8",
+ json_ensure_ascii: bool = True,
) -> str:
"""Download the table data in the specified format.
@@ -38,6 +42,10 @@ def download_as(
ext (str): The format to download the table data in.
drop_marimo_index (bool, optional): Whether to drop the marimo selection column.
Defaults to False.
+ csv_encoding (str | None, optional): Encoding used when generating CSV bytes.
+ Defaults to "utf-8". Ignored for non-CSV formats.
+ json_ensure_ascii (bool, optional): Whether to escape non-ASCII characters
+ in JSON output. Defaults to True.
Raises:
ValueError: If unrecognized format.
@@ -51,10 +59,14 @@ def download_as(
manager = manager.drop_columns([INDEX_COLUMN_NAME])
if ext == "csv":
- return mo_data.csv(manager.to_csv()).url
+ return mo_data.csv(manager.to_csv(encoding=csv_encoding)).url
elif ext == "json":
# Use strict JSON to ensure compliance with JSON spec
- return mo_data.json(manager.to_json(strict_json=True)).url
+ return mo_data.json(
+ manager.to_json(
+ encoding=None, ensure_ascii=json_ensure_ascii, strict_json=True
+ )
+ ).url
elif ext == "parquet":
return mo_data.parquet(manager.to_parquet()).url
else:
diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
index 71fce8d49c..b115b675a9 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
@@ -305,6 +305,35 @@ class TestDataframes:
assert subject._component_args["show-download"] is False
assert type(subject.value) is type(df)
+ @staticmethod
+ @pytest.mark.skipif(
+ not HAS_DEPS, reason="optional dependencies not installed"
+ )
+ def test_dataframe_download_encoding_utf8_sig_and_json_ensure_ascii() -> (
+ None
+ ):
+ df = pd.DataFrame({"A": [1, 2], "B": ["こんにちは", "世界"]})
+ subject = ui.dataframe(
+ df,
+ download_csv_encoding="utf-8-sig",
+ download_json_ensure_ascii=False,
+ )
+
+ # CSV should include BOM
+ csv_url = subject._download_as(DownloadAsArgs(format="csv"))
+ csv_bytes = from_data_uri(csv_url)[1]
+ assert csv_bytes.startswith(b"\xef\xbb\xbf")
+ assert "こんにちは" in csv_bytes.decode("utf-8-sig")
+
+ # JSON should preserve characters without BOM when ensure_ascii is False
+ json_url = subject._download_as(DownloadAsArgs(format="json"))
+ json_bytes = from_data_uri(json_url)[1]
+ assert not json_bytes.startswith(b"\xef\xbb\xbf")
+ json_text = json_bytes.decode("utf-8")
+ assert "こんにちは" in json_text
+ json_data = json.loads(json_text)
+ assert json_data[0]["B"] == "こんにちは"
+
@staticmethod
@pytest.mark.skipif(
not HAS_DEPS, reason="optional dependencies not installed"