A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: dataframe download encoding (#7385)
vavavavavavavavavacc71a4fda41b
diff --git a/marimo/_output/data/data.py b/marimo/_output/data/data.pyindex 0bfbd634d7..ac37370f66 100644--- a/marimo/_output/data/data.py+++ b/marimo/_output/data/data.py@@ -198,6 +198,7 @@ def is_bigint(value: int | float) -> bool:def sanitize_json_bigint(data: Union[str, dict[str, Any], list[dict[str, Any]]],+ ensure_ascii: bool = True,) -> str:"""Sanitize JSON big numbers to a string.@@ -239,4 +240,5 @@ def sanitize_json_bigint(indent=None,separators=(",", ":"),default=enc_hook,+ ensure_ascii=ensure_ascii,)diff --git a/marimo/_plugins/ui/_impl/charts/altair_transformer.py b/marimo/_plugins/ui/_impl/charts/altair_transformer.pyindex efc7130613..d32be13865 100644--- a/marimo/_plugins/ui/_impl/charts/altair_transformer.py+++ b/marimo/_plugins/ui/_impl/charts/altair_transformer.py@@ -105,7 +105,7 @@ def _data_to_json_string(data: _DataType) -> str:tm = get_table_manager_or_none(data)if tm:- return tm.to_json().decode("utf-8")+ return tm.to_json(ensure_ascii=True).decode("utf-8")raise NotImplementedError("to_marimo_json only works with data expressed as a DataFrame "diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.pyindex b51412fb0f..7f34376275 100644--- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py+++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py@@ -108,6 +108,10 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):dataframes via Ibis.show_download (bool, optional): Whether to show the download button.Defaults to True.+ download_csv_encoding (str, optional): Encoding used when downloading CSV.+ Defaults to "utf-8". Set to "utf-8-sig" to include BOM for Excel.+ download_json_ensure_ascii (bool, optional): Whether to escape non-ASCII characters+ in JSON downloads. Defaults to True.on_change (Optional[Callable[[DataFrameType], None]], optional): Optional callbackto run when this element's value changes.lazy (Optional[bool], optional): When lazy is True, an 'Apply' button will be shown to apply transformations.@@ -123,6 +127,9 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):page_size: Optional[int] = 5,limit: Optional[int] = None,show_download: bool = True,+ *,+ download_csv_encoding: str = "utf-8",+ download_json_ensure_ascii: bool = True,lazy: Optional[bool] = None,) -> None:validate_no_integer_columns(df)@@ -150,6 +157,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):self._limit = limitself._dataframe_name = dataframe_nameself._data = df+ self._download_csv_encoding = download_csv_encoding+ self._download_json_ensure_ascii = download_json_ensure_asciiself._handler = handlerself._manager = self._get_cached_table_manager(df, self._limit)self._transform_container = TransformsContainer(nw_df, handler)@@ -178,6 +187,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):"total": rows,"page-size": page_size,"show-download": show_download,+ "download-csv-encoding": download_csv_encoding,+ "download-json-ensure-ascii": download_json_ensure_ascii,"lazy": self._lazy,},functions=(@@ -315,7 +326,12 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):# Get the table manager for the transformed datamanager = self._get_cached_table_manager(df, self._limit)- return download_as(manager, args.format)+ return download_as(+ manager,+ args.format,+ csv_encoding=self._download_csv_encoding,+ json_ensure_ascii=self._download_json_ensure_ascii,+ )def _apply_filters_query_sort(self,diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.pyindex a13ae4455d..b24b234efd 100644--- a/marimo/_plugins/ui/_impl/table.py+++ b/marimo/_plugins/ui/_impl/table.py@@ -1118,7 +1118,9 @@ class table(# Fallback to JSONtry:- data_url = mo_data.json(table_manager.to_json({})).url+ data_url = mo_data.json(+ table_manager.to_json({}, ensure_ascii=True)+ ).urlreturn data_url, "json"except Exception as e:LOGGER.error(diff --git a/marimo/_plugins/ui/_impl/tables/default_table.py b/marimo/_plugins/ui/_impl/tables/default_table.pyindex d09c01f050..46ff8a31b0 100644--- a/marimo/_plugins/ui/_impl/tables/default_table.py+++ b/marimo/_plugins/ui/_impl/tables/default_table.py@@ -104,8 +104,10 @@ class DefaultTableManager(TableManager[JsonTableData]):self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False,+ ensure_ascii: bool = True,) -> str:del strict_json+ del ensure_asciinormalized = self._normalize_data(self.apply_formatting(format_mapping).data)diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.pyindex 4c142a8f50..3f96b33193 100644--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py@@ -96,10 +96,13 @@ class NarwhalsTableManager(self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False,+ ensure_ascii: bool = True,) -> str:del strict_jsonframe = self.apply_formatting(format_mapping).as_frame()- return sanitize_json_bigint(frame.rows(named=True))+ return sanitize_json_bigint(+ frame.rows(named=True), ensure_ascii=ensure_ascii+ )def to_parquet(self) -> bytes:stream = io.BytesIO()diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex 8b348f9e95..166397fb61 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -85,6 +85,7 @@ class PandasTableManagerFactory(TableManagerFactory):self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False,+ ensure_ascii: bool = True,) -> str:def to_json(result: pd.DataFrame,@@ -145,7 +146,9 @@ class PandasTableManagerFactory(TableManagerFactory):"Error handling complex or timedelta64 dtype",exc_info=e,)- return sanitize_json_bigint(to_json(result))+ return sanitize_json_bigint(+ to_json(result), ensure_ascii=ensure_ascii+ )# Flatten row multi-indexif isinstance(result.index, pd.MultiIndex) or (@@ -198,7 +201,9 @@ class PandasTableManagerFactory(TableManagerFactory):"Indexes with more than one level are not well supported, call reset_index() or use mo.plain(df)")- return sanitize_json_bigint(to_json(result))+ return sanitize_json_bigint(+ to_json(result), ensure_ascii=ensure_ascii+ )def _infer_dtype(self, column: ColumnName) -> str:# Typically, pandas dtypes returns a generic dtypediff --git a/marimo/_plugins/ui/_impl/tables/polars_table.py b/marimo/_plugins/ui/_impl/tables/polars_table.pyindex 3bcf82e157..5f26b2e726 100644--- a/marimo/_plugins/ui/_impl/tables/polars_table.py+++ b/marimo/_plugins/ui/_impl/tables/polars_table.py@@ -111,6 +111,7 @@ class PolarsTableManagerFactory(TableManagerFactory):self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False,+ ensure_ascii: bool = True,) -> str:def to_json(result: pl.DataFrame,@@ -146,7 +147,9 @@ class PolarsTableManagerFactory(TableManagerFactory):result = result.with_columns(pl.col(column.name).cast(pl.List(pl.String)))- return sanitize_json_bigint(to_json(result))+ return sanitize_json_bigint(+ to_json(result), ensure_ascii=ensure_ascii+ )except (BaseException): # Sometimes, polars throws a generic exception@@ -188,7 +191,9 @@ class PolarsTableManagerFactory(TableManagerFactory):", ".join(f"'{col}'" for col in converted_columns),)- return sanitize_json_bigint(to_json(result))+ return sanitize_json_bigint(+ to_json(result), ensure_ascii=ensure_ascii+ )def _convert_time_to_string(self, result: pl.DataFrame, column: pl.Seriesdiff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.pyindex 4828d75d45..d80791fc43 100644--- a/marimo/_plugins/ui/_impl/tables/table_manager.py+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py@@ -103,8 +103,10 @@ class TableManager(abc.ABC, Generic[T]):def to_csv(self,format_mapping: Optional[FormatMapping] = None,+ encoding: str | None = "utf-8",) -> bytes:- return self.to_csv_str(format_mapping).encode("utf-8")+ resolved_encoding = encoding or "utf-8"+ return self.to_csv_str(format_mapping).encode(resolved_encoding)def to_arrow_ipc(self) -> bytes:raise NotImplementedError("Arrow format not supported")@@ -114,6 +116,7 @@ class TableManager(abc.ABC, Generic[T]):self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False,+ ensure_ascii: bool = True,) -> str:pass@@ -121,8 +124,15 @@ class TableManager(abc.ABC, Generic[T]):self,format_mapping: Optional[FormatMapping] = None,strict_json: bool = False, # Whether the result should be strictly JSON compliant (eg. nan -> null)+ encoding: str | None = "utf-8",+ ensure_ascii: bool = True,) -> bytes:- return self.to_json_str(format_mapping, strict_json).encode("utf-8")+ resolved_encoding = encoding or "utf-8"+ return self.to_json_str(+ format_mapping=format_mapping,+ strict_json=strict_json,+ ensure_ascii=ensure_ascii,+ ).encode(resolved_encoding)@abc.abstractmethoddef to_parquet(self) -> bytes:diff --git a/marimo/_plugins/ui/_impl/utils/dataframe.py b/marimo/_plugins/ui/_impl/utils/dataframe.pyindex eaf20a473f..36466fea6c 100644--- a/marimo/_plugins/ui/_impl/utils/dataframe.py+++ b/marimo/_plugins/ui/_impl/utils/dataframe.py@@ -29,7 +29,11 @@ TableData = Union[def download_as(- manager: TableManager[Any], ext: str, drop_marimo_index: bool = False+ manager: TableManager[Any],+ ext: str,+ drop_marimo_index: bool = False,+ csv_encoding: str | None = "utf-8",+ json_ensure_ascii: bool = True,) -> str:"""Download the table data in the specified format.@@ -38,6 +42,10 @@ def download_as(ext (str): The format to download the table data in.drop_marimo_index (bool, optional): Whether to drop the marimo selection column.Defaults to False.+ csv_encoding (str | None, optional): Encoding used when generating CSV bytes.+ Defaults to "utf-8". Ignored for non-CSV formats.+ json_ensure_ascii (bool, optional): Whether to escape non-ASCII characters+ in JSON output. Defaults to True.Raises:ValueError: If unrecognized format.@@ -51,10 +59,14 @@ def download_as(manager = manager.drop_columns([INDEX_COLUMN_NAME])if ext == "csv":- return mo_data.csv(manager.to_csv()).url+ return mo_data.csv(manager.to_csv(encoding=csv_encoding)).urlelif ext == "json":# Use strict JSON to ensure compliance with JSON spec- return mo_data.json(manager.to_json(strict_json=True)).url+ return mo_data.json(+ manager.to_json(+ encoding=None, ensure_ascii=json_ensure_ascii, strict_json=True+ )+ ).urlelif ext == "parquet":return mo_data.parquet(manager.to_parquet()).urlelse:diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.pyindex 71fce8d49c..b115b675a9 100644--- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py+++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py@@ -305,6 +305,35 @@ class TestDataframes:assert subject._component_args["show-download"] is Falseassert type(subject.value) is type(df)+ @staticmethod+ @pytest.mark.skipif(+ not HAS_DEPS, reason="optional dependencies not installed"+ )+ def test_dataframe_download_encoding_utf8_sig_and_json_ensure_ascii() -> (+ None+ ):+ df = pd.DataFrame({"A": [1, 2], "B": ["こんにちは", "世界"]})+ subject = ui.dataframe(+ df,+ download_csv_encoding="utf-8-sig",+ download_json_ensure_ascii=False,+ )++ # CSV should include BOM+ csv_url = subject._download_as(DownloadAsArgs(format="csv"))+ csv_bytes = from_data_uri(csv_url)[1]+ assert csv_bytes.startswith(b"\xef\xbb\xbf")+ assert "こんにちは" in csv_bytes.decode("utf-8-sig")++ # JSON should preserve characters without BOM when ensure_ascii is False+ json_url = subject._download_as(DownloadAsArgs(format="json"))+ json_bytes = from_data_uri(json_url)[1]+ assert not json_bytes.startswith(b"\xef\xbb\xbf")+ json_text = json_bytes.decode("utf-8")+ assert "こんにちは" in json_text+ json_data = json.loads(json_text)+ assert json_data[0]["B"] == "こんにちは"+@staticmethod@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed"