Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix download_json in tables by adding strict_json arg (#7429)

Shahmir Varqha1bef461e0d5a
diff --git a/marimo/_plugins/ui/_impl/tables/default_table.py b/marimo/_plugins/ui/_impl/tables/default_table.py
index 5654a345c7..d09c01f050 100644
--- a/marimo/_plugins/ui/_impl/tables/default_table.py
+++ b/marimo/_plugins/ui/_impl/tables/default_table.py
@@ -101,8 +101,11 @@ class DefaultTableManager(TableManager[JsonTableData]):
return self._as_table_manager().to_csv_str(format_mapping)
def to_json_str(
- self, format_mapping: Optional[FormatMapping] = None
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False,
) -> str:
+ del strict_json
normalized = self._normalize_data(
self.apply_formatting(format_mapping).data
)
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
index 6d0b5f7222..f4b49f87b0 100644
--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py
+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
@@ -93,8 +93,11 @@ class NarwhalsTableManager(
return dataframe_to_csv(_data)
def to_json_str(
- self, format_mapping: Optional[FormatMapping] = None
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False,
) -> str:
+ del strict_json
frame = self.apply_formatting(format_mapping).as_frame()
return sanitize_json_bigint(frame.rows(named=True))
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index 348fbbf19c..8b348f9e95 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -82,11 +82,31 @@ class PandasTableManagerFactory(TableManagerFactory):
)._original_data.to_csv(index=has_headers)
def to_json_str(
- self, format_mapping: Optional[FormatMapping] = None
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False,
) -> str:
- def to_json(result: pd.DataFrame) -> list[dict[str, Any]]:
- # Use to_dict instead of to_json
- # nans, infs, -infs are properly serialized
+ def to_json(
+ result: pd.DataFrame,
+ ) -> list[dict[str, Any]] | str:
+ """
+ to_dict preserves nans, infs and is more accurate than to_json.
+ By default, we use to_dict unless strict_json is True
+ """
+ if strict_json:
+ try:
+ json_str = result.to_json(
+ orient="records",
+ date_format="iso",
+ default_handler=str,
+ )
+ assert json_str is not None
+ return json_str
+ except Exception as e:
+ LOGGER.warning(
+ "Error serializing to JSON. Falling back to to_dict. Error: %s",
+ e,
+ )
return result.to_dict(orient="records") # type: ignore
from pandas.api.types import (
diff --git a/marimo/_plugins/ui/_impl/tables/polars_table.py b/marimo/_plugins/ui/_impl/tables/polars_table.py
index ad1097aa34..3bcf82e157 100644
--- a/marimo/_plugins/ui/_impl/tables/polars_table.py
+++ b/marimo/_plugins/ui/_impl/tables/polars_table.py
@@ -108,11 +108,25 @@ class PolarsTableManagerFactory(TableManagerFactory):
return result.write_csv()
def to_json_str(
- self, format_mapping: Optional[FormatMapping] = None
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False,
) -> str:
- def to_json(result: pl.DataFrame) -> list[dict[str, Any]]:
- # Use to_dicts instead of write_json
- # Preserves certain types like nans, infs, -infs, etc.
+ def to_json(
+ result: pl.DataFrame,
+ ) -> list[dict[str, Any]] | str:
+ """
+ to_dicts preserves nans, infs and is more accurate than write_json.
+ By default, we use to_dicts unless strict_json is True
+ """
+ if strict_json:
+ try:
+ return result.write_json()
+ except Exception as e:
+ LOGGER.warning(
+ "Error serializing to JSON. Falling back to to_dicts. Error: %s",
+ e,
+ )
return result.to_dicts()
result = self.apply_formatting(format_mapping).collect()
diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.py
index 76ca38431b..4828d75d45 100644
--- a/marimo/_plugins/ui/_impl/tables/table_manager.py
+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py
@@ -111,12 +111,18 @@ class TableManager(abc.ABC, Generic[T]):
@abc.abstractmethod
def to_json_str(
- self, format_mapping: Optional[FormatMapping] = None
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False,
) -> str:
pass
- def to_json(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
- return self.to_json_str(format_mapping).encode("utf-8")
+ def to_json(
+ self,
+ format_mapping: Optional[FormatMapping] = None,
+ strict_json: bool = False, # Whether the result should be strictly JSON compliant (eg. nan -> null)
+ ) -> bytes:
+ return self.to_json_str(format_mapping, strict_json).encode("utf-8")
@abc.abstractmethod
def to_parquet(self) -> bytes:
diff --git a/marimo/_plugins/ui/_impl/utils/dataframe.py b/marimo/_plugins/ui/_impl/utils/dataframe.py
index 7366833c71..eaf20a473f 100644
--- a/marimo/_plugins/ui/_impl/utils/dataframe.py
+++ b/marimo/_plugins/ui/_impl/utils/dataframe.py
@@ -53,7 +53,8 @@ def download_as(
if ext == "csv":
return mo_data.csv(manager.to_csv()).url
elif ext == "json":
- return mo_data.json(manager.to_json()).url
+ # Use strict JSON to ensure compliance with JSON spec
+ return mo_data.json(manager.to_json(strict_json=True)).url
elif ext == "parquet":
return mo_data.parquet(manager.to_parquet()).url
else:
diff --git a/marimo/_smoke_tests/tables/complex_types.py b/marimo/_smoke_tests/tables/complex_types.py
index 2f0a74728e..e639fe33ae 100644
--- a/marimo/_smoke_tests/tables/complex_types.py
+++ b/marimo/_smoke_tests/tables/complex_types.py
@@ -1,6 +1,6 @@
import marimo
-__generated_with = "0.17.8"
+__generated_with = "0.18.3"
app = marimo.App(width="medium")
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/pandas.download.json b/tests/_plugins/ui/_impl/tables/snapshots/pandas.download.json
new file mode 100644
index 0000000000..6b1db83dc2
--- /dev/null
+++ b/tests/_plugins/ui/_impl/tables/snapshots/pandas.download.json
@@ -0,0 +1,191 @@
+[
+ {
+ "strings": "a",
+ "bool": true,
+ "int": 1,
+ "float": 1.0,
+ "datetime": "2021-01-01T00:00:00.000",
+ "date": "2021-01-01",
+ "nanosecond": "2017-03-22T15:16:45.000",
+ "datetime_index": [
+ "1960-01-02T00:00:00.000",
+ "1960-01-03T00:00:00.000",
+ "1960-01-04T00:00:00.000"
+ ],
+ "period_range": "2017Q1",
+ "interval_range": "(2017-01-01 00:00:00, 2017-01-02 00:00:00]",
+ "delta": [
+ "P1DT6H5M1.000030S"
+ ],
+ "none": null,
+ "infs": null,
+ "struct": {
+ "a": 1,
+ "b": 2
+ },
+ "list": [
+ 1,
+ 2
+ ],
+ "nested_lists": [
+ [
+ 1,
+ 2
+ ]
+ ],
+ "objects": {
+ "a": 1,
+ "b": 2
+ },
+ "nested_objects": [
+ {
+ "a": 1,
+ "b": 2
+ },
+ {
+ "a": 3,
+ "b": 4
+ }
+ ],
+ "nulls": null,
+ "category": "cat",
+ "set": [
+ 1,
+ 2
+ ],
+ "imaginary": "(1+2j)",
+ "time": "12:30:00",
+ "duration": "1 days 00:00:00",
+ "mixed_list": [
+ 1,
+ "two"
+ ]
+ },
+ {
+ "strings": "b",
+ "bool": false,
+ "int": 2,
+ "float": 2.0,
+ "datetime": "2021-01-02T00:00:00.000",
+ "date": "2021-01-02",
+ "nanosecond": "2017-07-14T02:40:00.000",
+ "datetime_index": [
+ "1970-01-02T00:00:00.000",
+ "1970-01-03T00:00:00.000",
+ "1970-01-04T00:00:00.000"
+ ],
+ "period_range": "2017Q2",
+ "interval_range": "(2017-01-02 00:00:00, 2017-01-03 00:00:00]",
+ "delta": [
+ "P1DT6H5M1.000030S",
+ "P0DT0H0M0.000015500S"
+ ],
+ "none": null,
+ "infs": null,
+ "struct": {
+ "a": 3,
+ "b": 4
+ },
+ "list": [
+ 3,
+ 4
+ ],
+ "nested_lists": [
+ [
+ 3,
+ 4
+ ]
+ ],
+ "objects": {
+ "a": 3,
+ "b": 4
+ },
+ "nested_objects": [
+ {
+ "a": 5,
+ "b": 6
+ },
+ {
+ "a": 7,
+ "b": 8
+ }
+ ],
+ "nulls": "data",
+ "category": "dog",
+ "set": [
+ 3,
+ 4
+ ],
+ "imaginary": "(3+4j)",
+ "time": "13:45:00",
+ "duration": "2 days 00:00:00",
+ "mixed_list": [
+ 3.0,
+ false
+ ]
+ },
+ {
+ "strings": "c",
+ "bool": true,
+ "int": 3,
+ "float": 3.0,
+ "datetime": "2021-01-03T00:00:00.000",
+ "date": "2021-01-03",
+ "nanosecond": "2017-11-06T20:26:40.000",
+ "datetime_index": [
+ "1970-01-02T00:00:00.000",
+ "1970-01-03T00:00:00.000",
+ "1970-01-04T00:00:00.000"
+ ],
+ "period_range": "2017Q3",
+ "interval_range": "(2017-01-03 00:00:00, 2017-01-04 00:00:00]",
+ "delta": [
+ "P1DT6H5M1.000030S",
+ "P0DT0H0M0.000015500S",
+ null
+ ],
+ "none": null,
+ "infs": null,
+ "struct": {
+ "a": 5,
+ "b": 6
+ },
+ "list": [
+ 5,
+ 6
+ ],
+ "nested_lists": [
+ [
+ 5,
+ 6
+ ]
+ ],
+ "objects": {
+ "a": 5,
+ "b": 6
+ },
+ "nested_objects": [
+ {
+ "a": 9,
+ "b": 10
+ },
+ {
+ "a": 11,
+ "b": 12
+ }
+ ],
+ "nulls": null,
+ "category": "mouse",
+ "set": [
+ 5,
+ 6
+ ],
+ "imaginary": "(5+6j)",
+ "time": "14:15:00",
+ "duration": "3 days 00:00:00",
+ "mixed_list": [
+ null,
+ "2021-01-01T00:00:00.000"
+ ]
+ }
+]
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/polars.download.json b/tests/_plugins/ui/_impl/tables/snapshots/polars.download.json
new file mode 100644
index 0000000000..ce09062932
--- /dev/null
+++ b/tests/_plugins/ui/_impl/tables/snapshots/polars.download.json
@@ -0,0 +1,205 @@
+[
+ {
+ "strings": "a",
+ "bool": true,
+ "int": 1,
+ "large_int": {
+ "$bigint": "18446744073709551616"
+ },
+ "float": 1.0,
+ "decimals": "1.23",
+ "datetime": "2021-01-01 00:00:00",
+ "date": "2021-01-01",
+ "struct": {
+ "a": 1,
+ "b": 2
+ },
+ "list": [
+ 1,
+ 2
+ ],
+ "nested_lists": [
+ [
+ 1,
+ 2
+ ]
+ ],
+ "nested_arrays": [
+ [
+ 1,
+ 2
+ ]
+ ],
+ "array": [
+ 1
+ ],
+ "nulls": null,
+ "category": "cat",
+ "set": [
+ 1,
+ 2
+ ],
+ "imaginary": "(1+2j)",
+ "time": "12:30:00",
+ "duration": "1d",
+ "nans":NaN,
+ "infs": "inf",
+ "mixed_list": [
+ "1",
+ "two"
+ ],
+ "structs_with_list": {
+ "a": [
+ 1,
+ 2
+ ],
+ "b": 2
+ },
+ "list_with_structs": [
+ {
+ "a": 1,
+ "c": null
+ },
+ {
+ "a": null,
+ "c": 3
+ }
+ ],
+ "enum_list": [
+ "A",
+ "B",
+ "C"
+ ],
+ "binary": "\u0000\u0000\u0000\u0000\u0001\u00c0U\u00e8\u00b1n1\u00c0T@D\u00f1?Bc\u0095\u0083"
+ },
+ {
+ "strings": "b",
+ "bool": false,
+ "int": 2,
+ "large_int": {
+ "$bigint": "36893488147419103233"
+ },
+ "float": 2.0,
+ "decimals": "4.56",
+ "datetime": "2021-01-02 00:00:00",
+ "date": "2021-01-02",
+ "struct": {
+ "a": 3,
+ "b": 4
+ },
+ "list": [
+ 3,
+ 4
+ ],
+ "nested_lists": [
+ [
+ 3,
+ 4
+ ]
+ ],
+ "nested_arrays": [
+ [
+ 3,
+ 4
+ ]
+ ],
+ "array": [
+ 2
+ ],
+ "nulls": "data",
+ "category": "dog",
+ "set": [
+ 3,
+ 4
+ ],
+ "imaginary": "(3+4j)",
+ "time": "13:45:00",
+ "duration": "315\u00b5s",
+ "nans":NaN,
+ "infs": "-inf",
+ "mixed_list": [
+ "3.0",
+ "0.0"
+ ],
+ "structs_with_list": {
+ "a": [
+ 3,
+ 4
+ ],
+ "b": 4
+ },
+ "list_with_structs": [
+ {
+ "a": null,
+ "c": null
+ }
+ ],
+ "enum_list": [
+ "A",
+ "B",
+ "C"
+ ],
+ "binary": "world"
+ },
+ {
+ "strings": "c",
+ "bool": true,
+ "int": 3,
+ "large_int": {
+ "$bigint": "73786976294838206466"
+ },
+ "float": 3.0,
+ "decimals": "7.89",
+ "datetime": "2021-01-03 00:00:00",
+ "date": "2021-01-03",
+ "struct": {
+ "a": 5,
+ "b": 6
+ },
+ "list": [
+ 5,
+ 6
+ ],
+ "nested_lists": [
+ [
+ 5,
+ 6
+ ]
+ ],
+ "nested_arrays": [
+ [
+ 5,
+ 6
+ ]
+ ],
+ "array": [
+ 3
+ ],
+ "nulls": null,
+ "category": "mouse",
+ "set": [
+ 5,
+ 6
+ ],
+ "imaginary": "(5+6j)",
+ "time": "14:15:00",
+ "duration": "2h 30m",
+ "nans":NaN,
+ "infs": "inf",
+ "mixed_list": [
+ null,
+ "2021-01-01 00:00:00.000000"
+ ],
+ "structs_with_list": {
+ "a": null,
+ "b": null
+ },
+ "list_with_structs": [],
+ "enum_list": [
+ "A",
+ "B",
+ "C"
+ ],
+ "binary": "foo"
+ }
+]
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index 1ab8851a6a..725881a66f 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -4,6 +4,7 @@ import datetime
import decimal
import json
import unittest
+import warnings
from math import isnan, nan
from typing import Any
from unittest.mock import Mock
@@ -1513,6 +1514,40 @@ class TestPandasTableManager(unittest.TestCase):
assert json_data[0]["date_series"] == "2000-01-01"
assert json_data[1]["date_series"] == "2000-01-02"
+ def test_to_json_str_strict_json(self) -> None:
+ import pandas as pd
+
+ df = pd.DataFrame({"A": [1, 2, 3]})
+ manager = self.factory.create()(df)
+ json_str = manager.to_json_str(strict_json=True)
+ assert json_str == '[{"A":1},{"A":2},{"A":3}]'
+
+ def test_to_json_str_strict_json_with_nans(self) -> None:
+ import pandas as pd
+
+ df_with_nans_infs = pd.DataFrame(
+ {"A": [1, 2, 3, float("nan"), float("inf")]}
+ )
+ manager = self.factory.create()(df_with_nans_infs)
+ json_str = manager.to_json_str(strict_json=True)
+ assert (
+ json_str == '[{"A":1.0},{"A":2.0},{"A":3.0},{"A":null},{"A":null}]'
+ )
+
+ def test_to_json_str_strict_json_with_complex_data(self) -> None:
+ # Ensure does not fail with complex data
+ complex_data = self.get_complex_data()
+ json_str = complex_data.to_json_str(strict_json=True)
+
+ # Snapshot complex data without erroring cols
+ with warnings.catch_warnings(record=True) as w:
+ complex_data_strict = complex_data.drop_columns(
+ ["bytes", "decimals"]
+ )
+ json_str_strict = complex_data_strict.to_json_str(strict_json=True)
+ snapshot("pandas.download.json", json_str_strict)
+ assert len(w) == 0, "Unexpected warnings raised"
+
def test_handle_integer_column_names_no_mutation(self) -> None:
import pandas as pd
diff --git a/tests/_plugins/ui/_impl/tables/test_polars_table.py b/tests/_plugins/ui/_impl/tables/test_polars_table.py
index e53717127d..6f0f78184e 100644
--- a/tests/_plugins/ui/_impl/tables/test_polars_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_polars_table.py
@@ -1045,3 +1045,28 @@ class TestPolarsTableManagerFactory(unittest.TestCase):
df_schema = pl.DataFrame(rows, schema=schema)
assert df_schema[0:5].write_json() == expected_first_five
assert df_schema[5:10].write_json() == expected_second_five # fails
+
+ def test_to_json_str_strict_json(self) -> None:
+ import polars as pl
+
+ data = pl.DataFrame({"A": [1, 2, 3]})
+ manager = self.factory.create()(data)
+ json_str = manager.to_json_str(strict_json=True)
+ assert json_str == '[{"A":1},{"A":2},{"A":3}]'
+
+ def test_to_json_str_strict_json_with_nans(self) -> None:
+ import polars as pl
+
+ data = pl.DataFrame(
+ {"A": [1, 2, 3, float("nan"), float("inf")]}, strict=False
+ )
+ manager = self.factory.create()(data)
+ json_str = manager.to_json_str(strict_json=True)
+ assert (
+ json_str == '[{"A":1.0},{"A":2.0},{"A":3.0},{"A":null},{"A":null}]'
+ )
+
+ def test_to_json_str_strict_json_with_complex_data(self) -> None:
+ data = self.get_complex_data()
+ json_str = data.to_json_str(strict_json=True)
+ snapshot("polars.download.json", json_str)