Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

pandas change to to_dict instead of to_json and fix rich elems (#7223)

Shahmir Varqhac8d94210762d
diff --git a/marimo/_output/data/data.py b/marimo/_output/data/data.py
index 2829c908d3..e5130cc801 100644
--- a/marimo/_output/data/data.py
+++ b/marimo/_output/data/data.py
@@ -5,6 +5,7 @@ import base64
import io
from typing import Any, Union
+from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.core.media import is_data_empty
from marimo._runtime.virtual_file import (
EMPTY_VIRTUAL_FILE,
@@ -237,5 +238,27 @@ def sanitize_json_bigint(
convert_bigint(as_json),
indent=None,
separators=(",", ":"),
- default=str,
+ default=custom_json,
)
+
+
+def custom_json(obj: Any) -> Any:
+ if hasattr(obj, "_mime_"):
+ mimetype, data = obj._mime_()
+ return {"mimetype": mimetype, "data": data}
+
+ if DependencyManager.pandas.imported():
+ import pandas as pd
+
+ # TODO: Port this logic over to msgspec encoder
+ if isinstance(
+ obj,
+ (
+ pd.TimedeltaIndex,
+ pd.DatetimeIndex,
+ pd.IntervalIndex,
+ pd.PeriodIndex,
+ ),
+ ):
+ return obj.astype(str).tolist()
+ return str(obj)
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index fe3c3b0491..91889492a6 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -84,6 +84,11 @@ class PandasTableManagerFactory(TableManagerFactory):
def to_json_str(
self, format_mapping: Optional[FormatMapping] = None
) -> str:
+ def to_json(result: pd.DataFrame) -> list[dict[str, Any]]:
+ # Use to_dict instead of to_json
+ # nans, infs, -infs are properly serialized
+ return result.to_dict(orient="records") # type: ignore
+
from pandas.api.types import (
is_complex_dtype,
is_object_dtype,
@@ -123,13 +128,7 @@ class PandasTableManagerFactory(TableManagerFactory):
"Error handling complex or timedelta64 dtype",
exc_info=e,
)
- return sanitize_json_bigint(
- result.to_json(
- orient="records",
- date_format="iso",
- default_handler=str,
- )
- )
+ return sanitize_json_bigint(to_json(result))
# Flatten row multi-index
if isinstance(result.index, pd.MultiIndex) or (
@@ -182,13 +181,7 @@ class PandasTableManagerFactory(TableManagerFactory):
"Indexes with more than one level are not well supported, call reset_index() or use mo.plain(df)"
)
- return sanitize_json_bigint(
- result.to_json(
- orient="records",
- date_format="iso",
- default_handler=str,
- )
- )
+ return sanitize_json_bigint(to_json(result))
def _infer_dtype(self, column: ColumnName) -> str:
# Typically, pandas dtypes returns a generic dtype
diff --git a/marimo/_smoke_tests/tables/big_ints.py b/marimo/_smoke_tests/tables/big_ints.py
index 038d4d680d..77b638b079 100644
--- a/marimo/_smoke_tests/tables/big_ints.py
+++ b/marimo/_smoke_tests/tables/big_ints.py
@@ -1,12 +1,14 @@
import marimo
-__generated_with = "0.15.5"
+__generated_with = "0.17.6"
app = marimo.App(width="medium")
@app.cell(hide_code=True)
def _(mo):
- mo.md(r"""## Big Ints""")
+ mo.md(r"""
+ ## Big Ints
+ """)
return
@@ -34,18 +36,18 @@ def _():
@app.cell
-def _(data):
+def _(data, mo):
import pandas as pd
- pd.DataFrame(data)
+ mo.vstack([mo.plain(pd.DataFrame(data)), pd.DataFrame(data)])
return
@app.cell
-def _(data):
+def _(data, mo):
import polars as pl
- pl.DataFrame(data)
+ mo.vstack([mo.plain(pl.DataFrame(data)), pl.DataFrame(data)])
return
diff --git a/marimo/_smoke_tests/tables/images.py b/marimo/_smoke_tests/tables/images.py
index 14b963ce2a..262cb06efb 100644
--- a/marimo/_smoke_tests/tables/images.py
+++ b/marimo/_smoke_tests/tables/images.py
@@ -1,6 +1,6 @@
import marimo
-__generated_with = "0.15.5"
+__generated_with = "0.17.8"
app = marimo.App(width="medium")
diff --git a/marimo/_smoke_tests/tables/nans.py b/marimo/_smoke_tests/tables/nans.py
new file mode 100644
index 0000000000..5db671834d
--- /dev/null
+++ b/marimo/_smoke_tests/tables/nans.py
@@ -0,0 +1,42 @@
+import marimo
+
+__generated_with = "0.17.6"
+app = marimo.App(width="medium")
+
+
+def _():
+ import marimo as mo
+ import polars as pl
+ import pandas as pd
+ import numpy as np
+ return mo, np, pd, pl
+
+
+def _(mo, np, pl):
+ polars_df = pl.DataFrame({"nans": [1.0, np.nan, np.inf, -np.inf, None]})
+ mo.vstack([mo.plain(polars_df), polars_df])
+ return
+
+
+def _(np, pd, pl):
+ pl.DataFrame(
+ {"nans_not_strict": [1.0, np.nan, np.inf, -np.inf, None, pd.NA, pd.NaT]},
+ strict=False,
+ )
+ return
+
+
+def _(mo, np, pd):
+ pandas_df = pd.DataFrame(
+ {"nans": [1, None, pd.NaT, np.nan, pd.NA, np.inf, -np.inf]}
+ )
+ mo.vstack([mo.plain(pandas_df), pandas_df])
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/narwhals.json b/tests/_plugins/ui/_impl/tables/snapshots/narwhals.json
index 4cc2c1fdb9..fc5129e88c 100644
--- a/tests/_plugins/ui/_impl/tables/snapshots/narwhals.json
+++ b/tests/_plugins/ui/_impl/tables/snapshots/narwhals.json
@@ -1,56 +1,89 @@
[
- {
- "strings": "a",
- "bool": true,
- "int": 1,
- "float": 1.0,
- "datetime": "2021-01-01 00:00:00",
- "date": "2021-01-01",
- "struct": { "a": 1, "b": 2 },
- "list": [1, 2],
- "array": [1],
- "nulls": null,
- "category": "cat",
- "set": "{1, 2}",
- "imaginary": "(1+2j)",
- "time": "12:30:00",
- "duration": "1 day, 0:00:00",
- "mixed_list": ["1", "two"]
- },
- {
- "strings": "b",
- "bool": false,
- "int": 2,
- "float": 2.0,
- "datetime": "2021-01-02 00:00:00",
- "date": "2021-01-02",
- "struct": { "a": 3, "b": 4 },
- "list": [3, 4],
- "array": [2],
- "nulls": "data",
- "category": "dog",
- "set": "{3, 4}",
- "imaginary": "(3+4j)",
- "time": "13:45:00",
- "duration": "2 days, 0:00:00",
- "mixed_list": ["3.0", "0.0"]
- },
- {
- "strings": "c",
- "bool": true,
- "int": 3,
- "float": 3.0,
- "datetime": "2021-01-03 00:00:00",
- "date": "2021-01-03",
- "struct": { "a": 5, "b": 6 },
- "list": [5, 6],
- "array": [3],
- "nulls": null,
- "category": "mouse",
- "set": "{5, 6}",
- "imaginary": "(5+6j)",
- "time": "14:15:00",
- "duration": "3 days, 0:00:00",
- "mixed_list": [null, "2021-01-01 00:00:00.000000"]
- }
-]
+ {
+ "strings": "a",
+ "bool": true,
+ "int": 1,
+ "float": 1.0,
+ "datetime": "2021-01-01 00:00:00",
+ "date": "2021-01-01",
+ "struct": {
+ "a": 1,
+ "b": 2
+ },
+ "list": [
+ 1,
+ 2
+ ],
+ "array": [
+ 1
+ ],
+ "nulls": null,
+ "category": "cat",
+ "set": "{1, 2}",
+ "imaginary": "(1+2j)",
+ "time": "12:30:00",
+ "duration": "1 day, 0:00:00",
+ "mixed_list": [
+ "1",
+ "two"
+ ]
+ },
+ {
+ "strings": "b",
+ "bool": false,
+ "int": 2,
+ "float": 2.0,
+ "datetime": "2021-01-02 00:00:00",
+ "date": "2021-01-02",
+ "struct": {
+ "a": 3,
+ "b": 4
+ },
+ "list": [
+ 3,
+ 4
+ ],
+ "array": [
+ 2
+ ],
+ "nulls": "data",
+ "category": "dog",
+ "set": "{3, 4}",
+ "imaginary": "(3+4j)",
+ "time": "13:45:00",
+ "duration": "2 days, 0:00:00",
+ "mixed_list": [
+ "3.0",
+ "0.0"
+ ]
+ },
+ {
+ "strings": "c",
+ "bool": true,
+ "int": 3,
+ "float": 3.0,
+ "datetime": "2021-01-03 00:00:00",
+ "date": "2021-01-03",
+ "struct": {
+ "a": 5,
+ "b": 6
+ },
+ "list": [
+ 5,
+ 6
+ ],
+ "array": [
+ 3
+ ],
+ "nulls": null,
+ "category": "mouse",
+ "set": "{5, 6}",
+ "imaginary": "(5+6j)",
+ "time": "14:15:00",
+ "duration": "3 days, 0:00:00",
+ "mixed_list": [
+ null,
+ "2021-01-01 00:00:00.000000"
+ ]
+ }
+]
\ No newline at end of file
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/pandas.csv b/tests/_plugins/ui/_impl/tables/snapshots/pandas.csv
index 204cb48224..c9264b19e4 100644
--- a/tests/_plugins/ui/_impl/tables/snapshots/pandas.csv
+++ b/tests/_plugins/ui/_impl/tables/snapshots/pandas.csv
@@ -1,4 +1,4 @@
-strings,bool,int,float,datetime,date,struct,list,nulls,category,set,imaginary,time,duration,mixed_list,bytes
-a,True,1,1.0,2021-01-01,2021-01-01,"{'a': 1, 'b': 2}","[1, 2]",,cat,"{1, 2}",(1+2j),12:30:00,1 days,"[1, 'two']",b'\x00\x00\x00\x00\x01\xc0U\xe8\xb1n1\xc0T@D\xf1?Bc\x95\x83'
-b,False,2,2.0,2021-01-02,2021-01-02,"{'a': 3, 'b': 4}","[3, 4]",data,dog,"{3, 4}",(3+4j),13:45:00,2 days,"[3.0, False]",b'world'
-c,True,3,3.0,2021-01-03,2021-01-03,"{'a': 5, 'b': 6}","[5, 6]",,mouse,"{5, 6}",(5+6j),14:15:00,3 days,"[None, datetime.datetime(2021, 1, 1, 0, 0)]",b'bytes'
+strings,bool,int,float,datetime,date,nanosecond,datetime_index,period_range,interval_range,delta,none,infs,struct,list,nulls,category,set,imaginary,time,duration,mixed_list,bytes
+a,True,1,1.0,2021-01-01,2021-01-01,2017-03-22 15:16:45,"DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'], dtype='datetime64[ns]', freq=None)",2017Q1,"(2017-01-01 00:00:00, 2017-01-02 00:00:00]","TimedeltaIndex(['1 days 06:05:01.000030'], dtype='timedelta64[ns]', freq=None)",,inf,"{'a': 1, 'b': 2}","[1, 2]",,cat,"{1, 2}",(1+2j),12:30:00,1 days,"[1, 'two']",b'\x00\x00\x00\x00\x01\xc0U\xe8\xb1n1\xc0T@D\xf1?Bc\x95\x83'
+b,False,2,2.0,2021-01-02,2021-01-02,2017-07-14 02:40:00,"DatetimeIndex(['1970-01-02', '1970-01-03', '1970-01-04'], dtype='datetime64[ns]', freq=None)",2017Q2,"(2017-01-02 00:00:00, 2017-01-03 00:00:00]","TimedeltaIndex(['1 days 06:05:01.000030', '0 days 00:00:00.000015500'], dtype='timedelta64[ns]', freq=None)",,inf,"{'a': 3, 'b': 4}","[3, 4]",data,dog,"{3, 4}",(3+4j),13:45:00,2 days,"[3.0, False]",b'world'
+c,True,3,3.0,2021-01-03,2021-01-03,2017-11-06 20:26:40,"DatetimeIndex(['1970-01-02', '1970-01-03', '1970-01-04'], dtype='datetime64[ns]', freq=None)",2017Q3,"(2017-01-03 00:00:00, 2017-01-04 00:00:00]","TimedeltaIndex(['1 days 06:05:01.000030', '0 days 00:00:00.000015500', NaT], dtype='timedelta64[ns]', freq=None)",,-inf,"{'a': 5, 'b': 6}","[5, 6]",,mouse,"{5, 6}",(5+6j),14:15:00,3 days,"[None, datetime.datetime(2021, 1, 1, 0, 0)]",b'bytes'
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/pandas.field_types.json b/tests/_plugins/ui/_impl/tables/snapshots/pandas.field_types.json
index e7ebe405bf..ab331e0cd2 100644
--- a/tests/_plugins/ui/_impl/tables/snapshots/pandas.field_types.json
+++ b/tests/_plugins/ui/_impl/tables/snapshots/pandas.field_types.json
@@ -41,6 +41,55 @@
"object"
]
],
+ [
+ "nanosecond",
+ [
+ "datetime",
+ "datetime64[ns]"
+ ]
+ ],
+ [
+ "datetime_index",
+ [
+ "string",
+ "object"
+ ]
+ ],
+ [
+ "period_range",
+ [
+ "unknown",
+ "period[Q-DEC]"
+ ]
+ ],
+ [
+ "interval_range",
+ [
+ "string",
+ "interval[datetime64[ns], right]"
+ ]
+ ],
+ [
+ "delta",
+ [
+ "string",
+ "object"
+ ]
+ ],
+ [
+ "none",
+ [
+ "string",
+ "object"
+ ]
+ ],
+ [
+ "infs",
+ [
+ "number",
+ "float64"
+ ]
+ ],
[
"struct",
[
diff --git a/tests/_plugins/ui/_impl/tables/snapshots/pandas.json b/tests/_plugins/ui/_impl/tables/snapshots/pandas.json
index 5a14590311..72e0eae27d 100644
--- a/tests/_plugins/ui/_impl/tables/snapshots/pandas.json
+++ b/tests/_plugins/ui/_impl/tables/snapshots/pandas.json
@@ -4,8 +4,21 @@
"bool": true,
"int": 1,
"float": 1.0,
- "datetime": "2021-01-01T00:00:00.000",
+ "datetime": "2021-01-01 00:00:00",
"date": "2021-01-01",
+ "nanosecond": "2017-03-22 15:16:45",
+ "datetime_index": [
+ "1960-01-02",
+ "1960-01-03",
+ "1960-01-04"
+ ],
+ "period_range": "2017Q1",
+ "interval_range": "(2017-01-01 00:00:00, 2017-01-02 00:00:00]",
+ "delta": [
+ "1 days 06:05:01.000030"
+ ],
+ "none": "NaT",
+ "infs": "inf",
"struct": {
"a": 1,
"b": 2
@@ -16,10 +29,7 @@
],
"nulls": null,
"category": "cat",
- "set": [
- 1,
- 2
- ],
+ "set": "{1, 2}",
"imaginary": "(1+2j)",
"time": "12:30:00",
"duration": "1 days 00:00:00",
@@ -34,8 +44,22 @@
"bool": false,
"int": 2,
"float": 2.0,
- "datetime": "2021-01-02T00:00:00.000",
+ "datetime": "2021-01-02 00:00:00",
"date": "2021-01-02",
+ "nanosecond": "2017-07-14 02:40:00",
+ "datetime_index": [
+ "1970-01-02",
+ "1970-01-03",
+ "1970-01-04"
+ ],
+ "period_range": "2017Q2",
+ "interval_range": "(2017-01-02 00:00:00, 2017-01-03 00:00:00]",
+ "delta": [
+ "1 days 06:05:01.000030",
+ "0 days 00:00:00.000015500"
+ ],
+ "none":NaN,
+ "infs": "inf",
"struct": {
"a": 3,
"b": 4
@@ -46,10 +70,7 @@
],
"nulls": "data",
"category": "dog",
- "set": [
- 3,
- 4
- ],
+ "set": "{3, 4}",
"imaginary": "(3+4j)",
"time": "13:45:00",
"duration": "2 days 00:00:00",
@@ -64,8 +85,23 @@
"bool": true,
"int": 3,
"float": 3.0,
- "datetime": "2021-01-03T00:00:00.000",
+ "datetime": "2021-01-03 00:00:00",
"date": "2021-01-03",
+ "nanosecond": "2017-11-06 20:26:40",
+ "datetime_index": [
+ "1970-01-02",
+ "1970-01-03",
+ "1970-01-04"
+ ],
+ "period_range": "2017Q3",
+ "interval_range": "(2017-01-03 00:00:00, 2017-01-04 00:00:00]",
+ "delta": [
+ "1 days 06:05:01.000030",
+ "0 days 00:00:00.000015500",
+ "NaT"
+ ],
+ "none": null,
+ "infs": "-inf",
"struct": {
"a": 5,
"b": 6
@@ -76,16 +112,13 @@
],
"nulls": null,
"category": "mouse",
- "set": [
- 5,
- 6
- ],
+ "set": "{5, 6}",
"imaginary": "(5+6j)",
"time": "14:15:00",
"duration": "3 days 00:00:00",
"mixed_list": [
null,
- "2021-01-01T00:00:00.000"
+ "2021-01-01 00:00:00"
],
"bytes": "b'bytes'"
}
diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.py
index 3a75ed755c..cc5689a834 100644
--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py
+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py
@@ -13,6 +13,7 @@ import pytest
from marimo._data.models import BinValue, ColumnStats
from marimo._dependencies.dependencies import DependencyManager
from marimo._output.data.data import BIGINT_KEY
+from marimo._plugins.ui._impl.input import button
from marimo._plugins.ui._impl.table import SortArgs
from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.narwhals_table import (
@@ -1714,3 +1715,55 @@ class TestSanitizeTableValue:
data = pl.DataFrame({"A": [1, 2, 3]})
return NarwhalsTableManager.from_dataframe(data)
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+class TestRichElements:
+ """Tests for rich elements."""
+
+ @pytest.fixture
+ def rich_data(self) -> dict[str, Any]:
+ return {"button": [button()]}
+
+ @pytest.mark.skipif(
+ not DependencyManager.pandas.has(), reason="Pandas not installed"
+ )
+ def test_pandas(self, rich_data: dict[str, Any]) -> None:
+ import pandas as pd
+
+ df = pd.DataFrame(rich_data)
+ manager = NarwhalsTableManager.from_dataframe(df)
+ json_data = json.loads(manager.to_json_str())
+
+ # Pandas uses mimetype and data instead of _serialized_mime_bundle
+ assert isinstance(json_data, list)
+ assert isinstance(json_data[0], dict)
+ assert isinstance(json_data[0]["button"], dict)
+
+ assert json_data[0]["button"]["mimetype"] == "text/html"
+ assert json_data[0]["button"]["data"].startswith("<marimo-ui-element")
+ assert json_data[0]["button"]["data"].endswith("</marimo-ui-element>")
+
+ @pytest.mark.parametrize(
+ "df",
+ create_dataframes(
+ {"button": [button()]}, include=["duckdb", "polars", "lazy-polars"]
+ ),
+ )
+ def test_rich_elements_default(self, df: Any) -> None:
+ manager = NarwhalsTableManager.from_dataframe(df)
+ json_data = json.loads(manager.to_json_str())
+
+ assert isinstance(json_data, list)
+ assert isinstance(json_data[0], dict)
+ assert isinstance(json_data[0]["button"], dict)
+ assert isinstance(
+ json_data[0]["button"]["_serialized_mime_bundle"], dict
+ )
+
+ serialized_mime_bundle = json_data[0]["button"][
+ "_serialized_mime_bundle"
+ ]
+ assert serialized_mime_bundle["mimetype"] == "text/html"
+ assert serialized_mime_bundle["data"].startswith("<marimo-ui-element")
+ assert serialized_mime_bundle["data"].endswith("</marimo-ui-element>")
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index 6d3687e78b..33b507a2bc 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import datetime
import json
import unittest
-from math import isnan
+from math import isnan, nan
from typing import Any
from unittest.mock import Mock
@@ -12,7 +12,7 @@ import pytest
from marimo._data.models import ColumnStats
from marimo._dependencies.dependencies import DependencyManager
-from marimo._output.data.data import BIGINT_KEY
+from marimo._output.data.data import BIGINT_KEY, sanitize_json_bigint
from marimo._plugins.ui._impl.table import SortArgs
from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.pandas_table import (
@@ -21,7 +21,7 @@ from marimo._plugins.ui._impl.tables.pandas_table import (
from marimo._plugins.ui._impl.tables.table_manager import TableManager
from tests.mocks import snapshotter
-HAS_DEPS = DependencyManager.pandas.has()
+HAS_DEPS = DependencyManager.pandas.has() and DependencyManager.numpy.has()
snapshot = snapshotter(__file__)
@@ -36,6 +36,24 @@ def assert_frame_equal(a: Any, b: Any) -> None:
pd.testing.assert_frame_equal(a, b)
+def assert_rows_equal_with_nan(
+ actual: list[dict[str, Any]], expected: list[dict[str, Any]]
+) -> None:
+ assert len(actual) == len(expected)
+ for actual_row, expected_row in zip(actual, expected, strict=True):
+ assert actual_row.keys() == expected_row.keys()
+ for key, expected_value in expected_row.items():
+ actual_value = actual_row[key]
+ if (
+ isinstance(actual_value, float)
+ and isinstance(expected_value, float)
+ and isnan(actual_value)
+ and isnan(expected_value)
+ ):
+ continue
+ assert actual_value == expected_value
+
+
try:
import pandas as pd
except ImportError:
@@ -45,6 +63,7 @@ except ImportError:
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
class TestPandasTableManager(unittest.TestCase):
def get_complex_data(self) -> TableManager[Any]:
+ import numpy as np
import pandas as pd
complex_data = pd.DataFrame(
@@ -63,6 +82,40 @@ class TestPandasTableManager(unittest.TestCase):
datetime.date(2021, 1, 2),
datetime.date(2021, 1, 3),
],
+ "nanosecond": [
+ pd.to_datetime(1490195805000000000, unit="ns"),
+ pd.to_datetime(1500000000000000000, unit="ns"),
+ pd.to_datetime(1510000000000000000, unit="ns"),
+ ],
+ "datetime_index": [
+ pd.to_datetime(
+ [1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01")
+ ),
+ pd.to_datetime([1, 2, 3], unit="D"),
+ pd.to_datetime([1, 2, 3], unit="D"),
+ ],
+ "period_range": pd.period_range(
+ start=pd.Period("2017Q1", freq="Q"),
+ end=pd.Period("2017Q3", freq="Q"),
+ freq="Q",
+ ),
+ "interval_range": pd.interval_range(
+ start=pd.Timestamp("2017-01-01"),
+ end=pd.Timestamp("2017-01-04"),
+ ),
+ "delta": [
+ pd.to_timedelta(["1 days 06:05:01.00003"]),
+ pd.to_timedelta(["1 days 06:05:01.00003", "15.5us"]),
+ pd.to_timedelta(
+ ["1 days 06:05:01.00003", "15.5us", "nan"]
+ ),
+ ],
+ "none": [
+ pd.NaT,
+ np.nan,
+ pd.NA,
+ ],
+ "infs": [float("inf"), np.inf, -np.inf],
"struct": [
{"a": 1, "b": 2},
{"a": 3, "b": 4},
@@ -214,21 +267,19 @@ class TestPandasTableManager(unittest.TestCase):
assert isinstance(self.manager.to_parquet(), bytes)
def test_to_json(self) -> None:
- expected_json = self.data.to_json(
- orient="records", date_format="iso"
- ).encode("utf-8")
- assert self.manager.to_json() == expected_json
+ expected_json = sanitize_json_bigint(
+ self.data.to_dict(orient="records")
+ )
+ actual_json = self.manager.to_json().decode("utf-8")
+ assert actual_json == expected_json, "JSON not equal"
def test_to_json_format_mapping(self) -> None:
- expected_json = (
- self.data.assign(A=self.data["A"] * 2)
- .to_json(orient="records", date_format="iso")
- .encode("utf-8")
+ expected_json = sanitize_json_bigint(
+ self.data.assign(A=self.data["A"] * 2).to_dict(orient="records")
)
format_mapping = {"A": lambda x: x * 2}
- assert self.manager.to_json(format_mapping) == expected_json, (
- "Format mapping not applied"
- )
+ actual_json = self.manager.to_json(format_mapping).decode("utf-8")
+ assert actual_json == expected_json, "Format mapping not applied"
def test_to_json_datetime_handling(self) -> None:
timestamps = pd.DataFrame(
@@ -241,10 +292,10 @@ class TestPandasTableManager(unittest.TestCase):
)
json_data = self.factory_create_json_from_df(timestamps)
- assert json_data[0]["timestamp"] == "2024-12-17T00:00:00.000"
+ assert json_data[0]["timestamp"] == "2024-12-17 00:00:00"
assert (
json_data[0]["timestamp_with_timezone"]
- == "2024-12-17T00:00:00.000Z"
+ == "2024-12-17 00:00:00+00:00"
)
def test_to_json_complex_number_handling(self) -> None:
@@ -367,12 +418,55 @@ class TestPandasTableManager(unittest.TestCase):
)
df = df.stack(future_stack=True)
json_data = self.factory_create_json_from_df(df)
- assert json_data == [
- {"": "cat", " ": "kg", "weight": 1.0, "height": None},
- {"": "cat", " ": "m", "weight": None, "height": 2.0},
- {"": "dog", " ": "kg", "weight": 3.0, "height": None},
- {"": "dog", " ": "m", "weight": None, "height": 4.0},
+ expected = [
+ {"": "cat", " ": "kg", "weight": 1.0, "height": nan},
+ {"": "cat", " ": "m", "weight": nan, "height": 2.0},
+ {"": "dog", " ": "kg", "weight": 3.0, "height": nan},
+ {"": "dog", " ": "m", "weight": nan, "height": 4.0},
+ ]
+ assert_rows_equal_with_nan(json_data, expected)
+
+ @pytest.mark.skipif(
+ not DependencyManager.numpy.has(), reason="numpy not installed"
+ )
+ def test_infs(self) -> None:
+ import numpy as np
+
+ df = pd.DataFrame(
+ {
+ "A": [
+ float("inf"),
+ float("-inf"),
+ np.inf,
+ -np.inf,
+ ]
+ }
+ )
+ json_data = self.factory_create_json_from_df(df)
+ expected = [
+ {"A": "inf"},
+ {"A": "-inf"},
+ {"A": "inf"},
+ {"A": "-inf"},
+ ]
+ assert_rows_equal_with_nan(json_data, expected)
+
+ @pytest.mark.skipif(
+ not DependencyManager.numpy.has(), reason="numpy not installed"
+ )
+ def test_nans(self) -> None:
+ import numpy as np
+
+ df = pd.DataFrame({"A": [float("nan"), np.nan, pd.NaT, pd.NA, None]})
+ json_data = self.factory_create_json_from_df(df)
+ expected = [
+ {"A": nan},
+ {"A": nan},
+ {"A": "NaT"},
+ {"A": None}, # pd.NA is treated as None
+ {"A": None},
]
+ assert_rows_equal_with_nan(json_data, expected)
def test_to_json_multi_col_index(self) -> None:
cols = pd.MultiIndex.from_arrays(