A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: json encode handle more human types, and more human readable for tables (#6334)
Myles Scolnick0bb08c5dfa58
diff --git a/marimo/_messaging/msgspec_encoder.py b/marimo/_messaging/msgspec_encoder.pyindex 04afe7b1a3..35e4bc1191 100644--- a/marimo/_messaging/msgspec_encoder.py+++ b/marimo/_messaging/msgspec_encoder.py@@ -3,6 +3,11 @@from __future__ import annotations+import collections+import datetime+import fractions+from math import isnan+from pathlib import PurePathfrom typing import Anyimport msgspec@@ -27,19 +32,35 @@ def enc_hook(obj: Any) -> Any:if isinstance(obj, complex):return str(obj)+ if isinstance(obj, fractions.Fraction):+ return str(obj)++ if isinstance(obj, PurePath):+ return str(obj)+if DependencyManager.numpy.imported():import numpy as np- if isinstance(obj, (np.datetime64, np.complexfloating)):+ if isinstance(+ obj, (np.datetime64, np.timedelta64, np.complexfloating)+ ):return str(obj)if isinstance(obj, np.integer):return int(obj)if isinstance(obj, np.floating):return float(obj)+ if isinstance(obj, np.bool_):+ return bool(obj)+ if isinstance(obj, (np.bytes_, np.str_)):+ return str(obj)if isinstance(obj, np.ndarray):if any(np.issubdtype(obj.dtype, dtype)- for dtype in (np.datetime64, np.complexfloating)+ for dtype in (+ np.datetime64,+ np.timedelta64,+ np.complexfloating,+ )):return obj.astype(str).tolist()return obj.tolist()@@ -57,11 +78,21 @@ def enc_hook(obj: Any) -> Any:return obj.tolist()if isinstance(obj,- (pd.CategoricalDtype, pd.Timestamp, pd.Timedelta, pd.Interval),+ (+ pd.CategoricalDtype,+ pd.Timestamp,+ pd.Timedelta,+ pd.Interval,+ pd.Period,+ ),):return str(obj)- if isinstance(obj, pd.TimedeltaIndex):+ if isinstance(obj, (pd.TimedeltaIndex, pd.DatetimeIndex)):return obj.astype(str).tolist()+ if isinstance(obj, pd.MultiIndex):+ return obj.to_list()+ if isinstance(obj, pd.Index):+ return obj.to_list()# Catch-all for other pandas objectstry:@@ -77,10 +108,19 @@ def enc_hook(obj: Any) -> Any:if isinstance(obj, pl.DataFrame):return obj.to_dict()-+ if isinstance(obj, pl.LazyFrame):+ return obj.collect().to_dict()if isinstance(obj, pl.Series):return obj.to_list()+ # Handle Polars data types+ if hasattr(pl, "datatypes") and hasattr(obj, "__class__"):+ # Check if it's a Polars data type+ if hasattr(pl.datatypes, "DataType") and isinstance(+ obj, pl.datatypes.DataType+ ):+ return str(obj)+# Handle objects with __slots__slots = getattr(obj, "__slots__", None)if slots is not None:@@ -105,7 +145,59 @@ def enc_hook(obj: Any) -> Any:# Convert the __dict__ using msgspec.to_builtins for proper handlingreturn msgspec.to_builtins(obj.__dict__, enc_hook=enc_hook)- raise NotImplementedError(f"Objects of type {type(obj)} are not supported")+ # Handle collections types+ if isinstance(obj, (list, tuple, set, frozenset)):+ return list([enc_hook(item) for item in obj])++ if isinstance(obj, collections.deque):+ return list([enc_hook(item) for item in obj])++ # Handle dict and dict-like types+ if isinstance(+ obj,+ (+ dict,+ collections.defaultdict,+ collections.OrderedDict,+ collections.Counter,+ ),+ ):+ return {enc_hook(k): enc_hook(v) for k, v in obj.items()}++ # Handle float('inf'), float('nan'), float('-inf')+ if isinstance(obj, float):+ if obj == float("inf"):+ return "Infinity"+ if obj == float("-inf"):+ return "-Infinity"+ if isnan(obj):+ return "NaN"+ return obj++ # Handle bytes objects+ if isinstance(obj, memoryview):+ obj = obj.tobytes()++ if isinstance(obj, bytes):+ try:+ return obj.decode("utf-8")+ except UnicodeDecodeError:+ # Fallback to latin1+ return obj.decode("latin1")++ # Handle primitive types+ if isinstance(obj, (int, str, bool)):+ return obj++ # Handle datetime types+ if isinstance(obj, (datetime.datetime, datetime.timedelta, datetime.date)):+ return str(obj)++ # Handle None+ if obj is None:+ return None++ return repr(obj)_encoder = msgspec.json.Encoder(enc_hook=enc_hook, decimal_format="number")diff --git a/marimo/_output/superjson.py b/marimo/_output/superjson.pynew file mode 100644index 0000000000..e35a65aec1--- /dev/null+++ b/marimo/_output/superjson.py@@ -0,0 +1,23 @@+# This data serializes+from typing import Any+++class SuperJson: # noqa: B903+ """+ This class bypasses the default msgspec encoder for the data provided and instead serializes+ the result to something that is more human readable and not information-lossy.++ The key differences from the default msgspec encoder are:+ - It serializes the b'hello' to 'hello' instead of base64 encoded+ - It serializes the float('inf') to Infinity instead of null+ - It serializes the float('nan') to NaN instead of null+ - It serializes the timedelta to a human readable string instead of a ISO 8601 duration (e.g. "1 day, 2:03:00")+ """++ def __init__(self, data: Any):+ self.data = data++ def _marimo_serialize_(self) -> Any:+ from marimo._messaging.msgspec_encoder import enc_hook++ return enc_hook(self.data)diff --git a/marimo/_plugins/ui/_impl/tables/default_table.py b/marimo/_plugins/ui/_impl/tables/default_table.pyindex 67f91560b0..7342d24cae 100644--- a/marimo/_plugins/ui/_impl/tables/default_table.py+++ b/marimo/_plugins/ui/_impl/tables/default_table.py@@ -10,6 +10,7 @@ from marimo._data.models import BinValue, ColumnStats, ExternalDataTypefrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.msgspec_encoder import encode_json_strfrom marimo._output.mime import MIME+from marimo._output.superjson import SuperJsonfrom marimo._plugins.core.web_component import JSONTypefrom marimo._plugins.ui._impl.tables.format import (FormatMapping,@@ -102,7 +103,7 @@ class DefaultTableManager(TableManager[JsonTableData]):normalized = self._normalize_data(self.apply_formatting(format_mapping).data)- return encode_json_str(normalized)+ return encode_json_str(SuperJson(normalized))def to_parquet(self) -> bytes:if isinstance(self.data, dict) and not self.is_column_oriented:diff --git a/tests/_plugins/core/test_json_encoder.py b/tests/_plugins/core/test_json_encoder.pyindex d16e1adf2b..f88ae9777e 100644--- a/tests/_plugins/core/test_json_encoder.py+++ b/tests/_plugins/core/test_json_encoder.py@@ -1,10 +1,16 @@# Copyright 2024 Marimo. All rights reserved.from __future__ import annotations+import collections+import datetime+import fractionsimport json+import pathlibimport sys+import uuidfrom collections import namedtuplefrom dataclasses import dataclass+from decimal import Decimalfrom typing import Any, Optionalimport pytest@@ -13,6 +19,7 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.mimetypes import KnownMimeTypefrom marimo._messaging.msgspec_encoder import encode_json_strfrom marimo._output.mime import MIME+from marimo._utils.platform import is_windowsHAS_DEPS = (DependencyManager.pandas.has()@@ -41,6 +48,23 @@ def test_numpy_encoding() -> None:encoded_complex_arr = encode_json_str(complex_arr)assert encoded_complex_arr == '["(1+2j)","(3+4j)"]'+ # Additional numpy types+ td64 = np.timedelta64(1, "D")+ encoded = encode_json_str(td64)+ assert encoded == '"1 days"'++ bool_val = np.bool_(True)+ encoded = encode_json_str(bool_val)+ assert encoded == "true"++ bytes_val = np.bytes_(b"hello")+ encoded = encode_json_str(bytes_val)+ assert encoded == "\"b'hello'\""++ str_val = np.str_("hello")+ encoded = encode_json_str(str_val)+ assert encoded == '"hello"'+@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")def test_pandas_encoding() -> None:@@ -92,6 +116,29 @@ def test_pandas_encoding() -> None:encoded_other = encode_json_str(other)assert encoded_other == '["a","b","c"]'+ # Additional pandas types+ period = pd.Period("2021-01", freq="M")+ encoded = encode_json_str(period)+ assert encoded == '"2021-01"'++ # DatetimeIndex+ dt_index = pd.date_range("2021-01-01", periods=3, freq="D")+ encoded = encode_json_str(dt_index)+ result = json.loads(encoded)+ assert len(result) == 3+ assert "2021-01-01" in result[0]++ # MultiIndex+ multi_index = pd.MultiIndex.from_tuples([("A", 1), ("B", 2)])+ encoded = encode_json_str(multi_index)+ result = json.loads(encoded)+ assert result == [["A", 1], ["B", 2]]++ # Index+ index = pd.Index([1, 2, 3])+ encoded = encode_json_str(index)+ assert encoded == "[1,2,3]"+@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")def test_polars_encoding() -> None:@@ -105,6 +152,17 @@ def test_polars_encoding() -> None:encoded_series = encode_json_str(series)assert encoded_series == "[1,2,3]"+ # LazyFrame+ lazy_df = pl.DataFrame({"a": [1, 2], "b": [3, 4]}).lazy()+ encoded = encode_json_str(lazy_df)+ result = json.loads(encoded)+ assert result == {"a": [1, 2], "b": [3, 4]}++ # Polars data types+ dtype = pl.Int64()+ encoded = encode_json_str(dtype)+ assert "Int64" in encoded+class MockMIMEObject(MIME):def _mime_(self) -> tuple[KnownMimeType, str]:@@ -222,7 +280,8 @@ def test_tuple_encoding() -> None:assert encoded_empty == "[]"-def test_frozen_set_encoding() -> None:+def test_collections_encoding() -> None:+ # frozensetfrozen_set_obj = frozenset(["a", "b"])encoded = encode_json_str(frozen_set_obj)assert encoded == '["a","b"]' or encoded == '["b","a"]'@@ -233,6 +292,29 @@ def test_frozen_set_encoding() -> None:encoded_number = encode_json_str(number_frozen_set)assert encoded_number == "[1,2]" or encoded_number == "[2,1]"+ # deque+ deque_obj = collections.deque([1, 2, 3])+ encoded = encode_json_str(deque_obj)+ assert encoded == "[1,2,3]"++ # defaultdict+ default_dict = collections.defaultdict(int, {"a": 1, "b": 2})+ encoded = encode_json_str(default_dict)+ result = json.loads(encoded)+ assert result == {"a": 1, "b": 2}++ # OrderedDict+ ordered_dict = collections.OrderedDict([("a", 1), ("b", 2)])+ encoded = encode_json_str(ordered_dict)+ result = json.loads(encoded)+ assert result == {"a": 1, "b": 2}++ # Counter+ counter = collections.Counter(["a", "b", "a"])+ encoded = encode_json_str(counter)+ result = json.loads(encoded)+ assert result == {"a": 2, "b": 1}+def test_null_encoding() -> None:null = None@@ -303,8 +385,7 @@ def test_nested_structure_encoding() -> None:def test_date_time_encoding() -> None:- import datetime-+ # Standard datetime typesdate_obj = datetime.date(2023, 1, 1)time_obj = datetime.time(12, 30, 45)datetime_obj = datetime.datetime(2023, 1, 1, 12, 30, 45)@@ -319,24 +400,46 @@ def test_date_time_encoding() -> None:def test_timedelta_encoding() -> None:- import datetime-+ # Standard timedeltatimedelta_obj = datetime.timedelta(days=1, seconds=2, microseconds=3)encoded = encode_json_str(timedelta_obj)assert encoded == '"P1DT2.000003S"'+ delta = datetime.timedelta(days=1, hours=2, minutes=3)+ encoded = encode_json_str(delta)+ assert encoded == '"P1DT7380S"' # ISO 8601 duration format+def test_enum_encoding() -> None:- from enum import Enum+ from enum import Enum, IntEnumclass Color(Enum):RED = 1GREEN = 2BLUE = 3+ class StringColor(Enum):+ RED = "red"+ GREEN = "green"+ BLUE = "blue"++ class Number(IntEnum):+ ONE = 1+ TWO = 2+ THREE = 3++ # Int enumencoded = encode_json_str(Color.RED)assert encoded == "1"+ # String enum+ encoded = encode_json_str(StringColor.RED)+ assert encoded == '"red"'++ # IntEnum+ encoded = encode_json_str(Number.TWO)+ assert encoded == "2"+@pytest.mark.skipif(sys.version_info < (3, 11), reason="StrEnum not supported in Python 3.10"@@ -354,8 +457,7 @@ def test_str_enum() -> None:def test_uuid_encoding() -> None:- import uuid-+ # Random UUIDuuid_obj = uuid.uuid4()encoded = encode_json_str(uuid_obj)assert encoded == f'"{str(uuid_obj)}"'@@ -464,13 +566,49 @@ def test_empty_slots() -> None:assert encoded == "{}"-def test_decimal_encoding() -> None:- from decimal import Decimal-+def test_numeric_types_encoding() -> None:+ # Decimaldecimal_obj = Decimal("123.45")encoded = encode_json_str(decimal_obj)assert encoded == "123.45"+ # More precise decimal+ decimal_val = Decimal("123.456")+ encoded = encode_json_str(decimal_val)+ assert encoded == "123.456"++ # Fraction+ fraction_val = fractions.Fraction(3, 4)+ encoded = encode_json_str(fraction_val)+ assert encoded == '"3/4"'++[email protected](is_windows(), reason="Test specific to POSIX")+def test_pathlib_encoding() -> None:+ """Test encoding of pathlib Path objects."""+ # PosixPath+ path = pathlib.Path("/tmp/test.txt")+ encoded = encode_json_str(path)+ assert encoded == '"/tmp/test.txt"'++ # PurePath+ pure_path = pathlib.PurePath("relative/path.txt")+ encoded = encode_json_str(pure_path)+ assert encoded == '"relative/path.txt"'+++def test_generator_encoding() -> None:+ """Test encoding of generator objects."""++ def gen():+ yield 1+ yield 2+ yield 3++ generator = gen()+ encoded = encode_json_str(generator)+ assert "<generator" in encoded+def test_html_encoding() -> None:from marimo._output.hypertext import Html@@ -537,3 +675,122 @@ def test_list_of_inf_encoding() -> None:nested_inf_list = [[float("inf")], [1, float("-inf")], [float("nan"), 3]]encoded_nested = encode_json_str(nested_inf_list)assert encoded_nested == "[[null],[1,null],[null,3]]"+++def test_superjson_with_custom_objects() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with objects that have __dict__ (custom classes)+ @dataclass+ class CustomData:+ name: str+ value: int++ custom_data = CustomData(name="test", value=42)+ superjson_obj = SuperJson(custom_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '{"name":"test","value":42}'+++def test_superjson_with_mime_objects() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with MIME objects (which have _mime_ method)+ mime_data = MockMIMEObject()+ superjson_obj = SuperJson(mime_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '{"mimetype":"text/plain","data":"data"}'+++def test_superjson_with_marimo_serializable() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with objects that have _marimo_serialize_+ serializable_data = MockMarimoSerializable("superjson test")+ superjson_obj = SuperJson(serializable_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '{"serialized_data":"superjson test","type":"mock"}'+++def test_superjson_with_ranges() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with range objects (handled by enc_hook)+ range_data = range(5)+ superjson_obj = SuperJson(range_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == "[0,1,2,3,4]"+++def test_superjson_with_complex_numbers() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with complex numbers (handled by enc_hook)+ complex_data = 3 + 4j+ superjson_obj = SuperJson(complex_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '"(3+4j)"'+++def test_superjson_with_inf_nan() -> None:+ from marimo._output.superjson import SuperJson++ # Test SuperJson with inf and nan (handled by enc_hook)+ superjson_obj = SuperJson(+ [float("inf"), float("nan"), 1.0, 2.0, float("-inf")]+ )+ encoded = encode_json_str(superjson_obj)+ assert encoded == '["Infinity","NaN",1.0,2.0,"-Infinity"]'+++def test_superjson_with_bytes() -> None:+ from marimo._output.superjson import SuperJson++ # ASCII bytes+ bytes_data = b"hello"+ superjson_obj = SuperJson(bytes_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '"hello"'++ # Unicode bytes+ bytes_data = b"hello\x80\x81\x82"+ superjson_obj = SuperJson(bytes_data)+ encoded = encode_json_str(superjson_obj)+ assert encoded == '"hello\x80\x81\x82"'+++ ("value", "expected"),+ [+ (1, "1"),+ (True, "true"),+ ("hello", '"hello"'),+ (None, "null"),+ ([1, 2, 3], "[1,2,3]"),+ ({"a": 1, "b": 2}, '{"a":1,"b":2}'),+ ((), "[]"),+ (set([1, 2, 3]), "[1,2,3]"),+ (frozenset([1, 2, 3]), "[1,2,3]"),+ (range(10), "[0,1,2,3,4,5,6,7,8,9]"),+ (datetime.datetime(2023, 1, 1, 12, 30, 45), '"2023-01-01 12:30:45"'),+ (datetime.timedelta(days=1, hours=2, minutes=3), '"1 day, 2:03:00"'),+ (datetime.date(2023, 1, 1), '"2023-01-01"'),+ (3 + 4j, '"(3+4j)"'),+ (+ [float("inf"), float("nan"), 1.0, 2.0, float("-inf")],+ '["Infinity","NaN",1.0,2.0,"-Infinity"]',+ ),+ (b"hello", '"hello"'),+ (memoryview(b"hello"), '"hello"'),+ (+ MockMarimoSerializable("test data"),+ '{"serialized_data":"test data","type":"mock"}',+ ),+ ],+)+def test_wrapped_in_superjson(value: Any, expected: str) -> None:+ from marimo._output.superjson import SuperJson++ obj = SuperJson(value)+ encoded = encode_json_str(obj)+ assert encoded == expecteddiff --git a/tests/_plugins/ui/_impl/tables/test_default_table.py b/tests/_plugins/ui/_impl/tables/test_default_table.pyindex 6afbaea2db..6282ced3c8 100644--- a/tests/_plugins/ui/_impl/tables/test_default_table.py+++ b/tests/_plugins/ui/_impl/tables/test_default_table.py@@ -1,13 +1,16 @@from __future__ import annotations+import datetimeimport jsonimport unittestfrom datetime import date+from pathlib import Pathfrom typing import Anyimport pytestfrom marimo._dependencies.dependencies import DependencyManager+from marimo._output.hypertext import Htmlfrom marimo._plugins.ui._impl.tables.default_table import DefaultTableManagerfrom marimo._plugins.ui._impl.tables.table_manager import (TableCell,@@ -980,3 +983,30 @@ class TestListDefaultTable(unittest.TestCase):)def test_to_parquet(self) -> None:assert isinstance(self.manager.to_parquet(), bytes)+++class TestDefaultTableWithComplexData(unittest.TestCase):+ def setUp(self) -> None:+ self.manager = DefaultTableManager(+ [+ {+ "inf": float("inf"),+ "nan": float("nan"),+ "timedelta": datetime.timedelta(+ days=1, hours=2, minutes=3+ ),+ "path": Path("test.txt"),+ "complex": 1 + 2j,+ "bytes": b"hello",+ "memoryview": memoryview(b"hello"),+ "range": range(10),+ "html": Html("<h1>Hello World</h1>"),+ }+ ]+ )++ def test_to_json(self) -> None:+ assert (+ self.manager.to_json_str()+ == '[{"inf":"Infinity","nan":"NaN","timedelta":"1 day, 2:03:00","path":"test.txt","complex":"(1+2j)","bytes":"hello","memoryview":"hello","range":[0,1,2,3,4,5,6,7,8,9],"html":{"mimetype":"text/html","data":"<h1>Hello World</h1>"}}]'+ )