Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

24.0 KB
Raw Blame
# SPDX-License-Identifier: MIT
# Copyright contributors to the kepler.gl project

"""Serializer tests."""

import json
import pytest
import pandas as pd
import geopandas as gpd
from shapely.geometry import Point, Polygon
from keplergl.serializers import serialize_dataset


class TestDataFrameSerialization:
    """Tests for DataFrame serialization (from DataFrame.ipynb)."""

    def test_serialize_dataframe(self, sample_df):
        result = serialize_dataset(sample_df, "test")
        assert result["id"] == "test"
        assert result["format"] == "df"
        assert "data" in result

    def test_serialize_dataframe_with_cities(self):
        """Test DataFrame with city data (from DataFrame.ipynb)."""
        df = pd.DataFrame({
            'City': ['Buenos Aires', 'Brasilia', 'Santiago', 'Bogota', 'Caracas'],
            'Country': ['Argentina', 'Brazil', 'Chile', 'Colombia', 'Venezuela'],
            'Latitude': [-34.58, -15.78, -33.45, 4.60, 10.48],
            'Longitude': [-58.66, -47.91, -70.66, -74.08, -66.86],
            'Time': ['2019-09-01 08:00', '2019-09-01 09:00', '2019-09-01 10:00',
                     '2019-09-01 11:00', '2019-09-01 12:00'],
        })
        result = serialize_dataset(df, "data_1")
        assert result["id"] == "data_1"
        assert result["format"] == "df"
        assert result["data"]["columns"] == ['City', 'Country', 'Latitude', 'Longitude', 'Time']
        assert len(result["data"]["data"]) == 5

    def test_serialize_dataframe_with_hex_data(self):
        """Test DataFrame with H3 hex IDs and mixed types (from Load kepler.gl.ipynb)."""
        df = pd.DataFrame({
            'hex_id': ['89283082c2fffff', '8928308288fffff', '89283082c07ffff'],
            'value': [64, 73, 65],
            'is_true': [True, True, True],
            'float_value': [64.1, 73.1, 65.1],
            'empty': ['', '', ''],
            'time': ['11/1/17 11:00', '11/1/17 11:00', '11/1/17 11:00'],
        })
        result = serialize_dataset(df, "data_1")
        assert result["id"] == "data_1"
        assert result["format"] == "df"
        assert 'hex_id' in result["data"]["columns"]
        assert 'value' in result["data"]["columns"]
        assert 'is_true' in result["data"]["columns"]

    def test_serialize_dataframe_with_nan_filled(self):
        """Test DataFrame with NaN values filled with empty string."""
        df = pd.DataFrame({
            'col1': [1, 2, 3],
            'col2': ['a', None, 'c'],
        })
        df = df.fillna('')
        result = serialize_dataset(df, "test")
        assert result["format"] == "df"
        assert result["data"]["data"][1][1] == ''


class TestGeoDataFrameSerialization:
    """Tests for GeoDataFrame serialization (from GeoDataFrame.ipynb)."""

    def test_serialize_geodataframe(self, sample_gdf):
        result = serialize_dataset(sample_gdf, "test")
        assert result["id"] == "test"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_geodataframe_with_timestamp(self):
        """Test GeoDataFrame with pd.Timestamp column.

        Regression test for issue where GeoDataFrame containing Timestamp
        columns would fail during Arrow serialization.
        """
        df = pd.DataFrame({
            'City': ['Buenos Aires'],
            'Country': ['Argentina'],
            'Latitude': [-34.58],
            'Longitude': [-58.66],
            'Timestamp': pd.Timestamp(2002, 3, 3),
        })
        gdf = gpd.GeoDataFrame(
            df,
            geometry=gpd.points_from_xy(df.Longitude, df.Latitude),
        )
        result = serialize_dataset(gdf, "cities")
        assert result["id"] == "cities"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_geodataframe_points_from_xy(self):
        """Test GeoDataFrame created with points_from_xy (from GeoDataFrame.ipynb)."""
        df = pd.DataFrame({
            'City': ['Buenos Aires', 'Brasilia', 'Santiago', 'Bogota', 'Caracas'],
            'Country': ['Argentina', 'Brazil', 'Chile', 'Colombia', 'Venezuela'],
            'Latitude': [-34.58, -15.78, -33.45, 4.60, 10.48],
            'Longitude': [-58.66, -47.91, -70.66, -74.08, -66.86],
        })
        gdf = gpd.GeoDataFrame(
            df,
            geometry=gpd.points_from_xy(df.Longitude, df.Latitude),
        )
        result = serialize_dataset(gdf, "cities")
        assert result["id"] == "cities"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_geodataframe_with_polygons(self):
        """Test GeoDataFrame with Polygon geometries (like zipcode boundaries)."""
        gdf = gpd.GeoDataFrame({
            'ZIP_CODE': ['94107', '94105'],
            'geometry': [
                Polygon([(-122.40, 37.78), (-122.39, 37.78),
                         (-122.39, 37.77), (-122.40, 37.77)]),
                Polygon([(-122.39, 37.79), (-122.38, 37.79),
                         (-122.38, 37.78), (-122.39, 37.78)]),
            ],
        })
        result = serialize_dataset(gdf, "zipcode")
        assert result["id"] == "zipcode"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_geodataframe_with_crs(self):
        """Test GeoDataFrame with explicit CRS."""
        gdf = gpd.GeoDataFrame(
            {"name": ["SF", "LA"]},
            geometry=[Point(-122.4194, 37.7749), Point(-118.2437, 34.0522)],
            crs="EPSG:4326",
        )
        result = serialize_dataset(gdf, "test")
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_empty_geodataframe(self):
        """Test that an empty GeoDataFrame serializes without error."""
        gdf = gpd.GeoDataFrame(
            {"name": pd.Series([], dtype="str")},
            geometry=gpd.GeoSeries([], crs="EPSG:4326"),
        )
        result = serialize_dataset(gdf, "empty")
        assert result["id"] == "empty"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_all_null_geometry_geodataframe(self):
        """Test that a GeoDataFrame with all-null geometries serializes without error."""
        gdf = gpd.GeoDataFrame(
            {"name": ["a", "b"]},
            geometry=[None, None],
        )
        result = serialize_dataset(gdf, "nulls")
        assert result["id"] == "nulls"
        assert result["format"] == "geoarrow"
        assert "data" in result

    def test_serialize_geodataframe_multiple_geometry_columns(self):
        """GeoDataFrame with multiple geometry columns must not raise.

        Regression test for the serialization error reported in
        https://github.com/keplergl/kepler.gl/issues/2283 — when a GeoDataFrame
        has more than one geometry-typed column, pa.array() would fail on the
        non-active geometry columns.  The active geometry column is encoded as
        GeoArrow; secondary ones are converted to WKT strings.
        """
        import base64
        import pyarrow as pa

        gdf = gpd.GeoDataFrame(
            {"name": ["SF", "LA"], "value": [1, 2]},
            geometry=[Point(-122.4194, 37.7749), Point(-118.2437, 34.0522)],
            crs="EPSG:4326",
        )
        # buffer() on a geographic CRS gives imprecise results — acceptable here
        # since we only care about serialization, not geometric accuracy.
        import warnings
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", UserWarning)
            gdf["buffer"] = gdf.geometry.buffer(0.01)

        result = serialize_dataset(gdf, "multi_geom")
        assert result["id"] == "multi_geom"
        assert result["format"] == "geoarrow"

        # Decode the Arrow table and verify all columns survived
        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert "name" in table.column_names
        assert "value" in table.column_names
        assert "geometry" in table.column_names
        assert "buffer" in table.column_names
        assert table.num_rows == 2

        # The secondary geometry column must be WKT strings, not raw geometry objects
        buffer_col = table.column("buffer")
        assert pa.types.is_string(buffer_col.type) or pa.types.is_large_string(buffer_col.type), (
            f"Expected secondary geometry column to be WKT string, got {buffer_col.type}"
        )
        assert buffer_col[0].as_py().startswith("POLYGON")

    def test_serialize_geodataframe_non_default_active_geometry(self):
        """Active geometry column with non-default name is encoded as GeoArrow.

        When a GeoDataFrame has two geometry columns and the active one is
        not named 'geometry', the active column must still be GeoArrow-encoded
        and the other must be WKT.
        """
        import base64
        import pyarrow as pa

        gdf = gpd.GeoDataFrame(
            {"name": ["A", "B"]},
            geometry=[Point(0, 0), Point(1, 1)],
        )
        gdf["secondary"] = gdf.geometry.buffer(0.1)
        # Switch the active geometry to the buffered column
        gdf = gdf.set_geometry("secondary")

        result = serialize_dataset(gdf, "swapped")
        assert result["format"] == "geoarrow"

        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert "geometry" in table.column_names
        assert "secondary" in table.column_names

        # Original (now non-active) column must be WKT
        orig_col = table.column("geometry")
        assert pa.types.is_string(orig_col.type) or pa.types.is_large_string(orig_col.type)

    def test_serialize_geodataframe_null_secondary_geometry(self):
        """A secondary geometry column that is all-null serializes as WKT nulls."""
        import base64
        import pyarrow as pa

        gdf = gpd.GeoDataFrame(
            {"name": ["A", "B"]},
            geometry=[Point(0, 0), Point(1, 1)],
        )
        gdf["empty_geom"] = gpd.GeoSeries([None, None])

        result = serialize_dataset(gdf, "null_secondary")
        assert result["format"] == "geoarrow"

        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert "empty_geom" in table.column_names
        assert table.num_rows == 2


class TestGeoJSONSerialization:
    """Tests for GeoJSON serialization (from GeoJSON.ipynb)."""

    def test_serialize_geojson_dict(self):
        """Test GeoJSON as dict."""
        geojson = {"type": "FeatureCollection", "features": []}
        result = serialize_dataset(geojson, "test")
        assert result["format"] == "geojson"
        assert result["data"] == geojson

    def test_serialize_geojson_feature_collection(self):
        """Test GeoJSON FeatureCollection with features."""
        geojson = {
            "type": "FeatureCollection",
            "features": [
                {
                    "type": "Feature",
                    "geometry": {"type": "Point", "coordinates": [-122.4, 37.8]},
                    "properties": {"name": "San Francisco"},
                },
                {
                    "type": "Feature",
                    "geometry": {"type": "Point", "coordinates": [-118.2, 34.0]},
                    "properties": {"name": "Los Angeles"},
                },
            ],
        }
        result = serialize_dataset(geojson, "geojson")
        assert result["id"] == "geojson"
        assert result["format"] == "geojson"
        assert result["data"]["type"] == "FeatureCollection"
        assert len(result["data"]["features"]) == 2

    def test_serialize_geojson_string(self):
        """Test GeoJSON as string (from GeoJSON.ipynb - reading from file)."""
        geojson_str = json.dumps({
            "type": "FeatureCollection",
            "features": [
                {
                    "type": "Feature",
                    "geometry": {"type": "Point", "coordinates": [-122.4, 37.8]},
                    "properties": {"name": "Test"},
                },
            ],
        })
        result = serialize_dataset(geojson_str, "geojson")
        assert result["id"] == "geojson"
        assert result["format"] == "geojson"
        assert result["data"]["type"] == "FeatureCollection"

    def test_serialize_geojson_polygon(self):
        """Test GeoJSON with Polygon geometry."""
        geojson = {
            "type": "Feature",
            "geometry": {
                "type": "Polygon",
                "coordinates": [[
                    [-122.4, 37.8], [-122.3, 37.8],
                    [-122.3, 37.7], [-122.4, 37.7], [-122.4, 37.8],
                ]],
            },
            "properties": {"name": "Test Area"},
        }
        result = serialize_dataset(geojson, "polygon")
        assert result["format"] == "geojson"


class TestCSVSerialization:
    """Tests for CSV string serialization."""

    def test_serialize_csv(self):
        csv_data = "lat,lng\n37.7749,-122.4194"
        result = serialize_dataset(csv_data, "test")
        assert result["format"] == "csv"
        assert result["data"] == csv_data

    def test_serialize_csv_multirow(self):
        """Test CSV with multiple rows."""
        csv_data = "City,Latitude,Longitude\nSF,37.77,-122.42\nLA,34.05,-118.24"
        result = serialize_dataset(csv_data, "cities")
        assert result["id"] == "cities"
        assert result["format"] == "csv"
        assert result["data"] == csv_data


class TestArrowSerialization:
    """Tests for Arrow IPC serialization of DataFrames."""

    def test_serialize_dataframe_arrow(self, sample_df):
        """Test basic DataFrame Arrow serialization."""
        result = serialize_dataset(sample_df, "test", use_arrow=True)
        assert result["id"] == "test"
        assert result["format"] == "arrow"
        assert isinstance(result["data"], str)

    def test_serialize_dataframe_arrow_roundtrip(self, sample_df):
        """Test that Arrow-serialized data can be deserialized back."""
        import base64
        import pyarrow as pa

        result = serialize_dataset(sample_df, "test", use_arrow=True)
        arrow_bytes = base64.b64decode(result["data"])
        reader = pa.ipc.open_stream(arrow_bytes)
        table = reader.read_all()
        assert table.num_rows == len(sample_df)
        assert table.num_columns == len(sample_df.columns)
        assert set(table.column_names) == set(sample_df.columns)

    def test_serialize_dataframe_arrow_preserves_types(self):
        """Test that Arrow serialization preserves numeric types accurately."""
        import base64
        import pyarrow as pa

        df = pd.DataFrame({
            'int_col': [1, 2, 3],
            'float_col': [1.1, 2.2, 3.3],
            'str_col': ['a', 'b', 'c'],
            'bool_col': [True, False, True],
        })
        result = serialize_dataset(df, "typed", use_arrow=True)
        assert result["format"] == "arrow"

        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert table.column('int_col').to_pylist() == [1, 2, 3]
        assert table.column('float_col').to_pylist() == pytest.approx([1.1, 2.2, 3.3])
        assert table.column('str_col').to_pylist() == ['a', 'b', 'c']
        assert table.column('bool_col').to_pylist() == [True, False, True]

    def test_serialize_dataframe_arrow_with_cities(self):
        """Test Arrow serialization with city data (parallel to JSON test)."""
        import base64
        import pyarrow as pa

        df = pd.DataFrame({
            'City': ['Buenos Aires', 'Brasilia', 'Santiago', 'Bogota', 'Caracas'],
            'Country': ['Argentina', 'Brazil', 'Chile', 'Colombia', 'Venezuela'],
            'Latitude': [-34.58, -15.78, -33.45, 4.60, 10.48],
            'Longitude': [-58.66, -47.91, -70.66, -74.08, -66.86],
        })
        result = serialize_dataset(df, "cities", use_arrow=True)
        assert result["format"] == "arrow"

        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert table.num_rows == 5
        assert 'City' in table.column_names
        assert 'Latitude' in table.column_names
        assert table.column('City').to_pylist()[0] == 'Buenos Aires'

    def test_serialize_dataframe_arrow_empty(self):
        """Test Arrow serialization of empty DataFrame."""
        import base64
        import pyarrow as pa

        df = pd.DataFrame({'col1': pd.Series([], dtype='float64'), 'col2': pd.Series([], dtype='str')})
        result = serialize_dataset(df, "empty", use_arrow=True)
        assert result["format"] == "arrow"

        arrow_bytes = base64.b64decode(result["data"])
        table = pa.ipc.open_stream(arrow_bytes).read_all()
        assert table.num_rows == 0
        assert set(table.column_names) == {'col1', 'col2'}

    def test_serialize_dataframe_arrow_default_off(self, sample_df):
        """Test that use_arrow=False (default) still produces JSON format."""
        result = serialize_dataset(sample_df, "test", use_arrow=False)
        assert result["format"] == "df"
        assert "columns" in result["data"]
        assert "data" in result["data"]

    def test_serialize_geodataframe_ignores_use_arrow(self, sample_gdf):
        """Test that GeoDataFrame always uses geoarrow regardless of use_arrow flag."""
        result = serialize_dataset(sample_gdf, "test", use_arrow=True)
        assert result["format"] == "geoarrow"


class TestEdgeCases:
    """Tests for edge cases and error handling."""

    def test_serialize_unsupported_type(self):
        """Test that unsupported types raise ValueError."""
        with pytest.raises(ValueError, match="Unsupported data type"):
            serialize_dataset([1, 2, 3], "test")

    def test_serialize_empty_dataframe(self):
        """Test serializing empty DataFrame."""
        df = pd.DataFrame({'col1': [], 'col2': []})
        result = serialize_dataset(df, "empty")
        assert result["format"] == "df"
        assert result["data"]["columns"] == ['col1', 'col2']
        assert result["data"]["data"] == []

    def test_serialize_single_row_dataframe(self):
        """Test serializing single-row DataFrame."""
        df = pd.DataFrame({'lat': [37.77], 'lng': [-122.42]})
        result = serialize_dataset(df, "single")
        assert result["format"] == "df"
        assert len(result["data"]["data"]) == 1

    def test_serialize_dataframe_with_various_dtypes(self):
        """Test DataFrame with various data types."""
        df = pd.DataFrame({
            'int_col': [1, 2, 3],
            'float_col': [1.1, 2.2, 3.3],
            'str_col': ['a', 'b', 'c'],
            'bool_col': [True, False, True],
        })
        result = serialize_dataset(df, "mixed")
        assert result["format"] == "df"
        assert len(result["data"]["columns"]) == 4

    def test_serialize_geojson_string_is_detected(self):
        """A valid GeoJSON string should be routed to geojson format, not csv."""
        import json
        geojson_str = json.dumps({
            "type": "FeatureCollection",
            "features": [],
        })
        result = serialize_dataset(geojson_str, "geo")
        assert result["format"] == "geojson"
        assert result["data"]["type"] == "FeatureCollection"

    def test_serialize_plain_csv_string_is_csv(self):
        """A plain CSV string should not be confused with GeoJSON."""
        csv = "lat,lng\n37.77,-122.42"
        result = serialize_dataset(csv, "csv")
        assert result["format"] == "csv"
        assert result["data"] == csv

    def test_serialize_dataframe_with_nan(self):
        """NaN values in a DataFrame should not raise during serialization,
        and the row containing NaN should be preserved (not dropped)."""
        df = pd.DataFrame({'a': [1.0, float('nan'), 3.0]})
        result = serialize_dataset(df, "nan_test")
        assert result["format"] == "df"
        rows = result["data"]["data"]
        assert len(rows) == 3
        assert rows[0][0] == pytest.approx(1.0)
        assert rows[1][0] is None
        assert rows[2][0] == pytest.approx(3.0)

    def test_serialize_dataframe_datetime_without_offset(self):
        """Naive datetimes must be sent as plain ISO strings, since an offset
        added by the kernel's time zone would shift them in the map."""
        import json
        df = pd.DataFrame({
            't': pd.to_datetime(['2024-01-01 00:00:00.000', '2024-06-01 12:30:00.500']),
            'v': [1, 2],
        })
        result = serialize_dataset(df, "dt_test")
        rows = result["data"]["data"]
        assert rows[0] == ['2024-01-01T00:00:00.000000', 1]
        assert rows[1] == ['2024-06-01T12:30:00.500000', 2]
        json.dumps(result, allow_nan=False)

    def test_serialize_dataframe_datetime_only(self):
        """A DataFrame with only datetime columns should be sent as ISO strings."""
        df = pd.DataFrame({'t': pd.to_datetime(['2024-01-01 00:00:00'])})
        rows = serialize_dataset(df, "dt_only")["data"]["data"]
        assert rows == [['2024-01-01T00:00:00.000000']]

    def test_serialize_dataframe_datetime_with_nat(self):
        """NaT should be sent as null instead of failing in JSON serialization."""
        import json
        df = pd.DataFrame({
            't': pd.to_datetime(['2024-01-01 00:00:00', None]),
            'v': [1.0, float('nan')],
        })
        result = serialize_dataset(df, "nat_test")
        rows = result["data"]["data"]
        assert rows == [['2024-01-01T00:00:00.000000', 1.0], [None, None]]
        json.dumps(result, allow_nan=False)

    def test_serialize_dataframe_datetime_with_timezone(self):
        """Timezone-aware datetimes keep their UTC offset."""
        df = pd.DataFrame({
            't': pd.to_datetime(['2024-01-01 00:00:00']).tz_localize('Asia/Taipei'),
        })
        rows = serialize_dataset(df, "tz_test")["data"]["data"]
        assert rows == [['2024-01-01T00:00:00.000000+0800']]

    @pytest.mark.parametrize("tz, offset", [(None, ""), ("Asia/Taipei", "+0800")])
    def test_serialize_dataframe_datetime_arrow_dtype(self, tz, offset):
        """Arrow-backed timestamp columns are formatted like numpy ones, with
        NaT sent as null."""
        import json
        pa = pytest.importorskip("pyarrow")
        t = pd.to_datetime(['2024-01-01 00:00:00.000', '2024-06-01 12:30:00.500', None])
        if tz:
            t = t.tz_localize(tz)
        df = pd.DataFrame({'t': pd.Series(t).astype(pd.ArrowDtype(pa.timestamp('us', tz=tz)))})
        result = serialize_dataset(df, "arrow_dt_test")
        assert result["data"]["data"] == [
            ['2024-01-01T00:00:00.000000' + offset],
            ['2024-06-01T12:30:00.500000' + offset],
            [None],
        ]
        json.dumps(result, allow_nan=False)

    def test_serialize_dataframe_arrow_with_nan(self):
        """Arrow serialization of a DataFrame containing NaN should not raise,
        and NaN should round-trip as null in Arrow."""
        import base64
        import pyarrow as pa
        df = pd.DataFrame({'a': [1.0, float('nan'), 3.0]})
        result = serialize_dataset(df, "nan_arrow", use_arrow=True)
        assert result["format"] == "arrow"
        table = pa.ipc.open_stream(base64.b64decode(result["data"])).read_all()
        assert table.num_rows == 3
        values = table.column('a').to_pylist()
        assert values[0] == pytest.approx(1.0)
        assert values[1] is None  # NaN becomes null in Arrow
        assert values[2] == pytest.approx(3.0)

    def test_data_to_json_returns_all_datasets(self, sample_df, sample_gdf):
        """data_to_json should serialize every dataset in the dict."""
        from keplergl.serializers import data_to_json

        class FakeWidget:
            _use_arrow = False

        result = data_to_json({"tab": sample_df, "geo": sample_gdf}, FakeWidget())
        assert "tab" in result
        assert "geo" in result
        assert result["tab"]["format"] == "df"
        assert result["geo"]["format"] == "geoarrow"

    def test_data_from_json_passthrough(self, sample_df):
        """data_from_json is a passthrough — returned value equals input."""
        from keplergl.serializers import data_from_json
        payload = {"foo": {"id": "foo", "format": "csv", "data": "a,b\n1,2"}}
        assert data_from_json(payload, None) is payload