Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

6.3 KB
Raw Blame
# SPDX-License-Identifier: MIT
# Copyright contributors to the kepler.gl project

"""Data serialization for Python <-> JavaScript communication."""

import base64
from typing import Any, Optional

import pandas as pd
import geopandas as gpd
import pyarrow as pa

DEBUG = False

def _debug(msg):
    if DEBUG:
        print(f"[keplergl-serializer] {msg}")


def _arrow_table_to_base64(table: pa.Table) -> str:
    """Serialize an Arrow table to a base64-encoded IPC stream."""
    sink = pa.BufferOutputStream()
    with pa.ipc.new_stream(sink, table.schema) as writer:
        writer.write_table(table)
    return base64.b64encode(sink.getvalue().to_pybytes()).decode("utf-8")


def _try_parse_geojson(data: str) -> Optional[dict]:
    """Try to parse a string as GeoJSON. Returns parsed dict if valid GeoJSON, None otherwise."""
    import json
    try:
        parsed = json.loads(data)
        if isinstance(parsed, dict):
            # Check for GeoJSON indicators
            geojson_types = {"FeatureCollection", "Feature", "Point", "MultiPoint",
                           "LineString", "MultiLineString", "Polygon", "MultiPolygon",
                           "GeometryCollection"}
            if parsed.get("type") in geojson_types:
                return parsed
    except (json.JSONDecodeError, TypeError):
        pass
    return None


def data_to_json(data: dict, widget) -> dict:
    """Serialize Python data for JavaScript consumption."""
    _debug(f"data_to_json called with {len(data)} datasets")
    use_arrow = getattr(widget, '_use_arrow', False) if widget else False
    result = {}
    for name, dataset in data.items():
        _debug(f"  Serializing dataset '{name}' of type {type(dataset).__name__}")
        result[name] = serialize_dataset(dataset, name, use_arrow=use_arrow)
        _debug(f"  Result format: {result[name].get('format')}")
    _debug(f"data_to_json result: {list(result.keys())}")
    return result


def data_from_json(data: dict, widget) -> dict:
    """Deserialize JavaScript data to Python (passthrough)."""
    return data


def serialize_dataset(data: Any, name: str, use_arrow: bool = False) -> dict:
    """Serialize a single dataset."""
    if isinstance(data, gpd.GeoDataFrame):
        return serialize_geodataframe(data, name)
    elif isinstance(data, pd.DataFrame):
        return serialize_dataframe(data, name, use_arrow=use_arrow)
    elif isinstance(data, str):
        geojson_data = _try_parse_geojson(data)
        if geojson_data is not None:
            _debug(f"  Detected GeoJSON string for '{name}'")
            return {"id": name, "data": geojson_data, "format": "geojson"}
        return {"id": name, "data": data, "format": "csv"}
    elif isinstance(data, dict):
        return {"id": name, "data": data, "format": "geojson"}
    else:
        raise ValueError(f"Unsupported data type: {type(data)}")


def serialize_dataframe(df: pd.DataFrame, name: str, use_arrow: bool = False) -> dict:
    """Serialize DataFrame for kepler.gl.

    When use_arrow=True, serializes as Arrow IPC (base64-encoded) which is more
    compact and preserves types better for large numeric datasets.
    When use_arrow=False (default), serializes as JSON columns+rows.
    """
    if use_arrow:
        return _serialize_dataframe_arrow(df, name)

    columns = df.columns.tolist()
    rows = df.values.tolist()
    # Jupyter adds the kernel's UTC offset to naive datetimes, which shifts them
    # in the map, so send datetime columns as ISO strings instead
    for c, dtype in enumerate(df.dtypes):
        if pd.api.types.is_datetime64_any_dtype(dtype):
            # Series.dt.strftime fails on Arrow-backed timestamps, so format each value
            times = [
                None if pd.isna(t) else t.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
                for t in df.iloc[:, c]
            ]
            for row, time in zip(rows, times):
                row[c] = time
    # NaN and NaT are not valid JSON, so send them as null
    for r, c in zip(*df.isna().to_numpy().nonzero()):
        rows[r][c] = None
    _debug(f"serialize_dataframe: {len(columns)} columns, {len(rows)} rows")
    _debug(f"  columns: {columns}")
    _debug(f"  first row: {rows[0] if rows else 'empty'}")
    return {
        "id": name,
        "data": {
            "columns": columns,
            "data": rows,
        },
        "format": "df",
    }


def _serialize_dataframe_arrow(df: pd.DataFrame, name: str) -> dict:
    """Serialize DataFrame to Arrow IPC format (base64-encoded)."""
    table = pa.Table.from_pandas(df)
    _debug(f"_serialize_dataframe_arrow: {table.num_columns} columns, {table.num_rows} rows")
    return {
        "id": name,
        "data": _arrow_table_to_base64(table),
        "format": "arrow",
    }


def _is_geometry_col(series: pd.Series) -> bool:
    """Return True if the series has a geopandas geometry dtype."""
    return isinstance(series.dtype, gpd.array.GeometryDtype)


def _geoseries_to_geoarrow(geom_series: gpd.GeoSeries):
    """Convert a GeoSeries to a GeoArrow array, handling all-null series."""
    import geoarrow.pyarrow as ga

    if geom_series.notna().any():
        return ga.as_geoarrow(geom_series)
    # GeoArrow cannot infer a concrete geometry type from an empty/all-null series.
    return ga.as_geoarrow(geom_series, type=ga.wkb())


def serialize_geodataframe(gdf: gpd.GeoDataFrame, name: str) -> dict:
    """Serialize GeoDataFrame to GeoArrow format.

    The active geometry column is encoded as GeoArrow. Any additional geometry
    columns (GeoDataFrames can have more than one) are converted to WKT strings
    so they round-trip through the Arrow table without serialization errors and
    remain readable in kepler.gl as text attributes.
    """
    active_geom_col = gdf.geometry.name

    arrays = []
    col_names = []
    for col in gdf.columns:
        series = gdf[col]
        if col == active_geom_col:
            arrays.append(_geoseries_to_geoarrow(gdf.geometry))
        elif _is_geometry_col(series):
            # Secondary geometry columns: convert to WKT strings to avoid
            # pa.array() failing on raw geometry objects.
            arrays.append(pa.array(series.to_wkt()))
        else:
            arrays.append(pa.array(series))
        col_names.append(col)

    table = pa.table(dict(zip(col_names, arrays)))
    return {
        "id": name,
        "data": _arrow_table_to_base64(table),
        "format": "geoarrow",
    }