Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

9.2 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import {tableFromIPC} from 'apache-arrow';
import {processCsvData, processGeojson} from '@kepler.gl/processors';
import type {ProcessorResult} from '@kepler.gl/types';
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import type {DatasetPayload} from '../types';

const DEBUG = true;
function debug(...args: unknown[]) {
  if (DEBUG) {
    console.log('[keplergl-data]', ...args);
  }
}

// GeoArrow extension metadata key
const GEOARROW_METADATA_KEY = 'ARROW:extension:name';

export async function processDataset(payload: DatasetPayload): Promise<ProcessorResult | null> {
  const {id, data, format} = payload;
  debug('processDataset called:', {id, format, dataType: typeof data});

  try {
    let result: ProcessorResult | null = null;
    switch (format) {
      case 'arrow':
      case 'geoarrow':
        debug('Processing as arrow/geoarrow');
        result = processGeoArrowData(data as string);
        break;
      case 'csv':
        debug('Processing as CSV');
        result = processCsvData(data as string);
        break;
      case 'geojson':
        debug('Processing as GeoJSON');
        result = processGeojson(data as object);
        break;
      case 'json':
      case 'df':
        debug('Processing as JSON/DataFrame', data);
        result = processJsonData(data);
        break;
      default:
        throw new Error(`Unknown data format: ${format}`);
    }
    debug(
      'processDataset result:',
      result ? {fields: result.fields?.length, rows: result.rows?.length} : null
    );
    return result;
  } catch (error) {
    console.error('[keplergl-data] Error processing dataset:', error);
    throw error;
  }
}

/**
 * Process GeoArrow data by converting to row-based format.
 * This avoids Arrow version mismatch issues between our code and kepler.gl's internal Arrow.
 */
function processGeoArrowData(base64Data: string): ProcessorResult | null {
  debug('processGeoArrowData: decoding base64 data, length:', base64Data.length);
  const bytes = Uint8Array.from(atob(base64Data), c => c.charCodeAt(0));
  debug('processGeoArrowData: decoded to', bytes.length, 'bytes');
  
  const table = tableFromIPC(bytes);
  debug('processGeoArrowData: created table with', table.numRows, 'rows and', table.numCols, 'columns');
  
  if (table.numRows === 0) {
    return null;
  }

  // Build fields from schema, detecting geoarrow columns from metadata
  const fields = table.schema.fields.map((field, fieldIndex) => {
    const extensionName = field.metadata?.get(GEOARROW_METADATA_KEY);
    const isGeoArrow = extensionName?.startsWith('geoarrow');
    
    debug(`Field ${field.name}: type=${field.type.toString()}, extensionName=${extensionName}, isGeoArrow=${isGeoArrow}`);

    let type: string;
    let analyzerType: string;

    if (isGeoArrow) {
      type = ALL_FIELD_TYPES.geoarrow;
      analyzerType = 'GEOMETRY';
    } else {
      type = arrowTypeToKeplerType(field.type);
      analyzerType = arrowTypeToAnalyzerType(field.type);
    }

    return {
      name: field.name,
      id: field.name,
      displayName: field.name,
      format: '',
      fieldIdx: fieldIndex,
      type,
      analyzerType
    };
  });

  // Convert Arrow table to row-based format
  // For geometry columns, convert to GeoJSON strings that kepler.gl can parse
  const rows: any[][] = [];
  for (let rowIdx = 0; rowIdx < table.numRows; rowIdx++) {
    const row: any[] = [];
    for (let colIdx = 0; colIdx < table.numCols; colIdx++) {
      const field = fields[colIdx];
      const value = table.getChildAt(colIdx)?.get(rowIdx);
      
      if (field.type === ALL_FIELD_TYPES.geoarrow && value !== null) {
        // Convert geoarrow geometry to GeoJSON string
        row.push(arrowGeometryToGeoJSON(value));
      } else {
        row.push(value);
      }
    }
    rows.push(row);
  }

  debug('processGeoArrowData: converted to', rows.length, 'rows');
  debug('processGeoArrowData: first row sample:', rows[0]?.slice(0, 3));

  // Update field types for geometry columns to geojson since we converted them
  const updatedFields = fields.map(f => {
    if (f.type === ALL_FIELD_TYPES.geoarrow) {
      return {...f, type: ALL_FIELD_TYPES.geojson, analyzerType: 'GEOMETRY'};
    }
    return f;
  });

  return {
    fields: updatedFields,
    rows
  };
}

/**
 * Convert Arrow geometry value to GeoJSON string.
 * Handles various geoarrow geometry types (point, polygon, etc.)
 */
function arrowGeometryToGeoJSON(geom: any): string {
  try {
    // The geometry is stored as nested Arrow structures
    // We need to extract coordinates and build GeoJSON
    
    if (geom === null || geom === undefined) {
      return '';
    }

    // For geoarrow polygon type, the structure is:
    // List<List<Struct<x: Float64, y: Float64>>> for polygon rings
    // We need to recursively extract the coordinates
    
    const coords = extractCoordinates(geom);
    
    if (!coords || coords.length === 0) {
      return '';
    }

    // Determine geometry type based on structure
    let geojson: any;
    
    if (typeof coords[0] === 'number') {
      // Point: [x, y]
      geojson = {type: 'Point', coordinates: coords};
    } else if (typeof coords[0][0] === 'number') {
      // LineString or Polygon ring: [[x, y], ...]
      // Check if it's closed (polygon) or open (linestring)
      const first = coords[0];
      const last = coords[coords.length - 1];
      if (coords.length >= 4 && first[0] === last[0] && first[1] === last[1]) {
        geojson = {type: 'Polygon', coordinates: [coords]};
      } else {
        geojson = {type: 'LineString', coordinates: coords};
      }
    } else if (Array.isArray(coords[0][0])) {
      // Polygon with rings or MultiLineString
      if (typeof coords[0][0][0] === 'number') {
        geojson = {type: 'Polygon', coordinates: coords};
      } else {
        geojson = {type: 'MultiPolygon', coordinates: coords};
      }
    } else {
      // Fallback - try to make a polygon
      geojson = {type: 'Polygon', coordinates: coords};
    }

    return JSON.stringify(geojson);
  } catch (e) {
    debug('arrowGeometryToGeoJSON error:', e);
    return '';
  }
}

/**
 * Recursively extract coordinates from Arrow geometry structure
 */
function extractCoordinates(value: any): any {
  if (value === null || value === undefined) {
    return null;
  }

  // If it's a simple object with x, y properties (Point struct)
  if (typeof value === 'object' && 'x' in value && 'y' in value) {
    return [value.x, value.y];
  }

  // If it's an array-like structure (Arrow Vector or Array)
  if (Array.isArray(value) || (typeof value === 'object' && typeof value.length === 'number')) {
    const result: any[] = [];
    const len = value.length;
    
    for (let i = 0; i < len; i++) {
      const item = Array.isArray(value) ? value[i] : value.get?.(i) ?? value[i];
      const extracted = extractCoordinates(item);
      if (extracted !== null) {
        result.push(extracted);
      }
    }
    return result;
  }

  // If it has a toArray method (Arrow Vector)
  if (typeof value.toArray === 'function') {
    return extractCoordinates(value.toArray());
  }

  // If it has a get method but no length (might be a struct)
  if (typeof value.get === 'function') {
    // Try to get x, y from struct
    const x = value.get('x') ?? value.get(0);
    const y = value.get('y') ?? value.get(1);
    if (x !== undefined && y !== undefined) {
      return [x, y];
    }
  }

  return null;
}

/**
 * Map Arrow data type to Kepler field type
 */
function arrowTypeToKeplerType(arrowType: any): string {
  const typeStr = arrowType.toString();
  
  if (typeStr.startsWith('Int') || typeStr.startsWith('Uint')) {
    return ALL_FIELD_TYPES.integer;
  }
  if (typeStr.startsWith('Float') || typeStr === 'Decimal') {
    return ALL_FIELD_TYPES.real;
  }
  if (typeStr === 'Bool') {
    return ALL_FIELD_TYPES.boolean;
  }
  if (typeStr.startsWith('Timestamp') || typeStr.startsWith('Date')) {
    return ALL_FIELD_TYPES.timestamp;
  }
  if (typeStr === 'Utf8' || typeStr === 'LargeUtf8') {
    return ALL_FIELD_TYPES.string;
  }
  
  return ALL_FIELD_TYPES.string;
}

/**
 * Map Arrow data type to type-analyzer type
 */
function arrowTypeToAnalyzerType(arrowType: any): string {
  const typeStr = arrowType.toString();
  
  if (typeStr.startsWith('Int') || typeStr.startsWith('Uint')) {
    return 'INT';
  }
  if (typeStr.startsWith('Float') || typeStr === 'Decimal') {
    return 'FLOAT';
  }
  if (typeStr === 'Bool') {
    return 'BOOLEAN';
  }
  if (typeStr.startsWith('Timestamp') || typeStr.startsWith('Date')) {
    return 'DATETIME';
  }
  
  return 'STRING';
}

function processJsonData(data: unknown): ProcessorResult | null {
  debug('processJsonData input:', data);
  if (typeof data === 'object' && data !== null && 'columns' in data) {
    const {columns, data: rows} = data as {columns: string[]; data: unknown[][]};
    debug('processJsonData parsed:', {columns, rowCount: rows?.length, firstRow: rows?.[0]});
    const result = {
      fields: columns.map(name => ({name})),
      rows: rows as any[][]
    };
    debug('processJsonData result:', {
      fieldCount: result.fields.length,
      rowCount: result.rows.length
    });
    return result;
  }
  debug('processJsonData: data does not have expected structure');
  return null;
}