Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

23.6 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import test from 'tape';
import moment from 'moment';
import * as arrow from 'apache-arrow';

import {
  testCreateCases,
  testFormatLayerDataCases,
  testRenderLayerCases,
  dataId,
  testRows,
  preparedDataset,
  preparedFilterDomain0,
  arcLayerMeta
} from 'test/helpers/layer-utils';
import testArcData, {arcFromHex, arcFromNeighbor} from 'test/fixtures/test-arc-data';
import {StateWArcNeighbors} from 'test/helpers/mock-state';
import {PROJECTED_PIXEL_SIZE_MULTIPLIER} from '@kepler.gl/constants';
import {KeplerGlLayers} from '@kepler.gl/layers';
import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';
import {arrowSchemaToFields} from '@kepler.gl/processors';
import {cellToLatLng} from 'h3-js';

const {ArcLayer, LineLayer} = KeplerGlLayers;
const columns = {
  lat0: 'lat',
  lng0: 'lng',
  lat1: 'lat_1',
  lng1: 'lng_1'
};

const geoArrowBoundsCases = [ArcLayer, LineLayer].flatMap(LayerClass =>
  [2, 3, 4].map(dimensions => ({LayerClass, dimensions}))
);

geoArrowBoundsCases.forEach(({LayerClass, dimensions}) => {
  test(`#${LayerClass.name} -> GeoArrow ${dimensions}D endpoint bounds across batches`, async t => {
    const pointType = new arrow.FixedSizeList(
      dimensions,
      new arrow.Field('coordinate', new arrow.Float64())
    );
    const coordinates = points =>
      arrow.vectorFromArray(
        points.map(point => (point === null ? null : [...point, 70, 80].slice(0, dimensions))),
        pointType
      );
    const makeTable = (source, target) => {
      const table = new arrow.Table({source: coordinates(source), target: coordinates(target)});
      table.schema.fields.forEach(field =>
        field.metadata.set('ARROW:extension:name', 'geoarrow.point')
      );
      return table;
    };
    const first = makeTable([[10, 20]], [[30, 40]]);
    const dataset = new KeplerTable({info: {id: 'arc-geoarrow-bounds'}, color: [0, 0, 0]});
    await dataset.importData({
      data: {
        fields: arrowSchemaToFields(first),
        cols: first.schema.fields.map((field, index) => first.getChildAt(index))
      }
    });
    const layer = new LayerClass({dataId: dataset.id, columnMode: 'geoarrow'});
    layer.config.columns.geoarrow0 = {value: 'source', fieldIdx: 0};
    layer.config.columns.geoarrow1 = {value: 'target', fieldIdx: 1};
    layer.updateLayerMeta(dataset);
    t.deepEqual(
      layer.getPositionAccessor(dataset.dataContainer)({index: 0}),
      [10, 20, 0, 30, 40, 0],
      'reads longitude and latitude from both point vectors'
    );
    t.deepEqual(layer.meta.bounds, [10, 20, 30, 40], 'bounds include the destination');

    const table = first.concat(
      makeTable(
        [
          [0, 5],
          [12, 22]
        ],
        [
          [50, 60],
          [null, null]
        ]
      )
    );
    // Progressive updates preserve batches; completed imports compact them.
    await dataset.update({
      cols: table.schema.fields.map((field, index) => table.getChildAt(index)),
      arrowTable: table
    });
    t.equal(dataset.dataContainer.getTable().batches.length, 2, 'uses two Arrow record batches');
    layer.updateLayerMeta(dataset);
    t.deepEqual(
      layer.getPositionAccessor(dataset.dataContainer)({index: 1}),
      [0, 5, 0, 50, 60, 0],
      'reads endpoints from the later batch with a global row index'
    );
    t.deepEqual(
      layer.meta.bounds,
      [0, 5, 50, 60],
      'bounds include later destinations and valid zero longitude, excluding null coordinates'
    );

    const nullableTable = table.concat(
      makeTable([null, [-10, -20], null], [[70, 80], null, null])
    );
    await dataset.update({
      cols: nullableTable.schema.fields.map((field, index) => nullableTable.getChildAt(index)),
      arrowTable: nullableTable
    });
    t.doesNotThrow(
      () => layer.updateLayerMeta(dataset),
      'null source, destination, and both endpoint cells do not crash bounds calculation'
    );
    t.deepEqual(
      layer.meta.bounds,
      [-10, -20, 70, 80],
      'valid endpoints still contribute when the other endpoint cell is null'
    );

    const emptyTable = makeTable([null], [null]);
    await dataset.update({
      cols: emptyTable.schema.fields.map((field, index) => emptyTable.getChildAt(index)),
      arrowTable: emptyTable
    });
    t.doesNotThrow(() => layer.updateLayerMeta(dataset), 'all-null endpoint cells are accepted');
    t.equal(layer.meta.bounds, null, 'all-null endpoints have no bounds');
    t.end();
  });
});

[ArcLayer, LineLayer].forEach(LayerClass => {
  ['geoarrow', 'points'].forEach(columnMode => {
    test(`#${LayerClass.name} -> ${columnMode} destination coordinate pairs`, async t => {
      const cases = [
        {target: [null, 80], bounds: [10, 20, 30, 40]},
        {target: [150, null], bounds: [10, 20, 30, 40]},
        {target: [NaN, 80], bounds: [10, 20, 30, 40]},
        {target: [150, NaN], bounds: [10, 20, 30, 40]},
        {target: [Infinity, 80], bounds: [10, 20, 30, 40]},
        {target: [150, Infinity], bounds: [10, 20, 30, 40]},
        {target: [-Infinity, -80], bounds: [10, 20, 30, 40]},
        {target: [-150, -Infinity], bounds: [10, 20, 30, 40]},
        {firstTarget: [150, null], target: [null, 80], bounds: [10, 20, 12, 22]},
        {target: [0, 45], bounds: [0, 20, 30, 45]},
        {target: [35, 0], bounds: [10, 0, 35, 40]},
        {target: [0, 0], bounds: [0, 0, 30, 40]}
      ];
      for (const {target, bounds, firstTarget = [30, 40]} of cases) {
        const pointType = new arrow.FixedSizeList(
          2,
          new arrow.Field('coordinate', new arrow.Float64())
        );
        const table = new arrow.Table({
          source: arrow.vectorFromArray(
            [
              [10, 20],
              [12, 22]
            ],
            pointType
          ),
          target: arrow.vectorFromArray([firstTarget, target], pointType)
        });
        table.schema.fields.forEach(field =>
          field.metadata.set('ARROW:extension:name', 'geoarrow.point')
        );
        const numericTable = new arrow.Table({
          lng0: arrow.vectorFromArray([10, 12], new arrow.Float64()),
          lat0: arrow.vectorFromArray([20, 22], new arrow.Float64()),
          lng1: arrow.vectorFromArray([firstTarget[0], target[0]], new arrow.Float64()),
          lat1: arrow.vectorFromArray([firstTarget[1], target[1]], new arrow.Float64())
        });
        const dataset = new KeplerTable({info: {id: 'endpoint-pairs'}, color: [0, 0, 0]});
        await dataset.importData({
          data:
            columnMode === 'geoarrow'
              ? {
                  fields: arrowSchemaToFields(table),
                  cols: table.schema.fields.map((field, index) => table.getChildAt(index))
                }
              : {
                  fields: arrowSchemaToFields(numericTable),
                  rows: [
                    [10, 20, ...firstTarget],
                    [12, 22, ...target]
                  ]
                }
        });
        const layer = new LayerClass({dataId: dataset.id, columnMode});
        const names =
          columnMode === 'geoarrow' ? ['source', 'target'] : ['lng0', 'lat0', 'lng1', 'lat1'];
        const keys = columnMode === 'geoarrow' ? ['geoarrow0', 'geoarrow1'] : names;
        keys.forEach((key, fieldIdx) => {
          layer.config.columns[key] = {value: names[fieldIdx], fieldIdx};
        });
        layer.updateLayerMeta(dataset);
        t.deepEqual(layer.meta.bounds, bounds, `bounds for target [${target.join(', ')}]`);
      }
      t.end();
    });
  });
});

test('#ArcLayer -> constructor', t => {
  const TEST_CASES = {
    CREATE: [
      {
        props: {
          dataId: 'smoothie',
          isVisible: true,
          label: 'test arc layer'
        },
        test: layer => {
          t.ok(layer.config.dataId === 'smoothie', 'ArcLayer dataId should be correct');
          t.ok(layer.type === 'arc', 'type should be arc');
          t.ok(layer.isAggregated === false, 'ArcLayer is not aggregated');
          t.ok(layer.config.label === 'test arc layer', 'label should be correct');
          t.ok(Object.keys(layer.columnPairs).length, 'should have columnPairs');
          t.equal(layer.config.visConfig.targetArrowEnabled, false, 'target arrow is off');
          t.equal(layer.config.visConfig.targetArrowShape, 'chevron', 'default arrow shape');
          t.equal(layer.config.visConfig.targetArrowSize, 16, 'default arrow size');
          t.equal(
            layer.visConfigSettings.targetArrowSize.allowCustomValue,
            false,
            'arrow size has no custom input'
          );
        }
      }
    ]
  };

  testCreateCases(t, ArcLayer, TEST_CASES.CREATE);
  t.end();
});

test('#ArcLayer -> formatLayerData', t => {
  const filteredIndex = [0, 2, 4];

  // filter.domain: [ 1474071056000, 1474071489000 ]
  // filterRange: [ [ 39000, 552000 ]
  // create a clone from preparedDataset
  const TEST_CASES = [
    {
      name: 'Arc trip data.1',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10]
        },
        type: 'arc',
        id: 'test_layer_0'
      },
      datasets: {
        [dataId]: copyTableAndUpdate(preparedDataset, {filteredIndex})
      },
      assert: result => {
        const {layerData, layer} = result;

        const expectedLayerData = {
          data: [
            {
              index: 0,
              sourcePosition: [testRows[0][2], testRows[0][1], 0],
              targetPosition: [testRows[0][4], testRows[0][3], 0]
            },
            {
              index: 4,
              sourcePosition: [testRows[4][2], testRows[4][1], 0],
              targetPosition: [testRows[4][4], testRows[4][3], 0]
            }
          ],
          getFilterValue: () => {},
          getFiltered: () => {},
          getSourceColor: () => {},
          getTargetColor: () => {},
          getWidth: () => {}
        };
        const expectedDataKeys = Object.keys(expectedLayerData).sort();

        t.deepEqual(
          Object.keys(layerData).sort(),
          expectedDataKeys,
          'layerData should have 6 keys'
        );
        t.deepEqual(
          layerData.data,
          expectedLayerData.data,
          'should format correct arc layerData data'
        );
        // getSourceColor
        t.deepEqual(
          layerData.getSourceColor,
          layer.config.color,
          'getSourceColor should be a constant'
        );
        // getSourceColor
        t.deepEqual(
          layerData.getTargetColor,
          layer.config.color,
          'getTargetColors should be a constant'
        );
        // getWidth
        t.equal(layerData.getWidth, 1, 'getWidth should be a constant');
        // getFilterValue
        t.deepEqual(
          layerData.data.map(layerData.getFilterValue),
          [
            [Number.MIN_SAFE_INTEGER, 0, 0, 0],
            [moment.utc(testRows[4][0]).valueOf() - preparedFilterDomain0, 0, 0, 0]
          ],
          'getFilterValue should return [value, 0, 0, 0]'
        );

        // layerMeta
        t.deepEqual(layer.meta, arcLayerMeta, 'should format correct arc layer meta');
      }
    },
    {
      name: 'Arc trip data.2 targetColor',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10],
          visConfig: {
            targetColor: [1, 2, 3]
          }
        },
        type: 'arc',
        id: 'test_layer_2'
      },
      datasets: {
        [dataId]: preparedDataset
      },
      assert: result => {
        const {layerData, layer} = result;

        const expectedLayerData = {
          data: [],
          getFilterValue: () => {},
          getFiltered: () => {},
          getSourceColor: () => {},
          getTargetColor: () => {},
          getWidth: () => {}
        };
        const expectedDataKeys = Object.keys(expectedLayerData).sort();

        t.deepEqual(
          Object.keys(layerData).sort(),
          expectedDataKeys,
          'layerData should have 6 keys'
        );

        // getSourceColor
        t.deepEqual(
          layerData.getSourceColor,
          layer.config.color,
          'getSourceColor should be a constant'
        );
        // getSourceColor
        t.deepEqual(layerData.getTargetColor, [1, 2, 3], 'getTargetColors should be a constant');
      }
    },
    {
      name: 'Arc trip data. with colorField and sizeField',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10],
          // color by types(string)
          colorField: {
            type: 'string',
            name: 'types'
          },
          // size by trip_distance(real)
          sizeField: {
            type: 'real',
            name: 'trip_distance'
          },
          visConfig: {
            colorRange: {
              colors: ['#010101', '#020202', '#030303']
            },
            sizeRange: [10, 20]
          }
        },
        type: 'arc',
        id: 'test_layer_1'
      },
      // modify preparedDataset
      datasets: {
        [dataId]: copyTableAndUpdate(preparedDataset, {filteredIndex})
      },
      assert: result => {
        const {layerData} = result;
        const expectedLayerData = {
          data: [
            {
              index: 0,
              sourcePosition: [testRows[0][2], testRows[0][1], 0],
              targetPosition: [testRows[0][4], testRows[0][3], 0]
            },
            {
              index: 4,
              sourcePosition: [testRows[4][2], testRows[4][1], 0],
              targetPosition: [testRows[4][4], testRows[4][3], 0]
            }
          ],
          getFilterValue: () => {},
          getFiltered: () => {},
          getSourceColor: () => {},
          getTargetColor: () => {},
          getWidth: () => {}
        };
        const expectedDataKeys = Object.keys(expectedLayerData).sort();

        t.deepEqual(
          Object.keys(layerData).sort(),
          expectedDataKeys,
          'layerData should have 6 keys'
        );
        t.deepEqual(
          layerData.data,
          expectedLayerData.data,
          'should format correct arc layerData data'
        );
        // getSourceColor
        // domain: ['driver_analytics', 'driver_analytics_0', 'driver_gps']
        // range ['#010101', '#020202', '#030303']
        t.deepEqual(
          layerData.data.map(layerData.getSourceColor),
          [
            [2, 2, 2],
            [1, 1, 1]
          ],
          'getSourceColor should be correct'
        );
        // getTargetColor
        // domain: ['driver_analytics', 'driver_analytics_0', 'driver_gps']
        // range ['#010101', '#020202', '#030303']
        t.deepEqual(
          layerData.data.map(layerData.getTargetColor),
          [
            [2, 2, 2],
            [1, 1, 1]
          ],
          'getTargetColors  be correct'
        );
        // getWidth
        // domain: [1.59, 11]
        // range: [10, 20]
        // value [1.59, 2.37]
        t.deepEqual(
          layerData.data.map(layerData.getWidth),
          [10, (2.37 - 1.59) * (10 / 9.41) + 10],
          'getWidth should be a constant'
        );
        // getFilterValue
        t.deepEqual(
          layerData.data.map(layerData.getFilterValue),
          [
            [Number.MIN_SAFE_INTEGER, 0, 0, 0],
            [moment.utc(testRows[4][0]).valueOf() - preparedFilterDomain0, 0, 0, 0]
          ],
          'getFilterValue should return [value, 0, 0, 0]'
        );
      }
    },
    {
      name: 'Arc data from neighbors',
      layer: arcFromNeighbor,
      datasets: StateWArcNeighbors.visState.datasets,
      assert: result => {
        const {layerData} = result;
        const expectedLayerData = {
          data: [
            {
              index: 0,
              sourcePosition: [testArcData[0].longitude, testArcData[0].latitude, 0],
              targetPosition: [testArcData[1].longitude, testArcData[1].latitude, 0]
            },
            {
              index: 0,
              sourcePosition: [testArcData[0].longitude, testArcData[0].latitude, 0],
              targetPosition: [testArcData[2].longitude, testArcData[2].latitude, 0]
            },
            {
              index: 1,
              sourcePosition: [testArcData[1].longitude, testArcData[1].latitude, 0],
              targetPosition: [testArcData[2].longitude, testArcData[2].latitude, 0]
            }
          ],
          getFilterValue: () => {},
          getFiltered: () => {},
          getSourceColor: () => {},
          getTargetColor: () => {},
          getWidth: () => {}
        };
        // index 18, 19 does not have valid neighbors
        t.equal(layerData.data.length, 38, 'should format 41 rows');

        for (let i = 0; i < 3; i++) {
          t.deepEqual(
            layerData.data[i],
            expectedLayerData.data[i],
            'should format correct arc layerData data'
          );
        }
      }
    },
    {
      name: 'Arc data from hex',
      layer: arcFromHex,
      datasets: StateWArcNeighbors.visState.datasets,
      assert: result => {
        const {layerData} = result;
        const expectedLayerData = {
          data: [
            {
              index: 0,
              sourcePosition: [...cellToLatLng(testArcData[0]['source hex_id']).reverse(), 0],
              targetPosition: [...cellToLatLng(testArcData[0]['target hex_id']).reverse(), 0]
            },
            {
              index: 1,
              sourcePosition: [...cellToLatLng(testArcData[1]['source hex_id']).reverse(), 0],
              targetPosition: [...cellToLatLng(testArcData[1]['target hex_id']).reverse(), 0]
            },
            {
              index: 2,
              sourcePosition: [...cellToLatLng(testArcData[2]['source hex_id']).reverse(), 0],
              targetPosition: [...cellToLatLng(testArcData[2]['target hex_id']).reverse(), 0]
            }
          ],
          getFilterValue: () => {},
          getFiltered: () => {},
          getSourceColor: () => {},
          getTargetColor: () => {},
          getWidth: () => {}
        };
        // index 18, 19 does not have valid neighbors
        t.equal(layerData.data.length, 20, 'should format 20 rows');

        for (let i = 0; i < 3; i++) {
          t.deepEqual(
            layerData.data[i],
            expectedLayerData.data[i],
            'should format correct arc layerData data'
          );
        }
      }
    }
  ];

  testFormatLayerDataCases(t, ArcLayer, TEST_CASES);
  t.end();
});

test('#ArcLayer -> renderLayer', t => {
  const filteredIndex = [0, 2, 4];

  const TEST_CASES = [
    {
      name: 'Arc render layer.1',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10]
        },
        type: 'arc',
        id: 'test_layer_0'
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        t.equal(deckLayers.length, 1, 'Should create 1 deck.gl layer');
        const {props} = deckLayers[0];

        const expectedProps = {
          opacity: layer.config.visConfig.opacity,
          widthScale: layer.config.visConfig.thickness * PROJECTED_PIXEL_SIZE_MULTIPLIER,
          filterRange: preparedDataset.gpuFilter.filterRange
        };
        Object.keys(expectedProps).forEach(key => {
          t.equal(props[key], expectedProps[key], `should have correct props.${key}`);
        });
      }
    },
    {
      name: 'Arc render layer.2 brushing',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10]
        },
        type: 'arc',
        id: 'test_layer_0'
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      renderArgs: {
        interactionConfig: {
          brush: {
            enabled: true,
            config: {
              size: 2.5
            }
          }
        }
      },
      assert: deckLayers => {
        t.equal(deckLayers.length, 1, 'Should create 1 deck.gl layer');
        const {props} = deckLayers[0];
        // test instancePositions
        const expectedProps = {
          brushingRadius: 2500,
          brushingEnabled: true,
          brushingTarget: 'source_target'
        };
        Object.keys(expectedProps).forEach(key => {
          t.equal(props[key], expectedProps[key], `should have correct props.${key}`);
        });
      }
    },
    {
      name: 'Arc render layer.3 target arrow',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10],
          visConfig: {
            targetArrowEnabled: true,
            targetArrowShape: 'chevron',
            targetArrowSize: 24,
            targetColor: [200, 30, 30]
          }
        },
        type: 'arc',
        id: 'test_layer_0'
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: deckLayers => {
        t.equal(deckLayers.length, 2, 'Should create the arc and an arrow head');
        const arrowLayer = deckLayers[deckLayers.length - 1];
        t.equal(arrowLayer.id, 'test_layer_0-arrow-head-chevron', 'arrow layer id');
        t.equal(arrowLayer.props.pickable, false, 'arrow head does not take the line hover');
        const radius =
          typeof arrowLayer.props.getRadius === 'function'
            ? arrowLayer.props.getRadius(arrowLayer.props.data[0])
            : arrowLayer.props.getRadius;
        t.equal(radius, 24, 'arrow size is the vis config value in pixels');
        t.equal(arrowLayer.props.iconGeometry.length, 18, 'chevron is two triangles');
        const angle = arrowLayer.props.getAngle(arrowLayer.props.data[0]);
        t.equal(typeof angle, 'number', 'arrow angle is computed');
        t.ok(Number.isFinite(angle), 'arrow angle is finite');
        const fill =
          typeof arrowLayer.props.getFillColor === 'function'
            ? arrowLayer.props.getFillColor(arrowLayer.props.data[0])
            : arrowLayer.props.getFillColor;
        t.deepEqual(fill, [200, 30, 30], 'arrow uses the target color');
      }
    },
    {
      name: 'Arc render layer.4 target arrow hover',
      layer: {
        config: {
          dataId,
          label: 'trip arcs',
          columns,
          color: [10, 10, 10],
          visConfig: {
            targetArrowEnabled: true,
            targetArrowShape: 'triangle',
            targetArrowSize: 24
          }
        },
        type: 'arc',
        id: 'test_layer_0'
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      renderArgs: {
        objectHovered: {
          picked: true,
          object: {
            index: 0,
            sourcePosition: [-122.4, 37.8, 0],
            targetPosition: [-122.3, 37.9, 0]
          },
          layer: {props: {id: 'test_layer_0'}}
        }
      },
      assert: deckLayers => {
        t.equal(deckLayers.length, 4, 'Should add a highlight stroke and a highlight arrow');
        const arrowLayer = deckLayers[deckLayers.length - 1];
        t.equal(arrowLayer.id, 'test_layer_0-arrow-head-triangle-hovered', 'highlight arrow id');
        t.equal(arrowLayer.props.data.length, 1, 'highlight arrow is only the hovered stroke');
        t.equal(arrowLayer.props.opacity, 1, 'highlight arrow is fully opaque');
        const color =
          typeof arrowLayer.props.getFillColor === 'function'
            ? arrowLayer.props.getFillColor(arrowLayer.props.data[0])
            : arrowLayer.props.getFillColor;
        t.deepEqual(color, [252, 242, 26, 255], 'highlight arrow uses the layer highlight color');
      }
    }
  ];

  testRenderLayerCases(t, ArcLayer, TEST_CASES);
  t.end();
});