Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

19.6 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import test from 'tape';
import moment from 'moment';
import global from 'global';
import sinon from 'sinon';
import sinonStubPromise from 'sinon-stub-promise';
sinonStubPromise(sinon);

import {
  testCreateCases,
  testFormatLayerDataCases,
  testRenderLayerCases,
  preparedDataset,
  dataId,
  testRows,
  pointLayerMeta,
  preparedFilterDomain0
} from 'test/helpers/layer-utils';
import {
  DEFAULT_SCENEGRAPH_MODEL,
  CUSTOM_SCENEGRAPH_MODEL_ID,
  LEGACY_DEFAULT_SCENEGRAPH_URL
} from '@kepler.gl/constants';
import {KeplerGlLayers, LayerClasses, toCorsSafeGcsUrl} from '@kepler.gl/layers';
import {KeplerTable, copyTableAndUpdate, getGpuFilterProps} from '@kepler.gl/table';
import {VisStateActions} from '@kepler.gl/actions';
import {
  INITIAL_VIS_STATE,
  INITIAL_MAP_STATE,
  visStateReducer,
  validateLayerWithData
} from '@kepler.gl/reducers';
import SchemaManager, {CURRENT_VERSION} from '@kepler.gl/schemas';
const {ScenegraphLayer} = KeplerGlLayers;
const columns = {lat: 'lat', lng: 'lng'};

test('#ScenegraphLayer -> constructor', t => {
  const TEST_CASES = {
    CREATE: [
      {
        props: {
          dataId: 'smoothie',
          isVisible: true,
          label: 'test 3d layer'
        },
        test: layer => {
          t.ok(layer.config.dataId === 'smoothie', 'ScenegraphLayer dataId should be correct');
          t.ok(layer.type === '3D', 'type should be 3D');
          t.ok(layer.isAggregated === false, 'ScenegraphLayer is not aggregated');
          t.ok(layer.config.label === 'test 3d layer', 'label should be correct');
          t.ok(Object.keys(layer.columnPairs).length, 'should have columnPairs');
          t.equal(
            layer.config.visConfig.scenegraph,
            DEFAULT_SCENEGRAPH_MODEL.id,
            'should default to the gallery duck'
          );
          t.equal(
            layer.config.visConfig.angleZ,
            0,
            'angle Z default lets the duck model supply yaw'
          );
          t.equal(layer.config.visConfig.scenegraphColorEnabled, false, 'tint starts off');
          t.deepEqual(
            layer.getLegendVisualChannels(),
            {},
            'should expose no legend channels (mesh has no fill color)'
          );
        }
      }
    ]
  };

  testCreateCases(t, ScenegraphLayer, TEST_CASES.CREATE);
  t.end();
});

test('#ScenegraphLayer -> formatLayerData', t => {
  const filteredIndex = [0, 2, 4];

  const TEST_CASES = [
    {
      name: 'Scenegraph gps point.1',
      layer: {
        type: '3D',
        id: 'test_layer_1',
        config: {
          dataId,
          label: 'gps 3d',
          columns
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: result => {
        const {layerData, layer} = result;
        const expectedLayerData = {
          data: [
            {
              index: 4,
              position: [testRows[4][2], testRows[4][1], 0]
            }
          ],
          getFilterValue: () => {},
          getPosition: () => {}
        };

        t.deepEqual(
          Object.keys(layerData).sort(),
          Object.keys(expectedLayerData).sort(),
          'layerData should have 3 keys'
        );
        t.deepEqual(
          layerData.data,
          expectedLayerData.data,
          'should format correct point layerData data'
        );
        t.deepEqual(
          layerData.data.map(layerData.getPosition),
          [[testRows[4][2], testRows[4][1], 0]],
          'getPosition should return correct lat lng'
        );
        // getFilterValue
        t.deepEqual(
          layerData.data.map(layerData.getFilterValue),
          [[moment.utc(testRows[4][0]).valueOf() - preparedFilterDomain0, 0, 0, 0]],
          'getFilterValue should return [value, 0, 0, 0]'
        );
        // layerMeta
        t.deepEqual(layer.meta, pointLayerMeta, 'should format correct point layer meta');
      }
    }
  ];

  testFormatLayerDataCases(t, ScenegraphLayer, TEST_CASES);
  t.end();
});

test('#ScenegraphLayer -> renderLayer', t => {
  // TODO: mock actual gltf response
  const mockSuccessResponse = {};
  const mockJsonPromise = sinon.stub().returnsPromise();
  mockJsonPromise.resolves(mockSuccessResponse);
  const stubedFetch = sinon.stub(global, 'fetch').returnsPromise();

  stubedFetch.resolves({
    json: mockJsonPromise
  });

  const filteredIndex = [0, 2, 4];

  const TEST_CASES = [
    {
      name: 'Scenegraph gps point.1',
      layer: {
        type: '3D',
        id: 'test_layer_1',
        config: {
          dataId,
          label: 'gps 3d',
          columns
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        t.equal(deckLayers.length, 1, 'Should create 1 deck.gl layer');
        const {props} = deckLayers[0];
        const {sizeScale, angleX, angleY, angleZ} = layer.config.visConfig;

        t.equal(
          layer.getScenegraph().url,
          DEFAULT_SCENEGRAPH_MODEL.url,
          'should load the duck model'
        );
        const expectedProps = {
          opacity: layer.config.visConfig.opacity,
          sizeScale: sizeScale * DEFAULT_SCENEGRAPH_MODEL.scale,
          filterRange: preparedDataset.gpuFilter.filterRange,
          getOrientation: [
            angleX + DEFAULT_SCENEGRAPH_MODEL.angles[0],
            angleY + DEFAULT_SCENEGRAPH_MODEL.angles[1],
            angleZ + DEFAULT_SCENEGRAPH_MODEL.angles[2]
          ],
          getColor: [255, 255, 255],
          _lighting: 'pbr'
        };
        Object.keys(expectedProps).forEach(key => {
          t.deepEqual(props[key], expectedProps[key], `should have correct props.${key}`);
        });
      }
    },
    {
      name: 'Scenegraph gallery model',
      layer: {
        type: '3D',
        id: 'test_layer_airplane',
        config: {
          dataId,
          label: 'gps 3d',
          columns,
          visConfig: {
            scenegraph: 'airplane'
          }
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        const model = layer.getScenegraph();
        t.equal(deckLayers.length, 1, 'should render a gallery model');
        t.equal(model.id, 'airplane');
        t.ok(model.url.endsWith('/Plane.glb'), 'should load the airplane model');
        t.equal(
          deckLayers[0].props.sizeScale,
          layer.config.visConfig.sizeScale * model.scale,
          'gallery models use a scale of 1'
        );
      }
    },
    {
      name: 'Scenegraph custom url and tint',
      layer: {
        type: '3D',
        id: 'test_layer_custom',
        config: {
          dataId,
          label: 'gps 3d',
          columns,
          visConfig: {
            scenegraph: CUSTOM_SCENEGRAPH_MODEL_ID,
            scenegraphCustomModelUrl: 'https://example.com/model.glb',
            scenegraphColorEnabled: true,
            scenegraphColor: [10, 20, 30]
          }
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        t.equal(layer.getScenegraph().url, 'https://example.com/model.glb');
        t.deepEqual(deckLayers[0].props.getColor, [10, 20, 30], 'should tint the model');
      }
    },
    {
      name: 'Scenegraph custom url missing',
      layer: {
        type: '3D',
        id: 'test_layer_custom_empty',
        config: {
          dataId,
          label: 'gps 3d',
          columns,
          visConfig: {
            scenegraph: CUSTOM_SCENEGRAPH_MODEL_ID,
            scenegraphCustomModelUrl: ''
          }
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: deckLayers => {
        t.equal(deckLayers.length, 0, 'should not render until a custom URL is set');
      }
    },
    {
      name: 'Scenegraph legacy file url',
      layer: {
        type: '3D',
        id: 'test_layer_legacy',
        config: {
          dataId,
          label: 'gps 3d',
          columns,
          visConfig: {
            scenegraph: 'blob:http://localhost/model.glb',
            angleZ: 90,
            sizeScale: 10
          }
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        const {props} = deckLayers[0];
        t.equal(layer.getScenegraph().url, 'blob:http://localhost/model.glb');
        t.equal(props.sizeScale, 10, 'legacy urls keep the saved size');
        t.deepEqual(props.getOrientation, [0, 0, 90], 'legacy urls do not add model angles');
      }
    },
    {
      name: 'Scenegraph legacy default duck',
      layer: {
        type: '3D',
        id: 'test_layer_legacy_default',
        config: {
          dataId,
          label: 'gps 3d',
          columns,
          visConfig: {
            scenegraph: null,
            angleZ: 90,
            sizeScale: 10
          }
        }
      },
      datasets: {
        [dataId]: {
          ...preparedDataset,
          filteredIndex
        }
      },
      assert: (deckLayers, layer) => {
        const {props} = deckLayers[0];
        t.equal(
          layer.config.visConfig.scenegraph,
          LEGACY_DEFAULT_SCENEGRAPH_URL,
          'null scenegraph keeps the pre-gallery duck'
        );
        t.equal(props.sizeScale, 10, 'legacy default duck keeps the saved size');
        t.deepEqual(
          props.getOrientation,
          [0, 0, 90],
          'legacy default duck does not add the gallery model yaw'
        );
      }
    }
  ];

  testRenderLayerCases(t, ScenegraphLayer, TEST_CASES);

  stubedFetch.restore();
  t.end();
});

test('#ScenegraphLayer -> geojson centroid', t => {
  const layer = new ScenegraphLayer({
    id: 'scenegraph-geojson',
    dataId: 'geo',
    columnMode: 'geojson',
    columns: {
      geojson: {value: 'geom', fieldIdx: 0}
    }
  });
  const rows = [
    [{type: 'Point', coordinates: [-122.4, 37.8]}],
    [
      {
        type: 'LineString',
        coordinates: [
          [0, 0],
          [2, 2]
        ]
      }
    ],
    [null]
  ];
  const dataset = {
    dataContainer: {
      numRows: () => rows.length,
      valueAt: (index, fieldIdx) => rows[index][fieldIdx]
    },
    fields: [{name: 'geom', fieldIdx: 0, type: 'geojson'}],
    gpuFilter: getGpuFilterProps([], 'geo', []),
    filteredIndex: [0, 1, 2]
  };

  layer.updateLayerMeta(dataset);
  const data = layer.calculateDataAttribute(dataset, () => null);

  t.equal(layer.config.columnMode, 'geojson', 'geojson column mode is selected');
  t.equal(data.length, 2, 'skips features without geometry');
  t.deepEqual(data[0].position, [-122.4, 37.8, 0], 'point features keep their coordinate');
  t.deepEqual(data[1].position, [1, 1, 0], 'lines and polygons use the vertex centroid');
  t.end();
});

test('#ScenegraphLayer -> reducer numeric filters without GPU extensions', t => {
  for (const columnMode of ['points', 'geojson']) {
    const table = new KeplerTable({info: {id: 'models'}, color: [1, 2, 3]});
    const rows = [0, 10, 20, 10].map((value, index) => [
      index === 3 ? null : 1 + index,
      30 + index,
      index === 3 ? null : {type: 'Point', coordinates: [30 + index, 1 + index]},
      value,
      [20, 0, 10, 10][index],
      index === 0 ? 'exclude' : 'keep'
    ]);
    const originalRows = JSON.stringify(rows);
    let expectedRows = rows;
    table.importData({
      data: {
        fields: ['lat', 'lng', 'geom', 'value', 'other', 'group'].map(name => ({
          name,
          type: name === 'geom' ? 'geojson' : name === 'group' ? 'string' : 'real'
        })),
        rows
      }
    });
    const layer = validateLayerWithData(
      table,
      {
        type: '3D',
        config: {
          dataId: 'models',
          columnMode,
          columns: columnMode === 'geojson' ? {geojson: 'geom'} : columns
        }
      },
      LayerClasses
    );
    let state = {
      ...INITIAL_VIS_STATE,
      datasets: {models: table},
      layers: [layer],
      layerData: [layer.formatLayerData({models: table})]
    };
    const apply = action => {
      state = visStateReducer(state, action);
    };
    const check = (label, expectedIndexes) => {
      const currentLayer = state.layers[0];
      const dataset = state.datasets.models;
      const [rendered] = currentLayer.renderLayer({
        data: state.layerData[0],
        gpuFilter: dataset.gpuFilter,
        mapState: INITIAL_MAP_STATE,
        idx: 0,
        visible: true,
        layerCallbacks: {},
        interactionConfig: INITIAL_VIS_STATE.interactionConfig
      });
      const {data, getPosition, getFilterValue, extensions} = rendered.props;
      t.deepEqual(extensions, [], `${columnMode}: GPU filtering remains disabled`);
      t.deepEqual(
        data.map(d => d.index),
        expectedIndexes,
        `${columnMode}: ${label}`
      );
      t.deepEqual(
        data.map(getPosition),
        expectedIndexes.map(index => [30 + index, 1 + index, 0]),
        `${columnMode}: positions remain aligned with source rows`
      );
      for (const d of data) {
        t.deepEqual(
          getFilterValue(d),
          dataset.gpuFilter.filterValueAccessor(dataset.dataContainer)()({index: d.index}),
          `${columnMode}: filter accessor uses the original row index`
        );
        t.deepEqual(
          currentLayer.getHoverData(d, dataset.dataContainer, dataset.fields).values(),
          expectedRows[d.index],
          `${columnMode}: picking resolves the original row`
        );
      }
    };
    check('no filter skips invalid positions', [0, 1, 2]);
    apply(VisStateActions.addFilter('models'));
    apply(VisStateActions.setFilter(0, 'name', 'value'));
    apply(VisStateActions.setFilter(0, 'value', [5, 15]));
    t.equal(state.filters[0].gpu, true, 'numeric filter uses GPU configuration');
    t.deepEqual(state.datasets.models.filteredIndex, [0, 1, 2, 3], 'CPU rows are unchanged');
    check('numeric range renders only the matching valid model', [1]);
    apply(VisStateActions.setFilter(0, 'name', 'other'));
    apply(VisStateActions.setFilter(0, 'value', [5, 15]));
    check('changing fields updates the original row lookup', [2]);
    apply(VisStateActions.setFilter(0, 'name', 'value'));
    apply(VisStateActions.setFilter(0, 'value', [0, 0]));
    check('domain minimum is active', [0]);
    apply(VisStateActions.addFilter('models'));
    apply(VisStateActions.setFilter(1, 'name', 'other'));
    apply(VisStateActions.setFilter(1, 'value', [0, 0]));
    check('multiple channels intersect to an empty result', []);
    apply(VisStateActions.setFilter(0, 'value', [0, 20]));
    check('expanding a range restores rows', [1]);
    apply(VisStateActions.removeFilter(1));
    check('removing a channel restores rows', [0, 1, 2]);
    apply(VisStateActions.addFilter('models'));
    apply(VisStateActions.setFilter(1, 'name', 'group'));
    apply(VisStateActions.setFilter(1, 'value', ['keep']));
    check('CPU category filtering is preserved', [1, 2]);
    apply(VisStateActions.setFilter(0, 'value', [5, 15]));
    check('CPU and GPU filters intersect', [1]);
    apply(VisStateActions.removeFilter(0));
    check('clearing GPU filters keeps CPU selection', [1, 2]);
    apply(VisStateActions.removeFilter(0));
    check('clearing all filters restores all valid models', [0, 1, 2]);
    t.equal(JSON.stringify(rows), originalRows, 'filtering does not mutate source rows');
    apply(VisStateActions.addFilter('models'));
    apply(VisStateActions.setFilter(0, 'name', 'value'));
    apply(VisStateActions.setFilter(0, 'value', [5, 15]));
    const oldContainer = state.datasets.models.dataContainer;
    const oldRevision = state.datasets.models.dataRevision;
    expectedRows = rows.map((row, index) => [
      ...row.slice(0, 3),
      [10, 20, 0, 10][index],
      ...row.slice(4)
    ]);
    apply(VisStateActions.addToDataset('models', expectedRows, {upsertBy: 'lng'}));
    t.equal(state.datasets.models.dataContainer, oldContainer, 'row edit keeps the container');
    t.equal(state.datasets.models.dataContainer.numRows(), 4, 'row count is unchanged');
    t.ok(state.datasets.models.dataRevision > oldRevision, 'row edit advances the data revision');
    check('same-size row edits refresh numeric filtering', [0]);
    apply(VisStateActions.removeFilter(0));
    state.datasets.models = copyTableAndUpdate(state.datasets.models, {
      filteredIndexByLayer: {[state.layers[0].id]: [2, 1]}
    });
    state.layerData[0] = state.layers[0].formatLayerData(state.datasets, state.layerData[0]);
    check('layer-specific CPU indexes retain their order', [2, 1]);
    const cachedData = state.layerData[0].data;
    t.equal(
      state.layers[0].formatLayerData(state.datasets, state.layerData[0]).data,
      cachedData,
      'unchanged inputs reuse formatted data'
    );
    state.datasets.models = copyTableAndUpdate(state.datasets.models, {
      filteredIndexByLayer: {[state.layers[0].id]: []}
    });
    state.layerData[0] = state.layers[0].formatLayerData(state.datasets, state.layerData[0]);
    check('empty CPU selection stays empty', []);
    apply(VisStateActions.removeFromDataset('models', [0, 1, 2, 3]));
    t.equal(state.datasets.models.dataContainer.numRows(), 0, 'dataset is empty');
    check('empty dataset stays empty', []);
  }
  t.end();
});

test('#ScenegraphLayer -> GCS model URL', t => {
  t.equal(
    toCorsSafeGcsUrl('https://storage.googleapis.com/kepler-examples/duck.glb'),
    'https://storage.googleapis.com/storage/v1/b/kepler-examples/o/duck.glb?alt=media',
    'rewrites public GCS object URLs to the CORS-enabled download API'
  );
  const signed =
    'https://storage.googleapis.com/kepler-examples/duck.glb?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Signature=abc';
  t.equal(toCorsSafeGcsUrl(signed), signed, 'leaves signed GCS URLs unchanged');
  const signedVirtualHost =
    'https://kepler-examples.storage.googleapis.com/duck.glb?GoogleAccessId=test&Expires=1&Signature=abc';
  t.equal(
    toCorsSafeGcsUrl(signedVirtualHost),
    signedVirtualHost,
    'leaves virtual-hosted signed GCS URLs unchanged'
  );
  t.equal(
    toCorsSafeGcsUrl('https://studio-public-data.foursquare.com/statics/keplergl/Duck.glb'),
    'https://studio-public-data.foursquare.com/statics/keplergl/Duck.glb',
    'leaves non-GCS URLs unchanged'
  );
  t.equal(
    toCorsSafeGcsUrl('blob:http://localhost/model.glb'),
    'blob:http://localhost/model.glb',
    'leaves local file URLs unchanged'
  );
  t.end();
});

test('#ScenegraphLayer -> saved map without a model', t => {
  const saved = {
    version: CURRENT_VERSION,
    config: {
      visState: {
        layers: [
          {
            id: 'legacy-3d',
            type: '3D',
            config: {
              dataId: 'd',
              label: 'old duck',
              visConfig: {scenegraph: null, angleZ: 90, sizeScale: 10}
            }
          },
          {
            id: 'trip-0',
            type: 'trip',
            config: {
              dataId: 'd',
              label: 'trip',
              visConfig: {scenegraph: null, scenegraphEnabled: false}
            }
          }
        ]
      }
    }
  };
  const layers = SchemaManager.parseSavedConfig(saved).visState.layers;
  const duck = layers.find(layer => layer.type === '3D');
  const trip = layers.find(layer => layer.type === 'trip');

  t.equal(
    duck.config.visConfig.scenegraph,
    LEGACY_DEFAULT_SCENEGRAPH_URL,
    'saved 3D layers with a null model keep the pre-gallery duck'
  );
  t.equal(duck.config.visConfig.angleZ, 90, 'saved angle Z is preserved');
  t.equal(trip.config.visConfig.scenegraph, null, 'trip layers keep an empty model');
  t.end();
});