Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

15.5 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import test from 'tape';
import {
  getAcceptedKeplerLoaderEntries,
  getAcceptedRemoteFileFormats,
  getKeplerLoaders,
  isKeplerFileFormatAccepted,
  KeplerCSVLoader,
  processFileData,
  readFileInBatches
} from '@kepler.gl/processors';
import {
  getDisplayedFileExtensions,
  getFileExtensions,
  getFileFormatNames
} from '@kepler.gl/reducers';
import {initApplicationConfig} from '@kepler.gl/utils';
import {tableFromJSON, tableToIPC} from 'apache-arrow';

test('#loader-registry -> resolves matching loaders lazily', async t => {
  const loaders = await getKeplerLoaders({name: 'data.csv', type: ''});

  t.equal(loaders.length, 1, 'only the matching default loader should be resolved');
  t.equal(loaders[0], KeplerCSVLoader, 'CSV should resolve to the Kepler CSV adapter');
  t.equal(typeof loaders[0].parseInBatches, 'function', 'adapter should expose parseInBatches');
  t.end();
});

test('#loader-registry -> resolves JSON loaders from MIME metadata', async t => {
  const loaders = await getKeplerLoaders({name: 'data', type: 'application/json; charset=utf-8'});

  t.equal(loaders.length, 1, 'only the matching default loader should be resolved');
  t.equal(loaders[0].id, 'json', 'JSON should be selected from the MIME type');
  t.end();
});

test('#loader-registry -> resolves NDJSON and GIS loaders by extension', async t => {
  const ndjson = await getKeplerLoaders({name: 'rows.jsonl', type: ''});
  t.equal(ndjson[0].id, 'ndjson', 'jsonl should resolve to NDJSON');

  const geojsonl = await getKeplerLoaders({name: 'places.geojsonl', type: ''});
  t.equal(geojsonl[0].id, 'ndjson', 'geojsonl should resolve to NDJSON');
  t.ok(geojsonl[0].extensions.includes('geojsonl'), 'NDJSON adapter should accept geojsonl');

  const kml = await getKeplerLoaders({name: 'places.kml', type: ''});
  t.equal(kml[0].id, 'kml', 'kml should resolve to the KML loader');

  const gpx = await getKeplerLoaders({name: 'track.gpx', type: ''});
  t.equal(gpx[0].id, 'gpx', 'gpx should resolve to the GPX loader');

  const tcx = await getKeplerLoaders({name: 'activity.tcx', type: ''});
  t.equal(tcx[0].id, 'tcx', 'tcx should resolve to the TCX loader');

  const shp = await getKeplerLoaders({name: 'places.shp', type: ''});
  t.equal(shp[0].id, 'shapefile', 'shp should resolve to the shapefile loader');
  t.notOk(shp[0].tests, 'shapefile magic tests should be stripped so .shx is not parsed as data');

  const xlsx = await getKeplerLoaders({name: 'table.xlsx', type: ''});
  t.equal(xlsx[0].id, 'excel', 'xlsx should resolve to the Excel loader');

  const fgb = await getKeplerLoaders({name: 'places.fgb', type: ''});
  t.equal(fgb[0].id, 'flatgeobuf', 'fgb should resolve to the FlatGeobuf loader');
  t.end();
});

test('#loader-registry -> generic octet-stream does not lock in shapefile', async t => {
  const extensionless = await getKeplerLoaders({
    name: 'abc123',
    type: 'application/octet-stream'
  });
  t.ok(
    extensionless.length > 1,
    'extensionless octet-stream should keep content-based loader selection'
  );
  t.ok(
    extensionless.some(loader => loader.id === 'arrow'),
    'Arrow should stay a candidate for generic binary MIME'
  );
  t.ok(
    extensionless.some(loader => loader.id === 'flatgeobuf'),
    'FlatGeobuf should stay a candidate for generic binary MIME'
  );
  t.ok(
    extensionless.every(loader => !(loader.mimeTypes || []).includes('application/octet-stream')),
    'no candidate should claim application/octet-stream (selectLoader matches MIME before magic)'
  );

  const fgb = await getKeplerLoaders({
    name: 'places.fgb',
    type: 'application/octet-stream'
  });
  t.equal(fgb[0].id, 'flatgeobuf', '.fgb should still win from the extension, not shapefile MIME');
  t.notOk(
    (fgb[0].mimeTypes || []).includes('application/octet-stream'),
    'FlatGeobuf should not advertise the generic octet-stream MIME'
  );

  const shpMime = await getKeplerLoaders({
    name: 'dataset',
    type: 'application/x-esri-shapefile'
  });
  t.equal(shpMime[0].id, 'shapefile', 'specific shapefile MIME should still resolve shapefile');
  t.end();
});

test('#loader-registry -> custom loaders take precedence', async t => {
  const customLoader = {...KeplerCSVLoader, name: 'Custom CSV'};
  const loaders = await getKeplerLoaders({name: 'data.csv', type: ''}, [customLoader]);

  t.equal(loaders.length, 1, 'the default loader should be deduplicated');
  t.equal(loaders[0], customLoader, 'custom loader should be first and authoritative');
  t.end();
});

test('#loader-registry -> CSV parser is async and dynamically loaded', async t => {
  const table = await KeplerCSVLoader.parseText('name,value\nalpha,1\n');

  t.equal(table.shape, 'object-row-table', 'CSV should return a loaders.gl table');
  t.deepEqual(table.data, [{name: 'alpha', value: '1'}], 'CSV values should remain strings');
  t.end();
});

async function readLastBatch(file) {
  const generator = await readFileInBatches({file, fileCache: [], loaders: [], loadOptions: {}});
  let last;
  for await (const batch of generator) {
    last = batch;
  }
  return last;
}

test('#loader-registry -> extensionless octet-stream Arrow is not parsed as FlatGeobuf', async t => {
  const bytes = tableToIPC(tableFromJSON([{name: 'alpha', value: 1}]), 'file');
  const file = new File([bytes], 'abc123', {type: 'application/octet-stream'});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'arrow', 'extensionless octet-stream Arrow should stay Arrow');
  t.ok(
    processed[0].data.fields.some(field => field.name === 'name'),
    'should keep Arrow columns, not GIS features'
  );
  t.end();
});

test('#loader-registry -> file loading uses the async loader path', async t => {
  const file = new File(['name,value\nalpha,1\n'], 'data.csv', {type: 'text/csv'});
  const generator = await readFileInBatches({file, fileCache: [], loaders: [], loadOptions: {}});
  const batches = [];

  for await (const batch of generator) {
    batches.push(batch);
  }

  const dataBatch = batches.find(batch => batch.batchType === 'data');
  t.ok(dataBatch, 'should yield a data batch');
  t.deepEqual(dataBatch.data, [{name: 'alpha', value: '1'}], 'should preserve CSV rows');
  t.end();
});

test('#loader-registry -> geojsonl files become GeoJSON datasets', async t => {
  const geojsonl = [
    '{"type":"Feature","properties":{"name":"alpha"},"geometry":{"type":"Point","coordinates":[-122.4,37.8]}}',
    '{"type":"Feature","properties":{"name":"beta"},"geometry":{"type":"Point","coordinates":[-122.5,37.9]}}'
  ].join('\n');
  const file = new File([geojsonl], 'places.geojsonl', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'geojson', 'geojsonl should process as geojson');
  t.equal(processed[0].data.rows.length, 2, 'should keep both features');
  t.end();
});

test('#loader-registry -> ndjson row files stay row datasets', async t => {
  const ndjson = '{"name":"alpha","value":1}\n{"name":"beta","value":2}\n';
  const file = new File([ndjson], 'rows.ndjson', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'row', 'generic NDJSON should process as rows');
  t.equal(processed[0].data.rows.length, 2, 'should keep both rows');
  t.end();
});

test('#loader-registry -> kml files become GeoJSON datasets', async t => {
  const kml = `<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
  <Placemark>
    <name>alpha</name>
    <Point><coordinates>-122.4,37.8</coordinates></Point>
  </Placemark>
</kml>`;
  const file = new File([kml], 'places.kml', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'geojson', 'KML should process as geojson');
  t.equal(processed[0].data.rows.length, 1, 'should parse the KML placemark');
  t.end();
});

test('#loader-registry -> gpx files become GeoJSON datasets', async t => {
  const gpx = `<?xml version="1.0" encoding="UTF-8"?>
<gpx version="1.1" creator="kepler-test">
  <wpt lat="37.8" lon="-122.4"><name>alpha</name></wpt>
</gpx>`;
  const file = new File([gpx], 'track.gpx', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'geojson', 'GPX should process as geojson');
  t.equal(processed[0].data.rows.length, 1, 'should parse the GPX waypoint');
  t.end();
});

test('#loader-registry -> tcx files become GeoJSON datasets', async t => {
  const tcx = `<?xml version="1.0" encoding="UTF-8"?>
<TrainingCenterDatabase xmlns="http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2">
  <Activities>
    <Activity Sport="Running">
      <Id>2020-01-01T00:00:00Z</Id>
      <Lap StartTime="2020-01-01T00:00:00Z">
        <Track>
          <Trackpoint>
            <Time>2020-01-01T00:00:00Z</Time>
            <Position>
              <LatitudeDegrees>37.8</LatitudeDegrees>
              <LongitudeDegrees>-122.4</LongitudeDegrees>
            </Position>
          </Trackpoint>
          <Trackpoint>
            <Time>2020-01-01T00:01:00Z</Time>
            <Position>
              <LatitudeDegrees>37.81</LatitudeDegrees>
              <LongitudeDegrees>-122.41</LongitudeDegrees>
            </Position>
          </Trackpoint>
        </Track>
      </Lap>
    </Activity>
  </Activities>
</TrainingCenterDatabase>`;
  const file = new File([tcx], 'activity.tcx', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'geojson', 'TCX should process as geojson');
  t.ok(processed[0].data.rows.length >= 1, 'should parse the TCX track');
  t.end();
});

test('#loader-registry -> excel files become row datasets', async t => {
  // eslint-disable-next-line @typescript-eslint/no-var-requires
  const XLSX = require('xlsx');
  const workbook = XLSX.utils.book_new();
  const sheet = XLSX.utils.json_to_sheet([{name: 'alpha', value: 1}]);
  XLSX.utils.book_append_sheet(workbook, sheet, 'Sheet1');
  const buffer = XLSX.write(workbook, {type: 'array', bookType: 'xlsx'});
  const file = new File([buffer], 'table.xlsx', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'row', 'Excel should process as rows');
  t.equal(processed[0].data.rows.length, 1, 'should keep the spreadsheet row');
  t.end();
});

test('#loader-registry -> flatgeobuf files become GeoJSON datasets', async t => {
  const {serialize} = await import('flatgeobuf/lib/mjs/geojson.js');
  const bytes = serialize({
    type: 'FeatureCollection',
    features: [
      {
        type: 'Feature',
        properties: {name: 'alpha'},
        geometry: {type: 'Point', coordinates: [-122.4, 37.8]}
      }
    ]
  });
  const file = new File([bytes], 'places.fgb', {type: ''});
  const batch = await readLastBatch(file);
  const processed = await processFileData({content: batch, fileCache: []});

  t.equal(processed[0].info.format, 'geojson', 'FlatGeobuf should process as geojson');
  t.equal(processed[0].data.rows.length, 1, 'should keep the FlatGeobuf feature');
  t.end();
});

test('#loader-registry -> acceptedFileFormats defaults to all formats', t => {
  t.ok(isKeplerFileFormatAccepted('kml'), 'kml is accepted by default');
  t.ok(isKeplerFileFormatAccepted('geojsonl'), 'geojsonl is accepted by default');
  t.ok(
    getAcceptedKeplerLoaderEntries().some(entry => entry.id === 'kml'),
    'KML stays in the default loader list'
  );
  t.ok(
    getAcceptedRemoteFileFormats().includes('kml'),
    'remote format list includes kml by default'
  );
  t.ok(getFileExtensions({loaders: []}).includes('kml'), 'file picker includes kml by default');
  t.ok(
    getFileExtensions({loaders: []}).includes('geojsonl'),
    'aliases stay accepted even when not shown as icons'
  );
  t.ok(isKeplerFileFormatAccepted('shp'), 'shp is accepted by default');
  t.ok(isKeplerFileFormatAccepted('xlsx'), 'xlsx is accepted by default');
  t.ok(isKeplerFileFormatAccepted('fgb'), 'fgb is accepted by default');
  t.ok(getFileExtensions({loaders: []}).includes('shp'), 'file picker includes shp');
  t.ok(getFileExtensions({loaders: []}).includes('zip'), 'file picker includes shapefile zip');
  t.ok(getFileExtensions({loaders: []}).includes('xlsx'), 'file picker includes xlsx');
  t.ok(getFileExtensions({loaders: []}).includes('fgb'), 'file picker includes fgb');
  t.ok(getAcceptedRemoteFileFormats().includes('shp'), 'remote format list includes shp');
  t.ok(getAcceptedRemoteFileFormats().includes('xlsx'), 'remote format list includes xlsx');
  t.ok(getAcceptedRemoteFileFormats().includes('fgb'), 'remote format list includes fgb');
  t.deepEqual(
    getDisplayedFileExtensions({loaders: []}),
    ['csv', 'json', 'geojson', 'arrow', 'parquet', 'geojsonl', 'kml', 'gpx', 'tcx'],
    'Add Data icons omit Shapefile, Excel, and FlatGeobuf'
  );
  t.deepEqual(
    getFileFormatNames({loaders: []}),
    [
      'CSV',
      'Json',
      'GeoJSON',
      'Arrow',
      'Parquet',
      'GeoJSONL',
      'KML',
      'GPX',
      'TCX',
      'Shapefile',
      'Zip',
      'Excel',
      'FlatGeobuf'
    ],
    'upload copy lists format families, not every alias'
  );
  t.end();
});

test('#loader-registry -> acceptedFileFormats restricts loaders and UI lists', async t => {
  initApplicationConfig({acceptedFileFormats: ['csv', 'geojson']});
  try {
    t.ok(isKeplerFileFormatAccepted('csv'), 'csv remains accepted');
    t.ok(isKeplerFileFormatAccepted('tsv'), 'csv aliases remain accepted');
    t.ok(isKeplerFileFormatAccepted('GeoJSON'), 'geojson remains accepted');
    t.notOk(isKeplerFileFormatAccepted('kml'), 'kml is rejected');
    t.notOk(isKeplerFileFormatAccepted('parquet'), 'parquet is rejected');
    t.notOk(isKeplerFileFormatAccepted('shp'), 'shp is rejected');
    t.ok(isKeplerFileFormatAccepted('zip'), 'a zip of the allowed formats stays accepted');
    t.notOk(isKeplerFileFormatAccepted('xlsx'), 'xlsx is rejected');

    const ids = getAcceptedKeplerLoaderEntries().map(entry => entry.id);
    t.deepEqual(ids.sort(), ['csv', 'json'], 'only matching built-in loaders remain');

    const remote = getAcceptedRemoteFileFormats();
    t.deepEqual(
      remote,
      ['auto', 'csv', 'geojson', 'json'],
      'remote dropdown keeps auto plus accepted formats'
    );

    t.deepEqual(
      getFileExtensions({loaders: []}),
      ['csv', 'tsv', 'dsv', 'json', 'geojson', 'zip'],
      'file picker extensions follow the allowlist'
    );
    t.deepEqual(
      getFileFormatNames({loaders: []}),
      ['CSV', 'Json', 'GeoJSON', 'Zip'],
      'file picker labels follow the allowlist'
    );
    t.deepEqual(
      getDisplayedFileExtensions({loaders: []}),
      ['csv', 'json', 'geojson'],
      'Add Data icons follow the allowlist without aliases'
    );

    const kmlLoaders = await getKeplerLoaders({name: 'places.kml', type: ''});
    t.ok(
      kmlLoaders.every(loader => loader.id !== 'kml'),
      'KML files do not resolve the KML loader when it is not accepted'
    );
    const csvLoaders = await getKeplerLoaders({name: 'data.csv', type: ''});
    t.equal(csvLoaders[0], KeplerCSVLoader, 'CSV still resolves when accepted');
  } finally {
    initApplicationConfig({acceptedFileFormats: null});
  }
  t.end();
});