Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
import test from 'tape';
import {crc32} from 'zlib';
import {
expandZipArchive,
getFilesToParse,
getDroppedFileExtension,
isShapefileSidecarFileName,
processFileData,
readFileInBatches,
unzipShapefileArchive
} from '@kepler.gl/processors';
import {initApplicationConfig} from '@kepler.gl/utils';
function createPointShapefileBuffers({x = -122.4, y = 37.8, name = 'alpha'} = {}) {
const shp = new ArrayBuffer(128);
const shpView = new DataView(shp);
shpView.setInt32(0, 9994, false);
shpView.setInt32(24, 64, false);
shpView.setInt32(28, 1000, true);
shpView.setInt32(32, 1, true);
shpView.setFloat64(36, x, true);
shpView.setFloat64(44, y, true);
shpView.setFloat64(52, x, true);
shpView.setFloat64(60, y, true);
shpView.setInt32(100, 1, false);
shpView.setInt32(104, 10, false);
shpView.setInt32(108, 1, true);
shpView.setFloat64(112, x, true);
shpView.setFloat64(120, y, true);
const shx = new ArrayBuffer(108);
const shxView = new DataView(shx);
shxView.setInt32(0, 9994, false);
shxView.setInt32(24, 54, false);
shxView.setInt32(28, 1000, true);
shxView.setInt32(32, 1, true);
shxView.setFloat64(36, x, true);
shxView.setFloat64(44, y, true);
shxView.setFloat64(52, x, true);
shxView.setFloat64(60, y, true);
shxView.setInt32(100, 50, false);
shxView.setInt32(104, 10, false);
const fieldLen = 10;
const headerLen = 65;
const recordLen = 1 + fieldLen;
const dbf = new ArrayBuffer(headerLen + recordLen);
const dbfBytes = new Uint8Array(dbf);
const dbfView = new DataView(dbf);
dbfBytes[0] = 0x03;
dbfBytes[1] = 124;
dbfBytes[2] = 1;
dbfBytes[3] = 1;
dbfView.setUint32(4, 1, true);
dbfView.setUint16(8, headerLen, true);
dbfView.setUint16(10, recordLen, true);
const fieldName = 'NAME';
for (let i = 0; i < fieldName.length; i++) {
dbfBytes[32 + i] = fieldName.charCodeAt(i);
}
dbfBytes[32 + 11] = 0x43;
dbfBytes[32 + 16] = fieldLen;
dbfBytes[64] = 0x0d;
dbfBytes[65] = 0x20;
const nameBytes = new TextEncoder().encode(name);
for (let i = 0; i < fieldLen; i++) {
dbfBytes[66 + i] = i < nameBytes.length ? nameBytes[i] : 0x20;
}
return {shp, shx, dbf};
}
function createStoredZip(files) {
const locals = [];
const centrals = [];
let offset = 0;
for (const file of files) {
const nameBuf = Buffer.from(file.name);
const data = Buffer.from(file.data);
const crc = crc32(data);
const local = Buffer.alloc(30 + nameBuf.length);
local.writeUInt32LE(0x04034b50, 0);
local.writeUInt16LE(20, 4);
local.writeUInt32LE(crc >>> 0, 14);
local.writeUInt32LE(data.length, 18);
local.writeUInt32LE(data.length, 22);
local.writeUInt16LE(nameBuf.length, 26);
nameBuf.copy(local, 30);
locals.push(local, data);
const central = Buffer.alloc(46 + nameBuf.length);
central.writeUInt32LE(0x02014b50, 0);
central.writeUInt16LE(20, 4);
central.writeUInt16LE(20, 6);
central.writeUInt32LE(crc >>> 0, 16);
central.writeUInt32LE(data.length, 20);
central.writeUInt32LE(data.length, 24);
central.writeUInt16LE(nameBuf.length, 28);
central.writeUInt32LE(offset, 42);
nameBuf.copy(central, 46);
centrals.push(central);
offset += local.length + data.length;
}
const centralBuf = Buffer.concat(centrals);
const end = Buffer.alloc(22);
end.writeUInt32LE(0x06054b50, 0);
end.writeUInt16LE(files.length, 8);
end.writeUInt16LE(files.length, 10);
end.writeUInt32LE(centralBuf.length, 12);
end.writeUInt32LE(offset, 16);
return Buffer.concat([...locals, centralBuf, end]);
}
async function readLastBatch(file, companionFiles) {
const generator = await readFileInBatches({
file,
fileCache: [],
loaders: [],
loadOptions: {},
companionFiles
});
let last;
for await (const batch of generator) {
last = batch;
}
return last;
}
test('#shapefile-files -> sidecar helpers', t => {
t.equal(getDroppedFileExtension('places.SHP'), 'shp');
t.ok(isShapefileSidecarFileName('places.dbf'));
t.ok(isShapefileSidecarFileName('places.prj'));
t.notOk(isShapefileSidecarFileName('places.shp'));
t.notOk(isShapefileSidecarFileName('places.csv'));
t.end();
});
test('#shapefile-files -> getFilesToParse skips sidecars when a shapefile is present', t => {
const files = [
{name: 'places.shp'},
{name: 'places.dbf'},
{name: 'places.shx'},
{name: 'extra.csv'}
];
t.deepEqual(
getFilesToParse(files).map(file => file.name),
['places.shp', 'extra.csv'],
'should parse the shapefile and other datasets, not sidecars'
);
t.deepEqual(
getFilesToParse([{name: 'places.dbf'}, {name: 'places.csv'}]).map(file => file.name),
['places.dbf', 'places.csv'],
'sidecars stay in the queue when no shapefile source is present'
);
t.deepEqual(
getFilesToParse([{name: 'places.zip'}, {name: 'places.prj'}]).map(file => file.name),
['places.zip'],
'a shapefile zip is a source file and its sidecars are skipped'
);
t.end();
});
test('#shapefile-files -> unzipShapefileArchive reads top-level shapefile files', async t => {
const {shp, shx, dbf} = createPointShapefileBuffers();
const zip = createStoredZip([
{name: 'places.shp', data: Buffer.from(shp)},
{name: 'places.shx', data: Buffer.from(shx)},
{name: 'places.dbf', data: Buffer.from(dbf)},
{name: 'nested/ignore.shp', data: Buffer.from(shp)}
]);
const files = await unzipShapefileArchive(new File([zip], 'places.zip'));
t.deepEqual(
files.map(file => file.name).sort(),
['places.dbf', 'places.shp', 'places.shx'],
'should keep top-level shapefile members and ignore nested folders'
);
t.end();
});
test('#shapefile-files -> shapefile with sidecars becomes a GeoJSON dataset', async t => {
const {shp, shx, dbf} = createPointShapefileBuffers();
const shapefile = new File([shp], 'places.shp');
const companions = [shapefile, new File([shx], 'places.shx'), new File([dbf], 'places.dbf')];
const batch = await readLastBatch(shapefile, companions);
const processed = await processFileData({content: batch, fileCache: []});
t.equal(processed[0].info.format, 'geojson', 'shapefile should process as geojson');
t.equal(processed[0].data.rows.length, 1, 'should keep the shapefile feature');
const nameField = processed[0].data.fields.findIndex(field => field.name === 'NAME');
t.ok(nameField >= 0, 'should include the DBF NAME field');
t.equal(
String(processed[0].data.rows[0][nameField]).trim(),
'alpha',
'should read DBF attributes'
);
t.end();
});
test('#shapefile-files -> shapefile zip unpacks before parse', async t => {
const {shp, shx, dbf} = createPointShapefileBuffers();
const zip = createStoredZip([
{name: 'places.shp', data: Buffer.from(shp)},
{name: 'places.shx', data: Buffer.from(shx)},
{name: 'places.dbf', data: Buffer.from(dbf)}
]);
const batch = await readLastBatch(new File([zip], 'places.zip'));
const processed = await processFileData({content: batch, fileCache: []});
t.equal(batch.fileName, 'places.zip', 'batches should keep the original zip name for progress');
t.equal(processed[0].info.format, 'geojson', 'shapefile zip should process as geojson');
t.equal(processed[0].info.label, 'places.zip', 'the dataset label should stay the zip name');
t.equal(processed[0].data.rows.length, 1, 'should keep the unzipped shapefile feature');
t.end();
});
test('#shapefile-files -> expandZipArchive keeps a single shapefile archive', async t => {
const {shp, shx, dbf} = createPointShapefileBuffers();
const zip = createStoredZip([
{name: 'places.shp', data: Buffer.from(shp)},
{name: 'places.shx', data: Buffer.from(shx)},
{name: 'places.dbf', data: Buffer.from(dbf)},
{name: 'readme.txt', data: Buffer.from('notes')},
{name: 'nested/ignore.shp', data: Buffer.from(shp)}
]);
const expansion = await expandZipArchive(new File([zip], 'places.zip'));
t.equal(expansion.kind, 'shapefile', 'one shapefile plus sidecars stays one archive');
if (expansion.kind !== 'shapefile') {
t.end();
return;
}
t.equal(expansion.shapefile.name, 'places.shp');
t.deepEqual(
expansion.companions.map(file => file.name).sort(),
['places.dbf', 'places.shx'],
'readme and nested files are not companions'
);
t.end();
});
test('#shapefile-files -> expandZipArchive splits mixed files into datasets', async t => {
const csv = 'name,lat,lng\nalpha,37.8,-122.4\n';
const zip = createStoredZip([
{name: 'places.csv', data: Buffer.from(csv)},
{
name: 'neighborhoods.geojson',
data: Buffer.from('{"type":"FeatureCollection","features":[]}')
},
{name: 'nested/ignore.csv', data: Buffer.from('a\n1\n')},
{name: 'readme.txt', data: Buffer.from('hi')},
{name: 'inner.zip', data: Buffer.from('not-a-zip')}
]);
const expansion = await expandZipArchive(new File([zip], 'bundle.zip'));
t.equal(expansion.kind, 'datasets');
if (expansion.kind !== 'datasets') {
t.end();
return;
}
t.deepEqual(
expansion.datasets.map(file => file.name),
['places.csv', 'neighborhoods.geojson'],
'each supported top-level file becomes a dataset'
);
t.deepEqual(expansion.companions, [], 'a csv zip has no shapefile sidecars');
const batch = await readLastBatch(expansion.datasets[0]);
const processed = await processFileData({content: batch, fileCache: []});
t.equal(processed[0].info.label, 'places.csv', 'the dataset label is the file inside the zip');
t.equal(processed[0].data.rows.length, 1, 'should parse the unzipped csv');
t.end();
});
test('#shapefile-files -> expandZipArchive loads every shapefile in the zip', async t => {
const alpha = createPointShapefileBuffers({name: 'alpha'});
const beta = createPointShapefileBuffers({name: 'beta', x: -122.5, y: 37.9});
const zip = createStoredZip([
{name: 'alpha.shp', data: Buffer.from(alpha.shp)},
{name: 'alpha.shx', data: Buffer.from(alpha.shx)},
{name: 'alpha.dbf', data: Buffer.from(alpha.dbf)},
{name: 'beta.shp', data: Buffer.from(beta.shp)},
{name: 'beta.shx', data: Buffer.from(beta.shx)},
{name: 'beta.dbf', data: Buffer.from(beta.dbf)}
]);
const expansion = await expandZipArchive(new File([zip], 'places.zip'));
t.equal(expansion.kind, 'datasets', 'several shapefiles are separate datasets');
if (expansion.kind !== 'datasets') {
t.end();
return;
}
t.deepEqual(
expansion.datasets.map(file => file.name),
['alpha.shp', 'beta.shp']
);
const companions = [...expansion.datasets, ...expansion.companions];
const names = [];
for (const file of expansion.datasets) {
const batch = await readLastBatch(file, companions);
const processed = await processFileData({content: batch, fileCache: []});
const nameField = processed[0].data.fields.findIndex(field => field.name === 'NAME');
names.push(String(processed[0].data.rows[0][nameField]).trim());
t.equal(processed[0].info.label, file.name, 'each shapefile keeps its own name');
}
t.deepEqual(names, ['alpha', 'beta'], 'each shapefile reads its own DBF');
t.end();
});
test('#shapefile-files -> expandZipArchive rejects a zip with nothing supported', async t => {
const zip = createStoredZip([
{name: 'readme.txt', data: Buffer.from('hi')},
{name: 'nested/places.csv', data: Buffer.from('a\n1\n')}
]);
try {
await expandZipArchive(new File([zip], 'notes.zip'));
t.fail('should reject a zip with no supported dataset');
} catch (error) {
t.equal(error.message, 'Zip archive does not contain a supported dataset');
}
t.end();
});
test('#shapefile-files -> expandZipArchive keeps custom loader extensions', async t => {
initApplicationConfig({acceptedFileFormats: ['csv']});
try {
const zip = createStoredZip([
{name: 'track.gpx', data: Buffer.from('<gpx></gpx>')},
{name: 'notes.custom', data: Buffer.from('x')}
]);
const expansion = await expandZipArchive(new File([zip], 'bundle.zip'), ['custom']);
t.equal(expansion.kind, 'datasets');
if (expansion.kind === 'datasets') {
t.deepEqual(
expansion.datasets.map(file => file.name),
['notes.custom'],
'custom extensions stay even when that built-in format is not allowed'
);
}
} finally {
initApplicationConfig({acceptedFileFormats: null});
}
t.end();
});
test('#shapefile-files -> expandZipArchive honors accepted file formats', async t => {
initApplicationConfig({acceptedFileFormats: ['csv']});
try {
const {shp} = createPointShapefileBuffers();
const zip = createStoredZip([
{name: 'places.csv', data: Buffer.from('name\nalpha\n')},
{name: 'places.geojson', data: Buffer.from('{"type":"FeatureCollection","features":[]}')},
{name: 'places.shp', data: Buffer.from(shp)}
]);
const expansion = await expandZipArchive(new File([zip], 'bundle.zip'));
t.equal(expansion.kind, 'datasets');
if (expansion.kind === 'datasets') {
t.deepEqual(
expansion.datasets.map(file => file.name),
['places.csv'],
'formats outside the allowlist stay in the zip'
);
}
} finally {
initApplicationConfig({acceptedFileFormats: null});
}
t.end();
});