Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
import test from 'tape';
import sinon from 'sinon';
import {WorkerFarm} from '@loaders.gl/worker-utils';
import {convertGeoArrowToBinaryFeatureCollection} from '@loaders.gl/gis';
import {KeplerTable} from '@kepler.gl/table';
import GeoJsonLayer from '../../../src/layers/src/geojson-layer/geojson-layer';
import {
isKeplerGlMap,
makeProgressIterator,
filesToDataPayload,
fileCacheToProgressiveSteps,
processFileData,
parseGeoArrowBatchesOnWorkers,
remoteDatasetFromUrl,
processArrowBatches,
getGeoJsonFromLoaderResult,
readBatch
} from '@kepler.gl/processors';
import {
GEOARROW_EXTENSIONS,
GEOARROW_METADATA_KEY,
getDatasetRefreshIntervalMs
} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
// The standalone bundle has distinct constructors, like DuckDB's bundled Arrow.
import * as foreignArrow from 'apache-arrow/Arrow.es2015.min';
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
import {parsedFields, parsedRows} from 'test/fixtures/row-object';
import {
savedStateV1InteractionCoordinate as keplerglMap,
parsedFields as parsedKeplerMapFields
} from 'test/fixtures/state-saved-v1-7';
test('#file-handler -> isKeplerGlMap', t => {
t.equal(
isKeplerGlMap('{datasets: [], info: {app: "kepler.gl"}, config: {}}'),
false,
'Should return false when passing a json string'
);
t.equal(
isKeplerGlMap({datasets: [], info: {app: 'kepler.gl'}, config: {}}),
true,
'Should return true when object is a kepler map'
);
t.equal(
isKeplerGlMap({datasets: [], info: {app: 'kepler.gl'}}),
false,
'Should return false when object is not a kepler map'
);
t.end();
});
test('#file-handler -> makeProgressIterator', async t => {
// mock AsyncIterator returned by loarder.gl patchInBatches
// Ideally should run this in browser-headless
async function* mock() {
let bytesUsed = 0;
let value = 0;
let b = 0;
await new Promise(resolve => setTimeout(resolve, 100));
while (b < 2) {
b += 1;
bytesUsed += 10;
value += 1;
yield {
data: [{value}],
bytesUsed
};
}
}
const asyncIterator = mock();
const info = {size: 100};
const progress = makeProgressIterator(asyncIterator, info);
const batch1 = await progress.next();
const batch2 = await progress.next();
const batch3 = await progress.next();
const expected1 = {
value: {
data: [{value: 1}],
bytesUsed: 10,
progress: {rowCount: 1, rowCountInBatch: 1, percent: 0.1}
},
done: false
};
const expected2 = {
value: {
data: [{value: 2}],
bytesUsed: 20,
progress: {rowCount: 2, rowCountInBatch: 1, percent: 0.2}
},
done: false
};
const expected3 = {
value: undefined,
done: true
};
t.deepEqual(batch1, expected1, 'batch1 should be correct');
t.deepEqual(batch2, expected2, 'batch2 should be correct');
t.deepEqual(batch3, expected3, 'batch3 should be correct');
t.end();
});
test('#file-handler -> filesToDataPayload', t => {
const fileCache = [
{
data: {
fields: parsedFields,
rows: parsedRows
},
info: {label: 'rows-data.json', format: 'row'}
},
{
data: {
datasets: [
{
data: {
fields: parsedKeplerMapFields,
rows: keplerglMap.datasets[0].data.allData
},
info: {id: 'a5ybmwl2d', label: 'geojson_as_string_small.csv', color: [53, 92, 125]}
}
],
config: keplerglMap.config
},
info: {label: 'keplergl-map.json', format: 'keplergl'}
}
];
const result = filesToDataPayload(fileCache);
// const expectedResults = [
// {
// datasets: [{data, info}],
// config: {
// version: 'v1',
// config: {}
// },
// options: {centerMap: true}
// },
// {datasets: [{data, info}]}
// ];
t.equal(result.length, 2, 'result shoud have 2 entries');
t.deepEqual(
Object.keys(result[0]),
['datasets', 'config', 'options'],
'result[0] should have 3 keys'
);
t.equal(result[0].datasets, fileCache[1].data.datasets, 'should save keplergl map datasets');
t.equal(result[0].config, fileCache[1].data.config, 'should save keplergl map config');
t.deepEqual(
result[0].options,
{centerMap: true},
'should save keplergl map set {centerMap: true}'
);
t.deepEqual(Object.keys(result[1]), ['datasets'], 'result[0] should have 1 key');
t.deepEqual(
result[1].datasets[0].data,
fileCache[0].data,
'should pass file data to datasets only'
);
t.deepEqual(
Object.keys(result[1].datasets[0].info),
['id', 'label', 'format'],
'result[0] datasets[0].info should have 3 key'
);
t.end();
});
test('#file-handler -> filesToDataPayload autoCreateLayers', t => {
const fileCache = [
{
data: {
fields: parsedFields,
rows: parsedRows
},
info: {label: 'rows-data.json', format: 'row'}
},
{
data: {
datasets: [],
config: keplerglMap.config
},
info: {label: 'keplergl-map.json', format: 'keplergl'}
}
];
const result = filesToDataPayload(fileCache, {autoCreateLayers: false});
t.deepEqual(
result[0].options,
{centerMap: true, autoCreateLayers: false},
'should keep kepler map centerMap and forward autoCreateLayers'
);
t.deepEqual(
result[1].options,
{autoCreateLayers: false},
'should forward autoCreateLayers for dataset files'
);
t.end();
});
test('#file-handler -> remoteDatasetFromUrl waits for the remote download', t => {
const dataset = remoteDatasetFromUrl('https://example.com/quakes.csv?sv=1', 'csv');
t.equal(dataset.info.type, 'externally-hosted', 'should be a remote dataset');
t.equal(dataset.info.label, 'quakes.csv', 'should name the card from the url');
t.deepEqual(dataset.data, {fields: [], rows: []}, 'should not include downloaded rows');
t.equal(dataset.metadata.source, 'https://example.com/quakes.csv?sv=1');
t.equal(dataset.metadata.sourceFormat, 'csv', 'should keep the selected format');
const payload = filesToDataPayload([dataset]);
t.equal(payload[0].datasets[0].info.type, 'externally-hosted', 'Add Data keeps the remote type');
t.deepEqual(payload[0].datasets[0].data, {fields: [], rows: []}, 'Add Data still has no rows');
t.end();
});
test('#file-handler -> filesToDataPayload remote metadata', t => {
const fileCache = [
{
data: {
fields: parsedFields,
rows: parsedRows
},
info: {
id: 'remote-ds',
label: 'quakes.csv',
format: 'row',
type: 'externally-hosted'
},
metadata: {
source: 'https://example.com/quakes.csv',
sourceFormat: 'csv'
}
}
];
const result = filesToDataPayload(fileCache);
t.equal(result.length, 1, 'result should have 1 entry');
t.equal(result[0].datasets[0].info.type, 'externally-hosted', 'should pass type');
t.deepEqual(
result[0].datasets[0].metadata,
{source: 'https://example.com/quakes.csv', sourceFormat: 'csv'},
'should pass remote source metadata'
);
t.end();
});
test('#file-handler -> processFileData one-shot load does not enable polling', async t => {
const rows = [{lat: 1, lng: 2}];
const local = await processFileData({
content: {fileName: 'local.csv', data: rows},
fileCache: []
});
t.equal(local[0].info.type, undefined, 'local files are not marked externally-hosted');
t.equal(local[0].metadata, undefined, 'local files get no remote refresh metadata');
const sized = await processFileData({
content: {fileName: 'local.csv', data: rows, fileSize: 50 * 1024 * 1024},
fileCache: []
});
t.equal(
sized[0].metadata.size,
50 * 1024 * 1024,
'local files keep their byte size for a later warning'
);
const remote = await processFileData({
content: {
fileName: 'remote.csv',
data: rows,
sourceUrl: 'https://example.com/remote.csv'
},
fileCache: []
});
t.equal(remote[0].info.type, 'externally-hosted', 'URL loads are still externally-hosted');
t.equal(
remote[0].metadata.refreshIntervalMs,
undefined,
'a one-shot URL load does not set a poll interval'
);
t.equal(
getDatasetRefreshIntervalMs(remote[0].metadata),
0,
'polling helper treats omitted interval as off'
);
t.end();
});
test('#file-handler -> processFileData persists remote file format', async t => {
const cache = await processFileData({
content: {
fileName: 'quakes.csv',
data: [{lat: 1, lng: 2}],
sourceUrl: 'https://example.com/abc123?sv=1',
keplerFormat: 'csv'
},
fileCache: []
});
t.equal(cache[0].info.type, 'externally-hosted', 'should mark the dataset as externally-hosted');
t.equal(cache[0].info.format, 'row', 'processor format stays row for CSV');
t.equal(cache[0].metadata.source, 'https://example.com/abc123?sv=1');
t.equal(
cache[0].metadata.sourceFormat,
'csv',
'should persist the file format, not the processor format'
);
t.equal(
typeof cache[0].metadata.lastFetchedAt,
'number',
'should record lastFetchedAt on remote load'
);
t.equal(cache[0].metadata.etag, undefined, 'should omit etag when the fetch did not provide one');
t.equal(
cache[0].metadata.lastModified,
undefined,
'should omit lastModified when the fetch did not provide one'
);
const withValidators = await processFileData({
content: {
fileName: 'quakes.csv',
data: [{lat: 1, lng: 2}],
sourceUrl: 'https://example.com/quakes.csv',
keplerFormat: 'csv',
etag: '"abc"',
lastModified: 'Wed, 21 Oct 2015 07:28:00 GMT'
},
fileCache: []
});
t.equal(withValidators[0].metadata.etag, '"abc"', 'should persist ETag for the next refresh');
t.equal(
withValidators[0].metadata.lastModified,
'Wed, 21 Oct 2015 07:28:00 GMT',
'should persist Last-Modified for the next refresh'
);
const parquetCache = await processFileData({
content: {
fileName: 'data.parquet',
data: [{lat: 1, lng: 2}],
sourceUrl: 'https://example.com/data.parquet'
},
fileCache: []
});
t.equal(
parquetCache[0].metadata.sourceFormat,
'parquet',
'should infer parquet from the filename when no format was selected'
);
t.end();
});
test('#file-handler -> processFileData remote ids do not collide on filename', async t => {
const rows = [{lat: 1, lng: 2}];
const local = await processFileData({
content: {fileName: 'quakes.csv', data: rows},
fileCache: []
});
const remoteA = await processFileData({
content: {
fileName: 'quakes.csv',
data: rows,
sourceUrl: 'https://a.example.com/quakes.csv'
},
fileCache: []
});
const remoteB = await processFileData({
content: {
fileName: 'quakes.csv',
data: rows,
sourceUrl: 'https://b.example.com/quakes.csv'
},
fileCache: []
});
const remoteAAgain = await processFileData({
content: {
fileName: 'quakes.csv',
data: rows,
sourceUrl: 'https://a.example.com/quakes.csv'
},
fileCache: []
});
t.equal(local[0].info.label, 'quakes.csv', 'local label stays the filename');
t.equal(remoteA[0].info.label, 'quakes.csv', 'remote label stays the filename');
t.notEqual(
local[0].info.id,
remoteA[0].info.id,
'local and remote files with the same name get different ids'
);
t.notEqual(
remoteA[0].info.id,
remoteB[0].info.id,
'two remote URLs with the same filename get different ids'
);
t.equal(
remoteA[0].info.id,
remoteAAgain[0].info.id,
'reloading the same URL keeps a stable id so progressive batches can update in place'
);
t.ok(!String(remoteA[0].info.id).includes('http'), 'id is a hash, not the raw URL');
t.end();
});
test('#file-handler -> convertArrowToSchema uses a single apache-arrow copy', t => {
// Parquet loading serializes the Arrow schema via loaders.gl. If kepler.gl
// and @loaders.gl/schema-utils resolve different apache-arrow packages, that
// switch-on-constructor check throws `arrow type not supported: <Class>`
// (minified to e.g. `tL` in exported HTML).
const table = new arrow.Table({
lat: arrow.vectorFromArray(new Float64Array([37.8])),
lng: arrow.vectorFromArray(new Float64Array([-122.4])),
name: arrow.vectorFromArray(['alpha'], new arrow.Utf8())
});
let schema;
t.doesNotThrow(() => {
schema = convertArrowToSchema(table.schema);
}, 'schema conversion should not fail with a constructor identity mismatch');
t.deepEqual(
schema.fields.map(field => field.type),
['float64', 'float64', 'utf8'],
'should serialize primitive Arrow types from the shared apache-arrow copy'
);
t.end();
});
test('#file-handler -> processArrowBatches skip compact for incremental loads', t => {
const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
const combined = batchA.concat(batchB);
t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');
const skipped = processArrowBatches(combined.batches, {compact: false});
t.equal(
skipped.cols[0].data.length,
combined.batches.length,
'compact:false should leave record batches unchanged'
);
const compacted = processArrowBatches(combined.batches);
t.equal(compacted.cols[0].data.length, 1, 'default processArrowBatches should compact');
t.equal(compacted.cols[0].length, 2, 'compacted table should keep all rows');
t.end();
});
test('#file-handler -> parseGeoArrowBatchesOnWorkers parses the latest chunk', async t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const batchA = new arrow.Table({
geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType)
});
const batchB = new arrow.Table({
geometry: arrow.vectorFromArray(
[
[-122.5, 37.9],
[-122.6, 38]
],
pointType
)
});
batchA.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
batchB.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
const combined = batchA.concat(batchB);
const processed = processArrowBatches(combined.batches, {compact: false});
const calls = [];
const binaryDataFromGeoArrow = {
binaryGeometries: [{shape: 'binary-feature-collection'}],
bounds: [-122.6, 37.9, -122.5, 38],
featureTypes: {point: true, line: false, polygon: false},
meanCenters: [
[-122.5, 37.9],
[-122.6, 38]
]
};
const parsed = await parseGeoArrowBatchesOnWorkers(processed, {
latestOnly: true,
parseOnWorker: async input => {
calls.push(input);
return {chunkIndex: input.chunkIndex, binaryDataFromGeoArrow};
}
});
t.equal(calls.length, 1, 'should schedule one worker for the newest chunk');
t.equal(calls[0].chunkIndex, 1, 'should parse the second record batch');
t.equal(calls[0].chunkOffset, 1, 'should offset global feature ids by earlier row counts');
t.equal(
calls[0].geometryEncoding,
GEOARROW_EXTENSIONS.POINT,
'should preserve the native GeoArrow encoding'
);
t.equal(parsed[0][1], binaryDataFromGeoArrow, 'should cache worker output by column and chunk');
t.end();
});
test('#file-handler -> separated points reach workers and fallback with all coordinates', async t => {
const type = new arrow.Struct(
['x', 'y', 'z'].map(name => new arrow.Field(name, new arrow.Float64(), false))
);
const batches = [
[
{x: 10, y: 20, z: 5},
{x: 30, y: 40, z: 15}
],
[{x: 50, y: 60, z: 25}]
].map(points => {
const table = new arrow.Table({geometry: arrow.vectorFromArray(points, type)});
table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
return table;
});
const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));
const bytes = arrow.tableToIPC(table, 'file');
const processed = processArrowBatches(table.batches, {compact: false});
for (const failWorker of [false, true]) {
const result = await parseGeoArrowBatchesOnWorkers(processed, {
parseOnWorker: async input => {
t.equal(input.chunkData.type.listSize, 3, 'worker input has XYZ coordinate size');
if (failWorker) throw new Error('worker unavailable');
return parseGeoArrowWorkerInput(input);
}
});
const {layer} = await layerFromFile({
info: {id: 'separated'},
data: {...processed, geoArrowBinaryData: result}
});
t.deepEqual(
layer.dataToFeature.flatMap(data => Array.from(data.points.positions.value)),
[10, 20, 5, 30, 40, 15, 50, 60, 25],
'worker/cache and fallback retain all axes'
);
t.deepEqual(
layer.meta.bounds,
[10, 20, 50, 60],
'cached bounds include latitude from every batch'
);
t.deepEqual(
layer.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)),
[0, 1, 2],
'cached feature ids retain unequal batch offsets'
);
}
t.deepEqual(arrow.tableToIPC(table, 'file'), bytes, 'worker preparation preserves source data');
t.end();
});
test('#file-handler -> worker fallback accepts points and lines from another Arrow runtime', async t => {
const pointType = new foreignArrow.FixedSizeList(
2,
new foreignArrow.Field('xy', new foreignArrow.Float64())
);
for (const encoding of [GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING]) {
const isPoint = encoding === GEOARROW_EXTENSIONS.POINT;
const chunks = [
[
[10, 20],
[30, 40]
],
[[50, 60]]
].map(points => {
// Exact child/offset lengths keep unrelated IPC padding out of line fixtures.
const coordinates = isPoint ? points : points.flatMap(point => [point, point]);
const child = foreignArrow
.vectorFromArray(coordinates, pointType)
.data[0].slice(0, coordinates.length);
return isPoint
? child
: foreignArrow.makeData({
type: new foreignArrow.List(new foreignArrow.Field('vertices', pointType)),
length: points.length,
valueOffsets: Int32Array.from(points.map((_, i) => i * 2).concat(points.length * 2)),
child
});
});
const geoColumn = foreignArrow.makeVector(chunks);
const field = {
name: 'geometry',
type: 'geoarrow',
fieldIdx: 0,
metadata: new Map([[GEOARROW_METADATA_KEY, encoding]])
};
const table = new foreignArrow.Table({geometry: geoColumn});
const originalBytes = foreignArrow.tableToIPC(table);
t.notOk(chunks[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime');
for (const latestOnly of [false, true]) {
const result = await parseGeoArrowBatchesOnWorkers(
{fields: [field], cols: [geoColumn]},
{
latestOnly,
parseOnWorker: async () => {
throw new Error('worker unavailable');
}
}
);
t.deepEqual(Object.keys(result[0]), latestOnly ? ['1'] : ['0', '1'], 'selected batches');
for (const [chunkIndex, binary] of Object.entries(result[0])) {
const last = chunkIndex === '1';
const points = last
? [[50, 60]]
: [
[10, 20],
[30, 40]
];
const rowIds = last ? [2] : [0, 1];
const geometry = binary.binaryGeometries[0][isPoint ? 'points' : 'lines'];
t.deepEqual(
Array.from(geometry.positions.value),
(isPoint ? points : points.flatMap(point => [point, point])).flat(),
'fallback preserves all coordinates'
);
t.deepEqual(
Array.from(geometry.globalFeatureIds.value),
isPoint ? rowIds : rowIds.flatMap(id => [id, id]),
'fallback preserves global row ids'
);
t.deepEqual(binary.bounds, last ? [50, 60, 50, 60] : [10, 20, 30, 40], 'bounds');
t.deepEqual(binary.meanCenters, points, 'centroids remain aligned with rows');
}
}
t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'fallback preserves source data');
}
t.end();
});
test('#file-handler -> parseGeoArrowBatchesOnWorkers combines compacted worker chunks', async t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const batchA = new arrow.Table({
geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType)
});
const batchB = new arrow.Table({
geometry: arrow.vectorFromArray([[-122.5, 37.9]], pointType)
});
batchA.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
batchB.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
const processed = processArrowBatches(batchA.concat(batchB).batches, {compact: false});
const calls = [];
const parsed = await parseGeoArrowBatchesOnWorkers(processed, {
targetChunkCount: 1,
parseOnWorker: async input => {
calls.push(input);
const x = input.chunkIndex === 0 ? -122.4 : -122.5;
const y = input.chunkIndex === 0 ? 37.8 : 37.9;
return {
chunkIndex: input.chunkIndex,
binaryDataFromGeoArrow: {
binaryGeometries: [{chunkIndex: input.chunkIndex}],
bounds: [x, y, x, y],
featureTypes: {point: true, line: false, polygon: false},
meanCenters: [[x, y]]
}
};
}
});
t.equal(calls.length, 2, 'should schedule one worker job per original record batch');
t.deepEqual(Object.keys(parsed[0]), ['0'], 'should expose one cache chunk after compaction');
t.deepEqual(
parsed[0][0].binaryGeometries,
[{chunkIndex: 0}, {chunkIndex: 1}],
'should retain worker geometry in record-batch order'
);
t.deepEqual(parsed[0][0].bounds, [-122.5, 37.8, -122.4, 37.9], 'should merge bounds');
t.deepEqual(
parsed[0][0].meanCenters,
[
[-122.4, 37.8],
[-122.5, 37.9]
],
'should merge mean centers'
);
t.end();
});
test('#file-handler -> compacted worker cache handles large centroid arrays', async t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const table = new arrow.Table({
geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType)
});
table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
const processed = processArrowBatches(table.batches, {compact: false});
const largeMeanCenters = Array.from({length: 150000}, () => [-122.4, 37.8]);
const parsed = await parseGeoArrowBatchesOnWorkers(processed, {
targetChunkCount: 1,
parseOnWorker: async input => ({
chunkIndex: input.chunkIndex,
binaryDataFromGeoArrow: {
binaryGeometries: [],
bounds: [-122.4, 37.8, -122.4, 37.8],
featureTypes: {point: true, line: false, polygon: false},
meanCenters: largeMeanCenters
}
})
});
t.equal(
parsed[0][0].meanCenters.length,
largeMeanCenters.length,
'should combine every centroid without spreading them into function arguments'
);
t.end();
});
function geoArrowPointBatches(pointBatches = [[[-122.4, 37.8]], [[-122.5, 37.9]]]) {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
let rowIndex = 0;
return pointBatches.map(points => {
const geometry = arrow.makeData({
type: pointType,
length: points.length,
child: arrow.makeData({type: new arrow.Float64(), data: Float64Array.from(points.flat())})
});
const row = arrow.makeData({
type: new arrow.Int32(),
data: Int32Array.from(points.map(() => rowIndex++))
});
const schema = new arrow.Schema([
new arrow.Field('geometry', pointType),
new arrow.Field('row', row.type)
]);
schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
return new arrow.RecordBatch(
schema,
arrow.makeData({
type: new arrow.Struct(schema.fields),
length: points.length,
children: [geometry, row]
})
);
});
}
// Replace only worker transport; successful jobs use the same converter as the
// loaders.gl worker. File processing, table import and layer cache reads stay real.
async function withGeoArrowWorker(parse, run) {
const originalWorker = globalThis.Worker;
globalThis.Worker = class {};
const stub = sinon.stub(WorkerFarm, 'getWorkerFarm').callsFake(() => ({
getWorkerPool: () => ({
startJob: async () => {
let resolve;
let reject;
const result = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return {
result,
postMessage: (_type, {input}) => {
Promise.resolve()
.then(() => parse(input))
.then(value => resolve({result: value}), reject);
}
};
}
})
}));
try {
await run();
} finally {
stub.restore();
if (originalWorker === undefined) {
delete globalThis.Worker;
} else {
globalThis.Worker = originalWorker;
}
}
}
function parseGeoArrowWorkerInput(input) {
const {chunkData, chunkIndex, chunkOffset, geometryEncoding} = input;
const vector = arrow.makeVector(
new arrow.Data(
chunkData.type,
chunkData.offset,
chunkData.length,
chunkData.nullCount,
chunkData.buffers,
chunkData.children,
chunkData.dictionary
)
);
return {
chunkIndex,
binaryDataFromGeoArrow: convertGeoArrowToBinaryFeatureCollection(vector, geometryEncoding, {
chunkIndex: 0,
chunkOffset,
calculateMeanCenters: true,
triangle: true
})
};
}
async function layerFromFile(file) {
const dataset = new KeplerTable({info: file.info, color: [0, 0, 0]});
await dataset.importData({data: file.data});
const layer = new GeoJsonLayer({dataId: dataset.id});
layer.config.columns.geojson = {value: 'geometry', fieldIdx: 0};
layer.updateLayerMeta(dataset);
return {dataset, layer};
}
async function assertCompleteGeoArrowLayer(
t,
file,
points = [
[-122.4, 37.8],
[-122.5, 37.9]
]
) {
const {dataset, layer} = await layerFromFile(file);
t.equal(dataset.dataContainer.numRows(), points.length, 'table keeps every row');
t.deepEqual(layer.centroids, points, 'centroids retain row order');
t.deepEqual(
layer.meta.bounds,
[
Math.min(...points.map(point => point[0])),
Math.min(...points.map(point => point[1])),
Math.max(...points.map(point => point[0])),
Math.max(...points.map(point => point[1]))
],
'bounds include every batch'
);
const featureIds = layer.dataToFeature.flatMap(data =>
Array.from(data.points.globalFeatureIds.value)
);
t.deepEqual(
featureIds,
points.map((_, index) => index),
'geometry retains every feature row id'
);
t.deepEqual(
featureIds.map(index => dataset.dataContainer.valueAt(index, 1)),
featureIds,
'geometry maps to the matching attribute row'
);
return {dataset, layer};
}
for (const failure of ['first', 'last', 'all', 'missing']) {
test(`#file-handler -> ${failure} worker failure preserves the complete compacted layer`, async t => {
await withGeoArrowWorker(
input => {
if (failure === 'missing' && input.chunkIndex === 1) {
return {chunkIndex: input.chunkIndex, binaryDataFromGeoArrow: null};
}
if (failure === 'all' || input.chunkIndex === (failure === 'first' ? 0 : 1)) {
throw new Error('worker failed');
}
return parseGeoArrowWorkerInput(input);
},
async () => {
const [file] = await processFileData({
content: {
fileName: 'points.arrow',
data: geoArrowPointBatches(),
prepareProgressiveReplay: true
},
fileCache: []
});
const {dataset} = await assertCompleteGeoArrowLayer(t, file);
t.equal(dataset.dataContainer.numChunks(), 1, 'final table is compacted');
await assertCompleteGeoArrowLayer(t, {...file, data: file.progressiveData[0]}, [
[-122.4, 37.8]
]);
}
);
t.end();
});
}
function deferred() {
let resolve;
let reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return {promise, resolve, reject};
}
test('#file-handler -> out-of-order workers retain the successful fast path', async t => {
const jobs = [deferred(), deferred()];
const started = deferred();
const outputs = [];
await withGeoArrowWorker(
input => {
outputs[input.chunkIndex] = parseGeoArrowWorkerInput(input);
if (outputs.filter(Boolean).length === 2) started.resolve();
return jobs[input.chunkIndex].promise;
},
async () => {
const pending = processFileData({
content: {fileName: 'points.arrow', data: geoArrowPointBatches()},
fileCache: []
});
await started.promise;
jobs[1].resolve(outputs[1]);
jobs[0].resolve(outputs[0]);
const [file] = await pending;
const {layer} = await assertCompleteGeoArrowLayer(t, file);
t.equal(
layer.dataToFeature[0],
outputs[0].binaryDataFromGeoArrow.binaryGeometries[0],
'first batch reuses the worker geometry'
);
t.equal(
layer.dataToFeature[1],
outputs[1].binaryDataFromGeoArrow.binaryGeometries[0],
'second batch reuses the worker geometry'
);
}
);
t.end();
});
for (const failEmpty of [false, true]) {
test(`#file-handler -> ${failEmpty ? 'failed' : 'successful'} empty worker batch`, async t => {
const outputs = [];
await withGeoArrowWorker(
input => {
if (failEmpty && input.chunkData.length === 0) throw new Error('empty worker failed');
const result = parseGeoArrowWorkerInput(input);
outputs[input.chunkIndex] = result;
return result;
},
async () => {
const [file] = await processFileData({
content: {
fileName: 'points.arrow',
data: geoArrowPointBatches([[[-122.4, 37.8]], [], [[-122.5, 37.9]]])
},
fileCache: []
});
const {layer} = await assertCompleteGeoArrowLayer(t, file);
t.equal(
layer.dataToFeature[0] === outputs[0].binaryDataFromGeoArrow.binaryGeometries[0],
!failEmpty,
'only a failed empty batch triggers complete fallback'
);
}
);
t.end();
});
}
test('#file-handler -> failed latest worker batch preserves incremental row ids', async t => {
await withGeoArrowWorker(
() => Promise.reject(new Error('worker failed')),
async () => {
const [file] = await processFileData({
content: {fileName: 'points.arrow', data: geoArrowPointBatches(), skipArrowCompact: true},
fileCache: []
});
const latest = file.data.geoArrowBinaryData[0][1];
t.deepEqual(latest.meanCenters, [[-122.5, 37.9]], 'latest batch has its centroid');
t.deepEqual(
Array.from(latest.binaryGeometries[0].points.globalFeatureIds.value),
[1],
'fallback keeps the original global row offset'
);
}
);
t.end();
});
test('#file-handler -> geometry errors still reject after worker fallback', async t => {
const malformed = arrow.tableFromArrays({geometry: new Int32Array([1])});
malformed.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
await withGeoArrowWorker(
() => Promise.reject(new Error('worker failed')),
async () => {
try {
await processFileData({
content: {fileName: 'invalid.arrow', data: malformed.batches},
fileCache: []
});
t.fail('invalid geometry must reject instead of returning a partial cache');
} catch (error) {
t.ok(error instanceof Error, 'synchronous geometry parse errors propagate');
}
}
);
t.end();
});
test('#file-handler -> decode errors after a valid batch still reject', async t => {
const error = new Error('Arrow decode failed');
async function* batches() {
yield {data: geoArrowPointBatches()};
throw error;
}
const iterator = readBatch(batches(), 'broken.arrow');
await iterator.next();
try {
await iterator.next();
t.fail('a decode error must not turn into a completed partial file');
} catch (caught) {
t.equal(caught, error, 'the original decode error propagates');
}
t.end();
});
test('#file-handler -> processArrowBatches Int64/Uint64 columns', async t => {
const table = arrow.tableFromArrays({
hexId: new BigUint64Array([610625465232654335n, 610625465081659391n]),
total: new BigInt64Array([29436887n, 40685227n])
});
let result;
t.doesNotThrow(() => {
result = processArrowBatches(table.batches);
}, 'should process Arrow Int64/Uint64 columns without BigInt TypeError');
t.equal(result.fields[0].name, 'hexId');
t.equal(result.fields[0].type, 'integer', 'uint64 columns should stay integer, not h3');
t.equal(result.fields[1].name, 'total');
t.equal(result.fields[1].type, 'integer', 'int64 counts should stay integer');
const processed = await processFileData({
content: {fileName: 'congo.parquet', data: table.batches},
fileCache: []
});
t.equal(processed.length, 1, 'processFileData should accept uint64 Arrow batches');
t.equal(processed[0].data.fields[0].type, 'integer');
t.end();
});
test('#file-handler -> processFileData Feature array is geojson', async t => {
const features = [
{
type: 'Feature',
properties: {name: 'alpha'},
geometry: {type: 'Point', coordinates: [-122.4, 37.8]}
},
{
type: 'Feature',
properties: {name: 'beta'},
geometry: {type: 'Point', coordinates: [-122.5, 37.9]}
}
];
const processed = await processFileData({
content: {fileName: 'places.geojsonl', data: features},
fileCache: []
});
t.equal(
processed[0].info.format,
'geojson',
'an array of Features should not be treated as rows'
);
t.equal(processed[0].data.rows.length, 2, 'should keep both features');
t.end();
});
test('#file-handler -> shapefile-shaped loader output becomes GeoJSON', async t => {
const data = {
data: [
{
type: 'Feature',
properties: {name: 'alpha'},
geometry: {type: 'Point', coordinates: [-122.4, 37.8]}
}
]
};
t.equal(
getGeoJsonFromLoaderResult(data).features.length,
1,
'should unwrap shapefile Feature arrays'
);
const processed = await processFileData({
content: {fileName: 'places.shp', data},
fileCache: []
});
t.equal(processed[0].info.format, 'geojson', 'shapefile output should process as geojson');
t.equal(processed[0].data.rows.length, 1, 'should keep the shapefile feature');
t.end();
});
async function readLastAggregatedBatch(iterator, fileName) {
const generator = readBatch(iterator, fileName);
let last;
for await (const batch of generator) {
last = batch;
}
return last;
}
test('#file-handler -> empty GIS loader output becomes an empty GeoJSON dataset', async t => {
const emptyShapefile = await processFileData({
content: {fileName: 'empty.shp', data: {data: []}},
fileCache: []
});
t.equal(emptyShapefile[0].info.format, 'geojson', 'empty shapefile output should be geojson');
t.equal(emptyShapefile[0].data.rows.length, 0, 'should keep zero features');
const emptyTable = await processFileData({
content: {
fileName: 'empty.fgb',
data: {shape: 'geojson-table', type: 'FeatureCollection', features: []}
},
fileCache: []
});
t.equal(emptyTable[0].info.format, 'geojson', 'empty geojson-table should be geojson');
t.equal(emptyTable[0].data.rows.length, 0, 'should keep zero geojson-table features');
const emptyExcel = await processFileData({
content: {fileName: 'empty.xlsx', data: {shape: 'object-row-table', data: []}},
fileCache: []
});
t.equal(emptyExcel[0].info.format, 'row', 'empty spreadsheet tables should stay rows');
t.end();
});
test('#file-handler -> readBatch does not wrap JSON streaming placeholders', async t => {
async function* geojsonStream() {
yield {batchType: 'metadata', shape: 'metadata', data: []};
yield {
shape: 'object-row-table',
batchType: 'partial-result',
container: {type: 'FeatureCollection', features: []},
data: [],
jsonpath: '$.features'
};
yield {
data: [
{
type: 'Feature',
properties: {name: 'alpha'},
geometry: {type: 'Point', coordinates: [0, 1]}
}
],
jsonpath: '$.features',
length: 1
};
}
const generator = readBatch(geojsonStream(), 'places.geojson');
await generator.next();
const partial = await generator.next();
t.equal(partial.value.batchType, 'partial-result', 'should yield the JSON partial-result');
t.equal(partial.value.data.length, 0, 'partial-result should keep an empty array');
const dataBatch = await generator.next();
t.equal(dataBatch.value.data.length, 1, 'data batch should stay a feature array');
t.end();
});
test('#file-handler -> readBatch preserves empty loader identity', async t => {
async function* emptyShapefileBatches() {
yield {header: {length: 50}, data: []};
}
const shapefileBatch = await readLastAggregatedBatch(emptyShapefileBatches(), 'empty.shp');
const emptyShapefile = await processFileData({content: shapefileBatch, fileCache: []});
t.equal(emptyShapefile[0].info.format, 'geojson', 'empty shapefile batches should stay geojson');
t.equal(emptyShapefile[0].data.rows.length, 0, 'should keep zero shapefile features');
async function* emptyExcelBatches() {
yield {shape: 'object-row-table', data: []};
}
const excelBatch = await readLastAggregatedBatch(emptyExcelBatches(), 'empty.xlsx');
const emptyExcel = await processFileData({content: excelBatch, fileCache: []});
t.equal(emptyExcel[0].info.format, 'row', 'empty spreadsheet batches should stay rows');
t.equal(emptyExcel[0].data.rows.length, 0, 'should keep zero spreadsheet rows');
async function* emptyFlatGeobufBatches() {
yield {batchType: 'metadata'};
yield {shape: 'geojson-table', type: 'FeatureCollection', features: []};
}
const fgbBatch = await readLastAggregatedBatch(emptyFlatGeobufBatches(), 'empty.fgb');
const emptyFgb = await processFileData({content: fgbBatch, fileCache: []});
t.equal(emptyFgb[0].info.format, 'geojson', 'empty FlatGeobuf batches should stay geojson');
t.equal(emptyFgb[0].data.rows.length, 0, 'should keep zero FlatGeobuf features');
t.end();
});
test('#file-handler -> object-row-table becomes a row dataset', async t => {
const processed = await processFileData({
content: {
fileName: 'table.xlsx',
data: {shape: 'object-row-table', data: [{name: 'alpha', value: 1}]}
},
fileCache: []
});
t.equal(processed[0].info.format, 'row', 'Excel-shaped tables should process as rows');
t.equal(processed[0].data.rows.length, 1, 'should keep the spreadsheet row');
t.end();
});
test('#file-handler -> rows with type Feature but no geometry stay tables', async t => {
const rows = [
{type: 'Feature', name: 'alpha', lat: 37.8, lng: -122.4},
{type: 'Feature', name: 'beta', lat: 37.9, lng: -122.5}
];
t.equal(
getGeoJsonFromLoaderResult(rows),
null,
'should not wrap tabular rows as a FeatureCollection'
);
const processed = await processFileData({
content: {fileName: 'places.csv', data: rows},
fileCache: []
});
t.equal(
processed[0].info.format,
'row',
'a type column equal to Feature should not force geojson'
);
t.equal(processed[0].data.rows.length, 2, 'should keep both rows');
t.end();
});
test('#file-handler -> processFileData skipArrowCompact', async t => {
const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
const batchC = arrow.tableFromJSON([{lng: -122.6, lat: 38.0}]);
const combined = batchA.concat(batchB).concat(batchC);
const incremental = await processFileData({
content: {
fileName: 'points.arrow',
data: combined.batches,
skipArrowCompact: true
},
fileCache: []
});
t.equal(
incremental[0].data.cols[0].data.length,
combined.batches.length,
'progressive processFileData should not compact Arrow batches'
);
const finished = await processFileData({
content: {
fileName: 'points.arrow',
data: combined.batches
},
fileCache: []
});
t.equal(
finished[0].data.cols[0].data.length,
1,
'final processFileData should compact Arrow batches'
);
const staged = await processFileData({
content: {
fileName: 'points.arrow',
data: combined.batches,
prepareProgressiveReplay: true
},
fileCache: []
});
t.equal(staged[0].progressiveData.length, 3, 'should keep one replay snapshot per batch');
t.deepEqual(
staged[0].progressiveData.map(data => data.cols[0].length),
[1, 2, 3],
'replay snapshots should grow by one batch at a time'
);
t.deepEqual(
staged[0].progressiveData.map(data => data.cols[0].data.length),
[1, 2, 1],
'only the completed replay snapshot should be compacted'
);
const csv = {
data: {fields: [], rows: [[1]]},
info: {id: 'csv', label: 'points.csv', format: 'csv'}
};
const steps = fileCacheToProgressiveSteps([staged[0], csv]);
t.equal(steps.length, 3, 'should create one confirmed Add Data update per Arrow batch');
t.equal(steps[0].length, 2, 'non-progressive files should be included in the first update');
t.equal(steps[1].length, 1, 'later updates should contain only the Arrow dataset');
t.notOk(steps[0][0].progressiveData, 'internal replay data should not enter map state');
t.end();
});