Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
import test from 'tape';
import sinon from 'sinon';
import {console as Console} from 'global/window';
import {DATA_TYPES} from 'type-analyzer';
import cloneDeep from 'es-toolkit/compat/cloneDeep';
import {convertGeometryToWKB} from '@loaders.gl/gis';
import testData, {
dataWithNulls,
parsedDataWithNulls,
testFields,
testAllData,
wktCsv,
wktCsvFields,
wktCsvRows
} from 'test/fixtures/test-csv-data';
import {
geojsonData,
geoJsonWithStyle,
geoStyleFields,
geoStyleRows,
fields as geojsonFields,
rows as geojsonRows
} from 'test/fixtures/geojson';
import testCsvObjectData, {objCsvFields, objCsvRows} from 'test/fixtures/test-csv-object';
import testCsvH3Data, {expectedFields as expectedHexFields} from 'test/fixtures/test-hex-id-data';
import {
parseCsvRowsByFieldType,
processCsvData,
processGeojson,
processKeplerglJSON,
processRowObject,
detectDelimiter
} from '@kepler.gl/processors';
import {validateInputData, createDataContainer} from '@kepler.gl/utils';
import {
ACCEPTED_ANALYZER_TYPES,
analyzerTypeToFieldType,
getFieldsFromData,
getSampleForTypeAnalyze
} from '@kepler.gl/common-utils';
import {formatCsv} from '@kepler.gl/reducers';
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {cmpFields} from '../../helpers/comparison-utils';
test('Processor -> getFieldsFromData', t => {
const data = [
{
time: '2016-09-17 00:09:55',
trip_epoch: '1472688000000',
time_str: 'January 1st 2017 11:00pm ',
value: '4',
novalue: null,
surge: '1.2',
isTrip: 'true',
zeroOnes: '0',
h3_9: '89268cd80b3ffff',
geojson: '{"type":"Point","coordinates":[-122.4194155,37.7749295]}',
wkt: 'POINT (-122.4194155 37.7749295)',
wkb: '0101000020E6100000E17A14AE47D25EC0F6F3F6F2F7F94040'
},
{
time: '2016-09-17 00:30:08',
trip_epoch: '1472860800000',
time_str: 'January 1st 2017 11:01pm ',
value: '3',
novalue: null,
surge: null,
isTrip: 'false',
zeroOnes: '1',
h3_9: '89268cd8103ffff',
geojson:
'{"type":"Polygon","coordinates":[[[-122.4194155,37.7749295],[-122.4194155,37.7749295],[-122.4194155,37.7749295]]]}',
wkt: 'POLYGON ((-122.4194155 37.7749295, -122.4194155 37.7749295, -122.4194155 37.7749295))',
wkb: '0103000020E61000000100000005000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F94040'
},
{
time: null,
trip_epoch: null,
time_str: 'January 1st, 2017 11:02pm ',
value: '2',
novalue: null,
surge: '1.3',
isTrip: null,
zeroOnes: '1',
h3_9: '89268cd8107ffff',
geojson:
'{"type":"LineString","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}',
wkt: 'LINESTRING (-122.4194155 37.7749295, -122.4194155 37.7749295)',
wkb: '0102000020E610000002000000E17A14AE47D25EC0F6F3F6F2F7F94040E17A14AE47D25EC0F6F3F6F2F7F94040'
},
{
time: null,
trip_epoch: null,
time_str: 'January 1st, 2017 11:02pm ',
value: '0',
novalue: null,
surge: '1.4',
isTrip: null,
zeroOnes: '0',
h3_9: '89268cd8113ffff',
geojson:
'{"type":"MultiPoint","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}',
wkt: 'MULTIPOINT (-122.4194155 37.7749295, -122.4194155 37.7749295)',
wkb: '0104000020E6100000020000000101000000E17A14AE47D25EC0F6F3F6F2F7F94040101000000E17A14AE47D25EC0F6F3F6F2F7F94040'
}
];
const headerRow = Object.keys(data[0]);
const fields = getFieldsFromData(data, headerRow);
const expectedFieldTypes = [
'timestamp',
'timestamp',
'timestamp',
'integer',
'string',
'real',
'boolean',
'integer',
'h3',
'geojson',
'geojson',
'geojson'
];
fields.forEach((f, i) =>
t.equal(f.type, expectedFieldTypes[i], `should find field type as ${expectedFieldTypes[i]}`)
);
t.end();
});
test('Processor -> getFieldsFromData BigInt columns', t => {
const data = [
{hexId: 610625465232654335n, total: 29436887n},
{hexId: 610625465081659391n, total: 40685227n}
];
t.doesNotThrow(
() => getFieldsFromData(data, ['hexId', 'total']),
'should not throw when samples contain BigInt (type-analyzer isNaN)'
);
const fields = getFieldsFromData(data, ['hexId', 'total']);
t.notEqual(
fields[0].type,
ALL_FIELD_TYPES.h3,
'integer BigInt columns should not be typed as h3'
);
t.equal(
fields[1].type,
ALL_FIELD_TYPES.integer,
'int64 counts within safe range should stay integer'
);
t.end();
});
test('Processor -> getFieldsFromData scientific notation', t => {
const data = [
{sci: '2.3e+10', small: '1.5E-05', id: '12345678901234567890', hex: '0x1A'},
{sci: '1.1e+09', small: null, id: '12345678901234567891', hex: '0xFF'},
{sci: '4E+07', small: '3e2', id: '12345678901234567892', hex: '0x10'}
];
const fields = getFieldsFromData(data, ['sci', 'small', 'id', 'hex']);
t.equal(fields[0].type, ALL_FIELD_TYPES.real, 'scientific notation should be real');
t.equal(
fields[0].analyzerType,
DATA_TYPES.FLOAT,
'scientific notation should be analyzed as float'
);
t.equal(fields[1].type, ALL_FIELD_TYPES.real, 'scientific notation with nulls should be real');
t.equal(
fields[2].type,
ALL_FIELD_TYPES.string,
'integers beyond the safe range should stay string to keep every digit'
);
t.equal(fields[3].type, ALL_FIELD_TYPES.string, 'hex literals should stay string');
t.end();
});
test('Processor -> geometry autodetection in custom columns', t => {
const point = {type: 'Point', coordinates: [10, 10]};
const line = {
type: 'LineString',
coordinates: [
[10, 10],
[10.25, 10.25]
]
};
const polygon = {
type: 'Polygon',
coordinates: [
[
[10, 10],
[10.25, 10],
[10.25, 10.25],
[10, 10]
]
]
};
const collection = {type: 'GeometryCollection', geometries: [point, line]};
const geometries = [
point,
line,
polygon,
{type: 'MultiPoint', coordinates: [point.coordinates]},
{type: 'MultiLineString', coordinates: [line.coordinates]},
{type: 'MultiPolygon', coordinates: [polygon.coordinates]},
collection
];
const values = [
...geometries.map(geometry => JSON.stringify(geometry)),
collection,
'GEOMETRYCOLLECTION (POINT (10 10), LINESTRING (10 10, 10.25 10.25))',
'GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (10 10)))',
'GEOMETRYCOLLECTION EMPTY',
'geometrycollection z (point z (10 10 5))',
'GEOMETRYCOLLECTION M (POINT M (10 10 5))',
'GEOMETRYCOLLECTION ZM (POINT ZM (10 10 5 7))',
'GEOMETRYCOLLECTION (POINT (-1.25e+1\t+2.5E-1), LINESTRING (.5 -0.25, 1.0 2.0))',
'GEOMETRYCOLLECTION ZM (POINT (1e1 2e1 3e1 4e1))',
'GEOMETRYCOLLECTION (MULTIPOINT ((10 10), (20 20)))',
'GEOMETRYCOLLECTION (MULTIPOINT (10 10, 20 20))',
'GEOMETRYCOLLECTION (MULTILINESTRING ((10 10, 20 20), (30 30, 40 40)))',
'GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 0)))',
'GEOMETRYCOLLECTION (MULTIPOLYGON (((0 0, 1 0, 1 1, 0 0))))',
JSON.stringify({type: 'Feature', geometry: point, properties: {}}),
JSON.stringify({
type: 'Feature',
geometry: {type: 'Point', coordinates: [10, 10, 5]},
properties: {}
}),
JSON.stringify({
type: 'GeometryCollection',
geometries: [collection, {type: 'GeometryCollection', geometries: []}]
}),
...geometries.map(geometry => Buffer.from(convertGeometryToWKB(geometry)).toString('hex'))
];
values.forEach(value => {
const data = processRowObject([{shape: null}, {shape: value}]);
t.equal(data.fields[0].type, ALL_FIELD_TYPES.geojson, `detects ${JSON.stringify(value)}`);
t.deepEqual(data.rows, [[null], [value]], 'preserves values for the geometry parser');
});
for (const values of [
[null, null],
['', null],
['ordinary text', 'more text'],
['0101000000', '0101000000'], // A WKB header without a point payload.
['010000000000000000000000000000000000000000', null],
['12345678901234567890', '12345678901234567891'],
['{"type":"invoice","coordinates":[1,2]}', null],
['{"type":"Point","coordinates":"invalid"}', null],
['{"type":"Point","coordinates":[10,10]}', 'ordinary text'],
[{type: 'GeometryCollection', geometries: [{type: 'invalid'}]}, null]
]) {
t.notEqual(
processRowObject(values.map(shape => ({shape}))).fields[0].type,
ALL_FIELD_TYPES.geojson,
`does not infer geometry from ${JSON.stringify(values)}`
);
}
t.end();
});
test('Processor -> geometry autodetection rejects malformed GeoJSON coordinates', t => {
const invalid = [
{type: 'Point', coordinates: ['x', 'y']},
{type: 'Point', coordinates: [null, null]},
{type: 'Point', coordinates: [10]},
{type: 'Point', coordinates: [[10, 10]]},
{type: 'Point', coordinates: [10, 10, '5']},
{type: 'MultiPoint', coordinates: [10, 10]},
{type: 'LineString', coordinates: [[10, 10]]},
{
type: 'LineString',
coordinates: [
[10, 10],
[null, 20]
]
},
{
type: 'MultiLineString',
coordinates: [
[10, 10],
[20, 20]
]
},
{type: 'Polygon', coordinates: [1, 2]},
{
type: 'Polygon',
coordinates: [
[
[0, 0],
[1, 0],
[0, 0]
]
]
},
{
type: 'Polygon',
coordinates: [
[
[0, 0],
[1, 0],
[1, 1],
[0, 1]
]
]
},
{
type: 'MultiPolygon',
coordinates: [
[
[0, 0],
[1, 0],
[1, 1],
[0, 0]
]
]
}
];
const valid = JSON.stringify({type: 'Point', coordinates: [10, 10]});
for (const geometry of invalid) {
for (const value of [
geometry,
{type: 'Feature', geometry, properties: {}},
{type: 'GeometryCollection', geometries: [geometry]}
].map(value => JSON.stringify(value))) {
for (const samples of [
[value, null],
[valid, value]
]) {
const data = processRowObject(samples.map(shape => ({shape})));
t.equal(
data.fields[0].type,
ALL_FIELD_TYPES.object,
`keeps malformed geometry as an object: ${value}`
);
}
}
}
const nonFinite = {
type: 'GeometryCollection',
geometries: [{type: 'Point', coordinates: [Infinity, NaN]}]
};
t.equal(
processRowObject([{shape: nonFinite}]).fields[0].type,
ALL_FIELD_TYPES.object,
'rejects non-finite coordinates in collection objects'
);
t.end();
});
test('Processor -> geometry autodetection rejects incomplete or invalid collection WKT', t => {
const valid = 'GEOMETRYCOLLECTION (POINT (10 10))';
for (const value of [
'GEOMETRYCOLLECTION (',
'GEOMETRYCOLLECTION (ordinary text)',
'GEOMETRYCOLLECTION EMPTY trailing text',
`${valid} trailing text`,
'GEOMETRYCOLLECTION (POINT (10 10)',
'GEOMETRYCOLLECTION (POINT (10 10)))',
'GEOMETRYCOLLECTION (POINT (10 10),)',
'GEOMETRYCOLLECTION (POINT (10 10) POINT (20 20))',
'GEOMETRYCOLLECTION (POINT (x y))',
'GEOMETRYCOLLECTION (POINT (1-2))',
'GEOMETRYCOLLECTION (POINT (1e999 10))',
'GEOMETRYCOLLECTION M (POINT (10 10 1e999))',
'GEOMETRYCOLLECTION (POINT (10 10, 20 20))',
'GEOMETRYCOLLECTION (LINESTRING (10 10))',
'GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 1)))',
'GEOMETRYCOLLECTION Z (POINT (10 10))',
'GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (ordinary text))',
`${'GEOMETRYCOLLECTION ('.repeat(34)}POINT (10 10)${')'.repeat(34)}`
]) {
for (const samples of [
[null, value],
[valid, value]
]) {
const data = processRowObject(samples.map(shape => ({shape})));
t.equal(data.fields[0].type, ALL_FIELD_TYPES.string, `keeps invalid WKT as text: ${value}`);
t.deepEqual(
data.rows,
samples.map(value => [value]),
'preserves the original text'
);
}
}
t.end();
});
test('Processor -> geometry autodetection scans complete large WKT collections', t => {
const coordinates = Array.from({length: 10000}, (_, index) => `${10 + index / 1000} 10`).join(
', '
);
const valid = `GEOMETRYCOLLECTION (LINESTRING (${coordinates}), POINT (20 20))`;
const data = processRowObject([{shape: valid}, {shape: valid}]);
t.equal(
data.fields[0].type,
ALL_FIELD_TYPES.geojson,
'recognizes all members after a large coordinate sequence'
);
t.deepEqual(data.rows, [[valid], [valid]], 'preserves the full large WKT values');
for (const invalid of [
valid.slice(0, -1),
`${valid} trailing text`,
valid.replace('POINT (20 20)', 'POINT (x y)'),
valid.replace(', POINT', ' POINT')
]) {
t.equal(
processRowObject([{shape: valid}, {shape: invalid}]).fields[0].type,
ALL_FIELD_TYPES.string,
'checks the end of large samples before promoting a mixed column'
);
}
t.end();
});
test('Processor -> processCsvData', t => {
t.throws(() => processCsvData(''), 'should throw if csv is empty');
// load sample dataset csv as text
const {fields, rows} = processCsvData(cloneDeep(testData));
t.equal(rows.length, testAllData.length, `should return ${testAllData.length} rows`);
cmpFields(t, testFields, fields, 'should parse rows correctly');
t.deepEqual(rows, testAllData, 'should parse rows correctly');
rows.forEach((r, i) => {
t.deepEqual(r, testAllData[i], `should parse row ${i} correctly`);
});
t.end();
});
test('Processor -> detectDelimiter', t => {
t.equal(detectDelimiter('a,b,c\n1,2,3'), ',', 'should detect comma delimiter');
t.equal(detectDelimiter('a\tb\tc\n1\t2\t3'), '\t', 'should detect tab delimiter');
t.equal(detectDelimiter('a;b;c\n1;2;3'), ';', 'should detect semicolon delimiter');
t.equal(detectDelimiter('a|b|c\n1|2|3'), '|', 'should detect pipe delimiter');
t.equal(
detectDelimiter('single_column\nvalue'),
',',
'should fall back to comma when no delimiter produces multiple columns'
);
t.equal(detectDelimiter(''), ',', 'should return comma for empty string');
t.equal(detectDelimiter('a,b,c'), ',', 'should handle input without newline (single line)');
t.equal(
detectDelimiter('"a\tb"\tc\td\n1\t2\t3'),
'\t',
'should handle quoted fields containing other delimiters'
);
t.equal(
detectDelimiter('"city, state"\tpopulation\tarea\n"New York, NY"\t8000000\t302'),
'\t',
'should detect tab even when commas appear inside quoted fields'
);
t.equal(
detectDelimiter('name;age;city\nAlice;30;"Berlin, Germany"'),
';',
'should detect semicolon with quoted fields containing commas'
);
t.equal(
detectDelimiter('a\tb\tc\td\te\n1\t2\t3\t4\t5'),
'\t',
'should prefer delimiter that produces more columns'
);
t.end();
});
test('Processor -> processCsvData -> tab-separated', t => {
const tsvData = 'name\tage\tcity\nAlice\t30\tBerlin\nBob\t25\tParis';
const result = processCsvData(tsvData);
t.equal(result.fields.length, 3, 'should parse 3 fields from TSV');
t.equal(result.fields[0].name, 'name', 'first field should be name');
t.equal(result.fields[1].name, 'age', 'second field should be age');
t.equal(result.fields[2].name, 'city', 'third field should be city');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.deepEqual(result.rows[0], ['Alice', 30, 'Berlin'], 'should parse first row correctly');
t.deepEqual(result.rows[1], ['Bob', 25, 'Paris'], 'should parse second row correctly');
t.end();
});
test('Processor -> processCsvData -> semicolon-separated', t => {
const ssvData = 'name;value;active\nfoo;100;true\nbar;200;false';
const result = processCsvData(ssvData);
t.equal(result.fields.length, 3, 'should parse 3 fields from semicolon-separated data');
t.equal(result.fields[0].name, 'name', 'first field should be name');
t.equal(result.fields[1].name, 'value', 'second field should be value');
t.equal(result.fields[2].name, 'active', 'third field should be active');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.deepEqual(result.rows[0], ['foo', 100, true], 'should parse first row correctly');
t.deepEqual(result.rows[1], ['bar', 200, false], 'should parse second row correctly');
t.end();
});
test('Processor -> processCsvData -> pipe-separated', t => {
const psvData = 'id|name|score\n1|Alice|95.5\n2|Bob|87.3';
const result = processCsvData(psvData);
t.equal(result.fields.length, 3, 'should parse 3 fields from pipe-separated data');
t.equal(result.fields[0].name, 'id', 'first field should be id');
t.equal(result.fields[1].name, 'name', 'second field should be name');
t.equal(result.fields[2].name, 'score', 'third field should be score');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.end();
});
test('Processor -> processCsvData -> semicolon with quoted commas', t => {
const data =
'"City, Country";Population;Area\n"Berlin, Germany";3600000;891\n"Paris, France";2100000;105';
const result = processCsvData(data);
t.equal(result.fields.length, 3, 'should parse 3 fields');
t.equal(result.fields[0].name, 'City, Country', 'should preserve comma inside quotes');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.equal(result.rows[0][0], 'Berlin, Germany', 'should preserve quoted value with comma');
t.end();
});
test('Processor -> detectDelimiter -> Windows line endings (CRLF)', t => {
t.equal(
detectDelimiter('a\tb\tc\r\n1\t2\t3\r\n'),
'\t',
'should detect tab delimiter with CRLF line endings'
);
t.equal(
detectDelimiter('a;b;c\r\n1;2;3\r\n'),
';',
'should detect semicolon delimiter with CRLF line endings'
);
t.end();
});
test('Processor -> detectDelimiter -> ambiguous cases', t => {
t.equal(
detectDelimiter('a,b\tc,d\n1,2\t3,4'),
',',
'should prefer comma when comma produces more columns than tab'
);
t.equal(
detectDelimiter('a\tb\tc\td,e\n1\t2\t3\t4,5'),
'\t',
'should prefer tab when tab produces more columns than comma'
);
t.equal(
detectDelimiter('a;b;c;d|e\n1;2;3;4|5'),
';',
'should prefer semicolon when it produces more columns than pipe'
);
t.end();
});
test('Processor -> detectDelimiter -> trailing and leading whitespace', t => {
t.equal(
detectDelimiter(' a\tb\tc \n 1\t2\t3 '),
'\t',
'should detect tab even with surrounding whitespace'
);
t.equal(
detectDelimiter(' a ; b ; c \n 1 ; 2 ; 3 '),
';',
'should detect semicolon even with spaces around values'
);
t.end();
});
test('Processor -> processCsvData -> tab-separated with empty fields', t => {
const tsvData = 'name\tage\tcity\nAlice\t\tBerlin\n\t25\t';
const result = processCsvData(tsvData);
t.equal(result.fields.length, 3, 'should parse 3 fields');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.equal(result.rows[0][0], 'Alice', 'first row first value should be Alice');
t.equal(result.rows[0][1], null, 'first row second value should be null (empty)');
t.equal(result.rows[0][2], 'Berlin', 'first row third value should be Berlin');
t.equal(result.rows[1][0], null, 'second row first value should be null (empty)');
t.end();
});
test('Processor -> processCsvData -> tab-separated with many columns', t => {
const headers = Array.from({length: 20}, (_, i) => `col${i}`).join('\t');
const row1 = Array.from({length: 20}, (_, i) => `val${i}`).join('\t');
const row2 = Array.from({length: 20}, (_, i) => `row2_${i}`).join('\t');
const tsvData = `${headers}\n${row1}\n${row2}`;
const result = processCsvData(tsvData);
t.equal(result.fields.length, 20, 'should parse 20 fields from wide TSV');
t.equal(result.fields[0].name, 'col0', 'first field should be col0');
t.equal(result.fields[19].name, 'col19', 'last field should be col19');
t.equal(result.rows.length, 2, 'should have 2 data rows');
t.equal(result.rows[0][0], 'val0', 'first cell should be val0');
t.equal(result.rows[0][19], 'val19', 'last cell should be val19');
t.end();
});
test('Processor -> processCsvData -> semicolon-separated with numeric data', t => {
const data = 'lat;lng;value\n52.52;13.405;1000.5\n48.8566;2.3522;2000.7\n40.4168;-3.7038;1500.3';
const result = processCsvData(data);
t.equal(result.fields.length, 3, 'should parse 3 fields');
t.equal(result.rows.length, 3, 'should have 3 data rows');
t.equal(result.rows[0][0], 52.52, 'should parse lat as number');
t.equal(result.rows[0][1], 13.405, 'should parse lng as number');
t.equal(result.rows[0][2], 1000.5, 'should parse value as number');
t.end();
});
test('Processor -> processCsvData -> scientific notation', t => {
const data = 'lat,lng,value\n37.1,-122.1,2.3e+10\n37.2,-122.2,1.1E+09\n37.3,-122.3,4E+07';
const {fields, rows} = processCsvData(data);
t.equal(fields[2].type, ALL_FIELD_TYPES.real, 'should type scientific notation as real');
t.deepEqual(
rows.map(r => r[2]),
[2.3e10, 1.1e9, 4e7],
'should parse scientific notation as numbers'
);
t.end();
});
test('Processor -> processCsvData -> pipe-separated with special characters in values', t => {
const data =
'id|description|url\n1|"hello, world"|http://example.com\n2|"foo; bar"|http://test.org';
const result = processCsvData(data);
t.equal(result.fields.length, 3, 'should parse 3 fields');
t.equal(
result.rows[0][1],
'hello, world',
'should handle commas inside quoted pipe-separated fields'
);
t.equal(
result.rows[1][1],
'foo; bar',
'should handle semicolons inside quoted pipe-separated fields'
);
t.end();
});
test('Processor -> processCsvData -> tab-separated preserves original comma data', t => {
const csvData = 'a,b,c\n1,2,3\n4,5,6';
const result = processCsvData(csvData);
t.equal(result.fields.length, 3, 'regular CSV should still parse correctly');
t.equal(result.fields[0].name, 'a', 'field name should be a');
t.deepEqual(result.rows[0], [1, 2, 3], 'first row should be [1,2,3]');
t.deepEqual(result.rows[1], [4, 5, 6], 'second row should be [4,5,6]');
t.end();
});
test('Processor -> processCsvData -> trailing newline does not break parsing', t => {
const tsvData = 'name\tage\nAlice\t30\nBob\t25\n';
const result = processCsvData(tsvData);
t.equal(result.fields.length, 2, 'should parse 2 fields');
t.equal(result.rows.length, 2, 'should have 2 data rows (trailing newline ignored)');
const ssvData = 'x;y;z\n1;2;3\n4;5;6\n';
const result2 = processCsvData(ssvData);
t.equal(result2.fields.length, 3, 'semicolon: should parse 3 fields');
t.equal(result2.rows.length, 2, 'semicolon: trailing newline should not add empty row');
t.end();
});
test('Processor -> processCsvData -> single data row (header + 1 row)', t => {
const tsvData = 'x\ty\tz\n10\t20\t30';
const result = processCsvData(tsvData);
t.equal(result.fields.length, 3, 'should parse 3 fields');
t.equal(result.rows.length, 1, 'should have 1 data row');
t.deepEqual(result.rows[0], [10, 20, 30], 'should parse the single row correctly');
t.end();
});
test('Processor -> processCsvData -> timestamps in TSV', t => {
const tsvData = 'timestamp\tvalue\n2023-01-15 10:30:00\t100\n2023-02-20 14:45:30\t200';
const result = processCsvData(tsvData);
t.equal(result.fields.length, 2, 'should parse 2 fields');
t.equal(result.fields[0].type, 'timestamp', 'should detect timestamp type');
t.equal(result.rows.length, 2, 'should have 2 rows');
t.end();
});
test('Processor -> processCsvData -> duplicated field name', t => {
const testData1 = `column1,column1,column1,column2\na,b,c,d\nc,d,e,f`;
// load sample dataset csv as text
const result = processCsvData(testData1);
const expectedResult = {
fields: [
{
name: 'column1',
id: 'column1',
format: '',
fieldIdx: 0,
displayName: 'column1',
type: 'string',
analyzerType: 'STRING',
valueAccessor: values => values[0]
},
{
name: 'column1-0',
id: 'column1-0',
format: '',
fieldIdx: 1,
displayName: 'column1-0',
type: 'string',
analyzerType: 'STRING',
valueAccessor: values => values[1]
},
{
name: 'column1-1',
id: 'column1-1',
format: '',
fieldIdx: 2,
displayName: 'column1-1',
type: 'string',
analyzerType: 'STRING',
valueAccessor: values => values[2]
},
{
name: 'column2',
id: 'column2',
format: '',
fieldIdx: 3,
displayName: 'column2',
type: 'string',
analyzerType: 'STRING',
valueAccessor: values => values[3]
}
],
rows: [
['a', 'b', 'c', 'd'],
['c', 'd', 'e', 'f']
]
};
cmpFields(t, result.fields, expectedResult.fields, 'should have created non duplicated fields');
t.deepEqual(
result.rows,
expectedResult.rows,
'should have computed rows with non duplicated fields'
);
t.end();
});
test('Processor -> processCsvData -> with nulls', t => {
const {fields, rows} = processCsvData(dataWithNulls);
cmpFields(t, fields, testFields, 'should parse fields correctly');
t.deepEqual(rows, parsedDataWithNulls, 'should parse rows correctly');
rows.forEach((r, i) => {
t.deepEqual(r, parsedDataWithNulls[i], `should parse row ${i} correctly`);
});
t.end();
});
test('Processor -> processCsvData -> wkt', t => {
const {fields, rows} = processCsvData(wktCsv);
cmpFields(t, fields, wktCsvFields, 'should find geometry fields as type:geojson');
rows.forEach((r, i) => {
t.deepEqual(r, wktCsvRows[i], `should process wkt rows[${i}] correctly`);
});
t.deepEqual(rows, wktCsvRows, 'should process wkt rows correctly');
t.end();
});
test('Processor -> processCsvData -> w/ array and object', t => {
const {fields, rows} = processCsvData(testCsvObjectData);
cmpFields(t, fields, objCsvFields, 'should find csv object fields as type:object');
t.equal(rows.length, objCsvRows.length, 'should have same row length');
rows.forEach((r, i) => {
t.deepEqual(r, objCsvRows[i], 'should format correct csv object rows');
});
t.end();
});
test('Processor -> processCsvData -> w/ hex id', t => {
const {fields, rows} = processCsvData(testCsvH3Data);
cmpFields(t, fields, expectedHexFields, 'should find csv object fields as h3');
t.equal(rows.length, 22, 'should have same row length');
t.end();
});
test('Processor => processGeojson', t => {
const {fields, rows} = processGeojson(cloneDeep(geojsonData));
cmpFields(t, fields, geojsonFields, 'should have same field length');
t.equal(rows.length, geojsonRows.length, 'should have same row length');
rows.forEach((r, i) => {
t.deepEqual(r, geojsonRows[i], 'should format correct geojson rows');
});
t.end();
});
test('Processor => processGeojson: with style property', t => {
const {fields, rows} = processGeojson(cloneDeep(geoJsonWithStyle));
cmpFields(t, fields, geoStyleFields, 'should preserve objects in geojson properties');
t.deepEqual(rows, geoStyleRows, 'should preserve objects in geojson properties');
t.end();
});
test('Processor => processGeojson:invalid geojson', t => {
t.throws(
() => processGeojson({fields: [], rows: []}),
'Should throw error with invalid geojson type'
);
t.end();
});
test('Processor => processGeojson: rejects explicit unsupported CRS declarations', t => {
for (const crs of [
{type: 'name', properties: {name: 'EPSG:3857'}},
{type: 'name', properties: {name: 'urn:ogc:def:crs:EPSG::2263'}},
{type: 'name', properties: {name: 'urn:ogc:def:crs:OGC:2:27'}},
{type: 'name', properties: {name: 'unknown-projection'}},
{type: 'link', properties: {href: 'https://example.com/projected-crs'}}
]) {
const geometry = {type: 'Point', coordinates: [1000000, 1000000]};
const feature = {type: 'Feature', properties: {}, geometry};
for (const input of [
{type: 'FeatureCollection', crs, features: [feature]},
{...feature, crs},
{...feature, geometry: {...geometry, crs}},
{type: 'FeatureCollection', features: [{...feature, crs}]},
{type: 'GeometryCollection', geometries: [{...geometry, crs}]}
]) {
t.throws(
() => processGeojson(input),
/WGS84.*EPSG:4326/,
'unsupported CRS should explain the required coordinate system'
);
}
}
t.end();
});
test('Processor => processGeojson: preserves supported and undeclared CRS data', t => {
const aliases = [
'EPSG:4326',
'epsg:4326',
'WGS84',
'WGS 84',
'CRS84',
'OGC:CRS84',
'CRS84h',
'OGC:CRS84h',
'urn:ogc:def:crs:EPSG::4326',
'urn:ogc:def:crs:EPSG:6.6:4326',
'urn:x-ogc:def:crs:EPSG:4326',
'urn:ogc:def:crs:OGC:1.3:CRS84',
'urn:ogc:def:crs:OGC:2:84',
'urn:x-ogc:def:crs:OGC:2:84',
'urn:ogc:def:crs:OGC:0:CRS84h',
'http://www.opengis.net/def/crs/EPSG/0/4326',
'https://www.opengis.net/def/crs/OGC/1.3/CRS84',
'http://www.opengis.net/def/crs/OGC/0/CRS84h',
'https://www.opengis.net/def/crs/OGC/0/CRS84h',
'http://www.opengis.net/gml/srs/epsg.xml#4326'
];
const features = [
{type: 'Feature', properties: {crs: 'an ordinary property'}, geometry: null},
{
type: 'Feature',
properties: {crs: {type: 'name', properties: {name: 'EPSG:3857'}}},
geometry: {type: 'Point', coordinates: [181, 20, 100]}
},
{type: 'Feature', properties: {}, geometry: {type: 'Polygon', coordinates: []}}
];
const expected = processGeojson({type: 'FeatureCollection', features: cloneDeep(features)});
for (const crs of [
undefined,
null,
...aliases.map(name => ({type: 'name', properties: {name}})),
{type: 'link', properties: {href: 'http://www.opengis.net/def/crs/EPSG/0/4326'}},
{type: 'link', properties: {href: 'https://www.opengis.net/def/crs/OGC/1.3/CRS84'}},
{type: 'link', properties: {href: 'http://www.opengis.net/def/crs/OGC/0/CRS84h'}},
{type: 'link', properties: {href: 'https://www.opengis.net/def/crs/OGC/0/CRS84h'}}
]) {
const data = processGeojson({type: 'FeatureCollection', crs, features: cloneDeep(features)});
t.deepEqual(data.rows, expected.rows, 'CRS recognition must not rewrite or drop geometry');
t.deepEqual(
processGeojson({type: 'FeatureCollection', crs, features: []}).rows,
[],
'empty collections remain valid'
);
}
t.end();
});
test('Processor => processGeojson: leaves malformed CRS schema validation unchanged', t => {
for (const crs of [{}, 'EPSG:3857', {type: 'name'}, {type: 'name', properties: {name: 3857}}]) {
t.doesNotThrow(
() => processGeojson({type: 'FeatureCollection', crs, features: []}),
'malformed CRS declarations retain their existing behavior'
);
}
t.end();
});
test('Processor => processGeojson: parse rows', t => {
const testGeoData = {
type: 'FeatureCollection',
features: [
{
type: 'Feature',
properties: {TRIPS: '11', RATE: 'a', TIME: '1570749707'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
{
type: 'Feature',
properties: {TRIPS: '10', RATE: 'b', TIME: '1570749707'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
{
type: 'Feature',
properties: {TRIPS: '9', RATE: 'b', TIME: '1570749707'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
}
]
};
const expectedFields = [
{
name: '_geojson',
id: '_geojson',
displayName: '_geojson',
format: '',
fieldIdx: 0,
type: 'geojson',
analyzerType: 'GEOMETRY',
valueAccessor: values => values[0]
},
{
name: 'TRIPS',
id: 'TRIPS',
displayName: 'TRIPS',
format: '',
fieldIdx: 1,
type: 'integer',
analyzerType: 'INT',
valueAccessor: values => values[1]
},
{
name: 'RATE',
id: 'RATE',
displayName: 'RATE',
format: '',
fieldIdx: 2,
type: 'string',
analyzerType: 'STRING',
valueAccessor: values => values[2]
},
{
name: 'TIME',
id: 'TIME',
displayName: 'TIME',
format: 'x',
fieldIdx: 3,
type: 'timestamp',
analyzerType: 'TIME',
valueAccessor: values => values[3]
}
];
const expectedRows = [
[
{
type: 'Feature',
properties: {TRIPS: 11, RATE: 'a', TIME: 1570749707},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
11,
'a',
1570749707
],
[
{
type: 'Feature',
properties: {TRIPS: 10, RATE: 'b', TIME: 1570749707},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
10,
'b',
1570749707
],
[
{
type: 'Feature',
properties: {TRIPS: 9, RATE: 'b', TIME: 1570749707},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
9,
'b',
1570749707
]
];
const result = processGeojson(testGeoData);
t.deepEqual(Object.keys(result), ['fields', 'rows'], 'should contain fields and rows');
cmpFields(
t,
result.fields,
expectedFields,
'should parse correct fields when geojson input contain string'
);
t.deepEqual(
result.rows,
expectedRows,
'should parse correct rows when geojson input contain string'
);
t.end();
});
test('Processor -> parseCsvRowsByFieldType -> real', t => {
const field = {
type: ALL_FIELD_TYPES.real
};
const rows = [
['0.0'],
['1.0'],
['-1.0'],
['155.0'],
['1.55e3'],
['1.55e-3'],
[' 1.55e-3 '],
[' 1.55e+3 xyz']
];
const expected = [[0.0], [1.0], [-1.0], [155.0], [1550.0], [0.00155], [0.00155], [1550.0]];
parseCsvRowsByFieldType(rows, -1, field, 0);
t.same(rows, expected, 'should parsed reals properly');
t.end();
});
test('Processor -> parseCsvRowsByFieldType -> integer', t => {
const field = {
type: ALL_FIELD_TYPES.integer
};
const rows = [['0'], ['1'], ['-1'], ['155'], [' 155 '], [' 155 xyz']];
const expected = [[0], [1], [-1], [155], [155], [155]];
parseCsvRowsByFieldType(rows, -1, field, 0);
t.same(rows, expected, 'should parsed ints properly');
t.end();
});
test('Processor -> parseCsvRowsByFieldType -> boolean', t => {
const field = {
type: ALL_FIELD_TYPES.boolean
};
const rows = [
[null],
['0'],
['1'],
['True'],
['False'],
['0'],
['1'],
['true'],
['yes'],
['Yes'],
['YES'],
['no'],
['No'],
['NO'],
['TRUE'],
['false'],
['FALSE']
];
const expected = [
[null],
[false],
[true],
[true],
[false],
[false],
[true],
[true],
[true],
[true],
[true],
[false],
[false],
[false],
[true],
[false],
[false]
];
parseCsvRowsByFieldType(rows, -1, field, 0);
t.same(rows, expected, 'should parsed boolean properly');
t.end();
});
test('Processor -> getSampleForTypeAnalyze', t => {
const fields = ['string', 'int', 'bool', 'time'];
const rows = [
['a', 0, true, null],
['b', 2, false, null],
['c', 3, true, '2017-01-01'],
[null, 1, false, '2017-01-02'],
['d', 6, false, '2017-01-03'],
['e', 4, true, null],
['f', 5, true, undefined],
['g', null, true, null],
['h', undefined, true, '2017-01-04']
];
const sample = getSampleForTypeAnalyze({fields, rows, sampleCount: 5});
const expected = [
{
string: 'a',
int: 0,
bool: true,
time: '2017-01-01'
},
{
string: 'b',
int: 2,
bool: false,
time: '2017-01-02'
},
{
string: 'c',
int: 3,
bool: true,
time: '2017-01-03'
},
{
string: 'd',
int: 1,
bool: false,
time: '2017-01-04'
},
{
string: 'e',
int: 6,
bool: false,
time: null
}
];
t.deepEqual(sample, expected, 'Should find correct sample for type analyzer');
t.end();
});
test('Processor -> validateInputData', t => {
t.equal(validateInputData(null), null, 'Should throw error if data is null');
t.equal(validateInputData({rows: 'hello'}), null, 'Should throw error if data.rows is null');
t.equal(
validateInputData({rows: [], fields: null}),
null,
'Should throw error if data.fields is null'
);
const cases = [
{
input: {
rows: [[1], [2], [3]],
fields: ['a']
},
expected: {
rows: [[1], [2], [3]],
fields: [{type: ALL_FIELD_TYPES.integer, format: '', name: 'column_0', analyzerType: 'INT'}]
},
msg: 'should re generate field if not an object'
},
{
input: {
rows: [[1], [2], [3]],
fields: [{type: ALL_FIELD_TYPES.integer, format: ''}]
},
expected: {
rows: [[1], [2], [3]],
fields: [{type: ALL_FIELD_TYPES.integer, format: '', name: 'column_0', analyzerType: 'INT'}]
},
msg: 'should reassign field name'
},
{
input: {
rows: [[1], [2], [3]],
fields: [{type: 'hello', format: '', name: 'taro'}]
},
expected: {
rows: [[1], [2], [3]],
fields: [
{
type: ALL_FIELD_TYPES.integer,
format: '',
name: 'taro',
analyzerType: 'INT'
}
]
},
msg: 'should reassign field type'
},
{
input: {
rows: [['2018-09-01 00:00'], ['2018-09-01 01:00'], ['2018-09-01 02:00']],
fields: [{type: ALL_FIELD_TYPES.timestamp, format: '', name: 'taro'}]
},
expected: {
rows: [['2018-09-01 00:00'], ['2018-09-01 01:00'], ['2018-09-01 02:00']],
fields: [
{
type: ALL_FIELD_TYPES.timestamp,
format: 'YYYY-M-D H:m',
name: 'taro',
analyzerType: 'DATETIME'
}
]
},
msg: 'should reassign field type'
}
];
cases.forEach(({input, expected, msg}) => {
t.deepEqual(validateInputData(input), expected, msg);
});
t.end();
});
test('Processor -> processRowObject', t => {
t.equal(processRowObject({}), null, 'Should return null when rawData is empty');
const cases = [
{
input: [
{a: 1, b: 'c', c: true, d: '1.2'},
{a: 2, b: 'c', c: false, d: '1.3'},
{a: 3, b: 'd', c: true, d: '1.4'}
],
expected: {
rows: [
[1, 'c', true, 1.2],
[2, 'c', false, 1.3],
[3, 'd', true, 1.4]
],
fields: [
{
name: 'a',
id: 'a',
displayName: 'a',
type: 'integer',
format: '',
fieldIdx: 0,
analyzerType: 'INT',
valueAccessor: values => values[0]
},
{
name: 'b',
id: 'b',
displayName: 'b',
type: 'string',
format: '',
fieldIdx: 1,
analyzerType: 'STRING',
valueAccessor: values => values[1]
},
{
name: 'c',
id: 'c',
displayName: 'c',
type: 'boolean',
format: '',
fieldIdx: 2,
analyzerType: 'BOOLEAN',
valueAccessor: values => values[2]
},
{
name: 'd',
id: 'd',
displayName: 'd',
type: 'real',
format: '',
fieldIdx: 3,
analyzerType: 'FLOAT',
valueAccessor: values => values[3]
}
]
},
msg: 'should parse correct row objects'
}
];
cases.forEach(({input, expected, msg}) => {
const {fields, rows} = processRowObject(input);
cmpFields(t, fields, expected.fields, `${msg} fields`);
t.deepEqual(rows, expected.rows, msg);
});
t.end();
});
test('Processor -> formatCsv', t => {
const geojsonFc = {
type: 'FeatureCollection',
features: [
{
type: 'Feature',
properties: {TRIPS: '11'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
{
type: 'Feature',
properties: {TRIPS: '10'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
},
{
type: 'Feature',
properties: {TRIPS: '9'},
geometry: {type: 'Point', coordinates: [[-122, 37]]}
}
]
};
const cases = [
{
input:
`gps_data.lat,gps_data.types,epoch,has_result,id,begintrip_ts_local,date\n` +
`29.9900937,driver_analytics,1472688000000,False,1,2016-10-01 09:41:39+00:00,2016-09-23\n` +
`29.9927699,driver_analytics,1472688000000,False,2,2016-10-01 16:46:37+00:00,2016-09-23\n` +
`29.9907261,driver_analytics,1472688000000,False,3,,2016-09-23`,
expected:
`gps_data.lat,gps_data.types,epoch,has_result,id,begintrip_ts_local,date\n` +
`29.9900937,driver_analytics,1472688000000,false,1,2016-10-01 09:41:39+00:00,2016-09-23\n` +
`29.9927699,driver_analytics,1472688000000,false,2,2016-10-01 16:46:37+00:00,2016-09-23\n` +
`29.9907261,driver_analytics,1472688000000,false,3,,2016-09-23`,
processor: processCsvData,
msg: 'should format correct csv'
},
{
input: geojsonFc,
expected:
'_geojson,TRIPS\n"{""type"":""Feature"",""properties"":{""TRIPS"":11},""geometry"":{""type"":""Point"",""coordinates"":[[-122,37]]}}",11\n"{""type"":""Feature"",""properties"":{""TRIPS"":10},""geometry"":{""type"":""Point"",""coordinates"":[[-122,37]]}}",10\n"{""type"":""Feature"",""properties"":{""TRIPS"":9},""geometry"":{""type"":""Point"",""coordinates"":[[-122,37]]}}",9',
processor: processGeojson,
msg: 'should format correct csv from geojsonInput'
}
];
cases.forEach(({input, expected, msg, processor}) => {
const {fields, rows} = processor(input);
const dataContainer = createDataContainer(rows, {fields});
t.deepEqual(formatCsv(dataContainer, fields), expected, msg);
});
const {fields, rows} = processCsvData(
`name,secret\n` + `Harbor,hidden-value\n` + `Pier,also-hidden`
);
const dataContainer = createDataContainer(rows, {fields});
t.deepEqual(
formatCsv(dataContainer, fields, ['secret']),
`name\nHarbor\nPier`,
'should omit hidden columns'
);
t.end();
});
test('Processor -> analyzerTypeToFieldType', t => {
Object.keys(DATA_TYPES).forEach(atype => {
const spy = sinon.spy(Console, 'warn');
if (!ACCEPTED_ANALYZER_TYPES.includes(atype)) {
t.equal(
analyzerTypeToFieldType(atype),
'string',
'should return string if type not recognized'
);
t.ok(spy.calledOnce, `should warn when pass unrecognized type ${atype}`);
} else {
const fieldType = analyzerTypeToFieldType(atype);
t.ok(ALL_FIELD_TYPES[fieldType], `should assign ${atype} to one of ALL_FIELD_TYPES`);
}
spy.restore();
});
t.end();
});
test('Processor -> processKeplerglJSON keeps display info', t => {
const parsed = processKeplerglJSON({
datasets: [],
config: {
version: 'v1',
config: {}
},
info: {
app: 'kepler.gl',
created_at: 'today',
title: 'Harbor',
description: 'Ferry routes',
source: 'kepler.gl'
}
});
t.equal(parsed.info.title, 'Harbor', 'should keep the map title');
t.equal(parsed.info.description, 'Ferry routes', 'should keep the map description');
t.equal(parsed.info.app, undefined, 'should not copy export metadata onto map info');
t.equal(parsed.info.source, undefined, 'should not copy the export source onto map info');
t.end();
});
test('Processor -> processKeplerglJSON replaces display info', t => {
const titleOnly = processKeplerglJSON({
datasets: [],
config: {version: 'v1', config: {}},
info: {app: 'kepler.gl', created_at: 'today', title: 'Harbor'}
});
t.ok(
'description' in titleOnly.info,
'should always return description so it clears the previous map'
);
t.equal(titleOnly.info.description, undefined, 'should not keep a missing description');
const noInfo = processKeplerglJSON({
datasets: [],
config: {version: 'v1', config: {}}
});
t.deepEqual(
noInfo.info,
{title: undefined, description: undefined},
'a saved map without display info should clear both values'
);
t.end();
});