Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(processors): detect geometry in custom columns (#3869)

Igor Dykhtad60bf45fc032
diff --git a/src/common-utils/src/data-type.ts b/src/common-utils/src/data-type.ts
index ecdf7a58c..3d5b937f8 100644
--- a/src/common-utils/src/data-type.ts
+++ b/src/common-utils/src/data-type.ts
@@ -12,7 +12,7 @@ import {h3IsValid} from './h3-utils';
const H3_ANALYZER_TYPE = 'H3';
// a plain decimal number, optionally in scientific notation: 12, -0.5, .5, 1.5e+10, 2E-4
-const DECIMAL_NUMBER_REGEX = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
+const DECIMAL_NUMBER_REGEX = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i;
const INTEGER_REGEX = /^[+-]?\d+$/;
/**
@@ -53,6 +53,187 @@ function toTypeAnalyzerValue(value: unknown): unknown {
return value;
}
+function isGeoJsonGeometry(value: unknown, depth = 0): boolean {
+ if (!value || typeof value !== 'object' || depth > 32) return false;
+ const geometry = value as Record<string, unknown>;
+ if (geometry.type === 'Feature') return isGeoJsonGeometry(geometry.geometry, depth + 1);
+ if (geometry.type === 'GeometryCollection') {
+ return (
+ Array.isArray(geometry.geometries) &&
+ geometry.geometries.every(child => isGeoJsonGeometry(child, depth + 1))
+ );
+ }
+ const position = (coordinates: unknown): coordinates is number[] =>
+ Array.isArray(coordinates) &&
+ coordinates.length >= 2 &&
+ coordinates.every(coordinate => typeof coordinate === 'number' && Number.isFinite(coordinate));
+ const positions = (coordinates: unknown, minimum: number): coordinates is number[][] =>
+ Array.isArray(coordinates) &&
+ (coordinates.length === 0 || coordinates.length >= minimum) &&
+ coordinates.every(position);
+ const ring = (coordinates: unknown): boolean =>
+ positions(coordinates, 4) &&
+ (coordinates.length === 0 ||
+ (coordinates[0].length === coordinates[coordinates.length - 1].length &&
+ coordinates[0].every(
+ (coordinate, index) => coordinate === coordinates[coordinates.length - 1][index]
+ )));
+ const polygon = (coordinates: unknown): boolean =>
+ Array.isArray(coordinates) && coordinates.every(ring);
+ switch (geometry.type) {
+ case 'Point':
+ return position(geometry.coordinates);
+ case 'MultiPoint':
+ return positions(geometry.coordinates, 1);
+ case 'LineString':
+ return positions(geometry.coordinates, 2);
+ case 'MultiLineString':
+ return (
+ Array.isArray(geometry.coordinates) &&
+ geometry.coordinates.every(line => positions(line, 2))
+ );
+ case 'Polygon':
+ return polygon(geometry.coordinates);
+ case 'MultiPolygon':
+ return Array.isArray(geometry.coordinates) && geometry.coordinates.every(polygon);
+ default:
+ return false;
+ }
+}
+
+/** Validate the complete WKT collection before changing a text column's type. */
+function isWktGeometryCollection(value: string): boolean {
+ let cursor = 0;
+ const consume = (pattern: RegExp): string | undefined => {
+ const match = pattern.exec(value.slice(cursor));
+ if (!match) return undefined;
+ cursor += match[0].length;
+ // Captures exclude leading whitespace; punctuation only needs a truthy result.
+ return match[1] ?? match[0];
+ };
+ const number = DECIMAL_NUMBER_REGEX.source.slice(1, -1);
+ const positionPattern = new RegExp(`^\\s*(${number})\\s+(${number})((?:\\s+${number})*)`, 'i');
+ const group = (read: () => unknown): unknown[] | null => {
+ if (!consume(/^\s*\(/)) return null;
+ const items: unknown[] = [];
+ do {
+ const item = read();
+ if (item == null) return null;
+ items.push(item);
+ } while (consume(/^\s*,/));
+ return consume(/^\s*\)/) ? items : null;
+ };
+ const readGeometry = (depth: number, inheritedDimensions?: number): unknown => {
+ if (depth > 32) return null;
+ const type = consume(
+ /^\s*(GEOMETRYCOLLECTION|MULTIPOLYGON|MULTILINESTRING|MULTIPOINT|POLYGON|LINESTRING|POINT)\b/i
+ )?.toUpperCase();
+ if (!type) return null;
+ const dimension = consume(/^\s+(ZM|Z|M)\b/i)?.toUpperCase();
+ const dimensions = dimension ? (dimension === 'ZM' ? 4 : 3) : inheritedDimensions;
+ if (consume(/^\s*EMPTY\b/i)) return {};
+ if (type === 'GEOMETRYCOLLECTION') {
+ return group(() => readGeometry(depth + 1, dimensions));
+ }
+ const readCoordinates = (level: number): unknown => {
+ if (level > 3) return null;
+ if (/^\s*\(/.test(value.slice(cursor))) return group(() => readCoordinates(level + 1));
+ const match = positionPattern.exec(value.slice(cursor));
+ if (!match) return null;
+ cursor += match[0].length;
+ // Capture common XY positions directly, without splitting/trimming each tuple.
+ const coordinates = [+match[1], +match[2]];
+ for (const coordinate of match[3].match(/\S+/g) || []) coordinates.push(+coordinate);
+ return coordinates.every(Number.isFinite) &&
+ (!dimensions || coordinates.length === dimensions)
+ ? coordinates
+ : null;
+ };
+ let coordinates = group(() => readCoordinates(0));
+ if (!coordinates) return null;
+ if (type === 'POINT') {
+ if (coordinates.length !== 1) return null;
+ coordinates = coordinates[0] as unknown[];
+ } else if (
+ type === 'MULTIPOINT' &&
+ coordinates.every(point => Array.isArray(point) && point.length === 1)
+ ) {
+ coordinates = coordinates.map(point => (point as unknown[])[0]);
+ }
+ const types = {
+ POINT: 'Point',
+ MULTIPOINT: 'MultiPoint',
+ LINESTRING: 'LineString',
+ MULTILINESTRING: 'MultiLineString',
+ POLYGON: 'Polygon',
+ MULTIPOLYGON: 'MultiPolygon'
+ };
+ return isGeoJsonGeometry({type: types[type], coordinates}) ? {} : null;
+ };
+ return readGeometry(0) != null && value.slice(cursor).trim() === '';
+}
+
+function isAdditionalGeometry(value: unknown): boolean {
+ if (typeof value !== 'string') return isGeoJsonGeometry(value);
+ if (/^GEOMETRYCOLLECTION\b/i.test(value)) return isWktGeometryCollection(value);
+ if (!value.startsWith('{')) return false;
+ try {
+ return isGeoJsonGeometry(JSON.parse(value));
+ } catch {
+ return false;
+ }
+}
+
+/** Validate numeric-looking WKB before overriding NUMBER (e.g. account IDs). */
+function isCompleteWkb(value: unknown): boolean {
+ if (typeof value !== 'string' || !isHexWkb(value)) return false;
+ const length = value.length / 2;
+ const uint32 = (offset: number, littleEndian: boolean): number => {
+ if (offset + 4 > length) throw new Error('Truncated WKB');
+ let number = 0;
+ for (let i = 0; i < 4; i++) {
+ const byte = parseInt(value.slice((offset + i) * 2, (offset + i + 1) * 2), 16);
+ number += byte * 256 ** (littleEndian ? i : 3 - i);
+ }
+ return number;
+ };
+ const endOfGeometry = (offset: number, depth: number): number => {
+ if (depth > 32 || offset + 5 > length) throw new Error('Invalid WKB');
+ const order = value.slice(offset * 2, offset * 2 + 2);
+ if (order !== '00' && order !== '01') throw new Error('Invalid WKB byte order');
+ const littleEndian = order === '01';
+ const rawType = uint32(offset + 1, littleEndian);
+ const type = rawType & 0x1fffffff;
+ const dimensions =
+ 2 +
+ Number(Boolean(rawType & 0x80000000) || [1, 3].includes(Math.floor(type / 1000))) +
+ Number(Boolean(rawType & 0x40000000) || [2, 3].includes(Math.floor(type / 1000)));
+ const baseType = type % 1000;
+ if (baseType < 1 || baseType > 7 || type >= 4000) throw new Error('Invalid WKB type');
+ let cursor = offset + 5 + (rawType & 0x20000000 ? 4 : 0);
+ if (baseType === 1) return cursor + dimensions * 8;
+ const count = uint32(cursor, littleEndian);
+ cursor += 4;
+ if (baseType === 2) return cursor + count * dimensions * 8;
+ if (count > (length - cursor) / (baseType === 3 ? 4 : 5)) throw new Error('Truncated WKB');
+ for (let i = 0; i < count; i++) {
+ if (baseType === 3) {
+ const points = uint32(cursor, littleEndian);
+ cursor += 4 + points * dimensions * 8;
+ } else {
+ cursor = endOfGeometry(cursor, depth + 1);
+ }
+ if (cursor > length) throw new Error('Truncated WKB');
+ }
+ return cursor;
+ };
+ try {
+ return endOfGeometry(0, 0) === length;
+ } catch {
+ return false;
+ }
+}
+
export const ACCEPTED_ANALYZER_TYPES = [
AnalyzerDATA_TYPES.DATE,
AnalyzerDATA_TYPES.TIME,
@@ -320,6 +501,27 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[]
type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type;
}
+ // The analyzer treats numeric-only WKB as NUMBER and GeoJSON strings as
+ // OBJECT. Only override these fallback types when every present sample
+ // describes geometry; unrelated objects, text and identifiers stay intact.
+ if (
+ [AnalyzerDATA_TYPES.STRING, AnalyzerDATA_TYPES.NUMBER, AnalyzerDATA_TYPES.OBJECT].includes(
+ type
+ )
+ ) {
+ const samples = analyzerData.map(row => row[field]).filter(notNullorUndefined);
+ if (
+ samples.length &&
+ samples.every(
+ value =>
+ isAdditionalGeometry(value) ||
+ (type === AnalyzerDATA_TYPES.NUMBER && isCompleteWkb(value))
+ )
+ ) {
+ type = AnalyzerDATA_TYPES.GEOMETRY;
+ }
+ }
+
// read a column of numbers in scientific notation as real numbers instead of strings
if (
type === AnalyzerDATA_TYPES.NUMBER &&
diff --git a/src/layers/src/geojson-layer/geojson-layer.spec.ts b/src/layers/src/geojson-layer/geojson-layer.spec.ts
index 15a1286a5..7c1d2cd1c 100644
--- a/src/layers/src/geojson-layer/geojson-layer.spec.ts
+++ b/src/layers/src/geojson-layer/geojson-layer.spec.ts
@@ -8,7 +8,13 @@ import {KeplerTable, copyTableAndUpdate} from '@kepler.gl/table';
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
import {Response} from 'node-fetch';
-import {processGeojson} from '@kepler.gl/processors';
+import {
+ processCsvData,
+ processFileData,
+ processGeojson,
+ processRowObject
+} from '@kepler.gl/processors';
+import {convertGeometryToWKB} from '@loaders.gl/gis';
const point = (x: number) => ({type: 'Point', coordinates: [x, x]});
const line = (x: number) => ({
@@ -872,3 +878,212 @@ describe('GeoJsonLayer source revisions', () => {
expect(layer.meta).toMatchObject({bounds: [30, 40, 70, 80]});
});
});
+
+describe('GeoJsonLayer inferred geometry fields', () => {
+ const originalResponse = globalThis.Response;
+ beforeAll(() => {
+ globalThis.Response = Response as any;
+ });
+ afterAll(() => {
+ globalThis.Response = originalResponse;
+ });
+ const point = {type: 'Point' as const, coordinates: [10, 10]};
+ const line = {
+ type: 'LineString' as const,
+ coordinates: [
+ [10, 10],
+ [10.25, 10.25]
+ ]
+ };
+ const polygon = {
+ type: 'Polygon' as const,
+ coordinates: [
+ [
+ [10, 10],
+ [10.25, 10],
+ [10.25, 10.25],
+ [10, 10]
+ ]
+ ]
+ };
+ const collection = {type: 'GeometryCollection' as const, geometries: [point, line]};
+ const cases = [
+ {name: 'GeoJSON Point string', value: JSON.stringify(point)},
+ {name: 'GeometryCollection object', value: collection},
+ {
+ name: 'GeometryCollection WKT',
+ value: 'GEOMETRYCOLLECTION (POINT (10 10), LINESTRING (10 10, 10.25 10.25))'
+ },
+ ...[
+ point,
+ line,
+ polygon,
+ {type: 'MultiPoint' as const, coordinates: [point.coordinates]},
+ {type: 'MultiLineString' as const, coordinates: [line.coordinates]},
+ {type: 'MultiPolygon' as const, coordinates: [polygon.coordinates]}
+ ].map(geometry => ({
+ name: `${geometry.type} numeric WKB`,
+ value: Buffer.from(convertGeometryToWKB(geometry)).toString('hex')
+ }))
+ ];
+
+ test.each(cases)(
+ 'imports $name from a custom column and keeps styling/filter cache reuse',
+ async ({value}) => {
+ const data = processRowObject([
+ {shape: value, score: 0.25},
+ {shape: value, score: 0.75}
+ ])!;
+ const dataset = new KeplerTable({info: {id: 'inferred'}, color: [0, 0, 0]});
+ await dataset.importData({data});
+ const {props} = GeoJsonLayer.findDefaultLayerProps(dataset);
+ expect(props).toHaveLength(1);
+ const layer = new GeoJsonLayer({...props[0], id: 'inferred-layer', dataId: dataset.id});
+ layer.setInitialLayerConfig(dataset);
+ const initial = layer.formatLayerData({inferred: dataset}, {}) as any;
+ expect(initial.data.map(feature => feature.properties.index)).toEqual([0, 1]);
+ expect(layer.centroids).toHaveLength(2);
+ expect(layer.centroids.flat().every(Number.isFinite)).toBe(true);
+ expect(
+ layer._getHoverOverlayData({
+ picked: true,
+ index: 1,
+ object: initial.data[1],
+ layer: {props: {id: layer.id}}
+ })
+ ).not.toBeNull();
+ const geometry = layer.dataToFeature;
+ layer.updateLayerVisConfig({opacity: 0.37});
+ const filtered = copyTableAndUpdate(dataset, {filteredIndex: [1]});
+ const updated = layer.formatLayerData({inferred: filtered}, initial) as any;
+ expect(updated.data.map(feature => feature.properties.index)).toEqual([1]);
+ expect(layer.dataToFeature).toBe(geometry);
+ expect(layer.config.visConfig.opacity).toBe(0.37);
+ }
+ );
+
+ test.each([
+ '{"type":"Point","coordinates":["x","y"]}',
+ '{"type":"Point","coordinates":[null,null]}',
+ '{"type":"Polygon","coordinates":[1,2]}',
+ 'GEOMETRYCOLLECTION (',
+ 'GEOMETRYCOLLECTION (ordinary text)'
+ ])('does not create a geometry layer for malformed custom-column values (%s)', async value => {
+ const data = processRowObject([{shape: value}])!;
+ const dataset = new KeplerTable({info: {id: 'invalid-geometry'}, color: [0, 0, 0]});
+ await dataset.importData({data});
+ expect(GeoJsonLayer.findDefaultLayerProps(dataset).props).toHaveLength(0);
+ });
+
+ test.each([
+ ['GeoJSON dataset', 'GeoJSON'],
+ ['CSV', 'GeoJSON'],
+ ['CSV', 'WKT'],
+ ['CSV', 'WKB hex'],
+ ['Arrow', 'GeoJSON'],
+ ['Arrow', 'WKT'],
+ ['Arrow', 'WKB hex'],
+ ['Arrow', 'WKB bytes'],
+ ['Parquet', 'GeoJSON'],
+ ['Parquet', 'WKT'],
+ ['Parquet', 'WKB hex'],
+ ['Parquet', 'WKB bytes']
+ ])('preserves initial %s imports with %s geometry', async (format, encoding) => {
+ const geometries = [10, 40].map(value => ({
+ type: 'Point' as const,
+ coordinates: [value, value]
+ }));
+ const values = geometries.map(geometry =>
+ encoding === 'GeoJSON'
+ ? JSON.stringify(geometry)
+ : encoding === 'WKT'
+ ? `POINT (${geometry.coordinates.join(' ')})`
+ : encoding === 'WKB hex'
+ ? Buffer.from(convertGeometryToWKB(geometry)).toString('hex')
+ : new Uint8Array(convertGeometryToWKB(geometry))
+ );
+ let data;
+ if (format === 'GeoJSON dataset') {
+ data = processGeojson({
+ type: 'FeatureCollection',
+ features: geometries.map((geometry, index) => ({
+ type: 'Feature',
+ geometry,
+ properties: {label: `row-${index}`}
+ }))
+ });
+ } else if (format === 'CSV') {
+ data = processCsvData(
+ `shape,label\n${values
+ .map(
+ (value, index) => `${JSON.stringify(String(value)).replace(/\\"/g, '""')},row-${index}`
+ )
+ .join('\n')}`
+ );
+ } else {
+ let table = new arrow.Table({
+ _geojson: arrow.vectorFromArray(
+ values as any[],
+ encoding === 'WKB bytes' ? new arrow.Binary() : new arrow.Utf8()
+ ),
+ label: arrow.vectorFromArray(['row-0', 'row-1'], new arrow.Utf8())
+ });
+ if (encoding === 'WKB bytes') {
+ table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB);
+ }
+ if (format === 'Parquet') {
+ if (encoding === 'WKB bytes') {
+ table.schema.metadata.set(
+ 'geo',
+ JSON.stringify({
+ version: '1.1.0',
+ primary_column: '_geojson',
+ columns: {_geojson: {encoding: 'WKB'}}
+ })
+ );
+ }
+ const specifier = 'parquet-wasm/node';
+ const imported = await import(specifier);
+ const wasm = imported.readParquet ? imported : imported.default;
+ const properties = new wasm.WriterPropertiesBuilder()
+ .setKeyValueMetadata(table.schema.metadata)
+ .build();
+ const parquet = wasm.writeParquet(
+ wasm.Table.fromIPCStream(arrow.tableToIPC(table, 'stream')),
+ properties
+ );
+ table = arrow.tableFromIPC(wasm.readParquet(parquet).intoIPCStream());
+ } else {
+ table = arrow.tableFromIPC(arrow.tableToIPC(table, 'stream'));
+ }
+ const [file] = await processFileData({
+ content: {
+ fileName: `points.${format === 'Parquet' ? 'parquet' : 'arrow'}`,
+ data: table.batches
+ },
+ fileCache: []
+ } as any);
+ data = file.data as any;
+ }
+ const dataset = new KeplerTable({info: {id: 'import-compatibility'}, color: [0, 0, 0]});
+ await dataset.importData({data});
+ const {props} = GeoJsonLayer.findDefaultLayerProps(dataset);
+ expect(props).toHaveLength(1);
+ const layer = new GeoJsonLayer({...props[0], id: 'import-compatibility', dataId: dataset.id});
+ layer.setInitialLayerConfig(dataset);
+ const formatted = layer.formatLayerData({[dataset.id]: dataset}, {}) as any;
+ expect(layer.centroids).toEqual([
+ [10, 10],
+ [40, 40]
+ ]);
+ expect(layer.meta).toMatchObject({bounds: [10, 10, 40, 40]});
+ expect(
+ layer._getHoverOverlayData({
+ picked: true,
+ index: 1,
+ object: layer.geoArrowMode ? undefined : formatted.data[1],
+ layer: {props: {id: layer.id}}
+ })?.[0].geometry
+ ).toEqual(geometries[1]);
+ });
+});
diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.js
index 499cd3b04..ee39e56cd 100644
--- a/test/node/utils/data-processor-test.js
+++ b/test/node/utils/data-processor-test.js
@@ -6,6 +6,7 @@ import sinon from 'sinon';
import {console as Console} from 'global/window';
import {DATA_TYPES} from 'type-analyzer';
import cloneDeep from 'es-toolkit/compat/cloneDeep';
+import {convertGeometryToWKB} from '@loaders.gl/gis';
import testData, {
dataWithNulls,
@@ -184,6 +185,246 @@ test('Processor -> getFieldsFromData scientific notation', t => {
t.end();
});
+test('Processor -> geometry autodetection in custom columns', t => {
+ const point = {type: 'Point', coordinates: [10, 10]};
+ const line = {
+ type: 'LineString',
+ coordinates: [
+ [10, 10],
+ [10.25, 10.25]
+ ]
+ };
+ const polygon = {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [10, 10],
+ [10.25, 10],
+ [10.25, 10.25],
+ [10, 10]
+ ]
+ ]
+ };
+ const collection = {type: 'GeometryCollection', geometries: [point, line]};
+ const geometries = [
+ point,
+ line,
+ polygon,
+ {type: 'MultiPoint', coordinates: [point.coordinates]},
+ {type: 'MultiLineString', coordinates: [line.coordinates]},
+ {type: 'MultiPolygon', coordinates: [polygon.coordinates]},
+ collection
+ ];
+ const values = [
+ ...geometries.map(geometry => JSON.stringify(geometry)),
+ collection,
+ 'GEOMETRYCOLLECTION (POINT (10 10), LINESTRING (10 10, 10.25 10.25))',
+ 'GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (10 10)))',
+ 'GEOMETRYCOLLECTION EMPTY',
+ 'geometrycollection z (point z (10 10 5))',
+ 'GEOMETRYCOLLECTION M (POINT M (10 10 5))',
+ 'GEOMETRYCOLLECTION ZM (POINT ZM (10 10 5 7))',
+ 'GEOMETRYCOLLECTION (POINT (-1.25e+1\t+2.5E-1), LINESTRING (.5 -0.25, 1.0 2.0))',
+ 'GEOMETRYCOLLECTION ZM (POINT (1e1 2e1 3e1 4e1))',
+ 'GEOMETRYCOLLECTION (MULTIPOINT ((10 10), (20 20)))',
+ 'GEOMETRYCOLLECTION (MULTIPOINT (10 10, 20 20))',
+ 'GEOMETRYCOLLECTION (MULTILINESTRING ((10 10, 20 20), (30 30, 40 40)))',
+ 'GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 0)))',
+ 'GEOMETRYCOLLECTION (MULTIPOLYGON (((0 0, 1 0, 1 1, 0 0))))',
+ JSON.stringify({type: 'Feature', geometry: point, properties: {}}),
+ JSON.stringify({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [10, 10, 5]},
+ properties: {}
+ }),
+ JSON.stringify({
+ type: 'GeometryCollection',
+ geometries: [collection, {type: 'GeometryCollection', geometries: []}]
+ }),
+ ...geometries.map(geometry => Buffer.from(convertGeometryToWKB(geometry)).toString('hex'))
+ ];
+
+ values.forEach(value => {
+ const data = processRowObject([{shape: null}, {shape: value}]);
+ t.equal(data.fields[0].type, ALL_FIELD_TYPES.geojson, `detects ${JSON.stringify(value)}`);
+ t.deepEqual(data.rows, [[null], [value]], 'preserves values for the geometry parser');
+ });
+
+ for (const values of [
+ [null, null],
+ ['', null],
+ ['ordinary text', 'more text'],
+ ['0101000000', '0101000000'], // A WKB header without a point payload.
+ ['010000000000000000000000000000000000000000', null],
+ ['12345678901234567890', '12345678901234567891'],
+ ['{"type":"invoice","coordinates":[1,2]}', null],
+ ['{"type":"Point","coordinates":"invalid"}', null],
+ ['{"type":"Point","coordinates":[10,10]}', 'ordinary text'],
+ [{type: 'GeometryCollection', geometries: [{type: 'invalid'}]}, null]
+ ]) {
+ t.notEqual(
+ processRowObject(values.map(shape => ({shape}))).fields[0].type,
+ ALL_FIELD_TYPES.geojson,
+ `does not infer geometry from ${JSON.stringify(values)}`
+ );
+ }
+ t.end();
+});
+
+test('Processor -> geometry autodetection rejects malformed GeoJSON coordinates', t => {
+ const invalid = [
+ {type: 'Point', coordinates: ['x', 'y']},
+ {type: 'Point', coordinates: [null, null]},
+ {type: 'Point', coordinates: [10]},
+ {type: 'Point', coordinates: [[10, 10]]},
+ {type: 'Point', coordinates: [10, 10, '5']},
+ {type: 'MultiPoint', coordinates: [10, 10]},
+ {type: 'LineString', coordinates: [[10, 10]]},
+ {
+ type: 'LineString',
+ coordinates: [
+ [10, 10],
+ [null, 20]
+ ]
+ },
+ {
+ type: 'MultiLineString',
+ coordinates: [
+ [10, 10],
+ [20, 20]
+ ]
+ },
+ {type: 'Polygon', coordinates: [1, 2]},
+ {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [0, 0],
+ [1, 0],
+ [0, 0]
+ ]
+ ]
+ },
+ {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [0, 0],
+ [1, 0],
+ [1, 1],
+ [0, 1]
+ ]
+ ]
+ },
+ {
+ type: 'MultiPolygon',
+ coordinates: [
+ [
+ [0, 0],
+ [1, 0],
+ [1, 1],
+ [0, 0]
+ ]
+ ]
+ }
+ ];
+ const valid = JSON.stringify({type: 'Point', coordinates: [10, 10]});
+ for (const geometry of invalid) {
+ for (const value of [
+ geometry,
+ {type: 'Feature', geometry, properties: {}},
+ {type: 'GeometryCollection', geometries: [geometry]}
+ ].map(value => JSON.stringify(value))) {
+ for (const samples of [
+ [value, null],
+ [valid, value]
+ ]) {
+ const data = processRowObject(samples.map(shape => ({shape})));
+ t.equal(
+ data.fields[0].type,
+ ALL_FIELD_TYPES.object,
+ `keeps malformed geometry as an object: ${value}`
+ );
+ }
+ }
+ }
+ const nonFinite = {
+ type: 'GeometryCollection',
+ geometries: [{type: 'Point', coordinates: [Infinity, NaN]}]
+ };
+ t.equal(
+ processRowObject([{shape: nonFinite}]).fields[0].type,
+ ALL_FIELD_TYPES.object,
+ 'rejects non-finite coordinates in collection objects'
+ );
+ t.end();
+});
+
+test('Processor -> geometry autodetection rejects incomplete or invalid collection WKT', t => {
+ const valid = 'GEOMETRYCOLLECTION (POINT (10 10))';
+ for (const value of [
+ 'GEOMETRYCOLLECTION (',
+ 'GEOMETRYCOLLECTION (ordinary text)',
+ 'GEOMETRYCOLLECTION EMPTY trailing text',
+ `${valid} trailing text`,
+ 'GEOMETRYCOLLECTION (POINT (10 10)',
+ 'GEOMETRYCOLLECTION (POINT (10 10)))',
+ 'GEOMETRYCOLLECTION (POINT (10 10),)',
+ 'GEOMETRYCOLLECTION (POINT (10 10) POINT (20 20))',
+ 'GEOMETRYCOLLECTION (POINT (x y))',
+ 'GEOMETRYCOLLECTION (POINT (1-2))',
+ 'GEOMETRYCOLLECTION (POINT (1e999 10))',
+ 'GEOMETRYCOLLECTION M (POINT (10 10 1e999))',
+ 'GEOMETRYCOLLECTION (POINT (10 10, 20 20))',
+ 'GEOMETRYCOLLECTION (LINESTRING (10 10))',
+ 'GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 1)))',
+ 'GEOMETRYCOLLECTION Z (POINT (10 10))',
+ 'GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (ordinary text))',
+ `${'GEOMETRYCOLLECTION ('.repeat(34)}POINT (10 10)${')'.repeat(34)}`
+ ]) {
+ for (const samples of [
+ [null, value],
+ [valid, value]
+ ]) {
+ const data = processRowObject(samples.map(shape => ({shape})));
+ t.equal(data.fields[0].type, ALL_FIELD_TYPES.string, `keeps invalid WKT as text: ${value}`);
+ t.deepEqual(
+ data.rows,
+ samples.map(value => [value]),
+ 'preserves the original text'
+ );
+ }
+ }
+ t.end();
+});
+
+test('Processor -> geometry autodetection scans complete large WKT collections', t => {
+ const coordinates = Array.from({length: 10000}, (_, index) => `${10 + index / 1000} 10`).join(
+ ', '
+ );
+ const valid = `GEOMETRYCOLLECTION (LINESTRING (${coordinates}), POINT (20 20))`;
+ const data = processRowObject([{shape: valid}, {shape: valid}]);
+ t.equal(
+ data.fields[0].type,
+ ALL_FIELD_TYPES.geojson,
+ 'recognizes all members after a large coordinate sequence'
+ );
+ t.deepEqual(data.rows, [[valid], [valid]], 'preserves the full large WKT values');
+ for (const invalid of [
+ valid.slice(0, -1),
+ `${valid} trailing text`,
+ valid.replace('POINT (20 20)', 'POINT (x y)'),
+ valid.replace(', POINT', ' POINT')
+ ]) {
+ t.equal(
+ processRowObject([{shape: valid}, {shape: invalid}]).fields[0].type,
+ ALL_FIELD_TYPES.string,
+ 'checks the end of large samples before promoting a mixed column'
+ );
+ }
+ t.end();
+});
+
test('Processor -> processCsvData', t => {
t.throws(() => processCsvData(''), 'should throw if csv is empty');