Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(processors): preserve vector export geometry and missing coordinates (#3873)

Igor Dykhta1d7ad445a1f0
diff --git a/src/processors/package.json b/src/processors/package.json
index e4685d54d..62eeadde3 100644
--- a/src/processors/package.json
+++ b/src/processors/package.json
@@ -31,6 +31,7 @@
"@deck.gl-community/editable-layers": "9.3.8",
"@kepler.gl/common-utils": "3.3.0-alpha.18",
"@kepler.gl/constants": "3.3.0-alpha.18",
+ "@kepler.gl/layers": "3.3.0-alpha.18",
"@kepler.gl/schemas": "3.3.0-alpha.18",
"@kepler.gl/table": "3.3.0-alpha.18",
"@kepler.gl/types": "3.3.0-alpha.18",
diff --git a/src/processors/src/export/dataset-to-geojson.ts b/src/processors/src/export/dataset-to-geojson.ts
index 49a8de5af..508b6a18d 100644
--- a/src/processors/src/export/dataset-to-geojson.ts
+++ b/src/processors/src/export/dataset-to-geojson.ts
@@ -3,10 +3,10 @@
import {h3IsValid, idToPolygonGeo} from '@kepler.gl/common-utils';
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {parseGeoJsonRawFeature} from '@kepler.gl/layers';
import {parseGeometry} from '@kepler.gl/table';
import {Field, FieldPair} from '@kepler.gl/types';
import {DataContainerInterface, parseFieldValue} from '@kepler.gl/utils';
-import {convertWKBToGeometry, convertWKTToGeometry} from '@loaders.gl/gis';
import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';
type GeometryPlan =
@@ -35,42 +35,6 @@ function geometryPlan(fields: Field[], fieldPairs: FieldPair[] = []): GeometryPl
return {kind: 'none'};
}
-function geometryFromBinary(raw: ArrayBuffer | ArrayBufferView): Geometry | null {
- try {
- const bytes =
- raw instanceof ArrayBuffer
- ? raw
- : raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength);
- return convertWKBToGeometry(bytes) as Geometry;
- } catch {
- return null;
- }
-}
-
-function geometryFromRaw(raw: unknown, fieldType: string): Geometry | null {
- if (raw == null) {
- return null;
- }
- if (raw instanceof ArrayBuffer || ArrayBuffer.isView(raw)) {
- return geometryFromBinary(raw);
- }
- const parsed = parseGeometry(raw);
- if (parsed?.geometry) {
- return parsed.geometry;
- }
- if (fieldType === ALL_FIELD_TYPES.geoarrow && typeof raw === 'string') {
- const text = raw.trim();
- if (/^(POINT|LINESTRING|POLYGON|MULTI|GEOMETRYCOLLECTION)\b/i.test(text)) {
- try {
- return convertWKTToGeometry(text) as Geometry;
- } catch {
- return null;
- }
- }
- }
- return null;
-}
-
function attributeValue(value: unknown, field: Field): string | number | boolean | null {
if (value == null) {
return null;
@@ -93,8 +57,17 @@ function pointGeometry(
rowIndex: number,
pair: FieldPair
): Geometry | null {
- const lat = Number(data.valueAt(rowIndex, pair.pair.lat.fieldIdx));
- const lng = Number(data.valueAt(rowIndex, pair.pair.lng.fieldIdx));
+ const latValue = data.valueAt(rowIndex, pair.pair.lat.fieldIdx);
+ const lngValue = data.valueAt(rowIndex, pair.pair.lng.fieldIdx);
+ if (
+ ![latValue, lngValue].every(
+ value => typeof value === 'number' || (typeof value === 'string' && value.trim() !== '')
+ )
+ ) {
+ return null;
+ }
+ const lat = Number(latValue);
+ const lng = Number(lngValue);
if (!Number.isFinite(lat) || !Number.isFinite(lng)) {
return null;
}
@@ -134,10 +107,17 @@ export function datasetToFeatureCollection(
let geometry: Geometry | null = null;
if (plan.kind === 'geo') {
- geometry = geometryFromRaw(
- data.valueAt(rowIndex, plan.fieldIndex),
- fields[plan.fieldIndex].type
- );
+ const raw = data.valueAt(rowIndex, plan.fieldIndex);
+ const encoding = fields[plan.fieldIndex].metadata?.get?.('ARROW:extension:name');
+ // Unannotated WKB may use any buffer view; native GeoArrow views contain coordinates.
+ const geometryValue =
+ !encoding && ArrayBuffer.isView(raw)
+ ? new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength)
+ : raw;
+ geometry =
+ parseGeometry(raw)?.geometry ??
+ parseGeoJsonRawFeature(geometryValue, encoding)?.geometry ??
+ null;
} else if (plan.kind === 'latlng') {
geometry = pointGeometry(data, rowIndex, plan.pair);
} else if (plan.kind === 'h3') {
diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.ts
index ae55a7681..1db2fa26e 100644
--- a/src/processors/src/export/vector-export.spec.ts
+++ b/src/processors/src/export/vector-export.spec.ts
@@ -1,13 +1,20 @@
+/** @jest-environment node */
+
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import {convertWKBToGeometry} from '@loaders.gl/gis';
+import {convertGeometryToWKB, convertWKBToGeometry} from '@loaders.gl/gis';
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
import {DBFLoader} from '@loaders.gl/shapefile';
import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';
import {Field} from '@kepler.gl/types';
-import {createDataContainer} from '@kepler.gl/utils';
+import {createDataContainer, validateInputData} from '@kepler.gl/utils';
+import {datasetSchema, VERSIONS} from '@kepler.gl/schemas';
+import * as arrow from 'apache-arrow';
import {tableFromIPC} from 'apache-arrow';
+import {ZipLoader} from '@loaders.gl/zip';
+import {KeplerTable} from '@kepler.gl/table';
+import {arrowSchemaToFields, processArrowTable, processRowObject} from '../data-processor';
import type {FeatureCollection} from 'geojson';
import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';
@@ -42,6 +49,187 @@ function container() {
}
describe('dataset export', () => {
+ test.each(['DataView', 'Uint16Array', 'Uint8Array', 'ArrayBuffer'])(
+ 'preserves WKB geometry from %s with bounded byte views',
+ async viewType => {
+ const geometry = {
+ type: 'MultiPoint' as const,
+ coordinates: [
+ [-122.4123456789, 37.8123456789],
+ [0, 0],
+ [1, 2]
+ ]
+ };
+ // Three WKB points occupy 72 bytes, so the Uint16Array has no padding inside its view.
+ const bytes = new Uint8Array(convertGeometryToWKB(geometry));
+ const padded = new Uint8Array(bytes.length + 16).fill(255);
+ padded.set(bytes, 8);
+ const value = {
+ DataView: new DataView(padded.buffer, 8, bytes.length),
+ Uint16Array: new Uint16Array(padded.buffer, 8, bytes.length / 2),
+ Uint8Array: padded.subarray(8, 8 + bytes.length),
+ ArrayBuffer: bytes.buffer
+ }[viewType];
+ const shared = {
+ data: createDataContainer([[value, 'p']]),
+ fields: [
+ {name: 'geometry', type: ALL_FIELD_TYPES.geojson},
+ {name: 'id', type: ALL_FIELD_TYPES.string}
+ ] as Field[],
+ layerName: 'views'
+ };
+ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});
+ expect(JSON.parse(String(geojson.data)).features).toEqual([
+ {type: 'Feature', geometry, properties: {id: 'p'}}
+ ]);
+
+ const shapefile = await encodeDatasetExport({
+ ...shared,
+ dataType: EXPORT_DATA_TYPE.SHAPEFILE
+ });
+ const files = await ZipLoader.parse(shapefile.data);
+ const shp = new DataView(files['views.shp']);
+ expect(shp.getInt32(108, true)).toBe(8);
+ expect(shp.getInt32(144, true)).toBe(geometry.coordinates.length);
+ expect(
+ geometry.coordinates.map((_, i) => [
+ shp.getFloat64(148 + i * 16, true),
+ shp.getFloat64(156 + i * 16, true)
+ ])
+ ).toEqual(geometry.coordinates);
+ }
+ );
+
+ test('preserves native GeoArrow coordinate views when decoding exports', () => {
+ const coordinates = new Float64Array([999, -122.4123456789, 37.8123456789, 999]);
+ const value = coordinates.subarray(1, 3);
+ const collection = datasetToFeatureCollection(createDataContainer([[value]]), [
+ {
+ name: 'geometry',
+ displayName: 'geometry',
+ type: ALL_FIELD_TYPES.geoarrow,
+ fieldIdx: 0,
+ format: '',
+ valueAccessor: () => value,
+ metadata: new Map([['ARROW:extension:name', 'geoarrow.point']])
+ }
+ ]);
+ expect(collection.features[0].geometry).toEqual({
+ type: 'Point',
+ coordinates: [-122.4123456789, 37.8123456789]
+ });
+ });
+
+ test.each(['WKT', 'hex WKB', 'Arrow WKT', 'native GeoArrow point'])(
+ 'preserves imported %s geometry in GeoJSON and shapefile exports',
+ async encoding => {
+ const geometry = {type: 'Point' as const, coordinates: [-122.4123456789, 37.8123456789]};
+ let data;
+ if (encoding === 'native GeoArrow point') {
+ const points = arrow.vectorFromArray(
+ [geometry.coordinates, null],
+ new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()))
+ );
+ const table = new arrow.Table({geometry: points});
+ table.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.point');
+ data = processArrowTable({shape: 'arrow-table', data: table} as any);
+ } else {
+ const value = encoding.endsWith('WKT')
+ ? `POINT (${geometry.coordinates.join(' ')})`
+ : Buffer.from(convertGeometryToWKB(geometry)).toString('hex');
+ if (encoding === 'Arrow WKT') {
+ const column = arrow.vectorFromArray([value, null], new arrow.Utf8());
+ const table = new arrow.Table({geometry: column});
+ data = {
+ fields: arrowSchemaToFields(table, {geometry: 'VARCHAR'}),
+ cols: [column],
+ rows: []
+ };
+ } else {
+ data = processRowObject([{geometry: value}, {geometry: null}]);
+ }
+ }
+ if (!data) throw new Error('Geometry import failed');
+ const dataset = new KeplerTable({color: [1, 2, 3]});
+ await dataset.importData({data});
+ const shared = {
+ data: dataset.dataContainer,
+ fields: dataset.fields,
+ hiddenColumns: ['geometry'],
+ layerName: 'geometry'
+ };
+ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});
+ const collection = JSON.parse(String(geojson.data));
+ expect(collection.features.map(feature => feature.geometry)).toEqual([geometry, null]);
+ expect(collection.features[0].properties).not.toHaveProperty('geometry');
+
+ const saved = JSON.parse(JSON.stringify(datasetSchema[VERSIONS.v1].save(dataset)));
+ const loaded = validateInputData(datasetSchema[VERSIONS.v1].load(saved).data);
+ if (!loaded) throw new Error('Saved geometry validation failed');
+ const restored = new KeplerTable({color: [1, 2, 3]}).updateSchema(loaded);
+ expect(datasetToFeatureCollection(restored.dataContainer, restored.fields)).toEqual(
+ collection
+ );
+
+ const shapefile = await encodeDatasetExport({
+ ...shared,
+ dataType: EXPORT_DATA_TYPE.SHAPEFILE
+ });
+ const files = await ZipLoader.parse(shapefile.data);
+ const shp = new DataView(files['geometry.shp']);
+ expect(shp.getInt32(0)).toBe(9994);
+ expect(shp.getInt32(108, true)).toBe(1);
+ expect([shp.getFloat64(112, true), shp.getFloat64(120, true)]).toEqual(geometry.coordinates);
+ expect(shp.getInt32(136, true)).toBe(0);
+ }
+ );
+
+ test.each([[[1, 2]], [['1', '2']], ['[1,2]']])(
+ 'preserves numeric point coordinates from %j',
+ value => {
+ const collection = datasetToFeatureCollection(createDataContainer([[value]]), [
+ {name: 'point', type: ALL_FIELD_TYPES.point} as Field
+ ]);
+ expect(collection.features[0].geometry).toEqual({type: 'Point', coordinates: [1, 2]});
+ }
+ );
+
+ test.each([
+ [null, null],
+ ['', ''],
+ [' ', '\t'],
+ [null, 10],
+ [10, undefined],
+ ['invalid', 10],
+ [Infinity, 10],
+ [false, true]
+ ])('exports missing or invalid coordinates [%s, %s] as null geometry', async (lat, lng) => {
+ const exported = await encodeDatasetExport({
+ data: createDataContainer([['missing', 'secret', lat, lng]]),
+ fields,
+ fieldPairs,
+ dataType: EXPORT_DATA_TYPE.GEOJSON,
+ layerName: 'missing'
+ });
+ expect(JSON.parse(String(exported.data)).features[0].geometry).toBeNull();
+ });
+
+ test.each([
+ [0, 0],
+ ['0', '0'],
+ [' 37.8 ', '-122.4']
+ ])('preserves valid coordinates [%s, %s]', (lat, lng) => {
+ const collection = datasetToFeatureCollection(
+ createDataContainer([['valid', 'secret', lat, lng]]),
+ fields,
+ {fieldPairs}
+ );
+ expect(collection.features[0].geometry).toEqual({
+ type: 'Point',
+ coordinates: [Number(lng), Number(lat)]
+ });
+ });
+
test('omits columns hidden in the attribute table', () => {
const collection = datasetToFeatureCollection(container(), fields, {
hiddenColumns: ['secret'],
diff --git a/yarn.lock b/yarn.lock
index a6b8e47d8..587afb606 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4422,6 +4422,7 @@ __metadata:
"@deck.gl-community/editable-layers": "npm:9.3.8"
"@kepler.gl/common-utils": "npm:3.3.0-alpha.18"
"@kepler.gl/constants": "npm:3.3.0-alpha.18"
+ "@kepler.gl/layers": "npm:3.3.0-alpha.18"
"@kepler.gl/schemas": "npm:3.3.0-alpha.18"
"@kepler.gl/table": "npm:3.3.0-alpha.18"
"@kepler.gl/types": "npm:3.3.0-alpha.18"