Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(processors): preserve vector export geometry and missing coordinates (#3873)
Igor Dykhta1d7ad445a1f0
diff --git a/src/processors/package.json b/src/processors/package.jsonindex e4685d54d..62eeadde3 100644--- a/src/processors/package.json+++ b/src/processors/package.json@@ -31,6 +31,7 @@"@deck.gl-community/editable-layers": "9.3.8","@kepler.gl/common-utils": "3.3.0-alpha.18","@kepler.gl/constants": "3.3.0-alpha.18",+ "@kepler.gl/layers": "3.3.0-alpha.18","@kepler.gl/schemas": "3.3.0-alpha.18","@kepler.gl/table": "3.3.0-alpha.18","@kepler.gl/types": "3.3.0-alpha.18",diff --git a/src/processors/src/export/dataset-to-geojson.ts b/src/processors/src/export/dataset-to-geojson.tsindex 49a8de5af..508b6a18d 100644--- a/src/processors/src/export/dataset-to-geojson.ts+++ b/src/processors/src/export/dataset-to-geojson.ts@@ -3,10 +3,10 @@import {h3IsValid, idToPolygonGeo} from '@kepler.gl/common-utils';import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {parseGeoJsonRawFeature} from '@kepler.gl/layers';import {parseGeometry} from '@kepler.gl/table';import {Field, FieldPair} from '@kepler.gl/types';import {DataContainerInterface, parseFieldValue} from '@kepler.gl/utils';-import {convertWKBToGeometry, convertWKTToGeometry} from '@loaders.gl/gis';import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';type GeometryPlan =@@ -35,42 +35,6 @@ function geometryPlan(fields: Field[], fieldPairs: FieldPair[] = []): GeometryPlreturn {kind: 'none'};}-function geometryFromBinary(raw: ArrayBuffer | ArrayBufferView): Geometry | null {- try {- const bytes =- raw instanceof ArrayBuffer- ? raw- : raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength);- return convertWKBToGeometry(bytes) as Geometry;- } catch {- return null;- }-}--function geometryFromRaw(raw: unknown, fieldType: string): Geometry | null {- if (raw == null) {- return null;- }- if (raw instanceof ArrayBuffer || ArrayBuffer.isView(raw)) {- return geometryFromBinary(raw);- }- const parsed = parseGeometry(raw);- if (parsed?.geometry) {- return parsed.geometry;- }- if (fieldType === ALL_FIELD_TYPES.geoarrow && typeof raw === 'string') {- const text = raw.trim();- if (/^(POINT|LINESTRING|POLYGON|MULTI|GEOMETRYCOLLECTION)\b/i.test(text)) {- try {- return convertWKTToGeometry(text) as Geometry;- } catch {- return null;- }- }- }- return null;-}-function attributeValue(value: unknown, field: Field): string | number | boolean | null {if (value == null) {return null;@@ -93,8 +57,17 @@ function pointGeometry(rowIndex: number,pair: FieldPair): Geometry | null {- const lat = Number(data.valueAt(rowIndex, pair.pair.lat.fieldIdx));- const lng = Number(data.valueAt(rowIndex, pair.pair.lng.fieldIdx));+ const latValue = data.valueAt(rowIndex, pair.pair.lat.fieldIdx);+ const lngValue = data.valueAt(rowIndex, pair.pair.lng.fieldIdx);+ if (+ ![latValue, lngValue].every(+ value => typeof value === 'number' || (typeof value === 'string' && value.trim() !== '')+ )+ ) {+ return null;+ }+ const lat = Number(latValue);+ const lng = Number(lngValue);if (!Number.isFinite(lat) || !Number.isFinite(lng)) {return null;}@@ -134,10 +107,17 @@ export function datasetToFeatureCollection(let geometry: Geometry | null = null;if (plan.kind === 'geo') {- geometry = geometryFromRaw(- data.valueAt(rowIndex, plan.fieldIndex),- fields[plan.fieldIndex].type- );+ const raw = data.valueAt(rowIndex, plan.fieldIndex);+ const encoding = fields[plan.fieldIndex].metadata?.get?.('ARROW:extension:name');+ // Unannotated WKB may use any buffer view; native GeoArrow views contain coordinates.+ const geometryValue =+ !encoding && ArrayBuffer.isView(raw)+ ? new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength)+ : raw;+ geometry =+ parseGeometry(raw)?.geometry ??+ parseGeoJsonRawFeature(geometryValue, encoding)?.geometry ??+ null;} else if (plan.kind === 'latlng') {geometry = pointGeometry(data, rowIndex, plan.pair);} else if (plan.kind === 'h3') {diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.tsindex ae55a7681..1db2fa26e 100644--- a/src/processors/src/export/vector-export.spec.ts+++ b/src/processors/src/export/vector-export.spec.ts@@ -1,13 +1,20 @@+/** @jest-environment node */+// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import {convertWKBToGeometry} from '@loaders.gl/gis';+import {convertGeometryToWKB, convertWKBToGeometry} from '@loaders.gl/gis';import {convertArrowToSchema} from '@loaders.gl/schema-utils';import {DBFLoader} from '@loaders.gl/shapefile';import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';import {Field} from '@kepler.gl/types';-import {createDataContainer} from '@kepler.gl/utils';+import {createDataContainer, validateInputData} from '@kepler.gl/utils';+import {datasetSchema, VERSIONS} from '@kepler.gl/schemas';+import * as arrow from 'apache-arrow';import {tableFromIPC} from 'apache-arrow';+import {ZipLoader} from '@loaders.gl/zip';+import {KeplerTable} from '@kepler.gl/table';+import {arrowSchemaToFields, processArrowTable, processRowObject} from '../data-processor';import type {FeatureCollection} from 'geojson';import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';@@ -42,6 +49,187 @@ function container() {}describe('dataset export', () => {+ test.each(['DataView', 'Uint16Array', 'Uint8Array', 'ArrayBuffer'])(+ 'preserves WKB geometry from %s with bounded byte views',+ async viewType => {+ const geometry = {+ type: 'MultiPoint' as const,+ coordinates: [+ [-122.4123456789, 37.8123456789],+ [0, 0],+ [1, 2]+ ]+ };+ // Three WKB points occupy 72 bytes, so the Uint16Array has no padding inside its view.+ const bytes = new Uint8Array(convertGeometryToWKB(geometry));+ const padded = new Uint8Array(bytes.length + 16).fill(255);+ padded.set(bytes, 8);+ const value = {+ DataView: new DataView(padded.buffer, 8, bytes.length),+ Uint16Array: new Uint16Array(padded.buffer, 8, bytes.length / 2),+ Uint8Array: padded.subarray(8, 8 + bytes.length),+ ArrayBuffer: bytes.buffer+ }[viewType];+ const shared = {+ data: createDataContainer([[value, 'p']]),+ fields: [+ {name: 'geometry', type: ALL_FIELD_TYPES.geojson},+ {name: 'id', type: ALL_FIELD_TYPES.string}+ ] as Field[],+ layerName: 'views'+ };+ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});+ expect(JSON.parse(String(geojson.data)).features).toEqual([+ {type: 'Feature', geometry, properties: {id: 'p'}}+ ]);++ const shapefile = await encodeDatasetExport({+ ...shared,+ dataType: EXPORT_DATA_TYPE.SHAPEFILE+ });+ const files = await ZipLoader.parse(shapefile.data);+ const shp = new DataView(files['views.shp']);+ expect(shp.getInt32(108, true)).toBe(8);+ expect(shp.getInt32(144, true)).toBe(geometry.coordinates.length);+ expect(+ geometry.coordinates.map((_, i) => [+ shp.getFloat64(148 + i * 16, true),+ shp.getFloat64(156 + i * 16, true)+ ])+ ).toEqual(geometry.coordinates);+ }+ );++ test('preserves native GeoArrow coordinate views when decoding exports', () => {+ const coordinates = new Float64Array([999, -122.4123456789, 37.8123456789, 999]);+ const value = coordinates.subarray(1, 3);+ const collection = datasetToFeatureCollection(createDataContainer([[value]]), [+ {+ name: 'geometry',+ displayName: 'geometry',+ type: ALL_FIELD_TYPES.geoarrow,+ fieldIdx: 0,+ format: '',+ valueAccessor: () => value,+ metadata: new Map([['ARROW:extension:name', 'geoarrow.point']])+ }+ ]);+ expect(collection.features[0].geometry).toEqual({+ type: 'Point',+ coordinates: [-122.4123456789, 37.8123456789]+ });+ });++ test.each(['WKT', 'hex WKB', 'Arrow WKT', 'native GeoArrow point'])(+ 'preserves imported %s geometry in GeoJSON and shapefile exports',+ async encoding => {+ const geometry = {type: 'Point' as const, coordinates: [-122.4123456789, 37.8123456789]};+ let data;+ if (encoding === 'native GeoArrow point') {+ const points = arrow.vectorFromArray(+ [geometry.coordinates, null],+ new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()))+ );+ const table = new arrow.Table({geometry: points});+ table.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.point');+ data = processArrowTable({shape: 'arrow-table', data: table} as any);+ } else {+ const value = encoding.endsWith('WKT')+ ? `POINT (${geometry.coordinates.join(' ')})`+ : Buffer.from(convertGeometryToWKB(geometry)).toString('hex');+ if (encoding === 'Arrow WKT') {+ const column = arrow.vectorFromArray([value, null], new arrow.Utf8());+ const table = new arrow.Table({geometry: column});+ data = {+ fields: arrowSchemaToFields(table, {geometry: 'VARCHAR'}),+ cols: [column],+ rows: []+ };+ } else {+ data = processRowObject([{geometry: value}, {geometry: null}]);+ }+ }+ if (!data) throw new Error('Geometry import failed');+ const dataset = new KeplerTable({color: [1, 2, 3]});+ await dataset.importData({data});+ const shared = {+ data: dataset.dataContainer,+ fields: dataset.fields,+ hiddenColumns: ['geometry'],+ layerName: 'geometry'+ };+ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});+ const collection = JSON.parse(String(geojson.data));+ expect(collection.features.map(feature => feature.geometry)).toEqual([geometry, null]);+ expect(collection.features[0].properties).not.toHaveProperty('geometry');++ const saved = JSON.parse(JSON.stringify(datasetSchema[VERSIONS.v1].save(dataset)));+ const loaded = validateInputData(datasetSchema[VERSIONS.v1].load(saved).data);+ if (!loaded) throw new Error('Saved geometry validation failed');+ const restored = new KeplerTable({color: [1, 2, 3]}).updateSchema(loaded);+ expect(datasetToFeatureCollection(restored.dataContainer, restored.fields)).toEqual(+ collection+ );++ const shapefile = await encodeDatasetExport({+ ...shared,+ dataType: EXPORT_DATA_TYPE.SHAPEFILE+ });+ const files = await ZipLoader.parse(shapefile.data);+ const shp = new DataView(files['geometry.shp']);+ expect(shp.getInt32(0)).toBe(9994);+ expect(shp.getInt32(108, true)).toBe(1);+ expect([shp.getFloat64(112, true), shp.getFloat64(120, true)]).toEqual(geometry.coordinates);+ expect(shp.getInt32(136, true)).toBe(0);+ }+ );++ test.each([[[1, 2]], [['1', '2']], ['[1,2]']])(+ 'preserves numeric point coordinates from %j',+ value => {+ const collection = datasetToFeatureCollection(createDataContainer([[value]]), [+ {name: 'point', type: ALL_FIELD_TYPES.point} as Field+ ]);+ expect(collection.features[0].geometry).toEqual({type: 'Point', coordinates: [1, 2]});+ }+ );++ test.each([+ [null, null],+ ['', ''],+ [' ', '\t'],+ [null, 10],+ [10, undefined],+ ['invalid', 10],+ [Infinity, 10],+ [false, true]+ ])('exports missing or invalid coordinates [%s, %s] as null geometry', async (lat, lng) => {+ const exported = await encodeDatasetExport({+ data: createDataContainer([['missing', 'secret', lat, lng]]),+ fields,+ fieldPairs,+ dataType: EXPORT_DATA_TYPE.GEOJSON,+ layerName: 'missing'+ });+ expect(JSON.parse(String(exported.data)).features[0].geometry).toBeNull();+ });++ test.each([+ [0, 0],+ ['0', '0'],+ [' 37.8 ', '-122.4']+ ])('preserves valid coordinates [%s, %s]', (lat, lng) => {+ const collection = datasetToFeatureCollection(+ createDataContainer([['valid', 'secret', lat, lng]]),+ fields,+ {fieldPairs}+ );+ expect(collection.features[0].geometry).toEqual({+ type: 'Point',+ coordinates: [Number(lng), Number(lat)]+ });+ });+test('omits columns hidden in the attribute table', () => {const collection = datasetToFeatureCollection(container(), fields, {hiddenColumns: ['secret'],diff --git a/yarn.lock b/yarn.lockindex a6b8e47d8..587afb606 100644--- a/yarn.lock+++ b/yarn.lock@@ -4422,6 +4422,7 @@ __metadata:"@deck.gl-community/editable-layers": "npm:9.3.8""@kepler.gl/common-utils": "npm:3.3.0-alpha.18""@kepler.gl/constants": "npm:3.3.0-alpha.18"+ "@kepler.gl/layers": "npm:3.3.0-alpha.18""@kepler.gl/schemas": "npm:3.3.0-alpha.18""@kepler.gl/table": "npm:3.3.0-alpha.18""@kepler.gl/types": "npm:3.3.0-alpha.18"