Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] parse WKT geometry in spatial joins (#3878)
Igor Dykhta593695cbb450
diff --git a/src/reducers/src/dataset-ops-updaters.spec.ts b/src/reducers/src/dataset-ops-updaters.spec.tsindex 2c9facd88..f32e003bf 100644--- a/src/reducers/src/dataset-ops-updaters.spec.ts+++ b/src/reducers/src/dataset-ops-updaters.spec.ts@@ -4,8 +4,10 @@import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {VisStateActions} from '@kepler.gl/actions';import {KeplerTable} from '@kepler.gl/table';+import {processCsvData} from '@kepler.gl/processors';import SchemaManager, {VisState} from '@kepler.gl/schemas';import {drainTasksForTesting, succeedTaskWithValues} from '@kepler.gl/tasks';+import {Response} from 'node-fetch';import visStateReducer from './vis-state';import {INITIAL_VIS_STATE} from './vis-state-updaters';@@ -44,6 +46,14 @@ function flushCreateTableTasks(state: VisState) {}describe('dataset ops vis-state', () => {+ const originalResponse = globalThis.Response;+ beforeAll(() => {+ globalThis.Response = Response as any;+ });+ afterAll(() => {+ globalThis.Response = originalResponse;+ });+beforeEach(() => {drainTasksForTesting();});@@ -165,6 +175,47 @@ describe('dataset ops vis-state', () => {expect(state.joins[0].predicate).toBe('within');});+ test('spatial joins imported WKT after repeated runs and source reloads without changing it', () => {+ const polygon = 'POLYGON ((-10 -10, 10 -10, 10 10, -10 10, -10 -10))';+ const polys = makeTable('polys', [], []);+ const pts = makeTable('pts', [], []);+ let state: VisState = {...INITIAL_VIS_STATE, datasets: {polys, pts}};++ for (const lng of [0, 0, 30]) {+ const polygonData = processCsvData(`geom\n"${polygon}"`);+ const point = `POINT (${lng} 0)`;+ const pointData = processCsvData(`geom,value\n"${point}",10`);+ if (!polygonData || !pointData) throw new Error('WKT CSV import failed');+ polys.updateSchema(polygonData);+ pts.updateSchema(pointData);+ expect(polys.fields[0].type).toBe(ALL_FIELD_TYPES.geojson);+ expect(pts.fields[0].type).toBe(ALL_FIELD_TYPES.geojson);++ state = reduceVisState(state, VisStateActions.addSpatialJoin('polys'));+ const op = state.joins[0];+ state = reduceVisState(+ state,+ VisStateActions.setSpatialJoinConfig(op.id, {+ rightDataId: 'pts',+ leftGeo: {kind: 'geojson', fieldName: 'geom'},+ rightGeo: {kind: 'geojson', fieldName: 'geom'},+ aggregations: {value: 'sum'}+ })+ );+ state = reduceVisState(state, VisStateActions.runSpatialJoin(op.id));+ state = flushCreateTableTasks(state);++ expect(state.joins).toHaveLength(0);+ expect(state.datasets[op.resultId].dataContainer.rowAsArray(0)).toEqual([+ polygon,+ lng === 0 ? 1 : 0,+ lng === 0 ? 10 : null+ ]);+ expect(polys.dataContainer.valueAt(0, 0)).toBe(polygon);+ expect(pts.dataContainer.valueAt(0, 0)).toBe(point);+ }+ });+test('delete spatial join source keeps result snapshot', () => {const polygon = {type: 'Polygon',diff --git a/src/table/package.json b/src/table/package.jsonindex f4129c7cd..7ae3d2f1a 100644--- a/src/table/package.json+++ b/src/table/package.json@@ -34,8 +34,10 @@"@kepler.gl/tasks-core": "3.3.0-alpha.18","@kepler.gl/types": "3.3.0-alpha.18","@kepler.gl/utils": "3.3.0-alpha.18",+ "@loaders.gl/core": "^4.4.1","@loaders.gl/mvt": "^4.4.1","@loaders.gl/pmtiles": "^4.4.1",+ "@loaders.gl/wkt": "^4.4.1","@turf/boolean-crosses": "^7.2.0","@turf/boolean-equal": "^7.2.0","@turf/boolean-intersects": "^7.2.0",diff --git a/src/table/src/dataset-ops.spec.ts b/src/table/src/dataset-ops.spec.tsindex fa6a4a756..c2f36e1f4 100644--- a/src/table/src/dataset-ops.spec.ts+++ b/src/table/src/dataset-ops.spec.ts@@ -4,6 +4,7 @@import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {latLngToCell} from '@kepler.gl/common-utils';import {Int64, Table, Uint64, vectorFromArray} from 'apache-arrow';+import {Response} from 'node-fetch';import KeplerTable from './kepler-table';import {aggregateValues} from './dataset-ops/aggregations';@@ -80,6 +81,14 @@ const square = (west: number, south: number, east: number, north: number) => ({});describe('dataset-ops engine', () => {+ const originalResponse = globalThis.Response;+ beforeAll(() => {+ globalThis.Response = Response as any;+ });+ afterAll(() => {+ globalThis.Response = originalResponse;+ });+test('groupByDataset aggregates numeric and unique string values', () => {const dataset = makeTable({id: 'cities',@@ -333,7 +342,24 @@ describe('dataset-ops engine', () => {expect(result.data.rows[0]).toEqual(['sf']);});- test('spatialJoinDatasets joins points contained by polygons', () => {+ test.each([+ {format: 'GeoJSON', west: square(-10, -10, 10, 10), east: square(20, -10, 40, 10)},+ {+ format: 'WKT',+ west: 'POLYGON ((-10 -10, 10 -10, 10 10, -10 10, -10 -10))',+ east: 'POLYGON ((20 -10, 40 -10, 40 10, 20 10, 20 -10))'+ },+ {+ format: 'MultiPolygon WKT',+ west: 'MULTIPOLYGON (((-10 -10, 10 -10, 10 10, -10 10, -10 -10)))',+ east: 'MULTIPOLYGON (((20 -10, 40 -10, 40 10, 20 10, 20 -10)))'+ },+ {+ format: 'GeoJSON string',+ west: JSON.stringify(square(-10, -10, 10, 10)),+ east: JSON.stringify(square(20, -10, 40, 10).geometry)+ }+ ])('spatialJoinDatasets joins points contained by $format polygons', ({west, east}) => {const polygons = makeTable({id: 'polys',fields: [@@ -341,8 +367,8 @@ describe('dataset-ops engine', () => {{name: 'geom', type: ALL_FIELD_TYPES.geojson}],rows: [- ['west', square(-10, -10, 10, 10)],- ['east', square(20, -10, 40, 10)]+ ['west', west],+ ['east', east]]});const points = makeTable({@@ -371,6 +397,9 @@ describe('dataset-ops engine', () => {expect(result.data.rows[0][3]).toBe(10);expect(result.data.rows[1][2]).toBe(1);expect(result.data.rows[1][3]).toBe(5);+ expect(result.data.rows.map(row => row[1])).toEqual([west, east]);+ expect(polygons.dataContainer.valueAt(0, 1)).toBe(west);+ expect(polygons.dataContainer.valueAt(1, 1)).toBe(east);expect(result.metadata.derivedDataset.type).toBe('spatialJoin');});@@ -697,6 +726,9 @@ describe('dataset-ops engine', () => {test.each([null,'invalid json',+ '',+ 'POLYGON EMPTY',+ 'POLYGON ((not coordinates))',{type: 'Feature', geometry: null, properties: {}},{type: 'Polygon', coordinates: []},{type: 'Polygon', coordinates: [[]]},diff --git a/src/table/src/dataset-ops/geometry.ts b/src/table/src/dataset-ops/geometry.tsindex bd032f60c..25c407517 100644--- a/src/table/src/dataset-ops/geometry.ts+++ b/src/table/src/dataset-ops/geometry.ts@@ -3,8 +3,13 @@import {cellToParent, h3GetResolution, h3IsValid, latLngToCell} from '@kepler.gl/common-utils';import {isInPolygon} from '@kepler.gl/utils';+import {parseSync} from '@loaders.gl/core';+import {WKTLoader} from '@loaders.gl/wkt';import type {Feature, Geometry, Position} from 'geojson';+const WKT_PREFIX_RE =+ /^\s*(POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)\s*[(ZME]/i;+function isLngLat(pos: unknown): pos is Position {return (Array.isArray(pos) &&@@ -32,7 +37,7 @@ export function parseGeometry(raw: unknown): Feature | null {}if (typeof raw === 'string') {try {- return parseGeometry(JSON.parse(raw));+ return parseGeometry(WKT_PREFIX_RE.test(raw) ? parseSync(raw, WKTLoader) : JSON.parse(raw));} catch {return null;}diff --git a/yarn.lock b/yarn.lockindex 587afb606..e0615565d 100644--- a/yarn.lock+++ b/yarn.lock@@ -4577,8 +4577,10 @@ __metadata:"@kepler.gl/tasks-core": "npm:3.3.0-alpha.18""@kepler.gl/types": "npm:3.3.0-alpha.18""@kepler.gl/utils": "npm:3.3.0-alpha.18"+ "@loaders.gl/core": "npm:^4.4.1""@loaders.gl/mvt": "npm:^4.4.1""@loaders.gl/pmtiles": "npm:^4.4.1"+ "@loaders.gl/wkt": "npm:^4.4.1""@turf/boolean-crosses": "npm:^7.2.0""@turf/boolean-equal": "npm:^7.2.0""@turf/boolean-intersects": "npm:^7.2.0"