Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] parse WKT geometry in spatial joins (#3878)

Igor Dykhta593695cbb450
diff --git a/src/reducers/src/dataset-ops-updaters.spec.ts b/src/reducers/src/dataset-ops-updaters.spec.ts
index 2c9facd88..f32e003bf 100644
--- a/src/reducers/src/dataset-ops-updaters.spec.ts
+++ b/src/reducers/src/dataset-ops-updaters.spec.ts
@@ -4,8 +4,10 @@
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {VisStateActions} from '@kepler.gl/actions';
import {KeplerTable} from '@kepler.gl/table';
+import {processCsvData} from '@kepler.gl/processors';
import SchemaManager, {VisState} from '@kepler.gl/schemas';
import {drainTasksForTesting, succeedTaskWithValues} from '@kepler.gl/tasks';
+import {Response} from 'node-fetch';
import visStateReducer from './vis-state';
import {INITIAL_VIS_STATE} from './vis-state-updaters';
@@ -44,6 +46,14 @@ function flushCreateTableTasks(state: VisState) {
}
describe('dataset ops vis-state', () => {
+ const originalResponse = globalThis.Response;
+ beforeAll(() => {
+ globalThis.Response = Response as any;
+ });
+ afterAll(() => {
+ globalThis.Response = originalResponse;
+ });
+
beforeEach(() => {
drainTasksForTesting();
});
@@ -165,6 +175,47 @@ describe('dataset ops vis-state', () => {
expect(state.joins[0].predicate).toBe('within');
});
+ test('spatial joins imported WKT after repeated runs and source reloads without changing it', () => {
+ const polygon = 'POLYGON ((-10 -10, 10 -10, 10 10, -10 10, -10 -10))';
+ const polys = makeTable('polys', [], []);
+ const pts = makeTable('pts', [], []);
+ let state: VisState = {...INITIAL_VIS_STATE, datasets: {polys, pts}};
+
+ for (const lng of [0, 0, 30]) {
+ const polygonData = processCsvData(`geom\n"${polygon}"`);
+ const point = `POINT (${lng} 0)`;
+ const pointData = processCsvData(`geom,value\n"${point}",10`);
+ if (!polygonData || !pointData) throw new Error('WKT CSV import failed');
+ polys.updateSchema(polygonData);
+ pts.updateSchema(pointData);
+ expect(polys.fields[0].type).toBe(ALL_FIELD_TYPES.geojson);
+ expect(pts.fields[0].type).toBe(ALL_FIELD_TYPES.geojson);
+
+ state = reduceVisState(state, VisStateActions.addSpatialJoin('polys'));
+ const op = state.joins[0];
+ state = reduceVisState(
+ state,
+ VisStateActions.setSpatialJoinConfig(op.id, {
+ rightDataId: 'pts',
+ leftGeo: {kind: 'geojson', fieldName: 'geom'},
+ rightGeo: {kind: 'geojson', fieldName: 'geom'},
+ aggregations: {value: 'sum'}
+ })
+ );
+ state = reduceVisState(state, VisStateActions.runSpatialJoin(op.id));
+ state = flushCreateTableTasks(state);
+
+ expect(state.joins).toHaveLength(0);
+ expect(state.datasets[op.resultId].dataContainer.rowAsArray(0)).toEqual([
+ polygon,
+ lng === 0 ? 1 : 0,
+ lng === 0 ? 10 : null
+ ]);
+ expect(polys.dataContainer.valueAt(0, 0)).toBe(polygon);
+ expect(pts.dataContainer.valueAt(0, 0)).toBe(point);
+ }
+ });
+
test('delete spatial join source keeps result snapshot', () => {
const polygon = {
type: 'Polygon',
diff --git a/src/table/package.json b/src/table/package.json
index f4129c7cd..7ae3d2f1a 100644
--- a/src/table/package.json
+++ b/src/table/package.json
@@ -34,8 +34,10 @@
"@kepler.gl/tasks-core": "3.3.0-alpha.18",
"@kepler.gl/types": "3.3.0-alpha.18",
"@kepler.gl/utils": "3.3.0-alpha.18",
+ "@loaders.gl/core": "^4.4.1",
"@loaders.gl/mvt": "^4.4.1",
"@loaders.gl/pmtiles": "^4.4.1",
+ "@loaders.gl/wkt": "^4.4.1",
"@turf/boolean-crosses": "^7.2.0",
"@turf/boolean-equal": "^7.2.0",
"@turf/boolean-intersects": "^7.2.0",
diff --git a/src/table/src/dataset-ops.spec.ts b/src/table/src/dataset-ops.spec.ts
index fa6a4a756..c2f36e1f4 100644
--- a/src/table/src/dataset-ops.spec.ts
+++ b/src/table/src/dataset-ops.spec.ts
@@ -4,6 +4,7 @@
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {latLngToCell} from '@kepler.gl/common-utils';
import {Int64, Table, Uint64, vectorFromArray} from 'apache-arrow';
+import {Response} from 'node-fetch';
import KeplerTable from './kepler-table';
import {aggregateValues} from './dataset-ops/aggregations';
@@ -80,6 +81,14 @@ const square = (west: number, south: number, east: number, north: number) => ({
});
describe('dataset-ops engine', () => {
+ const originalResponse = globalThis.Response;
+ beforeAll(() => {
+ globalThis.Response = Response as any;
+ });
+ afterAll(() => {
+ globalThis.Response = originalResponse;
+ });
+
test('groupByDataset aggregates numeric and unique string values', () => {
const dataset = makeTable({
id: 'cities',
@@ -333,7 +342,24 @@ describe('dataset-ops engine', () => {
expect(result.data.rows[0]).toEqual(['sf']);
});
- test('spatialJoinDatasets joins points contained by polygons', () => {
+ test.each([
+ {format: 'GeoJSON', west: square(-10, -10, 10, 10), east: square(20, -10, 40, 10)},
+ {
+ format: 'WKT',
+ west: 'POLYGON ((-10 -10, 10 -10, 10 10, -10 10, -10 -10))',
+ east: 'POLYGON ((20 -10, 40 -10, 40 10, 20 10, 20 -10))'
+ },
+ {
+ format: 'MultiPolygon WKT',
+ west: 'MULTIPOLYGON (((-10 -10, 10 -10, 10 10, -10 10, -10 -10)))',
+ east: 'MULTIPOLYGON (((20 -10, 40 -10, 40 10, 20 10, 20 -10)))'
+ },
+ {
+ format: 'GeoJSON string',
+ west: JSON.stringify(square(-10, -10, 10, 10)),
+ east: JSON.stringify(square(20, -10, 40, 10).geometry)
+ }
+ ])('spatialJoinDatasets joins points contained by $format polygons', ({west, east}) => {
const polygons = makeTable({
id: 'polys',
fields: [
@@ -341,8 +367,8 @@ describe('dataset-ops engine', () => {
{name: 'geom', type: ALL_FIELD_TYPES.geojson}
],
rows: [
- ['west', square(-10, -10, 10, 10)],
- ['east', square(20, -10, 40, 10)]
+ ['west', west],
+ ['east', east]
]
});
const points = makeTable({
@@ -371,6 +397,9 @@ describe('dataset-ops engine', () => {
expect(result.data.rows[0][3]).toBe(10);
expect(result.data.rows[1][2]).toBe(1);
expect(result.data.rows[1][3]).toBe(5);
+ expect(result.data.rows.map(row => row[1])).toEqual([west, east]);
+ expect(polygons.dataContainer.valueAt(0, 1)).toBe(west);
+ expect(polygons.dataContainer.valueAt(1, 1)).toBe(east);
expect(result.metadata.derivedDataset.type).toBe('spatialJoin');
});
@@ -697,6 +726,9 @@ describe('dataset-ops engine', () => {
test.each([
null,
'invalid json',
+ '',
+ 'POLYGON EMPTY',
+ 'POLYGON ((not coordinates))',
{type: 'Feature', geometry: null, properties: {}},
{type: 'Polygon', coordinates: []},
{type: 'Polygon', coordinates: [[]]},
diff --git a/src/table/src/dataset-ops/geometry.ts b/src/table/src/dataset-ops/geometry.ts
index bd032f60c..25c407517 100644
--- a/src/table/src/dataset-ops/geometry.ts
+++ b/src/table/src/dataset-ops/geometry.ts
@@ -3,8 +3,13 @@
import {cellToParent, h3GetResolution, h3IsValid, latLngToCell} from '@kepler.gl/common-utils';
import {isInPolygon} from '@kepler.gl/utils';
+import {parseSync} from '@loaders.gl/core';
+import {WKTLoader} from '@loaders.gl/wkt';
import type {Feature, Geometry, Position} from 'geojson';
+const WKT_PREFIX_RE =
+ /^\s*(POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)\s*[(ZME]/i;
+
function isLngLat(pos: unknown): pos is Position {
return (
Array.isArray(pos) &&
@@ -32,7 +37,7 @@ export function parseGeometry(raw: unknown): Feature | null {
}
if (typeof raw === 'string') {
try {
- return parseGeometry(JSON.parse(raw));
+ return parseGeometry(WKT_PREFIX_RE.test(raw) ? parseSync(raw, WKTLoader) : JSON.parse(raw));
} catch {
return null;
}
diff --git a/yarn.lock b/yarn.lock
index 587afb606..e0615565d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4577,8 +4577,10 @@ __metadata:
"@kepler.gl/tasks-core": "npm:3.3.0-alpha.18"
"@kepler.gl/types": "npm:3.3.0-alpha.18"
"@kepler.gl/utils": "npm:3.3.0-alpha.18"
+ "@loaders.gl/core": "npm:^4.4.1"
"@loaders.gl/mvt": "npm:^4.4.1"
"@loaders.gl/pmtiles": "npm:^4.4.1"
+ "@loaders.gl/wkt": "npm:^4.4.1"
"@turf/boolean-crosses": "npm:^7.2.0"
"@turf/boolean-equal": "npm:^7.2.0"
"@turf/boolean-intersects": "npm:^7.2.0"