Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(filters): retain native GeoArrow MultiPoints in polygon filters (#3881)
Igor Dykhtaf66e4141fe6d
diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.tsindex 1d54d96a9..466974924 100644--- a/src/utils/src/filter-utils.ts+++ b/src/utils/src/filter-utils.ts@@ -15,7 +15,9 @@ import {ANIMATION_WINDOW,PLOT_TYPES,LAYER_TYPES,- FILTER_VIEW_TYPES+ FILTER_VIEW_TYPES,+ GEOARROW_EXTENSIONS,+ GEOARROW_METADATA_KEY} from '@kepler.gl/constants';// import {VisState} from '@kepler.gl/schemas';import * as ScaleUtils from './data-scale-utils';@@ -562,6 +564,25 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {switch (layer.type) {case LAYER_TYPES.point:case LAYER_TYPES.icon:+ if (+ layer.config?.columnMode === 'geoarrow' &&+ dataContainer+ .getField?.(layer.config.columns.geoarrow.fieldIdx)+ ?.metadata?.get(GEOARROW_METADATA_KEY) === GEOARROW_EXTENSIONS.MULTIPOINT+ ) {+ return data => {+ const points = dataContainer.valueAt(data.index, layer.config.columns.geoarrow.fieldIdx);+ // Match GeoJSON MultiPoint semantics: any valid point retains the whole feature.+ for (const point of points ?? []) {+ const coord =+ typeof point?.get === 'function'+ ? [point.get(0), point.get(1)]+ : [point?.x, point?.y];+ if (hasFiniteLngLat(coord) && isInPolygon(coord, filter.value)) return true;+ }+ return false;+ };+ }if (layer.config?.columnMode === 'geojson' && layer.dataToFeature?.length) {return data => {const coordinates = layer.dataToFeature[data.index];diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 4af3a2542..c280fe0b0 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -4,6 +4,7 @@/* eslint-disable max-statements */import test from 'tape-catch';import sinon from 'sinon';+import * as arrow from 'apache-arrow';import {console as Console} from 'global/window';import {drainTasksForTesting, succeedTaskInTest, errorTaskInTest} from '@kepler.gl/tasks';@@ -26,14 +27,19 @@ import {calculateLayerData} from '@kepler.gl/reducers';-import {processCsvData, processGeojson, remoteDatasetFromUrl} from '@kepler.gl/processors';+import {+ processCsvData,+ processGeojson,+ processArrowBatches,+ remoteDatasetFromUrl+} from '@kepler.gl/processors';import {Layer,KeplerGlLayers,COLUMN_MODE_TABLE,createPointCountLabelField} from '@kepler.gl/layers';-import {copyTableAndUpdate, maybeToDate} from '@kepler.gl/table';+import {copyTableAndUpdate, maybeToDate, KeplerTable} from '@kepler.gl/table';import {createDataContainer,applyFilterFieldName,@@ -6263,6 +6269,196 @@ test('#visStateReducer -> POLYGON: Create polygon filter', t => {t.end();});+test('#visStateReducer -> POLYGON: native GeoArrow MultiPoint membership', async t => {+ const previousMaxBatches = getApplicationConfig().maxArrowBatches;+ const polygon = {+ type: 'Feature',+ id: 'east',+ properties: {},+ geometry: {+ type: 'Polygon',+ coordinates: [+ [+ [-98, 39],+ [-95, 39],+ [-95, 41],+ [-98, 41],+ [-98, 39]+ ]+ ]+ }+ };+ const coordinates = [+ [+ [-103, 40],+ [-102, 40]+ ], // outside+ [],+ [+ [-110, 40],+ [-97, 40]+ ], // only the second point is inside+ [+ [-97, 40],+ [-96, 40]+ ], // both inside, in the later batch+ [+ [NaN, 40],+ [Infinity, 40]+ ],+ null+ ];+ const values = [10, 50, 20, 90, 30, 40];+ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()));+ const multiPointType = new arrow.List(new arrow.Field('points', pointType));+ const makeTable = (rows, numbers, type, extension) => {+ const table = new arrow.Table({+ geometry: arrow.vectorFromArray(rows, type),+ value: arrow.vectorFromArray(numbers, new arrow.Float64())+ });+ table.schema.fields[0].metadata.set('ARROW:extension:name', extension);+ return table;+ };+ const multiPointTable = (rows, numbers) =>+ makeTable(rows, numbers, multiPointType, 'geoarrow.multipoint');+ const cases = [+ {+ name: 'single batch MultiPoint',+ table: multiPointTable(coordinates, values),+ batches: [6],+ expected: [2, 3]+ },+ {+ name: 'multiple batch MultiPoint',+ table: multiPointTable(coordinates.slice(0, 3), values.slice(0, 3)).concat(+ multiPointTable(coordinates.slice(3), values.slice(3))+ ),+ batches: [3, 3],+ expected: [2, 3]+ },+ {+ name: 'separated-coordinate MultiPoint',+ table: makeTable(+ coordinates.map(points => points?.map(([x, y]) => ({x, y})) ?? null),+ values,+ new arrow.List(+ new arrow.Field(+ 'points',+ new arrow.Struct([+ new arrow.Field('x', new arrow.Float64()),+ new arrow.Field('y', new arrow.Float64())+ ])+ )+ ),+ 'geoarrow.multipoint'+ ),+ batches: [6],+ expected: [2, 3]+ },+ {+ name: 'native Point control',+ table: makeTable(+ [+ [-103, 40],+ [-97, 40]+ ],+ [10, 90],+ pointType,+ 'geoarrow.point'+ ),+ batches: [2],+ expected: [1]+ },+ {+ name: 'GeoJSON MultiPoint control',+ data: processGeojson({+ type: 'FeatureCollection',+ features: coordinates.slice(0, 4).map((coords, index) => ({+ type: 'Feature',+ properties: {value: values[index]},+ geometry: {type: 'MultiPoint', coordinates: coords}+ }))+ }),+ expected: [2, 3]+ }+ ];+ initApplicationConfig({maxArrowBatches: 2});+ try {+ for (const fixture of cases) {+ const dataset = new KeplerTable({info: {id: 'points'}, color: [0, 0, 0]});+ // Round-trip IPC, then use the same processor/table import as a file upload.+ const table = fixture.table && arrow.tableFromIPC(arrow.tableToIPC(fixture.table, 'file'));+ await dataset.importData({data: table ? processArrowBatches(table.batches) : fixture.data});+ const columnMode = table ? 'geoarrow' : 'geojson';+ const layer = new PointLayer({id: 'points-layer', dataId: 'points', columnMode});+ const geometryField = dataset.fields.find(field => field.type === columnMode);+ layer.config.columns[columnMode] = {+ value: geometryField.name,+ fieldIdx: geometryField.fieldIdx+ };+ const datasets = {points: dataset};+ layer.updateLayerDomain(datasets);+ const initialData = layer.formatLayerData(datasets);+ const readSource = () =>+ table+ ? Array.from(arrow.tableToIPC(dataset.dataContainer.getTable(), 'file'))+ : CloneDeep(dataset.dataContainer.flattenData());+ const originalValues = readSource();+ const allRows = dataset.dataContainer.getPlainIndex();+ let state = {+ ...INITIAL_VIS_STATE,+ datasets,+ layers: [layer],+ layerData: [initialData],+ layerOrder: [layer.id]+ };+ if (table) {+ t.deepEqual(+ dataset.dataContainer.getTable().batches.map(batch => batch.numRows),+ fixture.batches,+ `${fixture.name}: retains input batches`+ );+ }+ state = reducer(state, VisStateActions.setPolygonFilterLayer(layer, polygon));+ t.deepEqual(+ state.datasets.points.filteredIndexByLayer[layer.id],+ fixture.expected,+ `${fixture.name}: retains any valid inside point, rejects outside/empty/invalid rows`+ );+ t.deepEqual(state.datasets.points.filteredIndex, allRows, 'polygon membership is per layer');+ if (table) {+ // Test the layer's row mask independently of the separate #3879 exploded-batch fix.+ t.deepEqual(+ Array.from(state.layers[0].filteredIndex),+ allRows.map(index => Number(fixture.expected.includes(index))),+ `${fixture.name}: sends matching row mask to the Arrow layer`+ );+ } else {+ t.deepEqual(+ state.layerData[0].data.map(point => point.index),+ [2, 2, 3, 3],+ 'GeoJSON control retains both points of a partially matching feature'+ );+ }+ state = reducer(state, VisStateActions.removeFilter(0));+ t.deepEqual(state.datasets.points.filteredIndexByLayer, {}, 'clearing removes layer filter');+ if (table) {+ t.deepEqual(+ Array.from(state.layers[0].filteredIndex),+ allRows.map(() => 1),+ 'restores all rows'+ );+ } else {+ t.equal(state.layerData[0].data.length, 6, 'restores all nonempty GeoJSON points');+ }+ t.deepEqual(readSource(), originalValues, 'source values are unchanged');+ }+ } finally {+ initApplicationConfig({maxArrowBatches: previousMaxBatches});+ }+ t.end();+});+test('#visStateReducer -> POLYGON: unselecting all layers returns sketch', t => {const initialState = CloneDeep(StateWFiles.visState);const layer = initialState.layers[0];