Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(arrow): preserve batch offsets for MultiPoint accessors (#3879)
Igor Dykhtaa84b9724150a
diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.tsindex 58c07c1ce..7b1c194f7 100644--- a/src/deckgl-arrow-layers/src/utils/utils.ts+++ b/src/deckgl-arrow-layers/src/utils/utils.ts@@ -137,7 +137,7 @@ function wrapAccessorFunction<In, Out>(const {index, data} = objectInfo;let newIndex = index + batchOffset;if (data.invertedGeomOffsets !== undefined) {- newIndex = data.invertedGeomOffsets[index];+ newIndex = data.invertedGeomOffsets[index] + batchOffset;}const newObjectData = {data: data.data,diff --git a/test/browser/layer-tests/point-layer-specs.js b/test/browser/layer-tests/point-layer-specs.jsindex 2d2288af6..91bbe1c56 100644--- a/test/browser/layer-tests/point-layer-specs.js+++ b/test/browser/layer-tests/point-layer-specs.js@@ -6,13 +6,14 @@ import moment from 'moment';import {getDistanceScales, zoomToScale} from '@math.gl/web-mercator';import {scaleQuantize} from 'd3-scale';import cloneDeep from 'es-toolkit/compat/cloneDeep';+import * as arrow from 'apache-arrow';import {DEFAULT_TEXT_LABEL, PROJECTED_PIXEL_SIZE_MULTIPLIER} from '@kepler.gl/constants';import {EnhancedMultiIconLayer, EnhancedTextBackgroundLayer} from '@kepler.gl/deckgl-layers';import {KeplerGlLayers} from '@kepler.gl/layers';-import {processGeojson} from '@kepler.gl/processors';+import {arrowSchemaToFields, processGeojson} from '@kepler.gl/processors';import {INITIAL_MAP_STATE, INITIAL_VIS_STATE, renderDeckGlLayer} from '@kepler.gl/reducers';-import {copyTableAndUpdate} from '@kepler.gl/table';+import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';import {hexToRgb} from '@kepler.gl/utils';import {geoJsonWithStyle, geojsonData} from 'test/fixtures/geojson';@@ -35,6 +36,182 @@ import {const {PointLayer} = KeplerGlLayers;+['point', 'multipoint'].forEach(geometryType => {+ [false, true].forEach(gpu => {+ test(`#PointLayer -> batched Arrow ${geometryType} accessors (${+ gpu ? 'GPU' : 'CPU'+ })`, async t => {+ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()));+ const type =+ geometryType === 'multipoint'+ ? new arrow.List(new arrow.Field('points', pointType))+ : pointType;+ const coordinates =+ geometryType === 'multipoint'+ ? [+ [+ [+ [0, 0],+ [1, 1]+ ],+ [],+ [[2, 2]]+ ],+ [+ [+ [3, 3],+ [4, 4]+ ],+ [[5, 5]]+ ]+ ]+ : [+ [+ [0, 0],+ [1, 1],+ [2, 2]+ ],+ [+ [3, 3],+ [4, 4]+ ]+ ];+ const values = [+ [10, 50, 20],+ [90, 30]+ ];+ const batches = coordinates.map((coords, index) => {+ const built = arrow.vectorFromArray(coords, type);+ // Use exact-length offsets, as in an Arrow IPC record batch, without+ // the builder's trailing allocation padding.+ const geometry =+ geometryType === 'multipoint'+ ? arrow.makeVector(+ arrow.makeData({+ type,+ length: coords.length,+ valueOffsets: built.data[0].valueOffsets.slice(0, coords.length + 1),+ child: built.data[0].children[0]+ })+ )+ : built;+ const table = new arrow.Table({+ geometry,+ value: arrow.vectorFromArray(values[index], new arrow.Float64())+ });+ table.schema.fields[0].metadata.set('ARROW:extension:name', `geoarrow.${geometryType}`);+ return table;+ });+ const table = batches[0].concat(batches[1]);+ const dataset = new KeplerTable({info: {id: 'batched'}, color: [0, 0, 0]});+ await dataset.importData({+ data: {+ fields: arrowSchemaToFields(table),+ cols: batches[0].schema.fields.map((field, index) => batches[0].getChildAt(index))+ }+ });+ // Progressive updates preserve batches; completed imports compact them.+ await dataset.update({+ cols: table.schema.fields.map((field, index) => table.getChildAt(index)),+ arrowTable: table+ });+ const layer = new PointLayer({dataId: dataset.id});+ layer.config.columnMode = 'geoarrow';+ layer.config.columns.geoarrow = {value: 'geometry', fieldIdx: 0};+ layer.config.colorField = dataset.fields[1];+ layer.config.sizeField = dataset.fields[1];+ layer.config.sizeScale = 'linear';+ layer.config.visConfig.radiusRange = [10, 90];+ layer.updateLayerDomain({[dataset.id]: dataset});+ dataset.filterTable(+ [+ {+ id: 'later-batch',+ dataId: ['unrelated', dataset.id],+ name: ['other', 'value'],+ fieldIdx: [0, 1],+ type: 'range',+ value: [80, 100],+ domain: [10, 90],+ enabled: true,+ fixedDomain: true,+ gpu,+ gpuChannel: [2, 0]+ }+ ],+ [layer]+ );+ const formatted = layer.formatLayerData({[dataset.id]: dataset});+ const [composite] = layer.renderLayer({+ data: formatted,+ dataset,+ idx: 0,+ layerInteraction: {},+ mapState: INITIAL_MAP_STATE,+ gpuFilter: dataset.gpuFilter,+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,+ visible: true,+ layerCallbacks: {}+ });+ const sublayers = composite.renderLayers();+ const rowIndices =+ geometryType === 'multipoint'+ ? [+ [0, 0, 2],+ [3, 3, 4]+ ]+ : [+ [0, 1, 2],+ [3, 4]+ ];+ t.equal(sublayers.length, 2, 'renders both Arrow record batches');+ sublayers.forEach((sublayer, batchIndex) => {+ const {props} = sublayer;+ t.equal(props.data.length, rowIndices[batchIndex].length, 'empty features add no points');+ rowIndices[batchIndex].forEach((rowIndex, index) => {+ const context = {index, data: props.data, target: []};+ const expected = {index: rowIndex};+ t.equal(+ props.getRadius(null, context),+ formatted.getRadius(expected),+ 'radius uses the feature row'+ );+ t.deepEqual(+ props.getFillColor(null, context),+ formatted.getFillColor(expected),+ 'color uses the feature row'+ );+ t.equal(+ props.getFiltered(null, context),+ formatted.getFiltered(expected),+ 'CPU mask uses the feature row'+ );+ const filterValues = props.getFilterValue(null, context);+ t.deepEqual(+ filterValues,+ formatted.getFilterValue(expected),+ 'GPU values use the feature row'+ );+ const [min, max] = dataset.gpuFilter.filterRange[0];+ const visible =+ Boolean(props.getFiltered(null, context)) &&+ filterValues[0] >= min &&+ filterValues[0] <= max;+ t.equal(visible, rowIndex === 3, 'only the matching later-batch feature is visible');+ const picked = composite.getPickingInfo({info: {index}, sourceLayer: sublayer});+ t.equal(picked.index, rowIndex, 'rendered attributes agree with the picked feature');+ });+ });+ t.equal(+ sublayers[1].props.getRadius(null, {index: 0, data: sublayers[1].props.data, target: []}),+ 90,+ 'later batch reads 90 rather than the first batch value 10'+ );+ t.end();+ });+ });+});+test('#PointLayer -> constructor', t => {const TEST_CASES = {CREATE: [