Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(arrow): preserve batch offsets for MultiPoint accessors (#3879)

Igor Dykhtaa84b9724150a
diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.ts
index 58c07c1ce..7b1c194f7 100644
--- a/src/deckgl-arrow-layers/src/utils/utils.ts
+++ b/src/deckgl-arrow-layers/src/utils/utils.ts
@@ -137,7 +137,7 @@ function wrapAccessorFunction<In, Out>(
const {index, data} = objectInfo;
let newIndex = index + batchOffset;
if (data.invertedGeomOffsets !== undefined) {
- newIndex = data.invertedGeomOffsets[index];
+ newIndex = data.invertedGeomOffsets[index] + batchOffset;
}
const newObjectData = {
data: data.data,
diff --git a/test/browser/layer-tests/point-layer-specs.js b/test/browser/layer-tests/point-layer-specs.js
index 2d2288af6..91bbe1c56 100644
--- a/test/browser/layer-tests/point-layer-specs.js
+++ b/test/browser/layer-tests/point-layer-specs.js
@@ -6,13 +6,14 @@ import moment from 'moment';
import {getDistanceScales, zoomToScale} from '@math.gl/web-mercator';
import {scaleQuantize} from 'd3-scale';
import cloneDeep from 'es-toolkit/compat/cloneDeep';
+import * as arrow from 'apache-arrow';
import {DEFAULT_TEXT_LABEL, PROJECTED_PIXEL_SIZE_MULTIPLIER} from '@kepler.gl/constants';
import {EnhancedMultiIconLayer, EnhancedTextBackgroundLayer} from '@kepler.gl/deckgl-layers';
import {KeplerGlLayers} from '@kepler.gl/layers';
-import {processGeojson} from '@kepler.gl/processors';
+import {arrowSchemaToFields, processGeojson} from '@kepler.gl/processors';
import {INITIAL_MAP_STATE, INITIAL_VIS_STATE, renderDeckGlLayer} from '@kepler.gl/reducers';
-import {copyTableAndUpdate} from '@kepler.gl/table';
+import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';
import {hexToRgb} from '@kepler.gl/utils';
import {geoJsonWithStyle, geojsonData} from 'test/fixtures/geojson';
@@ -35,6 +36,182 @@ import {
const {PointLayer} = KeplerGlLayers;
+['point', 'multipoint'].forEach(geometryType => {
+ [false, true].forEach(gpu => {
+ test(`#PointLayer -> batched Arrow ${geometryType} accessors (${
+ gpu ? 'GPU' : 'CPU'
+ })`, async t => {
+ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()));
+ const type =
+ geometryType === 'multipoint'
+ ? new arrow.List(new arrow.Field('points', pointType))
+ : pointType;
+ const coordinates =
+ geometryType === 'multipoint'
+ ? [
+ [
+ [
+ [0, 0],
+ [1, 1]
+ ],
+ [],
+ [[2, 2]]
+ ],
+ [
+ [
+ [3, 3],
+ [4, 4]
+ ],
+ [[5, 5]]
+ ]
+ ]
+ : [
+ [
+ [0, 0],
+ [1, 1],
+ [2, 2]
+ ],
+ [
+ [3, 3],
+ [4, 4]
+ ]
+ ];
+ const values = [
+ [10, 50, 20],
+ [90, 30]
+ ];
+ const batches = coordinates.map((coords, index) => {
+ const built = arrow.vectorFromArray(coords, type);
+ // Use exact-length offsets, as in an Arrow IPC record batch, without
+ // the builder's trailing allocation padding.
+ const geometry =
+ geometryType === 'multipoint'
+ ? arrow.makeVector(
+ arrow.makeData({
+ type,
+ length: coords.length,
+ valueOffsets: built.data[0].valueOffsets.slice(0, coords.length + 1),
+ child: built.data[0].children[0]
+ })
+ )
+ : built;
+ const table = new arrow.Table({
+ geometry,
+ value: arrow.vectorFromArray(values[index], new arrow.Float64())
+ });
+ table.schema.fields[0].metadata.set('ARROW:extension:name', `geoarrow.${geometryType}`);
+ return table;
+ });
+ const table = batches[0].concat(batches[1]);
+ const dataset = new KeplerTable({info: {id: 'batched'}, color: [0, 0, 0]});
+ await dataset.importData({
+ data: {
+ fields: arrowSchemaToFields(table),
+ cols: batches[0].schema.fields.map((field, index) => batches[0].getChildAt(index))
+ }
+ });
+ // Progressive updates preserve batches; completed imports compact them.
+ await dataset.update({
+ cols: table.schema.fields.map((field, index) => table.getChildAt(index)),
+ arrowTable: table
+ });
+ const layer = new PointLayer({dataId: dataset.id});
+ layer.config.columnMode = 'geoarrow';
+ layer.config.columns.geoarrow = {value: 'geometry', fieldIdx: 0};
+ layer.config.colorField = dataset.fields[1];
+ layer.config.sizeField = dataset.fields[1];
+ layer.config.sizeScale = 'linear';
+ layer.config.visConfig.radiusRange = [10, 90];
+ layer.updateLayerDomain({[dataset.id]: dataset});
+ dataset.filterTable(
+ [
+ {
+ id: 'later-batch',
+ dataId: ['unrelated', dataset.id],
+ name: ['other', 'value'],
+ fieldIdx: [0, 1],
+ type: 'range',
+ value: [80, 100],
+ domain: [10, 90],
+ enabled: true,
+ fixedDomain: true,
+ gpu,
+ gpuChannel: [2, 0]
+ }
+ ],
+ [layer]
+ );
+ const formatted = layer.formatLayerData({[dataset.id]: dataset});
+ const [composite] = layer.renderLayer({
+ data: formatted,
+ dataset,
+ idx: 0,
+ layerInteraction: {},
+ mapState: INITIAL_MAP_STATE,
+ gpuFilter: dataset.gpuFilter,
+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,
+ visible: true,
+ layerCallbacks: {}
+ });
+ const sublayers = composite.renderLayers();
+ const rowIndices =
+ geometryType === 'multipoint'
+ ? [
+ [0, 0, 2],
+ [3, 3, 4]
+ ]
+ : [
+ [0, 1, 2],
+ [3, 4]
+ ];
+ t.equal(sublayers.length, 2, 'renders both Arrow record batches');
+ sublayers.forEach((sublayer, batchIndex) => {
+ const {props} = sublayer;
+ t.equal(props.data.length, rowIndices[batchIndex].length, 'empty features add no points');
+ rowIndices[batchIndex].forEach((rowIndex, index) => {
+ const context = {index, data: props.data, target: []};
+ const expected = {index: rowIndex};
+ t.equal(
+ props.getRadius(null, context),
+ formatted.getRadius(expected),
+ 'radius uses the feature row'
+ );
+ t.deepEqual(
+ props.getFillColor(null, context),
+ formatted.getFillColor(expected),
+ 'color uses the feature row'
+ );
+ t.equal(
+ props.getFiltered(null, context),
+ formatted.getFiltered(expected),
+ 'CPU mask uses the feature row'
+ );
+ const filterValues = props.getFilterValue(null, context);
+ t.deepEqual(
+ filterValues,
+ formatted.getFilterValue(expected),
+ 'GPU values use the feature row'
+ );
+ const [min, max] = dataset.gpuFilter.filterRange[0];
+ const visible =
+ Boolean(props.getFiltered(null, context)) &&
+ filterValues[0] >= min &&
+ filterValues[0] <= max;
+ t.equal(visible, rowIndex === 3, 'only the matching later-batch feature is visible');
+ const picked = composite.getPickingInfo({info: {index}, sourceLayer: sublayer});
+ t.equal(picked.index, rowIndex, 'rendered attributes agree with the picked feature');
+ });
+ });
+ t.equal(
+ sublayers[1].props.getRadius(null, {index: 0, data: sublayers[1].props.data, target: []}),
+ 90,
+ 'later batch reads 90 rather than the first batch value 10'
+ );
+ t.end();
+ });
+ });
+});
+
test('#PointLayer -> constructor', t => {
const TEST_CASES = {
CREATE: [