Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(arrow): ignore MultiPoint offset buffer padding (#3884)

Igor Dykhta7e56773e8ac1
diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
index 46cc854a2..9f96e95d5 100644
--- a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
+++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
@@ -224,7 +224,8 @@ export class GeoArrowScatterplotLayer<ExtraProps extends object = object> extend
for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) {
const multiPointData = geometryColumn.data[recordBatchIdx];
const pointData = ga.child.getMultiPointChild(multiPointData);
- const geomOffsets = multiPointData.valueOffsets;
+ // Arrow buffers may include alignment padding after the logical offsets.
+ const geomOffsets = multiPointData.valueOffsets.subarray(0, multiPointData.length + 1);
const flatCoordsData = ga.child.getPointChild(pointData);
const flatCoordinateArray = flatCoordsData.values;
diff --git a/test/browser/layer-tests/point-layer-specs.js b/test/browser/layer-tests/point-layer-specs.js
index 91bbe1c56..7963949cb 100644
--- a/test/browser/layer-tests/point-layer-specs.js
+++ b/test/browser/layer-tests/point-layer-specs.js
@@ -10,8 +10,9 @@ import * as arrow from 'apache-arrow';
import {DEFAULT_TEXT_LABEL, PROJECTED_PIXEL_SIZE_MULTIPLIER} from '@kepler.gl/constants';
import {EnhancedMultiIconLayer, EnhancedTextBackgroundLayer} from '@kepler.gl/deckgl-layers';
+import {GeoArrowScatterplotLayer} from '@kepler.gl/deckgl-arrow-layers';
import {KeplerGlLayers} from '@kepler.gl/layers';
-import {arrowSchemaToFields, processGeojson} from '@kepler.gl/processors';
+import {arrowSchemaToFields, processArrowBatches, processGeojson} from '@kepler.gl/processors';
import {INITIAL_MAP_STATE, INITIAL_VIS_STATE, renderDeckGlLayer} from '@kepler.gl/reducers';
import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';
import {hexToRgb} from '@kepler.gl/utils';
@@ -81,28 +82,16 @@ const {PointLayer} = KeplerGlLayers;
[90, 30]
];
const batches = coordinates.map((coords, index) => {
- const built = arrow.vectorFromArray(coords, type);
- // Use exact-length offsets, as in an Arrow IPC record batch, without
- // the builder's trailing allocation padding.
- const geometry =
- geometryType === 'multipoint'
- ? arrow.makeVector(
- arrow.makeData({
- type,
- length: coords.length,
- valueOffsets: built.data[0].valueOffsets.slice(0, coords.length + 1),
- child: built.data[0].children[0]
- })
- )
- : built;
const table = new arrow.Table({
- geometry,
+ geometry: arrow.vectorFromArray(coords, type),
value: arrow.vectorFromArray(values[index], new arrow.Float64())
});
table.schema.fields[0].metadata.set('ARROW:extension:name', `geoarrow.${geometryType}`);
return table;
});
- const table = batches[0].concat(batches[1]);
+ // IPC buffers can contain alignment padding after the logical offsets.
+ const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));
+ const originalBytes = arrow.tableToIPC(table, 'file');
const dataset = new KeplerTable({info: {id: 'batched'}, color: [0, 0, 0]});
await dataset.importData({
data: {
@@ -207,11 +196,109 @@ const {PointLayer} = KeplerGlLayers;
90,
'later batch reads 90 rather than the first batch value 10'
);
+ const vectorLayer = new GeoArrowScatterplotLayer({
+ id: 'batched-vector-radius',
+ data: table,
+ getPosition: table.getChild('geometry'),
+ getRadius: table.getChild('value')
+ });
+ vectorLayer.renderLayers().forEach((sublayer, batchIndex) => {
+ t.deepEqual(
+ Array.from(sublayer.props.data.attributes.getRadius.value),
+ rowIndices[batchIndex].map(index => values.flat()[index]),
+ 'vector radius uses the correct row across unequal batches'
+ );
+ });
+ t.deepEqual(
+ arrow.tableToIPC(table, 'file'),
+ originalBytes,
+ 'rendering preserves source data'
+ );
t.end();
});
});
});
+test('#PointLayer -> MultiPoint IPC offset padding and vector attributes', async t => {
+ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64()));
+ const type = new arrow.List(new arrow.Field('points', pointType));
+ const colorType = new arrow.FixedSizeList(3, new arrow.Field('rgb', new arrow.Uint8()));
+ const cases = [
+ {name: 'two rows', coords: [[[0, 0]], [[1, 1]]]},
+ {name: 'three rows', coords: [[[0, 0]], [[1, 1]], [[2, 2]]]},
+ {name: 'empty first row', coords: [[], [[1, 1]]]},
+ {name: 'empty last row', coords: [[[0, 0]], []]},
+ {name: 'all empty', coords: [[], []]}
+ ];
+ for (const {name, coords} of cases) {
+ const values = coords.map((_, index) => 10 + index * 40);
+ const colors = values.map(value => [value, value + 1, value + 2]);
+ const input = new arrow.Table({
+ geometry: arrow.vectorFromArray(coords, type),
+ value: arrow.vectorFromArray(values, new arrow.Float32()),
+ color: arrow.vectorFromArray(colors, colorType)
+ });
+ input.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.multipoint');
+ const bytes = arrow.tableToIPC(input, 'file');
+ const table = arrow.tableFromIPC(bytes);
+ const originalBytes = bytes.slice();
+ const dataset = new KeplerTable({info: {id: name}, color: [0, 0, 0]});
+ await dataset.importData({data: processArrowBatches(table.batches)});
+ const layer = new PointLayer({dataId: dataset.id, columnMode: 'geoarrow'});
+ layer.config.columns.geoarrow = {value: 'geometry', fieldIdx: 0};
+ layer.config.sizeField = dataset.fields[1];
+ layer.config.sizeScale = 'linear';
+ layer.config.visConfig.radiusRange = [10, values[values.length - 1]];
+ layer.updateLayerDomain({[dataset.id]: dataset});
+ const formatted = layer.formatLayerData({[dataset.id]: dataset});
+ const [composite] = layer.renderLayer({
+ data: formatted,
+ dataset,
+ idx: 0,
+ layerInteraction: {},
+ mapState: INITIAL_MAP_STATE,
+ gpuFilter: dataset.gpuFilter,
+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,
+ visible: true,
+ layerCallbacks: {}
+ });
+ const [sublayer] = composite.renderLayers();
+ const rowIndices = coords.flatMap((points, index) => points.map(() => index));
+ t.equal(sublayer.props.data.length, rowIndices.length, `${name}: empty rows add no points`);
+ rowIndices.forEach((row, index) => {
+ const context = {index, data: sublayer.props.data, target: []};
+ t.equal(sublayer.props.getRadius(null, context), values[row], `${name}: correct radius`);
+ t.equal(sublayer.props.getFiltered(null, context), 1, `${name}: point remains visible`);
+ t.equal(
+ composite.getPickingInfo({info: {index}, sourceLayer: sublayer}).index,
+ row,
+ `${name}: correct picked row`
+ );
+ });
+ const vectorLayer = new GeoArrowScatterplotLayer({
+ id: `${name}-vectors`,
+ data: table,
+ getPosition: table.getChild('geometry'),
+ getRadius: table.getChild('value'),
+ getFillColor: table.getChild('color')
+ });
+ const [vectorSublayer] = vectorLayer.renderLayers();
+ t.deepEqual(
+ Array.from(vectorSublayer.props.data.attributes.getRadius.value),
+ rowIndices.map(index => values[index]),
+ `${name}: expands numeric Arrow attributes`
+ );
+ t.deepEqual(
+ Array.from(vectorSublayer.props.data.attributes.getFillColor.value),
+ rowIndices.flatMap(index => colors[index]),
+ `${name}: expands vector Arrow attributes`
+ );
+ t.deepEqual(bytes, originalBytes, `${name}: leaves the source IPC buffer unchanged`);
+ t.deepEqual(arrow.tableToIPC(table, 'file'), originalBytes, `${name}: preserves source values`);
+ }
+ t.end();
+});
+
test('#PointLayer -> constructor', t => {
const TEST_CASES = {
CREATE: [