Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] preserve point layers when Arrow refresh returns zero rows (#3882)

Igor Dykhta4b5d4b4d9d4f
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts
index 550ee0f6e..3fad2b155 100644
--- a/src/layers/src/layer-utils.ts
+++ b/src/layers/src/layer-utils.ts
@@ -378,7 +378,8 @@ export function createGeoArrowPointVector(
fixedSizeListBuilder.append(getPosition(indexData));
}
- return arrow.makeVector(numElements > 0 ? [fixedSizeListBuilder.flush()] : []);
+ // Flush even for zero rows so Arrow receives typed Data instead of an untyped empty array.
+ return arrow.makeVector([fixedSizeListBuilder.flush()]);
}
/**
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index 178e83737..6c6002ebb 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -10152,6 +10152,89 @@ test('VisStateUpdater -> refreshDataset', async t => {
t.end();
});
+test('VisStateUpdater -> refreshDataset keeps Arrow point layers valid through empty snapshots', async t => {
+ const snapshot = values => {
+ const table = arrow.tableFromArrays({
+ value: Float64Array.from(values),
+ lat: Float64Array.from(values, (_, i) => 37.77 + i / 100),
+ lng: Float64Array.from(values, (_, i) => -122.42 - i / 100)
+ });
+ return processArrowBatches(arrow.tableFromIPC(arrow.tableToIPC(table, 'file')).batches);
+ };
+
+ for (const withFilter of [false, true]) {
+ drainTasksForTesting();
+ const dataId = `empty-arrow-${withFilter}`;
+ const datasets = await createNewDataEntryMock({
+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},
+ data: snapshot([1, 2, 3]),
+ metadata: {source: 'https://example.com/live.arrow', sourceFormat: 'arrow'}
+ });
+ const {props} = PointLayer.findDefaultLayerProps(datasets[dataId]);
+ const point = new PointLayer({id: `point-${dataId}`, dataId, ...props[0]});
+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};
+ const initial = calculateLayerData(point, state);
+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};
+ t.equal(state.layers[0].isValid, true, 'initial point layer is valid');
+ if (withFilter) {
+ state = reducer(state, VisStateActions.addFilter(dataId));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, 'value', [2, 3]));
+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', true));
+ }
+ const schema = datasets[dataId].fields.map(({name, type}) => ({name, type}));
+
+ for (const values of [[], [3, 2, 1], [], [1]]) {
+ const loading = reducer(state, VisStateActions.refreshDataset(dataId));
+ const progressed = reducer(loading, VisStateActions.refreshDatasetProgress(dataId, 50));
+ const [task] = drainTasksForTesting();
+ state = reducer(
+ progressed,
+ succeedTaskInTest(task, {data: snapshot(values), notModified: false})
+ );
+ const layer = state.layers[0];
+ t.equal(layer.isValid, true, `refresh to ${values.length} rows keeps the point layer valid`);
+ t.equal(layer.errorMessage, null, 'refresh does not report a layer exception');
+ t.equal(layer.config.isVisible, true, 'refresh keeps the layer visible');
+ t.equal(state.layerData[0].data?.numRows, values.length, 'layer data follows the snapshot');
+ t.equal(layer.geoArrowVector.length, values.length, 'coordinate vector follows the snapshot');
+ t.deepEqual(
+ state.datasets[dataId].fields.map(({name, type}) => ({name, type})),
+ schema,
+ 'empty IPC snapshots preserve the schema'
+ );
+ if (values.length) {
+ t.deepEqual(
+ Array.from(layer.geoArrowVector.get(0)),
+ [-122.42, 37.77, 0],
+ 'restores positions'
+ );
+ }
+ const expected = values.flatMap((value, index) =>
+ !withFilter || value < 2 || value > 3 ? [index] : []
+ );
+ t.deepEqual(state.datasets[dataId].filteredIndex, expected, 'refresh reapplies active filters');
+ t.deepEqual(
+ values.flatMap((_, index) => (state.layerData[0].getFiltered({index}) ? [index] : [])),
+ expected,
+ 'point layer visibility follows the refreshed filter result'
+ );
+ state.datasets[dataId].filterTableCPU(state.filters, state.layers);
+ t.deepEqual(
+ state.datasets[dataId].filteredIdxCPU,
+ expected,
+ 'CPU/export evaluation uses the refreshed filtered rows'
+ );
+ if (withFilter) {
+ t.deepEqual(state.filters[0].value, [2, 3], 'refresh preserves the filter selection');
+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');
+ t.equal(state.filters[0].inverted, true, 'refresh preserves Filter out mode');
+ }
+ }
+ }
+ t.end();
+});
+
test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapshots', async t => {
drainTasksForTesting();
const dataId = 'remote-arrow';
diff --git a/test/node/utils/layer-utils-test.js b/test/node/utils/layer-utils-test.js
index a7877dd23..c308a5e92 100644
--- a/test/node/utils/layer-utils-test.js
+++ b/test/node/utils/layer-utils-test.js
@@ -4,7 +4,12 @@
import test from 'tape';
import cloneDeep from 'es-toolkit/compat/cloneDeep';
import {processCsvData, processGeojson} from '@kepler.gl/processors';
-import {getGeojsonLayerMetaFromArrow, LayerClasses, KeplerGlLayers} from '@kepler.gl/layers';
+import {
+ createGeoArrowPointVector,
+ getGeojsonLayerMetaFromArrow,
+ LayerClasses,
+ KeplerGlLayers
+} from '@kepler.gl/layers';
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, GEOJSON_FIELDS} from '@kepler.gl/constants';
import {findDefaultLayer, getLayerHoverProp, getLayerOrderFromLayers} from '@kepler.gl/reducers';
import {StateWTripGeojson, StateWFiles} from 'test/helpers/mock-state';
@@ -21,6 +26,57 @@ import {KeplerTable, findPointFieldPairs} from '@kepler.gl/table';
import {ArrowDataContainer, createDataContainer, findDefaultColorField} from '@kepler.gl/utils';
import * as arrow from 'apache-arrow';
+test('layerUtils -> createGeoArrowPointVector handles empty and nonempty coordinates', t => {
+ for (const coordinates of [
+ [],
+ [
+ [-122.42, 37.77],
+ [0, 0]
+ ],
+ [
+ [-122.42, 37.77, 10],
+ [0, 0, -5]
+ ]
+ ]) {
+ const dataContainer = new ArrowDataContainer({
+ cols: [
+ arrow.vectorFromArray(
+ coordinates.map((_, i) => i),
+ new arrow.Float64()
+ )
+ ],
+ fields: [{name: 'row', type: 'real', fieldIdx: 0}]
+ });
+ let accessorCalls = 0;
+ let vector;
+ t.doesNotThrow(() => {
+ vector = createGeoArrowPointVector(dataContainer, ({index}) => {
+ accessorCalls++;
+ return coordinates[index];
+ });
+ }, 'should create a coordinate vector, including for zero rows');
+ if (vector) {
+ t.equal(vector.length, coordinates.length, 'preserves the row count');
+ t.ok(arrow.DataType.isFixedSizeList(vector.type), 'coordinates have a fixed-size-list type');
+ t.equal(vector.type.listSize, coordinates[0]?.length || 2, 'retains coordinate dimensions');
+ t.equal(
+ String(vector.type.children[0].type),
+ 'Float64',
+ 'coordinates retain Float64 precision'
+ );
+ t.deepEqual(
+ Array.from({length: vector.length}, (_, i) => Array.from(vector.get(i))),
+ coordinates,
+ 'preserves nonempty coordinate values and order'
+ );
+ }
+ if (!coordinates.length) {
+ t.equal(accessorCalls, 0, 'does not read a nonexistent row');
+ }
+ }
+ t.end();
+});
+
test('layerUtils -> getGeojsonLayerMetaFromArrow uses worker-prepared data', t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const geoColumn = arrow.vectorFromArray([[-122.4, 37.8]], pointType);