Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] preserve point layers when Arrow refresh returns zero rows (#3882)
Igor Dykhta4b5d4b4d9d4f
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.tsindex 550ee0f6e..3fad2b155 100644--- a/src/layers/src/layer-utils.ts+++ b/src/layers/src/layer-utils.ts@@ -378,7 +378,8 @@ export function createGeoArrowPointVector(fixedSizeListBuilder.append(getPosition(indexData));}- return arrow.makeVector(numElements > 0 ? [fixedSizeListBuilder.flush()] : []);+ // Flush even for zero rows so Arrow receives typed Data instead of an untyped empty array.+ return arrow.makeVector([fixedSizeListBuilder.flush()]);}/**diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 178e83737..6c6002ebb 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -10152,6 +10152,89 @@ test('VisStateUpdater -> refreshDataset', async t => {t.end();});+test('VisStateUpdater -> refreshDataset keeps Arrow point layers valid through empty snapshots', async t => {+ const snapshot = values => {+ const table = arrow.tableFromArrays({+ value: Float64Array.from(values),+ lat: Float64Array.from(values, (_, i) => 37.77 + i / 100),+ lng: Float64Array.from(values, (_, i) => -122.42 - i / 100)+ });+ return processArrowBatches(arrow.tableFromIPC(arrow.tableToIPC(table, 'file')).batches);+ };++ for (const withFilter of [false, true]) {+ drainTasksForTesting();+ const dataId = `empty-arrow-${withFilter}`;+ const datasets = await createNewDataEntryMock({+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},+ data: snapshot([1, 2, 3]),+ metadata: {source: 'https://example.com/live.arrow', sourceFormat: 'arrow'}+ });+ const {props} = PointLayer.findDefaultLayerProps(datasets[dataId]);+ const point = new PointLayer({id: `point-${dataId}`, dataId, ...props[0]});+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};+ const initial = calculateLayerData(point, state);+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};+ t.equal(state.layers[0].isValid, true, 'initial point layer is valid');+ if (withFilter) {+ state = reducer(state, VisStateActions.addFilter(dataId));+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));+ state = reducer(state, VisStateActions.setFilter(0, 'value', [2, 3]));+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', true));+ }+ const schema = datasets[dataId].fields.map(({name, type}) => ({name, type}));++ for (const values of [[], [3, 2, 1], [], [1]]) {+ const loading = reducer(state, VisStateActions.refreshDataset(dataId));+ const progressed = reducer(loading, VisStateActions.refreshDatasetProgress(dataId, 50));+ const [task] = drainTasksForTesting();+ state = reducer(+ progressed,+ succeedTaskInTest(task, {data: snapshot(values), notModified: false})+ );+ const layer = state.layers[0];+ t.equal(layer.isValid, true, `refresh to ${values.length} rows keeps the point layer valid`);+ t.equal(layer.errorMessage, null, 'refresh does not report a layer exception');+ t.equal(layer.config.isVisible, true, 'refresh keeps the layer visible');+ t.equal(state.layerData[0].data?.numRows, values.length, 'layer data follows the snapshot');+ t.equal(layer.geoArrowVector.length, values.length, 'coordinate vector follows the snapshot');+ t.deepEqual(+ state.datasets[dataId].fields.map(({name, type}) => ({name, type})),+ schema,+ 'empty IPC snapshots preserve the schema'+ );+ if (values.length) {+ t.deepEqual(+ Array.from(layer.geoArrowVector.get(0)),+ [-122.42, 37.77, 0],+ 'restores positions'+ );+ }+ const expected = values.flatMap((value, index) =>+ !withFilter || value < 2 || value > 3 ? [index] : []+ );+ t.deepEqual(state.datasets[dataId].filteredIndex, expected, 'refresh reapplies active filters');+ t.deepEqual(+ values.flatMap((_, index) => (state.layerData[0].getFiltered({index}) ? [index] : [])),+ expected,+ 'point layer visibility follows the refreshed filter result'+ );+ state.datasets[dataId].filterTableCPU(state.filters, state.layers);+ t.deepEqual(+ state.datasets[dataId].filteredIdxCPU,+ expected,+ 'CPU/export evaluation uses the refreshed filtered rows'+ );+ if (withFilter) {+ t.deepEqual(state.filters[0].value, [2, 3], 'refresh preserves the filter selection');+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');+ t.equal(state.filters[0].inverted, true, 'refresh preserves Filter out mode');+ }+ }+ }+ t.end();+});+test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapshots', async t => {drainTasksForTesting();const dataId = 'remote-arrow';diff --git a/test/node/utils/layer-utils-test.js b/test/node/utils/layer-utils-test.jsindex a7877dd23..c308a5e92 100644--- a/test/node/utils/layer-utils-test.js+++ b/test/node/utils/layer-utils-test.js@@ -4,7 +4,12 @@import test from 'tape';import cloneDeep from 'es-toolkit/compat/cloneDeep';import {processCsvData, processGeojson} from '@kepler.gl/processors';-import {getGeojsonLayerMetaFromArrow, LayerClasses, KeplerGlLayers} from '@kepler.gl/layers';+import {+ createGeoArrowPointVector,+ getGeojsonLayerMetaFromArrow,+ LayerClasses,+ KeplerGlLayers+} from '@kepler.gl/layers';import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, GEOJSON_FIELDS} from '@kepler.gl/constants';import {findDefaultLayer, getLayerHoverProp, getLayerOrderFromLayers} from '@kepler.gl/reducers';import {StateWTripGeojson, StateWFiles} from 'test/helpers/mock-state';@@ -21,6 +26,57 @@ import {KeplerTable, findPointFieldPairs} from '@kepler.gl/table';import {ArrowDataContainer, createDataContainer, findDefaultColorField} from '@kepler.gl/utils';import * as arrow from 'apache-arrow';+test('layerUtils -> createGeoArrowPointVector handles empty and nonempty coordinates', t => {+ for (const coordinates of [+ [],+ [+ [-122.42, 37.77],+ [0, 0]+ ],+ [+ [-122.42, 37.77, 10],+ [0, 0, -5]+ ]+ ]) {+ const dataContainer = new ArrowDataContainer({+ cols: [+ arrow.vectorFromArray(+ coordinates.map((_, i) => i),+ new arrow.Float64()+ )+ ],+ fields: [{name: 'row', type: 'real', fieldIdx: 0}]+ });+ let accessorCalls = 0;+ let vector;+ t.doesNotThrow(() => {+ vector = createGeoArrowPointVector(dataContainer, ({index}) => {+ accessorCalls++;+ return coordinates[index];+ });+ }, 'should create a coordinate vector, including for zero rows');+ if (vector) {+ t.equal(vector.length, coordinates.length, 'preserves the row count');+ t.ok(arrow.DataType.isFixedSizeList(vector.type), 'coordinates have a fixed-size-list type');+ t.equal(vector.type.listSize, coordinates[0]?.length || 2, 'retains coordinate dimensions');+ t.equal(+ String(vector.type.children[0].type),+ 'Float64',+ 'coordinates retain Float64 precision'+ );+ t.deepEqual(+ Array.from({length: vector.length}, (_, i) => Array.from(vector.get(i))),+ coordinates,+ 'preserves nonempty coordinate values and order'+ );+ }+ if (!coordinates.length) {+ t.equal(accessorCalls, 0, 'does not read a nonexistent row');+ }+ }+ t.end();+});+test('layerUtils -> getGeojsonLayerMetaFromArrow uses worker-prepared data', t => {const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));const geoColumn = arrow.vectorFromArray([[-122.4, 37.8]], pointType);