Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] reapply active filters after Arrow refresh (#3877)

Igor Dykhtac3758f462cb0
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index fee0da7d1..1e7917acc 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -291,8 +291,8 @@ class KeplerTable<F extends Field = Field> {
*/
async update(data: ProtoDataset['data']) {
// Arrow/DuckDB incremental loads pass `cols` (and empty `rows`). Row snapshots
- // for remote refresh pass `rows` only. Keep the column path close to the
- // pre-refresh update() so filters/gpuFilter are not wiped on a normal load.
+ // for remote refresh pass `rows` only. Preserve the GPU filter configuration
+ // on column updates, but invalidate cached filter results for every snapshot.
const isRowSnapshot = !data.cols && Array.isArray(data.rows);
if (data.fields?.length && !fieldNamesMatch(this.fields, data.fields)) {
@@ -323,9 +323,6 @@ class KeplerTable<F extends Field = Field> {
valueAccessor: getFieldValueAccessor(f, i, this.dataContainer)
};
}) as F[];
- this.filterRecord = undefined;
- this.filterRecordCPU = undefined;
- this.changedFilters = undefined;
this.gpuFilter = getGpuFilterProps([], this.id, this.fields, undefined);
}
@@ -333,6 +330,10 @@ class KeplerTable<F extends Field = Field> {
this.filteredIndex = this.allIndexes;
this.filteredIndexForDomain = this.allIndexes;
this.filteredIndexByLayer = {};
+ this.filteredIdxCPU = undefined;
+ this.filterRecord = undefined;
+ this.filterRecordCPU = undefined;
+ this.changedFilters = undefined;
this.dataRevision += 1;
return this;
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index c280fe0b0..178e83737 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -10152,6 +10152,56 @@ test('VisStateUpdater -> refreshDataset', async t => {
t.end();
});
+test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapshots', async t => {
+ drainTasksForTesting();
+ const dataId = 'remote-arrow';
+ const snapshot = values => {
+ const arrowTable = arrow.tableFromArrays({value: new Float64Array(values)});
+ return {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}],
+ rows: [],
+ cols: [arrowTable.getChildAt(0)],
+ arrowTable
+ };
+ };
+ const datasets = await createNewDataEntryMock({
+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},
+ data: snapshot([1, 2, 3]),
+ metadata: {source: 'https://example.com/live.arrow', sourceFormat: 'arrow'}
+ });
+ let state = reducer({...INITIAL_VIS_STATE, datasets}, VisStateActions.addFilter(dataId));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, 'value', [2, 3]));
+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', true));
+ t.deepEqual(state.datasets[dataId].filteredIndex, [0], 'Filter out 2–3 initially keeps only 1');
+ t.equal(state.filters[0].gpu, false, 'inverted filtering runs on the CPU');
+
+ for (const [values, expected] of [
+ [[1, 2, 3], [0]],
+ [[3, 1, 2], [1]],
+ [[2, 1, 3, 4], [1, 3]],
+ [[], []],
+ [[3, 2, 1], [2]],
+ [[2], []]
+ ]) {
+ const loading = reducer(state, VisStateActions.refreshDataset(dataId));
+ const progressed = reducer(loading, VisStateActions.refreshDatasetProgress(dataId, 50));
+ const [task] = drainTasksForTesting();
+ const data = snapshot(values);
+ state = reducer(progressed, succeedTaskInTest(task, {data, notModified: false}));
+ const table = state.datasets[dataId];
+ t.deepEqual(table.filteredIndex, expected, `refresh filters snapshot [${values}]`);
+ t.deepEqual(table.filteredIndexForDomain, expected, 'domain uses the same filtered snapshot');
+ table.filterTableCPU(state.filters, state.layers);
+ t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export evaluation keeps the filtered rows');
+ t.deepEqual(state.filters[0].value, [2, 3], 'refresh preserves the selected range');
+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');
+ t.equal(state.filters[0].inverted, true, 'refresh preserves Filter out');
+ t.deepEqual(Array.from(data.arrowTable.getChildAt(0)), values, 'source values are unchanged');
+ }
+ t.end();
+});
+
test('VisStateUpdater -> refreshDataset rebuilds filter domains and preserves selections', async t => {
drainTasksForTesting();
const dataId = 'remote-filter';
diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.js
index 22f9ddd75..47d7d0f76 100644
--- a/test/node/utils/kepler-table-test.js
+++ b/test/node/utils/kepler-table-test.js
@@ -466,11 +466,27 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>
}
});
+ const filters = [
+ {
+ id: 'keep-one',
+ dataId: [table.id],
+ name: ['lat'],
+ fieldIdx: [0],
+ type: FILTER_TYPES.range,
+ domain: [1, 3],
+ value: [1, 1],
+ enabled: true,
+ gpu: true,
+ gpuChannel: [0]
+ }
+ ];
+ table.filterTable(filters, []);
+ table.filterTableCPU(filters, []);
+ t.deepEqual(table.filteredIdxCPU, [0], 'initial CPU evaluation keeps the matching row');
const gpuFilter = table.gpuFilter;
- table.filterRecord = {name: 'keep-me'};
const next = arrow.tableFromArrays({
- lat: [1, 2, 3],
+ lat: [3, 1, 2],
lng: [3, 4, 5]
});
await table.update({
@@ -485,8 +501,12 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>
t.equal(table.length, 3, 'arrow incremental update should grow rows');
t.equal(table.gpuFilter, gpuFilter, 'arrow incremental update should keep gpuFilter');
- t.equal(table.filterRecord?.name, 'keep-me', 'arrow incremental update should keep filterRecord');
t.equal(table.dataRevision, 1, 'arrow incremental update should bump dataRevision');
+ table.filterTable(filters, []);
+ table.filterTableCPU(filters, []);
+ t.deepEqual(table.filteredIndex, [0, 1, 2], 'GPU filtering keeps all rows available to the GPU');
+ t.deepEqual(table.filteredIndexForDomain, [1], 'domain filtering uses the refreshed values');
+ t.deepEqual(table.filteredIdxCPU, [1], 'CPU evaluation uses the refreshed matching row');
t.end();
});