Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] reapply active filters after Arrow refresh (#3877)
Igor Dykhtac3758f462cb0
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex fee0da7d1..1e7917acc 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -291,8 +291,8 @@ class KeplerTable<F extends Field = Field> {*/async update(data: ProtoDataset['data']) {// Arrow/DuckDB incremental loads pass `cols` (and empty `rows`). Row snapshots- // for remote refresh pass `rows` only. Keep the column path close to the- // pre-refresh update() so filters/gpuFilter are not wiped on a normal load.+ // for remote refresh pass `rows` only. Preserve the GPU filter configuration+ // on column updates, but invalidate cached filter results for every snapshot.const isRowSnapshot = !data.cols && Array.isArray(data.rows);if (data.fields?.length && !fieldNamesMatch(this.fields, data.fields)) {@@ -323,9 +323,6 @@ class KeplerTable<F extends Field = Field> {valueAccessor: getFieldValueAccessor(f, i, this.dataContainer)};}) as F[];- this.filterRecord = undefined;- this.filterRecordCPU = undefined;- this.changedFilters = undefined;this.gpuFilter = getGpuFilterProps([], this.id, this.fields, undefined);}@@ -333,6 +330,10 @@ class KeplerTable<F extends Field = Field> {this.filteredIndex = this.allIndexes;this.filteredIndexForDomain = this.allIndexes;this.filteredIndexByLayer = {};+ this.filteredIdxCPU = undefined;+ this.filterRecord = undefined;+ this.filterRecordCPU = undefined;+ this.changedFilters = undefined;this.dataRevision += 1;return this;diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex c280fe0b0..178e83737 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -10152,6 +10152,56 @@ test('VisStateUpdater -> refreshDataset', async t => {t.end();});+test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapshots', async t => {+ drainTasksForTesting();+ const dataId = 'remote-arrow';+ const snapshot = values => {+ const arrowTable = arrow.tableFromArrays({value: new Float64Array(values)});+ return {+ fields: [{name: 'value', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}],+ rows: [],+ cols: [arrowTable.getChildAt(0)],+ arrowTable+ };+ };+ const datasets = await createNewDataEntryMock({+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},+ data: snapshot([1, 2, 3]),+ metadata: {source: 'https://example.com/live.arrow', sourceFormat: 'arrow'}+ });+ let state = reducer({...INITIAL_VIS_STATE, datasets}, VisStateActions.addFilter(dataId));+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));+ state = reducer(state, VisStateActions.setFilter(0, 'value', [2, 3]));+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', true));+ t.deepEqual(state.datasets[dataId].filteredIndex, [0], 'Filter out 2–3 initially keeps only 1');+ t.equal(state.filters[0].gpu, false, 'inverted filtering runs on the CPU');++ for (const [values, expected] of [+ [[1, 2, 3], [0]],+ [[3, 1, 2], [1]],+ [[2, 1, 3, 4], [1, 3]],+ [[], []],+ [[3, 2, 1], [2]],+ [[2], []]+ ]) {+ const loading = reducer(state, VisStateActions.refreshDataset(dataId));+ const progressed = reducer(loading, VisStateActions.refreshDatasetProgress(dataId, 50));+ const [task] = drainTasksForTesting();+ const data = snapshot(values);+ state = reducer(progressed, succeedTaskInTest(task, {data, notModified: false}));+ const table = state.datasets[dataId];+ t.deepEqual(table.filteredIndex, expected, `refresh filters snapshot [${values}]`);+ t.deepEqual(table.filteredIndexForDomain, expected, 'domain uses the same filtered snapshot');+ table.filterTableCPU(state.filters, state.layers);+ t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export evaluation keeps the filtered rows');+ t.deepEqual(state.filters[0].value, [2, 3], 'refresh preserves the selected range');+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');+ t.equal(state.filters[0].inverted, true, 'refresh preserves Filter out');+ t.deepEqual(Array.from(data.arrowTable.getChildAt(0)), values, 'source values are unchanged');+ }+ t.end();+});+test('VisStateUpdater -> refreshDataset rebuilds filter domains and preserves selections', async t => {drainTasksForTesting();const dataId = 'remote-filter';diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.jsindex 22f9ddd75..47d7d0f76 100644--- a/test/node/utils/kepler-table-test.js+++ b/test/node/utils/kepler-table-test.js@@ -466,11 +466,27 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>}});+ const filters = [+ {+ id: 'keep-one',+ dataId: [table.id],+ name: ['lat'],+ fieldIdx: [0],+ type: FILTER_TYPES.range,+ domain: [1, 3],+ value: [1, 1],+ enabled: true,+ gpu: true,+ gpuChannel: [0]+ }+ ];+ table.filterTable(filters, []);+ table.filterTableCPU(filters, []);+ t.deepEqual(table.filteredIdxCPU, [0], 'initial CPU evaluation keeps the matching row');const gpuFilter = table.gpuFilter;- table.filterRecord = {name: 'keep-me'};const next = arrow.tableFromArrays({- lat: [1, 2, 3],+ lat: [3, 1, 2],lng: [3, 4, 5]});await table.update({@@ -485,8 +501,12 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>t.equal(table.length, 3, 'arrow incremental update should grow rows');t.equal(table.gpuFilter, gpuFilter, 'arrow incremental update should keep gpuFilter');- t.equal(table.filterRecord?.name, 'keep-me', 'arrow incremental update should keep filterRecord');t.equal(table.dataRevision, 1, 'arrow incremental update should bump dataRevision');+ table.filterTable(filters, []);+ table.filterTableCPU(filters, []);+ t.deepEqual(table.filteredIndex, [0, 1, 2], 'GPU filtering keeps all rows available to the GPU');+ t.deepEqual(table.filteredIndexForDomain, [1], 'domain filtering uses the refreshed values');+ t.deepEqual(table.filteredIdxCPU, [1], 'CPU evaluation uses the refreshed matching row');t.end();});