Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: synced filter dataset replacement state (#3839)

Igor Dykhtac384e288f62c
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 875d137c5..44c28fd4e 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -69,6 +69,7 @@ import {
combineSplitMapsByIndex,
computeSplitMapLayers,
adjustValueToFilterDomain,
+ areFilterFieldTypesCompatible,
disableGpuIfInverted,
filterTypeUsesGpu,
getPlotBaseIndexByDataId,
@@ -1619,9 +1620,92 @@ export function setFilterUpdater<S extends VisState>(
return newState;
}
-function _updateFilterDataIdAtValueIndex(filter, valueIndex, value, datasets) {
+function _updateFilterDataIdAtValueIndex(filter, valueIndex, value, datasets, filters) {
+ const currentDataId = filter.dataId[valueIndex];
+ if (currentDataId && value && filter.dataId.length > 1) {
+ const fieldName = filter.name[valueIndex];
+ const endFieldName = toArray(filter.endName)[valueIndex];
+ const previousValue = filter.value;
+ const nextDataIds = filter.dataId.slice();
+ nextDataIds[valueIndex] = value;
+
+ let nextFilter = removeFilterPlot(filter, currentDataId);
+ nextFilter = set(['dataId'], nextDataIds, nextFilter);
+
+ for (const prop of ['gpuChannel', 'gpuEndChannel']) {
+ if (Array.isArray(nextFilter[prop])) {
+ const nextChannels = nextFilter[prop].slice();
+ nextChannels[valueIndex] = undefined;
+ nextFilter = set([prop], nextChannels, nextFilter);
+ }
+ }
+
+ const dataset = datasets[value];
+ const field = dataset?.fields.find(candidate => candidate.name === fieldName);
+ const canRebindField =
+ field &&
+ (!filter.fieldType || areFilterFieldTypesCompatible(filter.fieldType, field.type));
+ const {filter: reboundFilter} = canRebindField
+ ? applyFilterFieldName(nextFilter, datasets, value, fieldName, valueIndex, {
+ mergeDomain: true
+ })
+ : {filter: null};
+
+ if (reboundFilter) {
+ nextFilter = reboundFilter;
+ } else {
+ const nextNames = nextFilter.name.slice();
+ const nextFieldIdx = nextFilter.fieldIdx.slice();
+ nextNames[valueIndex] = '';
+ nextFieldIdx[valueIndex] = -1;
+ nextFilter = {
+ ...nextFilter,
+ name: nextNames,
+ fieldIdx: nextFieldIdx,
+ gpu: false
+ };
+ }
+
+ if (endFieldName && nextFilter.type === FILTER_TYPES.timeRange) {
+ const {filter: reboundEndFilter} = applyTimeFilterEndFieldName(
+ nextFilter,
+ datasets,
+ value,
+ endFieldName,
+ valueIndex
+ );
+ if (reboundEndFilter) {
+ nextFilter = reboundEndFilter;
+ } else {
+ const {filter: clearedEndFilter} = applyTimeFilterEndFieldName(
+ nextFilter,
+ datasets,
+ value,
+ null,
+ valueIndex
+ );
+ nextFilter = clearedEndFilter || nextFilter;
+ }
+ }
+
+ const domainSteps = mergeFilterDomain(nextFilter, datasets);
+ nextFilter = {
+ ...nextFilter,
+ ...(domainSteps ? {domain: domainSteps.domain, step: domainSteps.step} : {})
+ };
+
+ if (nextFilter.gpu) {
+ nextFilter = assignGpuChannel(setFilterGpuMode(nextFilter, filters), filters);
+ }
+
+ return {
+ ...nextFilter,
+ value: adjustValueToFilterDomain(previousValue, nextFilter)
+ };
+ }
+
let newFilter = filter;
- if (filter.dataId[valueIndex]) {
+ if (currentDataId) {
// if dataId already exist
newFilter = _removeFilterDataIdAtValueIndex(filter, valueIndex, datasets);
}
@@ -1689,7 +1773,13 @@ function _updateFilterProp(state, filter, prop, value, valueIndex, datasetIds?)
// 2. Add a new dataset id
case FILTER_UPDATER_PROPS.dataId: {
const oldDataId = [...filter.dataId];
- filter = _updateFilterDataIdAtValueIndex(filter, valueIndex, value, state.datasets);
+ filter = _updateFilterDataIdAtValueIndex(
+ filter,
+ valueIndex,
+ value,
+ state.datasets,
+ state.filters
+ );
datasetIdsToFilter = uniq([...oldDataId, ...filter.dataId]);
break;
}
diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts
index d0ece6939..3277550fd 100644
--- a/src/utils/src/filter-utils.ts
+++ b/src/utils/src/filter-utils.ts
@@ -250,7 +250,10 @@ export function getDefaultFilter({
*/
export function shouldApplyFilter(filter: Filter, datasetId: string): boolean {
const dataIds = toArray(filter.dataId);
- return dataIds.includes(datasetId) && filter.value !== null;
+ const datasetIndex = dataIds.indexOf(datasetId);
+ const hasSelectedField =
+ filter.type === FILTER_TYPES.polygon || Boolean(toArray(filter.name)[datasetIndex]);
+ return datasetIndex >= 0 && hasSelectedField && filter.value !== null;
}
/**
@@ -1502,8 +1505,11 @@ export function mergeFilterDomain(
}
filter.dataId.forEach((filterDataId, idx) => {
const dataset = datasets[filterDataId];
- const filterProps = dataset.getColumnFilterProps(filter.name[idx]);
- domainSteps = mergeFilterDomainStep(domainSteps ?? ({} as Filter), filterProps);
+ const fieldName = filter.name[idx];
+ if (fieldName) {
+ const filterProps = dataset.getColumnFilterProps(fieldName);
+ domainSteps = mergeFilterDomainStep(domainSteps ?? ({} as Filter), filterProps);
+ }
const endName = toArray((filter as TimeRangeFilter).endName)[idx];
if (endName) {
const endProps = dataset.getColumnFilterProps(endName);
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index e0902f6fb..58c07f823 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -58,6 +58,7 @@ import {
BINS,
INTERVAL,
LAYER_TYPES,
+ MAX_GPU_FILTERS,
SYNC_TIMELINE_MODES,
GEOCODER_DATASET_NAME,
GEOCODER_ICON_COLOR,
@@ -2875,6 +2876,236 @@ test('#visStateReducer -> SET_FILTER.dataId', t => {
t.end();
});
+test('#visStateReducer -> SET_FILTER.dataId replaces synced datasets in place', async t => {
+ const firstDataset = await createNewDataEntryMock({
+ info: {id: 'first', label: 'first'},
+ data: {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[0], [10]]
+ }
+ });
+ const secondDataset = await createNewDataEntryMock({
+ info: {id: 'second', label: 'second'},
+ data: {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[20], [30]]
+ }
+ });
+ const replacementDataset = await createNewDataEntryMock({
+ info: {id: 'replacement', label: 'replacement'},
+ data: {
+ fields: [
+ {name: 'other', type: ALL_FIELD_TYPES.integer},
+ {name: 'value', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ [400, 40],
+ [500, 50]
+ ]
+ }
+ });
+ let state = {
+ ...INITIAL_VIS_STATE,
+ datasets: {...firstDataset, ...secondDataset, ...replacementDataset}
+ };
+
+ state = reducer(state, VisStateActions.addFilter('first'));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, ['dataId', 'name'], ['second', 'value'], 1));
+ state = reducer(state, VisStateActions.addFilter('replacement'));
+ state = reducer(state, VisStateActions.setFilter(1, 'name', 'other'));
+
+ const replaceFirstState = reducer(
+ state,
+ VisStateActions.setFilter(0, 'dataId', 'replacement', 0)
+ );
+ const replaceFirst = replaceFirstState.filters[0];
+ t.deepEqual(replaceFirst.dataId, ['replacement', 'second'], 'replaces the first dataset');
+ t.deepEqual(replaceFirst.name, ['value', 'value'], 'keeps field names aligned');
+ t.deepEqual(replaceFirst.fieldIdx, [1, 0], 'rebinds the first field index by name');
+ t.deepEqual(replaceFirst.gpuChannel, [1, 0], 'reallocates the replaced GPU channel');
+ t.equal(
+ replaceFirstState.datasets.replacement.gpuFilter.filterValueUpdateTriggers.gpuFilter_0.name,
+ 'other',
+ 'keeps the existing replacement-dataset GPU filter'
+ );
+ t.equal(
+ replaceFirstState.datasets.replacement.gpuFilter.filterValueUpdateTriggers.gpuFilter_1.name,
+ 'value',
+ 'adds the rebound filter on a free GPU channel'
+ );
+ t.deepEqual(
+ replaceFirst.domain,
+ [20, 50],
+ 'rebuilds the domain after replacing the first dataset'
+ );
+
+ const replaceSecond = reducer(
+ state,
+ VisStateActions.setFilter(0, 'dataId', 'replacement', 1)
+ ).filters[0];
+ t.deepEqual(replaceSecond.dataId, ['first', 'replacement'], 'replaces a later dataset');
+ t.deepEqual(replaceSecond.name, ['value', 'value'], 'keeps later field names aligned');
+ t.deepEqual(replaceSecond.fieldIdx, [0, 1], 'rebinds a later field index by name');
+ t.deepEqual(replaceSecond.gpuChannel, [0, 1], 'reallocates a later GPU channel');
+ t.deepEqual(replaceSecond.domain, [0, 50], 'rebuilds the domain after replacing a later dataset');
+
+ t.end();
+});
+
+test('#visStateReducer -> SET_FILTER.dataId rebinds interval fields on replacement', async t => {
+ const makeDataset = async (id, csv) => {
+ const {fields, rows} = processCsvData(csv);
+ return createNewDataEntryMock({info: {id, label: id}, data: {fields, rows}});
+ };
+ const firstDataset = await makeDataset(
+ 'first',
+ 'start,end\n2020-01-01 00:00:00,2020-01-01 00:30:00\n2020-01-01 01:00:00,2020-01-01 01:30:00'
+ );
+ const secondDataset = await makeDataset(
+ 'second',
+ 'start,end\n2020-01-02 00:00:00,2020-01-02 00:30:00\n2020-01-02 01:00:00,2020-01-02 01:30:00'
+ );
+ const replacementDataset = await makeDataset(
+ 'replacement',
+ 'other,end,start\n1,2020-01-03 00:30:00,2020-01-03 00:00:00\n2,2020-01-03 01:30:00,2020-01-03 01:00:00'
+ );
+ const missingEndDataset = await makeDataset(
+ 'missing-end',
+ 'other,start\n1,2020-01-04 00:00:00\n2,2020-01-04 01:00:00'
+ );
+ let state = {
+ ...INITIAL_VIS_STATE,
+ datasets: {
+ ...firstDataset,
+ ...secondDataset,
+ ...replacementDataset,
+ ...missingEndDataset
+ }
+ };
+
+ state = reducer(state, VisStateActions.addFilter('first'));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'start'));
+ state = reducer(state, VisStateActions.setFilter(0, 'endName', 'end'));
+ state = reducer(state, VisStateActions.setFilter(0, ['dataId', 'name'], ['second', 'start'], 1));
+ state = reducer(state, VisStateActions.setFilter(0, 'endName', 'end', 1));
+ state = reducer(state, VisStateActions.addFilter('replacement'));
+ state = reducer(state, VisStateActions.setFilter(1, 'name', 'other'));
+
+ const replaced = reducer(
+ state,
+ VisStateActions.setFilter(0, 'dataId', 'replacement', 1)
+ ).filters[0];
+ const replacementEndProps = replacementDataset.replacement.getColumnFilterProps('end');
+ t.deepEqual(replaced.fieldIdx, [0, 2], 'rebinds the interval start field index');
+ t.deepEqual(replaced.endFieldIdx, [1, 1], 'rebinds the interval end field index');
+ t.deepEqual(
+ replaced.endMappedValue[1],
+ replacementEndProps.mappedValue,
+ 'rebuilds the replacement interval end values'
+ );
+ t.deepEqual(replaced.gpuChannel, [0, 1], 'allocates a free replacement start channel');
+ t.deepEqual(replaced.gpuEndChannel, [1, 2], 'allocates a free replacement end channel');
+
+ const missingEnd = reducer(
+ state,
+ VisStateActions.setFilter(0, 'dataId', 'missing-end', 1)
+ ).filters[0];
+ t.equal(missingEnd.name[1], 'start', 'keeps an available start selection');
+ t.equal(missingEnd.endName[1], null, 'clears a missing end selection');
+ t.equal(missingEnd.endFieldIdx[1], null, 'clears a missing end field index');
+ t.equal(missingEnd.endMappedValue[1], null, 'clears missing interval values');
+ t.equal(missingEnd.gpuEndChannel[1], undefined, 'clears the missing end GPU channel');
+
+ t.end();
+});
+
+test('#visStateReducer -> SET_FILTER.dataId leaves missing replacement fields empty', async t => {
+ const firstDataset = await createNewDataEntryMock({
+ info: {id: 'first', label: 'first'},
+ data: {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[0], [10]]
+ }
+ });
+ const secondDataset = await createNewDataEntryMock({
+ info: {id: 'second', label: 'second'},
+ data: {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[20], [30]]
+ }
+ });
+ const replacementDataset = await createNewDataEntryMock({
+ info: {id: 'replacement', label: 'replacement'},
+ data: {
+ fields: [{name: 'other', type: ALL_FIELD_TYPES.integer}],
+ rows: [[100], [200]]
+ }
+ });
+ let state = {
+ ...INITIAL_VIS_STATE,
+ datasets: {...firstDataset, ...secondDataset, ...replacementDataset}
+ };
+
+ state = reducer(state, VisStateActions.addFilter('first'));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, ['dataId', 'name'], ['second', 'value'], 1));
+ state = reducer(state, VisStateActions.setFilter(0, 'value', [5, 15]));
+ state = reducer(state, VisStateActions.setFilter(0, 'dataId', 'replacement', 1));
+
+ t.deepEqual(state.filters[0].name, ['value', ''], 'leaves the unavailable field unselected');
+ t.deepEqual(state.filters[0].fieldIdx, [0, -1], 'clears the unavailable field index');
+ t.equal(state.filters[0].gpu, false, 'uses CPU filtering while a selection is missing');
+ t.equal(state.filters[0].gpuChannel[1], undefined, 'clears the replaced GPU channel');
+ t.equal(
+ state.datasets.replacement.filteredIndex.length,
+ 2,
+ 'does not apply the synced filter until a replacement field is selected'
+ );
+
+ t.end();
+});
+
+test('#visStateReducer -> SET_FILTER.dataId falls back to CPU when GPU channels are exhausted', async t => {
+ const createDataset = (id, rows) =>
+ createNewDataEntryMock({
+ info: {id, label: id},
+ data: {fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}], rows}
+ });
+ const firstDataset = await createDataset('first', [[0], [10]]);
+ const secondDataset = await createDataset('second', [[20], [30]]);
+ const replacementDataset = await createDataset('replacement', [[0], [10], [20]]);
+ let state = {
+ ...INITIAL_VIS_STATE,
+ datasets: {...firstDataset, ...secondDataset, ...replacementDataset}
+ };
+
+ state = reducer(state, VisStateActions.addFilter('first'));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, ['dataId', 'name'], ['second', 'value'], 1));
+ state = reducer(state, VisStateActions.setFilter(0, 'value', [5, 15]));
+ for (let i = 0; i < MAX_GPU_FILTERS; i++) {
+ state = reducer(state, VisStateActions.addFilter('replacement'));
+ state = reducer(state, VisStateActions.setFilter(i + 1, 'name', 'value'));
+ }
+
+ state = reducer(state, VisStateActions.setFilter(0, 'dataId', 'replacement', 1));
+
+ t.equal(state.filters[0].gpu, false, 'falls back to CPU filtering');
+ t.equal(state.filters[0].gpuChannel[1], undefined, 'does not retain a conflicting channel');
+ t.ok(
+ state.datasets.replacement.filterRecord.cpu.some(filter => filter.id === state.filters[0].id),
+ 'records the replaced filter as a CPU filter'
+ );
+ t.equal(
+ state.datasets.replacement.filteredIndex.length,
+ 1,
+ 'applies the replacement filter on CPU'
+ );
+
+ t.end();
+});
+
test('#visStateReducer -> SET_FILTER synced', t => {
const initialState = StateWSyncedTimeFilter.visState;
const oldFilter = {...initialState.filters[0]};
diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.js
index 38e0b0acc..6f5c67335 100644
--- a/test/node/utils/filter-utils-test.js
+++ b/test/node/utils/filter-utils-test.js
@@ -20,6 +20,7 @@ import {
scaleSourceDomainToDestination,
mergeFilterWithTimeline,
createDataContainer,
+ shouldApplyFilter,
isTimeIntervalFilter,
timeWindowOverlapsInterval,
applyTimeFilterEndFieldName,
@@ -34,6 +35,20 @@ import {ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';
import {mockPolygonFeature, mockPolygonData} from '../../fixtures/polygon';
/* eslint-disable max-statements */
+test('filterUtils -> shouldApplyFilter skips an unselected synced field', t => {
+ const filter = {
+ ...getDefaultFilter({dataId: ['first', 'second']}),
+ type: FILTER_TYPES.range,
+ name: ['value', ''],
+ value: [0, 1]
+ };
+
+ t.equal(shouldApplyFilter(filter, 'first'), true, 'applies the selected dataset field');
+ t.equal(shouldApplyFilter(filter, 'second'), false, 'skips the empty dataset field');
+ t.equal(shouldApplyFilter(filter, 'missing'), false, 'skips an unrelated dataset');
+ t.end();
+});
+
test('filterUtils -> adjustValueToFilterDomain', t => {
// TODO: needs id
const rangeFilter = getDefaultFilter();