Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: hide Grid and Hexbin cells excluded by numeric filters (#3951)

Igor Dykhtad6561902e197
diff --git a/src/layers/src/aggregation-layer.ts b/src/layers/src/aggregation-layer.ts
index 448fc2cb8..160ed2907 100644
--- a/src/layers/src/aggregation-layer.ts
+++ b/src/layers/src/aggregation-layer.ts
@@ -709,14 +709,21 @@ export default class AggregationLayer extends Layer {
getElevationValue = wrapOrdinalAccessor(getElevationValue);
}
- // Wrap accessors to filter points within each bin before aggregating.
+ // Mask bins emptied by filtering with NaN: deck.gl skips them in cell
+ // rendering and scale domains. Retained points may legitimately aggregate to 0.
const getFilteredColorValue =
filterData && getColorValue
- ? points => getColorValue(points.filter(filterData))
+ ? points => {
+ const filteredPoints = points.filter(filterData);
+ return filteredPoints.length ? getColorValue(filteredPoints) : NaN;
+ }
: getColorValue;
const getFilteredElevationValue =
filterData && getElevationValue
- ? points => getElevationValue(points.filter(filterData))
+ ? points => {
+ const filteredPoints = points.filter(filterData);
+ return filteredPoints.length ? getElevationValue(filteredPoints) : NaN;
+ }
: getElevationValue;
const {data} = this.updateData(datasets, oldLayerData);
diff --git a/test/browser/layer-tests/grid-layer-specs.js b/test/browser/layer-tests/grid-layer-specs.js
index 02cfbd536..eb1d11a9d 100644
--- a/test/browser/layer-tests/grid-layer-specs.js
+++ b/test/browser/layer-tests/grid-layer-specs.js
@@ -3,20 +3,30 @@
import test from 'tape';
import sinon from 'sinon';
+import {LayerManager, MapView} from '@deck.gl/core';
+import {device} from '@deck.gl/test-utils';
+import {KeplerTable} from '@kepler.gl/table';
+import {VisStateActions} from '@kepler.gl/actions';
import {
testCreateCases,
testFormatLayerDataCases,
testRenderLayerCases,
+ testRenderDeckLayer,
preparedDataset,
dataId,
testRows,
pointLayerMeta
} from 'test/helpers/layer-utils';
-import {pointToPolygonGeo, KeplerGlLayers} from '@kepler.gl/layers';
+import {pointToPolygonGeo, KeplerGlLayers, LayerClasses} from '@kepler.gl/layers';
const {GridLayer} = KeplerGlLayers;
-import {INITIAL_MAP_STATE} from '@kepler.gl/reducers';
+import {
+ INITIAL_MAP_STATE,
+ INITIAL_VIS_STATE,
+ validateLayerWithData,
+ visStateReducer
+} from '@kepler.gl/reducers';
const columns = {
lat: 'lat',
@@ -331,20 +341,20 @@ test('#GridLayer -> renderLayer', t => {
// Verify actual attribute array values (deck.gl 9 equivalent of instanceFillColors/instanceElevations)
// cellData.attributes contain SCALED values (processed through AttributeWithScale),
- // not raw aggregated weights. For quantize scale with 3 bins [0, 1, 2],
- // values are mapped to percentile-like range [0, 50, 99].
+ // not raw aggregated weights. Quantile scaling excludes the empty bin
+ // and maps the two retained counts to [0, 99], preserving the NaN mask.
const colorValues = Array.from(cellData.attributes.getColorValue.value);
const elevationValues = Array.from(cellData.attributes.getElevationValue.value);
t.deepEqual(
- colorValues.slice().sort((a, b) => a - b),
- [0, 50, 99],
- 'getColorValue typed array should contain scaled values [0, 50, 99] (sorted)'
+ colorValues.slice().sort(),
+ [0, 99, NaN],
+ 'scaled color values should mask the empty bin'
);
// Elevation uses linear scale (default), so raw values pass through
t.deepEqual(
- elevationValues.slice().sort((a, b) => a - b),
- [0, 1, 2],
- 'getElevationValue typed array should contain raw counts [0, 1, 2] (sorted)'
+ elevationValues.slice().sort(),
+ [1, 2, NaN],
+ 'raw elevation counts should mask the empty bin'
);
// getBin contains [col, row] integer IDs per bin (size=2)
@@ -358,7 +368,7 @@ test('#GridLayer -> renderLayer', t => {
// onSetColorDomain fires twice: first the parent's [min, max], then our
// enriched {domain, aggregatedBins}. The last call is the one that sticks.
- // count aggregation: bins have 0, 2, and 1 filtered points
+ // count aggregation: only bins with 2 and 1 filtered points contribute
t.ok(spyLayerCallbacks.called, 'should call onSetLayerDomain');
const lastIdx0 = spyLayerCallbacks.args.length - 1;
const enrichedArg0 = spyLayerCallbacks.args[lastIdx0][0];
@@ -370,7 +380,7 @@ test('#GridLayer -> renderLayer', t => {
// sorted array of per-bin values (not just [min, max]).
t.deepEqual(
enrichedArg0.domain,
- [0, 1, 2],
+ [1, 2],
'enriched domain should be full sorted bin values for quantile scale'
);
t.ok(enrichedArg0.aggregatedBins, 'enriched call should include aggregatedBins');
@@ -386,10 +396,10 @@ test('#GridLayer -> renderLayer', t => {
);
const colorDomain = aggregator.getResultDomain(0);
- t.deepEqual(colorDomain, [0, 2], 'color domain should be [0, 2]');
+ t.deepEqual(colorDomain, [1, 2], 'color domain should exclude empty bins');
const elevationDomain = aggregator.getResultDomain(1);
- t.deepEqual(elevationDomain, [0, 2], 'elevation domain should be [0, 2]');
+ t.deepEqual(elevationDomain, [1, 2], 'elevation domain should exclude empty bins');
// Verify per-bin data: collect bins and match by count
const binCounts = [];
@@ -409,13 +419,13 @@ test('#GridLayer -> renderLayer', t => {
);
t.deepEqual(
binColorValues.slice().sort(),
- [0, 1, 2],
- 'bin color values (filtered count agg) should be [0, 1, 2] (sorted)'
+ [1, 2, NaN],
+ 'bin color counts should mask empty bins'
);
t.deepEqual(
binElevationValues.slice().sort(),
- [0, 1, 2],
- 'bin elevation values (filtered count agg) should be [0, 1, 2] (sorted)'
+ [1, 2, NaN],
+ 'bin elevation counts should mask empty bins'
);
// Verify raw result arrays
@@ -424,7 +434,7 @@ test('#GridLayer -> renderLayer', t => {
const sortedColorResult = Array.from(
colorResult.value.slice(0, aggregator.binCount)
).sort();
- t.deepEqual(sortedColorResult, [0, 1, 2], 'color result values should be [0, 1, 2]');
+ t.deepEqual(sortedColorResult, [1, 2, NaN], 'color results should mask empty bins');
}
}
},
@@ -605,3 +615,179 @@ test('#GridLayer -> pointToPolygonGeo', t => {
t.deepEqual(polygonGeo, expected, 'should create correct polygonGeo');
t.end();
});
+
+// Grid and Hexbin share per-bin filtering, but have separate deck.gl cell layers.
+for (const type of ['grid', 'hexagon']) {
+ test(`#${type} -> numeric filters mask empty cells and exclude them from domains`, t => {
+ const dataset = new KeplerTable({info: {id: dataId}, color: [1, 2, 3]});
+ dataset.importData({
+ data: {
+ fields: [
+ {name: 'lat', type: 'real'},
+ {name: 'lng', type: 'real'},
+ {name: 'value', type: 'integer'},
+ {name: 'weight', type: 'integer'},
+ {name: 'other', type: 'integer'},
+ {name: 'category', type: 'string'}
+ ],
+ rows: [
+ [37, -122, 1, 7, 0, 'keep'],
+ [38, -121, 5, 0, 10, 'keep'],
+ [39, -120, 10, 10, 10, 'drop']
+ ]
+ }
+ });
+ const layer = validateLayerWithData(
+ dataset,
+ {type, config: {dataId, columns, visConfig: {worldUnitSize: 1}}},
+ LayerClasses
+ );
+ let state = {
+ ...INITIAL_VIS_STATE,
+ datasets: {[dataId]: dataset},
+ layers: [layer],
+ layerData: [layer.formatLayerData({[dataId]: dataset})]
+ };
+ const viewport = new MapView().makeViewport({
+ width: 800,
+ height: 600,
+ viewState: INITIAL_MAP_STATE
+ });
+ const layerManager = new LayerManager(device, {viewport});
+ const onSetLayerDomain = sinon.spy();
+ const apply = action => {
+ state = visStateReducer(state, action);
+ };
+ const sorted = values =>
+ Array.from(values)
+ .filter(Number.isFinite)
+ .sort((a, b) => a - b);
+ const check = (label, expectedColors, expectedElevations = expectedColors) => {
+ const current = state.layers[0];
+ const deckLayers = current.renderLayer({
+ data: state.layerData[0],
+ gpuFilter: state.datasets[dataId].gpuFilter,
+ mapState: INITIAL_MAP_STATE,
+ idx: 0,
+ visible: true,
+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,
+ layerCallbacks: {onSetLayerDomain}
+ });
+ const [rendered, cells] = testRenderDeckLayer(t, type, deckLayers, {layerManager});
+ const {aggregator} = rendered.state;
+ const attributes = cells.props.data.attributes;
+ // Both deck.gl cell shaders discard NaN instanceColorValues. Assert on the
+ // final scaled attributes, not just the custom aggregation accessors.
+ t.equal(
+ sorted(attributes.getColorValue.value.subarray(0, cells.props.data.length)).length,
+ expectedColors.length,
+ `${label}: only retained bins have drawable color values`
+ );
+ t.equal(
+ sorted(attributes.getElevationValue.value.subarray(0, cells.props.data.length)).length,
+ expectedElevations.length,
+ `${label}: only retained bins have elevation values`
+ );
+ for (const [channel, expected] of [expectedColors, expectedElevations].entries()) {
+ t.deepEqual(
+ sorted(aggregator.getResult(channel).value.subarray(0, aggregator.binCount)),
+ expected,
+ `${label}: channel ${channel} aggregates retained rows`
+ );
+ t.deepEqual(
+ aggregator.getResultDomain(channel),
+ expected.length ? [expected[0], expected[expected.length - 1]] : [Infinity, -Infinity],
+ `${label}: channel ${channel} domain excludes empty bins`
+ );
+ }
+ t.deepEqual(
+ onSetLayerDomain.lastCall.args[0].domain,
+ expectedColors,
+ `${label}: quantile legend domain excludes empty bins`
+ );
+ };
+
+ try {
+ check('unfiltered', [1, 1, 1]);
+ apply(VisStateActions.addFilter(dataId));
+ apply(VisStateActions.setFilter(0, 'name', 'value'));
+ apply(VisStateActions.setFilter(0, 'value', [5, 10]));
+ t.equal(state.filters[0].gpu, true, 'numeric filters use GPU mode by default');
+ const cpuIndex = state.datasets[dataId].filteredIndex;
+ const cachedData = state.layerData[0].data;
+ t.deepEqual(cpuIndex, [0, 1, 2], 'GPU filtering leaves all source points for binning');
+ check('range [5, 10]', [1, 1]);
+
+ apply(VisStateActions.setFilter(0, 'gpu', false));
+ check('equivalent CPU filter', [1, 1]);
+ apply(VisStateActions.setFilter(0, 'gpu', true));
+ check('GPU filter restored', [1, 1]);
+ const gpuData = state.layerData[0].data;
+ apply(VisStateActions.setFilter(0, 'value', [1, 1]));
+ t.deepEqual(state.datasets[dataId].gpuFilter.filterRange[0], [0, 0], 'minimum is active');
+ t.equal(state.layerData[0].data, gpuData, 'range updates reuse the binned input data');
+ check('minimum-only range', [1]);
+ apply(VisStateActions.setFilter(0, 'value', [5, 10]));
+ apply(VisStateActions.addFilter(dataId));
+ apply(VisStateActions.setFilter(1, 'name', 'other'));
+ apply(VisStateActions.setFilter(1, 'value', [0, 0]));
+ check('all bins excluded by intersecting numeric filters', []);
+ apply(VisStateActions.removeFilter(1));
+ check('removing a filter restores bins', [1, 1]);
+ apply(VisStateActions.layerVisConfigChange(state.layers[0], {opacity: 0.4, enable3d: true}));
+ check('style and extrusion update preserve filtering', [1, 1]);
+
+ const weight = dataset.fields.find(field => field.name === 'weight');
+ for (const channel of ['color', 'size']) {
+ apply(
+ VisStateActions.layerVisualChannelConfigChange(
+ state.layers[0],
+ {[`${channel}Field`]: weight},
+ channel
+ )
+ );
+ }
+ for (const [aggregation, combined] of [
+ ['count', 2],
+ ['sum', 10],
+ ['average', 5],
+ ['minimum', 0],
+ ['maximum', 10]
+ ]) {
+ apply(
+ VisStateActions.layerVisConfigChange(state.layers[0], {
+ colorAggregation: aggregation,
+ sizeAggregation: aggregation
+ })
+ );
+ check(aggregation, aggregation === 'count' ? [1, 1] : [0, 10]);
+ for (const accessor of ['getColorValue', 'getElevationValue']) {
+ t.equal(
+ state.layerData[0][accessor](cachedData),
+ combined,
+ `${accessor}: ${aggregation} filters a partially retained bin before aggregation`
+ );
+ }
+ }
+ // Different aggregation modes must retain their own semantics.
+ apply(
+ VisStateActions.layerVisConfigChange(state.layers[0], {
+ colorAggregation: 'sum',
+ sizeAggregation: 'count'
+ })
+ );
+ check('zero-valued color with count height', [0, 10], [1, 1]);
+ apply(VisStateActions.addFilter(dataId));
+ apply(VisStateActions.setFilter(1, 'name', 'category'));
+ apply(VisStateActions.setFilter(1, 'value', ['keep']));
+ check('mixed CPU and GPU filters retain the zero bin', [0], [1]);
+ apply(VisStateActions.removeFilter(0));
+ check('clearing GPU filter keeps CPU selection', [0, 7], [1, 1]);
+ apply(VisStateActions.removeFilter(0));
+ check('clearing all filters restores every bin', [0, 7, 10], [1, 1, 1]);
+ } finally {
+ layerManager.finalize();
+ }
+ t.end();
+ });
+}
diff --git a/test/browser/layer-tests/heatmap-layer-specs.js b/test/browser/layer-tests/heatmap-layer-specs.js
index 0d6fe11bc..082f63adb 100644
--- a/test/browser/layer-tests/heatmap-layer-specs.js
+++ b/test/browser/layer-tests/heatmap-layer-specs.js
@@ -844,7 +844,10 @@ test('#Aggregation layers -> reducer filters at the domain minimum', t => {
const aggregator = rendered.state.aggregator;
const counts = [0, 0];
for (let j = 0; j < aggregator.binCount; j++) {
- aggregator.getBin(j).value.forEach((count, channel) => (counts[channel] += count));
+ aggregator.getBin(j).value.forEach((count, channel) => {
+ // Masked empty bins contribute no retained points to the total.
+ if (!Number.isNaN(count)) counts[channel] += count;
+ });
}
t.deepEqual(counts, [expected, expected], message);
}
diff --git a/test/browser/layer-tests/hexagon-layer-specs.js b/test/browser/layer-tests/hexagon-layer-specs.js
index a7533d598..629a9ac62 100644
--- a/test/browser/layer-tests/hexagon-layer-specs.js
+++ b/test/browser/layer-tests/hexagon-layer-specs.js
@@ -335,20 +335,20 @@ test('#HexagonLayer -> renderLayer', t => {
// Verify actual attribute array values (deck.gl 9 equivalent of instanceFillColors/instanceElevations)
// cellData.attributes contain SCALED values (processed through AttributeWithScale),
- // not raw aggregated weights. For quantize scale with 3 bins [0, 1, 2],
- // values are mapped to percentile-like range [0, 50, 99].
+ // not raw aggregated weights. Quantile scaling excludes the empty bin
+ // and maps the two retained counts to [0, 99], preserving the NaN mask.
const colorValues = Array.from(cellData.attributes.getColorValue.value);
const elevationValues = Array.from(cellData.attributes.getElevationValue.value);
t.deepEqual(
- colorValues.slice().sort((a, b) => a - b),
- [0, 50, 99],
- 'getColorValue typed array should contain scaled values [0, 50, 99] (sorted)'
+ colorValues.slice().sort(),
+ [0, 99, NaN],
+ 'scaled color values should mask the empty bin'
);
// Elevation uses linear scale (default), so raw values pass through
t.deepEqual(
- elevationValues.slice().sort((a, b) => a - b),
- [0, 1, 2],
- 'getElevationValue typed array should contain raw counts [0, 1, 2] (sorted)'
+ elevationValues.slice().sort(),
+ [1, 2, NaN],
+ 'raw elevation counts should mask the empty bin'
);
// getBin contains [col, row] integer IDs per bin (size=2)
@@ -366,7 +366,7 @@ test('#HexagonLayer -> renderLayer', t => {
// onSetColorDomain fires twice: first the parent's [min, max], then our
// enriched {domain, aggregatedBins}. The last call is the one that sticks.
- // count aggregation: bins have 0, 2, and 1 filtered points
+ // count aggregation: only bins with 2 and 1 filtered points contribute
t.ok(spyLayerCallbacks.called, 'should call onSetLayerDomain');
const lastIdx = spyLayerCallbacks.args.length - 1;
const enrichedArg = spyLayerCallbacks.args[lastIdx][0];
@@ -378,7 +378,7 @@ test('#HexagonLayer -> renderLayer', t => {
// sorted array of per-bin values (not just [min, max]).
t.deepEqual(
enrichedArg.domain,
- [0, 1, 2],
+ [1, 2],
'enriched domain should be full sorted bin values for quantile scale'
);
t.ok(enrichedArg.aggregatedBins, 'enriched call should include aggregatedBins');
@@ -394,10 +394,10 @@ test('#HexagonLayer -> renderLayer', t => {
);
const colorDomain = aggregator.getResultDomain(0);
- t.deepEqual(colorDomain, [0, 2], 'color domain should be [0, 2]');
+ t.deepEqual(colorDomain, [1, 2], 'color domain should exclude empty bins');
const elevationDomain = aggregator.getResultDomain(1);
- t.deepEqual(elevationDomain, [0, 2], 'elevation domain should be [0, 2]');
+ t.deepEqual(elevationDomain, [1, 2], 'elevation domain should exclude empty bins');
// Verify per-bin data
const binCounts = [];
@@ -417,13 +417,13 @@ test('#HexagonLayer -> renderLayer', t => {
);
t.deepEqual(
binColorValues.slice().sort(),
- [0, 1, 2],
- 'bin color values (filtered count agg) should be [0, 1, 2] (sorted)'
+ [1, 2, NaN],
+ 'bin color counts should mask empty bins'
);
t.deepEqual(
binElevationValues.slice().sort(),
- [0, 1, 2],
- 'bin elevation values (filtered count agg) should be [0, 1, 2] (sorted)'
+ [1, 2, NaN],
+ 'bin elevation counts should mask empty bins'
);
// Verify raw result arrays
@@ -432,7 +432,7 @@ test('#HexagonLayer -> renderLayer', t => {
const sortedColorResult = Array.from(
colorResult.value.slice(0, aggregator.binCount)
).sort();
- t.deepEqual(sortedColorResult, [0, 1, 2], 'color result values should be [0, 1, 2]');
+ t.deepEqual(sortedColorResult, [1, 2, NaN], 'color results should mask empty bins');
}
}
},