Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(layers): refresh cached GeoJSON geometry after data updates (#3905)

Igor Dykhta0f3bfd8e5cf6
diff --git a/src/layers/src/aggregation-layer.ts b/src/layers/src/aggregation-layer.ts
index 8a7588b06..448fc2cb8 100644
--- a/src/layers/src/aggregation-layer.ts
+++ b/src/layers/src/aggregation-layer.ts
@@ -155,6 +155,8 @@ export default class AggregationLayer extends Layer {
dataToFeature: any[] = [];
centroids: Array<number[] | null> = [];
private _geojsonFieldIdx = -1;
+ private _geojsonDataContainer: DataContainerInterface | null = null;
+ private _geojsonDataRevision: number | undefined;
private _geojsonBounds: [number, number, number, number] | null = null;
constructor(
@@ -536,7 +538,7 @@ export default class AggregationLayer extends Layer {
geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'
? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY)
: (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];
- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);
+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);
this.updateMeta({bounds: this._geojsonBounds});
} else {
this.dataToFeature = [];
@@ -550,18 +552,23 @@ export default class AggregationLayer extends Layer {
}
private _buildGeojsonDataToFeature(
- dataContainer: DataContainerInterface,
+ dataset: KeplerTable,
getFeature: any,
geoArrowEncoding?: string | null
) {
+ const {dataContainer, dataRevision} = dataset;
const fieldIdx = this.config.columns.geojson.fieldIdx;
if (
this.dataToFeature.length === dataContainer.numRows() &&
- this._geojsonFieldIdx === fieldIdx
+ this._geojsonFieldIdx === fieldIdx &&
+ this._geojsonDataContainer === dataContainer &&
+ this._geojsonDataRevision === dataRevision
) {
return;
}
this._geojsonFieldIdx = fieldIdx;
+ this._geojsonDataContainer = dataContainer;
+ this._geojsonDataRevision = dataRevision;
this.dataToFeature = [];
this.centroids = [];
diff --git a/src/layers/src/heatmap-layer/heatmap-layer.ts b/src/layers/src/heatmap-layer/heatmap-layer.ts
index 530117440..e3ee23c88 100644
--- a/src/layers/src/heatmap-layer/heatmap-layer.ts
+++ b/src/layers/src/heatmap-layer/heatmap-layer.ts
@@ -244,6 +244,8 @@ class HeatmapLayer extends Layer {
dataToFeature: any[] = [];
centroids: Array<number[] | null> = [];
private _geojsonFieldIdx = -1;
+ private _geojsonDataContainer: DataContainerInterface | null = null;
+ private _geojsonDataRevision: number | undefined;
private _geojsonBounds: [number, number, number, number] | null = null;
constructor(props) {
@@ -442,7 +444,7 @@ class HeatmapLayer extends Layer {
geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'
? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY)
: (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];
- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);
+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);
this.updateMeta({bounds: this._geojsonBounds});
} else {
this.dataToFeature = [];
@@ -455,21 +457,26 @@ class HeatmapLayer extends Layer {
/**
* Parse raw GeoJSON values into Feature objects and store them.
- * Re-parses when the data container size changes (new data load or column switch).
+ * Re-parses when the source container, revision, size or geometry column changes.
*/
private _buildGeojsonDataToFeature(
- dataContainer: DataContainerInterface,
+ dataset: KeplerTable,
getFeature: any,
geoArrowEncoding?: string | null
) {
+ const {dataContainer, dataRevision} = dataset;
const fieldIdx = this.config.columns.geojson.fieldIdx;
if (
this.dataToFeature.length === dataContainer.numRows() &&
- this._geojsonFieldIdx === fieldIdx
+ this._geojsonFieldIdx === fieldIdx &&
+ this._geojsonDataContainer === dataContainer &&
+ this._geojsonDataRevision === dataRevision
) {
return;
}
this._geojsonFieldIdx = fieldIdx;
+ this._geojsonDataContainer = dataContainer;
+ this._geojsonDataRevision = dataRevision;
this.dataToFeature = [];
this.centroids = [];
diff --git a/src/layers/src/scenegraph-layer/scenegraph-layer.ts b/src/layers/src/scenegraph-layer/scenegraph-layer.ts
index 466586e28..765cc9a39 100644
--- a/src/layers/src/scenegraph-layer/scenegraph-layer.ts
+++ b/src/layers/src/scenegraph-layer/scenegraph-layer.ts
@@ -271,6 +271,8 @@ export default class ScenegraphLayer extends Layer {
dataToFeature: any[] = [];
centroids: Array<number[] | null> = [];
private _geojsonFieldIdx = -1;
+ private _geojsonDataContainer: DataContainerInterface | null = null;
+ private _geojsonDataRevision: number | undefined;
private _geojsonBounds: [number, number, number, number] | null = null;
constructor(props) {
@@ -458,7 +460,7 @@ export default class ScenegraphLayer extends Layer {
geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'
? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY)
: (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];
- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);
+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);
this.updateMeta({bounds: this._geojsonBounds});
return;
}
@@ -470,18 +472,23 @@ export default class ScenegraphLayer extends Layer {
}
private _buildGeojsonDataToFeature(
- dataContainer: DataContainerInterface,
+ dataset: KeplerTable,
getFeature: (d: {index: number}) => unknown,
geoArrowEncoding?: string | null
) {
+ const {dataContainer, dataRevision} = dataset;
const fieldIdx = this.config.columns.geojson?.fieldIdx ?? -1;
if (
this.dataToFeature.length === dataContainer.numRows() &&
- this._geojsonFieldIdx === fieldIdx
+ this._geojsonFieldIdx === fieldIdx &&
+ this._geojsonDataContainer === dataContainer &&
+ this._geojsonDataRevision === dataRevision
) {
return;
}
this._geojsonFieldIdx = fieldIdx;
+ this._geojsonDataContainer = dataContainer;
+ this._geojsonDataRevision = dataRevision;
this.dataToFeature = [];
this.centroids = [];
diff --git a/test/browser/layer-tests/aggregation-layer-geojson-specs.js b/test/browser/layer-tests/aggregation-layer-geojson-specs.js
index e551ffc67..2094e52fb 100644
--- a/test/browser/layer-tests/aggregation-layer-geojson-specs.js
+++ b/test/browser/layer-tests/aggregation-layer-geojson-specs.js
@@ -6,7 +6,9 @@ import test from 'tape';
import {testFormatLayerDataCases, prepareGeojsonDataset, dataId} from 'test/helpers/layer-utils';
import {KeplerGlLayers} from '@kepler.gl/layers';
-const {GridLayer, HexagonLayer, ClusterLayer} = KeplerGlLayers;
+import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';
+import {tableFromArrays} from 'apache-arrow';
+const {GridLayer, HexagonLayer, ClusterLayer, HeatmapLayer, ScenegraphLayer} = KeplerGlLayers;
const geojsonColumns = {
geojson: '_geojson'
@@ -497,3 +499,122 @@ test('#AggregationLayer -> isInPolygon with geojson mode', t => {
t.end();
});
+
+// These layers share the same GeoJSON centroid cache behavior.
+for (const LayerClass of [GridLayer, HexagonLayer, ClusterLayer, HeatmapLayer, ScenegraphLayer]) {
+ for (const storage of ['rows', 'arrow']) {
+ test(`#${LayerClass.name} -> GeoJSON ${storage} refresh invalidates cached geometry`, async t => {
+ const feature = coordinates => ({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates},
+ properties: {}
+ });
+ const rowsFor = coordinates => [[7, JSON.stringify(feature(coordinates))]];
+ const fields = [
+ {name: 'id', type: 'integer'},
+ {name: 'geometry', type: 'geojson'}
+ ];
+ const payload = rows => {
+ if (storage === 'rows') return {fields, rows};
+ const arrowTable = tableFromArrays({
+ id: rows.map(row => row[0]),
+ geometry: rows.map(row => row[1])
+ });
+ return {
+ fields,
+ rows: [],
+ cols: [arrowTable.getChildAt(0), arrowTable.getChildAt(1)],
+ arrowTable
+ };
+ };
+ const createTable = coordinates =>
+ new KeplerTable({info: {id: 'refresh'}, color: [1, 2, 3]}).updateSchema(
+ payload(rowsFor(coordinates))
+ );
+ const dataset = createTable([1, 2]);
+ const layer = new LayerClass({
+ dataId: dataset.id,
+ columnMode: 'geojson',
+ columns: {geojson: {value: 'geometry', fieldIdx: 1}}
+ });
+ const container = dataset.dataContainer;
+ let layerData = layer.formatLayerData({refresh: dataset}, {});
+ const assertPosition = (coordinates, message) => {
+ t.deepEqual(
+ layerData.data.map(d => layerData.getPosition(d).slice(0, 2)),
+ [coordinates],
+ message
+ );
+ t.deepEqual(
+ layerData.data.map(d => d.index),
+ [0],
+ 'preserves source indexes for accessors and picking'
+ );
+ t.deepEqual(layer.centroids, [coordinates], 'updates centroids');
+ t.deepEqual(layer.meta.bounds, [...coordinates, ...coordinates], 'updates bounds');
+ };
+ assertPosition([1, 2], 'renders initial geometry');
+ const oldData = layerData.data;
+ const initialRevision = dataset.dataRevision;
+ await dataset.update(payload(rowsFor([30, 40])));
+ t.equal(dataset.dataContainer, container, 'snapshot retains the container');
+ t.ok(dataset.dataRevision > initialRevision, 'snapshot advances revision');
+ layerData = layer.formatLayerData({refresh: dataset}, layerData);
+ assertPosition([30, 40], 'renders same-length refreshed geometry');
+ t.deepEqual(
+ oldData.map(d => d.position.slice(0, 2)),
+ [[1, 2]],
+ 'does not mutate previous rendered data'
+ );
+ t.equal(
+ container.valueAt(0, 1),
+ rowsFor([30, 40])[0][1],
+ 'formatting preserves source geometry'
+ );
+
+ const cachedFeatures = layer.dataToFeature;
+ const cachedCentroids = layer.centroids;
+ layer.updateLayerVisConfig({opacity: 0.3});
+ layer.updateLayerMeta(copyTableAndUpdate(dataset, {metadata: {refreshProgress: 50}}));
+ const filtered = copyTableAndUpdate(dataset, {filteredIndex: []});
+ const filteredData = layer.formatLayerData({refresh: filtered}, layerData);
+ t.deepEqual(filteredData.data, [], 'respects CPU-filtered indexes');
+ t.equal(
+ layer.dataToFeature,
+ cachedFeatures,
+ 'reuses geometry for style, metadata and filter changes'
+ );
+ t.equal(layer.centroids, cachedCentroids, 'reuses centroids when the source is unchanged');
+ layerData = layer.formatLayerData({refresh: dataset}, filteredData);
+ assertPosition([30, 40], 'restores all geometry when CPU filter is removed');
+
+ const mutationRevision = dataset.dataRevision;
+ t.ok(dataset.upsertRows(rowsFor([50, 60]), 'id'), 'row replacement succeeds');
+ t.equal(dataset.dataContainer, container, 'row edit retains the container');
+ t.ok(dataset.dataRevision > mutationRevision, 'row edit advances revision');
+ layerData = layer.formatLayerData({refresh: dataset}, layerData);
+ assertPosition([50, 60], 'renders same-length row edits');
+
+ const replacement = createTable([70, 80]);
+ replacement.dataRevision = dataset.dataRevision;
+ t.notEqual(replacement.dataContainer, container, 'replacement changes container identity');
+ layerData = layer.formatLayerData({refresh: replacement}, layerData);
+ assertPosition([70, 80], 'renders a new container with the same revision');
+
+ await replacement.update(payload([[7, 'invalid geometry']]));
+ layerData = layer.formatLayerData({refresh: replacement}, layerData);
+ t.deepEqual(layerData.data, [], 'invalid replacement clears rendered geometry');
+ t.deepEqual(layer.centroids, [null], 'invalid replacement clears old centroid');
+ t.equal(layer.meta.bounds, null, 'invalid replacement clears bounds');
+ await replacement.update(payload([]));
+ layerData = layer.formatLayerData({refresh: replacement}, layerData);
+ t.deepEqual(layerData.data, [], 'empty snapshot renders no geometry');
+ t.deepEqual(layer.centroids, [], 'empty snapshot clears centroids');
+ t.equal(layer.meta.bounds, null, 'empty snapshot clears bounds');
+ await replacement.update(payload(rowsFor([10, 20])));
+ layerData = layer.formatLayerData({refresh: replacement}, layerData);
+ assertPosition([10, 20], 'recovers when geometry arrives after an empty snapshot');
+ t.end();
+ });
+ }
+}