Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(layers): refresh cached GeoJSON geometry after data updates (#3905)
Igor Dykhta0f3bfd8e5cf6
diff --git a/src/layers/src/aggregation-layer.ts b/src/layers/src/aggregation-layer.tsindex 8a7588b06..448fc2cb8 100644--- a/src/layers/src/aggregation-layer.ts+++ b/src/layers/src/aggregation-layer.ts@@ -155,6 +155,8 @@ export default class AggregationLayer extends Layer {dataToFeature: any[] = [];centroids: Array<number[] | null> = [];private _geojsonFieldIdx = -1;+ private _geojsonDataContainer: DataContainerInterface | null = null;+ private _geojsonDataRevision: number | undefined;private _geojsonBounds: [number, number, number, number] | null = null;constructor(@@ -536,7 +538,7 @@ export default class AggregationLayer extends Layer {geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY): (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);this.updateMeta({bounds: this._geojsonBounds});} else {this.dataToFeature = [];@@ -550,18 +552,23 @@ export default class AggregationLayer extends Layer {}private _buildGeojsonDataToFeature(- dataContainer: DataContainerInterface,+ dataset: KeplerTable,getFeature: any,geoArrowEncoding?: string | null) {+ const {dataContainer, dataRevision} = dataset;const fieldIdx = this.config.columns.geojson.fieldIdx;if (this.dataToFeature.length === dataContainer.numRows() &&- this._geojsonFieldIdx === fieldIdx+ this._geojsonFieldIdx === fieldIdx &&+ this._geojsonDataContainer === dataContainer &&+ this._geojsonDataRevision === dataRevision) {return;}this._geojsonFieldIdx = fieldIdx;+ this._geojsonDataContainer = dataContainer;+ this._geojsonDataRevision = dataRevision;this.dataToFeature = [];this.centroids = [];diff --git a/src/layers/src/heatmap-layer/heatmap-layer.ts b/src/layers/src/heatmap-layer/heatmap-layer.tsindex 530117440..e3ee23c88 100644--- a/src/layers/src/heatmap-layer/heatmap-layer.ts+++ b/src/layers/src/heatmap-layer/heatmap-layer.ts@@ -244,6 +244,8 @@ class HeatmapLayer extends Layer {dataToFeature: any[] = [];centroids: Array<number[] | null> = [];private _geojsonFieldIdx = -1;+ private _geojsonDataContainer: DataContainerInterface | null = null;+ private _geojsonDataRevision: number | undefined;private _geojsonBounds: [number, number, number, number] | null = null;constructor(props) {@@ -442,7 +444,7 @@ class HeatmapLayer extends Layer {geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY): (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);this.updateMeta({bounds: this._geojsonBounds});} else {this.dataToFeature = [];@@ -455,21 +457,26 @@ class HeatmapLayer extends Layer {/*** Parse raw GeoJSON values into Feature objects and store them.- * Re-parses when the data container size changes (new data load or column switch).+ * Re-parses when the source container, revision, size or geometry column changes.*/private _buildGeojsonDataToFeature(- dataContainer: DataContainerInterface,+ dataset: KeplerTable,getFeature: any,geoArrowEncoding?: string | null) {+ const {dataContainer, dataRevision} = dataset;const fieldIdx = this.config.columns.geojson.fieldIdx;if (this.dataToFeature.length === dataContainer.numRows() &&- this._geojsonFieldIdx === fieldIdx+ this._geojsonFieldIdx === fieldIdx &&+ this._geojsonDataContainer === dataContainer &&+ this._geojsonDataRevision === dataRevision) {return;}this._geojsonFieldIdx = fieldIdx;+ this._geojsonDataContainer = dataContainer;+ this._geojsonDataRevision = dataRevision;this.dataToFeature = [];this.centroids = [];diff --git a/src/layers/src/scenegraph-layer/scenegraph-layer.ts b/src/layers/src/scenegraph-layer/scenegraph-layer.tsindex 466586e28..765cc9a39 100644--- a/src/layers/src/scenegraph-layer/scenegraph-layer.ts+++ b/src/layers/src/scenegraph-layer/scenegraph-layer.ts@@ -271,6 +271,8 @@ export default class ScenegraphLayer extends Layer {dataToFeature: any[] = [];centroids: Array<number[] | null> = [];private _geojsonFieldIdx = -1;+ private _geojsonDataContainer: DataContainerInterface | null = null;+ private _geojsonDataRevision: number | undefined;private _geojsonBounds: [number, number, number, number] | null = null;constructor(props) {@@ -458,7 +460,7 @@ export default class ScenegraphLayer extends Layer {geoField?.metadata && typeof (geoField.metadata as Map<string, string>).get === 'function'? (geoField.metadata as Map<string, string>).get(GEOARROW_METADATA_KEY): (geoField?.metadata as Record<string, string> | undefined)?.[GEOARROW_METADATA_KEY];- this._buildGeojsonDataToFeature(dataContainer, getFeature, encoding);+ this._buildGeojsonDataToFeature(dataset, getFeature, encoding);this.updateMeta({bounds: this._geojsonBounds});return;}@@ -470,18 +472,23 @@ export default class ScenegraphLayer extends Layer {}private _buildGeojsonDataToFeature(- dataContainer: DataContainerInterface,+ dataset: KeplerTable,getFeature: (d: {index: number}) => unknown,geoArrowEncoding?: string | null) {+ const {dataContainer, dataRevision} = dataset;const fieldIdx = this.config.columns.geojson?.fieldIdx ?? -1;if (this.dataToFeature.length === dataContainer.numRows() &&- this._geojsonFieldIdx === fieldIdx+ this._geojsonFieldIdx === fieldIdx &&+ this._geojsonDataContainer === dataContainer &&+ this._geojsonDataRevision === dataRevision) {return;}this._geojsonFieldIdx = fieldIdx;+ this._geojsonDataContainer = dataContainer;+ this._geojsonDataRevision = dataRevision;this.dataToFeature = [];this.centroids = [];diff --git a/test/browser/layer-tests/aggregation-layer-geojson-specs.js b/test/browser/layer-tests/aggregation-layer-geojson-specs.jsindex e551ffc67..2094e52fb 100644--- a/test/browser/layer-tests/aggregation-layer-geojson-specs.js+++ b/test/browser/layer-tests/aggregation-layer-geojson-specs.js@@ -6,7 +6,9 @@ import test from 'tape';import {testFormatLayerDataCases, prepareGeojsonDataset, dataId} from 'test/helpers/layer-utils';import {KeplerGlLayers} from '@kepler.gl/layers';-const {GridLayer, HexagonLayer, ClusterLayer} = KeplerGlLayers;+import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';+import {tableFromArrays} from 'apache-arrow';+const {GridLayer, HexagonLayer, ClusterLayer, HeatmapLayer, ScenegraphLayer} = KeplerGlLayers;const geojsonColumns = {geojson: '_geojson'@@ -497,3 +499,122 @@ test('#AggregationLayer -> isInPolygon with geojson mode', t => {t.end();});++// These layers share the same GeoJSON centroid cache behavior.+for (const LayerClass of [GridLayer, HexagonLayer, ClusterLayer, HeatmapLayer, ScenegraphLayer]) {+ for (const storage of ['rows', 'arrow']) {+ test(`#${LayerClass.name} -> GeoJSON ${storage} refresh invalidates cached geometry`, async t => {+ const feature = coordinates => ({+ type: 'Feature',+ geometry: {type: 'Point', coordinates},+ properties: {}+ });+ const rowsFor = coordinates => [[7, JSON.stringify(feature(coordinates))]];+ const fields = [+ {name: 'id', type: 'integer'},+ {name: 'geometry', type: 'geojson'}+ ];+ const payload = rows => {+ if (storage === 'rows') return {fields, rows};+ const arrowTable = tableFromArrays({+ id: rows.map(row => row[0]),+ geometry: rows.map(row => row[1])+ });+ return {+ fields,+ rows: [],+ cols: [arrowTable.getChildAt(0), arrowTable.getChildAt(1)],+ arrowTable+ };+ };+ const createTable = coordinates =>+ new KeplerTable({info: {id: 'refresh'}, color: [1, 2, 3]}).updateSchema(+ payload(rowsFor(coordinates))+ );+ const dataset = createTable([1, 2]);+ const layer = new LayerClass({+ dataId: dataset.id,+ columnMode: 'geojson',+ columns: {geojson: {value: 'geometry', fieldIdx: 1}}+ });+ const container = dataset.dataContainer;+ let layerData = layer.formatLayerData({refresh: dataset}, {});+ const assertPosition = (coordinates, message) => {+ t.deepEqual(+ layerData.data.map(d => layerData.getPosition(d).slice(0, 2)),+ [coordinates],+ message+ );+ t.deepEqual(+ layerData.data.map(d => d.index),+ [0],+ 'preserves source indexes for accessors and picking'+ );+ t.deepEqual(layer.centroids, [coordinates], 'updates centroids');+ t.deepEqual(layer.meta.bounds, [...coordinates, ...coordinates], 'updates bounds');+ };+ assertPosition([1, 2], 'renders initial geometry');+ const oldData = layerData.data;+ const initialRevision = dataset.dataRevision;+ await dataset.update(payload(rowsFor([30, 40])));+ t.equal(dataset.dataContainer, container, 'snapshot retains the container');+ t.ok(dataset.dataRevision > initialRevision, 'snapshot advances revision');+ layerData = layer.formatLayerData({refresh: dataset}, layerData);+ assertPosition([30, 40], 'renders same-length refreshed geometry');+ t.deepEqual(+ oldData.map(d => d.position.slice(0, 2)),+ [[1, 2]],+ 'does not mutate previous rendered data'+ );+ t.equal(+ container.valueAt(0, 1),+ rowsFor([30, 40])[0][1],+ 'formatting preserves source geometry'+ );++ const cachedFeatures = layer.dataToFeature;+ const cachedCentroids = layer.centroids;+ layer.updateLayerVisConfig({opacity: 0.3});+ layer.updateLayerMeta(copyTableAndUpdate(dataset, {metadata: {refreshProgress: 50}}));+ const filtered = copyTableAndUpdate(dataset, {filteredIndex: []});+ const filteredData = layer.formatLayerData({refresh: filtered}, layerData);+ t.deepEqual(filteredData.data, [], 'respects CPU-filtered indexes');+ t.equal(+ layer.dataToFeature,+ cachedFeatures,+ 'reuses geometry for style, metadata and filter changes'+ );+ t.equal(layer.centroids, cachedCentroids, 'reuses centroids when the source is unchanged');+ layerData = layer.formatLayerData({refresh: dataset}, filteredData);+ assertPosition([30, 40], 'restores all geometry when CPU filter is removed');++ const mutationRevision = dataset.dataRevision;+ t.ok(dataset.upsertRows(rowsFor([50, 60]), 'id'), 'row replacement succeeds');+ t.equal(dataset.dataContainer, container, 'row edit retains the container');+ t.ok(dataset.dataRevision > mutationRevision, 'row edit advances revision');+ layerData = layer.formatLayerData({refresh: dataset}, layerData);+ assertPosition([50, 60], 'renders same-length row edits');++ const replacement = createTable([70, 80]);+ replacement.dataRevision = dataset.dataRevision;+ t.notEqual(replacement.dataContainer, container, 'replacement changes container identity');+ layerData = layer.formatLayerData({refresh: replacement}, layerData);+ assertPosition([70, 80], 'renders a new container with the same revision');++ await replacement.update(payload([[7, 'invalid geometry']]));+ layerData = layer.formatLayerData({refresh: replacement}, layerData);+ t.deepEqual(layerData.data, [], 'invalid replacement clears rendered geometry');+ t.deepEqual(layer.centroids, [null], 'invalid replacement clears old centroid');+ t.equal(layer.meta.bounds, null, 'invalid replacement clears bounds');+ await replacement.update(payload([]));+ layerData = layer.formatLayerData({refresh: replacement}, layerData);+ t.deepEqual(layerData.data, [], 'empty snapshot renders no geometry');+ t.deepEqual(layer.centroids, [], 'empty snapshot clears centroids');+ t.equal(layer.meta.bounds, null, 'empty snapshot clears bounds');+ await replacement.update(payload(rowsFor([10, 20])));+ layerData = layer.formatLayerData({refresh: replacement}, layerData);+ assertPosition([10, 20], 'recovers when geometry arrives after an empty snapshot');+ t.end();+ });+ }+}