Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(layers): select default GeoJSON color field (#3857)
Igor Dykhta1b7de9f9bdfc
diff --git a/src/layers/src/geojson-layer/geojson-layer.spec.ts b/src/layers/src/geojson-layer/geojson-layer.spec.tsindex 8cd156119..6eb24e41d 100644--- a/src/layers/src/geojson-layer/geojson-layer.spec.ts+++ b/src/layers/src/geojson-layer/geojson-layer.spec.ts@@ -1,9 +1,319 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import GeoJsonLayer, {getTextLabelPosition} from './geojson-layer';-import {LAYER_VIS_CONFIGS} from '@kepler.gl/constants';+import GeoJsonLayer, {GeoJsonLayerConfig, getTextLabelPosition} from './geojson-layer';+import {ALL_FIELD_TYPES, LAYER_VIS_CONFIGS} from '@kepler.gl/constants';import {valueToPosition} from '@kepler.gl/utils';+import {processGeojson} from '@kepler.gl/processors';+import {KeplerTable} from '@kepler.gl/table';++const point = (x: number) => ({type: 'Point', coordinates: [x, x]});+const line = (x: number) => ({+ type: 'LineString',+ coordinates: [+ [x, x],+ [x + 0.25, x + 0.25]+ ]+});+const polygon = (x: number) => ({+ type: 'Polygon',+ coordinates: [+ [+ [x, x],+ [x + 0.25, x],+ [x + 0.25, x + 0.25],+ [x, x]+ ]+ ]+});++async function initializeGeoJsonLayer(+ features: Array<{geometry: Record<string, unknown>; properties: Record<string, unknown>}>,+ rowCount?: number+) {+ const dataset = new KeplerTable({+ info: {id: 'geojson-default-color'},+ color: [0, 0, 0]+ });+ const data = processGeojson({+ type: 'FeatureCollection',+ features: features.map(feature => ({type: 'Feature', ...feature}))+ } as any);+ if (!data) {+ throw new Error('Expected processed GeoJSON data');+ }+ await dataset.importData({data});++ const {props} = GeoJsonLayer.findDefaultLayerProps(dataset);+ if (!props.length) {+ throw new Error('Expected GeoJSON layer props');+ }+ const layer = new GeoJsonLayer({...props[0], dataId: dataset.id});+ if (rowCount !== undefined) {+ // Cache geometry metadata before simulating a large dataset's row count.+ layer.updateLayerMeta(dataset);+ jest.spyOn(dataset.dataContainer, 'numRows').mockReturnValue(rowCount);+ }+ layer.setInitialLayerConfig(dataset);+ return {dataset, layer};+}++async function initializeGeoJsonColumnLayer(+ features: Array<{geometry: Record<string, unknown>; properties: Record<string, unknown>}>+) {+ const dataset = new KeplerTable({+ info: {id: 'geojson-column-default-color'},+ color: [0, 0, 0]+ });+ await dataset.importData({+ data: {+ fields: [+ {name: 'geometry', type: ALL_FIELD_TYPES.geojson, analyzerType: 'GEOMETRY'},+ {name: 'score', type: ALL_FIELD_TYPES.real}+ ],+ rows: features.map((feature, index) => [{type: 'Feature', ...feature}, index + 0.25])+ } as any+ });++ const {props} = GeoJsonLayer.findDefaultLayerProps(dataset);+ if (!props.length) {+ throw new Error('Expected GeoJSON column layer props');+ }+ const layer = new GeoJsonLayer({...props[0], dataId: dataset.id});+ layer.setInitialLayerConfig(dataset);+ return {dataset, layer};+}++describe('GeoJsonLayer default color field', () => {+ test('keeps string and integer-only polygon data uniformly colored', async () => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: polygon(0), properties: {category: 'a', count: 1}},+ {geometry: polygon(1), properties: {category: 'b', count: 2}}+ ]);++ expect(layer.config.colorField).toBeNull();+ expect(layer.config.strokeColorField).toBeNull();+ });++ test.each([+ ['point', point],+ ['polygon', polygon]+ ])('selects an eligible real field for %s fill color', async (_name, geometry) => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: geometry(0), properties: {score: 0.25}},+ {geometry: geometry(1), properties: {score: 0.75}}+ ]);++ expect(layer.config.colorField?.name).toBe('score');+ expect(layer.config.colorDomain).toEqual([0.25, 0.75]);+ expect(layer.config.strokeColorField).toBeNull();+ });++ test.each([+ ['point', point, 'colorField', 'colorDomain'],+ ['polygon', polygon, 'colorField', 'colorDomain'],+ ['line', line, 'strokeColorField', 'strokeColorDomain']+ ] as const)(+ 'skips constant real fields and selects the next preferred field for %s color',+ async (_name, geometry, field, domain) => {+ const {layer} = await initializeGeoJsonLayer([+ {+ geometry: geometry(0),+ properties: {score: 0.5, metric: 1.25, depth: 2.25, value: 0.25}+ },+ {+ geometry: geometry(1),+ properties: {score: 0.5, metric: 1.25, depth: 3.75, value: 0.75}+ }+ ]);++ expect(layer.config[field]?.name).toBe('value');+ expect(layer.config[domain]).toEqual([0.25, 0.75]);+ }+ );++ test.each([+ ['point', point],+ ['polygon', polygon],+ ['line', line]+ ] as const)(+ 'keeps %s color uniform when all eligible real fields are constant',+ async (_name, geometry) => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: geometry(0), properties: {score: 0.5, metric: 1.25, count: 1}},+ {geometry: geometry(1), properties: {score: 0.5, metric: 1.25, count: 2}}+ ]);++ expect(layer.config.colorField).toBeNull();+ expect(layer.config.strokeColorField).toBeNull();+ }+ );++ test('ignores missing values when skipping a constant real field', async () => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: polygon(0), properties: {score: 0.5, value: 0.25}},+ {geometry: polygon(1), properties: {score: null, value: 0.75}},+ {geometry: polygon(2), properties: {score: 0.5, value: 1.25}}+ ]);++ expect(layer.config.colorField?.name).toBe('value');+ expect(layer.config.colorDomain).toEqual([0.25, 0.75, 1.25]);+ });++ test('keeps a single-feature dataset uniformly colored', async () => {+ const {layer} = await initializeGeoJsonLayer([{geometry: polygon(0), properties: {score: 0.5}}]);++ expect(layer.config.colorField).toBeNull();+ expect(layer.config.strokeColorField).toBeNull();+ });++ test.each([+ {rowCount: 20_000, varyingColumn: 9, expected: 'score_9'},+ {rowCount: 300_000, varyingColumn: 2, expected: 'score_2'},+ {rowCount: 300_000, varyingColumn: 3, expected: null},+ {rowCount: 250_000, varyingColumn: 3, expected: 'score_3'},+ {rowCount: 250_001, varyingColumn: 3, expected: null},+ {rowCount: 1_000_000, varyingColumn: 0, expected: 'score_0'},+ {rowCount: 1_000_001, varyingColumn: 0, expected: null}+ ])(+ 'limits candidate checks for $rowCount rows with varying column $varyingColumn',+ async ({rowCount, varyingColumn, expected}) => {+ const properties = Object.fromEntries(+ Array.from({length: 10}, (_, index) => [`score_${index}`, 0.5])+ );+ const {layer} = await initializeGeoJsonLayer(+ [+ {geometry: point(0), properties: {...properties, [`score_${varyingColumn}`]: 0.25}},+ {geometry: point(1), properties: {...properties, [`score_${varyingColumn}`]: 0.75}}+ ],+ rowCount+ );++ expect(layer.config.colorField?.name ?? null).toBe(expected);+ expect(layer.config.strokeColorField).toBeNull();+ }+ );++ describe.each([+ ['point', point],+ ['polygon', polygon],+ ['line', line]+ ] as const)('%s initialization after automatic color failure', (name, geometry) => {+ test.each(['field values', 'color domain'])(+ 'continues with uniform color when reading %s throws',+ async failure => {+ const {dataset, layer} = await initializeGeoJsonLayer([+ {geometry: geometry(0), properties: {score: 0.25}},+ {geometry: geometry(1), properties: {score: 0.75}}+ ]);+ layer.updateLayerConfig<GeoJsonLayerConfig>({+ colorField: null,+ strokeColorField: null,+ colorDomain: [0, 1],+ strokeColorDomain: [0, 1]+ });+ layer.updateLayerVisConfig({filled: false, stroked: false});++ if (failure === 'field values') {+ const field = dataset.fields.find(candidate => candidate.name === 'score')!;+ jest.spyOn(field, 'valueAccessor').mockImplementation(() => {+ throw new Error('Cannot read field values');+ });+ } else {+ jest.spyOn(dataset, 'getColumnLayerDomain').mockImplementation(() => {+ throw new Error('Cannot calculate color domain');+ });+ }++ expect(() => layer.setInitialLayerConfig(dataset)).not.toThrow();+ expect(layer.config.colorField).toBeNull();+ expect(layer.config.strokeColorField).toBeNull();+ expect(layer.config.colorDomain).toEqual([0, 1]);+ expect(layer.config.strokeColorDomain).toEqual([0, 1]);+ expect(layer.config.visConfig).toMatchObject({+ filled: name !== 'line',+ stroked: name !== 'point'+ });+ }+ );+ });++ test('uses stroke color for line-only data', async () => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: line(0), properties: {score: 0.25}},+ {geometry: line(1), properties: {score: 0.75}}+ ]);++ expect(layer.config.colorField).toBeNull();+ expect(layer.config.strokeColorField?.name).toBe('score');+ expect(layer.config.strokeColorDomain).toEqual([0.25, 0.75]);+ expect(layer.config.visConfig.filled).toBe(false);+ expect(layer.config.visConfig.stroked).toBe(true);+ });++ test('uses fill color for mixed polygon and line data without changing outlines', async () => {+ const {layer} = await initializeGeoJsonLayer([+ {geometry: polygon(0), properties: {score: 0.25}},+ {geometry: line(1), properties: {score: 0.75}}+ ]);++ expect(layer.config.colorField?.name).toBe('score');+ expect(layer.config.strokeColorField).toBeNull();+ });++ test('preserves embedded fill and line colors', async () => {+ const {layer: pointLayer} = await initializeGeoJsonLayer([+ {geometry: point(0), properties: {score: 0.25, fillColor: [1, 2, 3]}},+ {geometry: point(1), properties: {score: 0.75, fillColor: [4, 5, 6]}}+ ]);+ const {layer: lineLayer} = await initializeGeoJsonLayer([+ {geometry: line(0), properties: {score: 0.25, lineColor: [1, 2, 3]}},+ {geometry: line(1), properties: {score: 0.75, lineColor: [4, 5, 6]}}+ ]);++ expect(pointLayer.config.colorField).toBeNull();+ expect(lineLayer.config.strokeColorField).toBeNull();+ });++ test('preserves embedded colors stored only inside a GeoJSON column', async () => {+ const {dataset, layer} = await initializeGeoJsonColumnLayer([+ {geometry: point(0), properties: {fillColor: [1, 2, 3]}},+ {geometry: point(1), properties: {fillColor: [4, 5, 6]}}+ ]);++ expect(dataset.fields.map(field => field.name)).toEqual(['geometry', 'score']);+ expect(layer.config.colorField).toBeNull();+ });++ test('selects a real stroke field for manually initialized table mode data', async () => {+ const dataset = new KeplerTable({+ info: {id: 'geojson-table-default-color'},+ color: [0, 0, 0]+ });+ await dataset.importData({+ data: {+ fields: [+ {name: 'id', type: ALL_FIELD_TYPES.string},+ {name: 'latitude', type: ALL_FIELD_TYPES.real},+ {name: 'longitude', type: ALL_FIELD_TYPES.real},+ {name: 'timestamp', type: ALL_FIELD_TYPES.timestamp},+ {name: 'score', type: ALL_FIELD_TYPES.real},+ {name: 'lineColor', type: ALL_FIELD_TYPES.string}+ ],+ rows: [+ ['trip', 0, 0, '2024-01-01 00:00:00', 0.25, '#ff0000'],+ ['trip', 1, 1, '2024-01-01 00:00:01', 0.75, '#00ff00']+ ]+ } as any+ });++ const layer = new GeoJsonLayer({id: 'geojson-table', dataId: dataset.id});+ layer.setInitialLayerConfig(dataset);++ expect(layer.config.columnMode).toBe('table');+ expect(layer.config.strokeColorField?.name).toBe('score');+ });+});describe('GeoJsonLayer default deck parameters', () => {const gpuFilter = null as any;diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.tsindex 599dc089f..8675bbd24 100644--- a/src/layers/src/geojson-layer/geojson-layer.ts+++ b/src/layers/src/geojson-layer/geojson-layer.ts@@ -29,7 +29,8 @@ import {COLUMN_MODE_GEOJSON,applyFiltersToTableColumns,fieldIsGeoArrow,- featureToHoverOutline+ featureToHoverOutline,+ applyDefaultGeojsonColorField} from './geojson-utils';import {getGeojsonLayerMetaFromArrow,@@ -788,6 +789,7 @@ export default class GeoJsonLayer extends Layer {}this.updateLayerMeta(dataset);+ applyDefaultGeojsonColorField(this, dataset);const {featureTypes} = this.meta;// default settings is stroke: true, filled: falsediff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.tsindex b0ab78988..c413a5035 100644--- a/src/layers/src/geojson-layer/geojson-utils.ts+++ b/src/layers/src/geojson-layer/geojson-utils.ts@@ -5,20 +5,21 @@ import {Buffer} from 'buffer';import {Feature, Position, BBox} from 'geojson';import normalize from '@mapbox/geojson-normalize';import {bbox} from '@turf/bbox';-import {ascending} from 'd3-array';+import {ascending, extent} from 'd3-array';import {center} from '@turf/center';import {AllGeoJSON} from '@turf/helpers';import {parseSync} from '@loaders.gl/core';import {WKTLoader} from '@loaders.gl/wkt';import {convertWKBToGeometry, convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';import {BinaryFeatureCollection} from '@loaders.gl/schema';-import {DataContainerInterface, getSampleData} from '@kepler.gl/utils';+import {DataContainerInterface, findDefaultColorField, getSampleData} from '@kepler.gl/utils';import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {LayerColumns, ProtoDatasetField} from '@kepler.gl/types';import {KeplerTable} from '@kepler.gl/table';import {GeojsonLayerMetaProps, assignPointPairToLayerColumn} from '../layer-utils';import {LayerBaseConfig} from '../base-layer';+import type GeoJsonLayer from './geojson-layer';export type GetFeature = (d: any) => Feature;export type GeojsonDataMaps = (Feature | BinaryFeatureCollection | null)[];@@ -26,6 +27,9 @@ export type GeojsonPointDataMaps = (number[] | number[][] | null)[];export const COLUMN_MODE_GEOJSON = 'geojson';+// Bound the extra column scans used to reject constant default color fields.+const DEFAULT_COLOR_FIELD_VALUE_CHECK_LIMIT = 1_000_000;+/* eslint-disable */// TODO: Re-enable eslint when we upgrade to handle enums and type mapsexport enum FeatureTypes {@@ -53,6 +57,59 @@ export function fieldIsGeoArrow(geoField?: ProtoDatasetField | null): boolean {return Boolean(geoField?.metadata?.get('ARROW:extension:name'));}+/** Apply automatic color settings without preventing normal layer initialization. */+export function applyDefaultGeojsonColorField(layer: GeoJsonLayer, dataset: KeplerTable): void {+ const previousConfig = layer.config;+ try {+ const {featureTypes} = layer.meta;+ const colorChannel =+ featureTypes?.polygon || featureTypes?.point+ ? 'color'+ : featureTypes?.line+ ? 'strokeColor'+ : null;+ const embeddedColorField =+ colorChannel === 'color' ? 'fillColor' : colorChannel === 'strokeColor' ? 'lineColor' : null;+ const hasEmbeddedColor = Boolean(+ embeddedColorField &&+ layer.dataToFeature.some(+ feature => feature && 'properties' in feature && feature.properties?.[embeddedColorField]+ )+ );+ if (!colorChannel || hasEmbeddedColor) {+ return;+ }++ const rowCount = dataset.dataContainer.numRows();+ if (!rowCount) {+ return;+ }++ let fields = dataset.fields;+ let defaultColorField = findDefaultColorField(dataset);+ const maxFieldChecks = Math.floor(DEFAULT_COLOR_FIELD_VALUE_CHECK_LIMIT / rowCount);+ let fieldsChecked = 0;+ while (defaultColorField && fieldsChecked < maxFieldChecks) {+ fieldsChecked++;+ const {valueAccessor} = defaultColorField;+ const [min, max] = extent(dataset.filteredIndexForDomain, index => valueAccessor({index}));+ if (min !== max) {+ layer.updateLayerConfig({+ [layer.visualChannels[colorChannel].field]: defaultColorField+ });+ layer.updateLayerVisualChannel(dataset, colorChannel);+ return;+ }++ fields = fields.filter(field => field !== defaultColorField);+ defaultColorField = findDefaultColorField({fields, fieldPairs: dataset.fieldPairs});+ }+ } catch {+ // Automatic coloring is optional; discard partially applied settings on failure.+ layer.config = previousConfig;+ }+}+function toArrayBuffer(rawFeature: ArrayBuffer | {buffer: ArrayBufferLike; byteOffset: number; byteLength: number}): ArrayBuffer {diff --git a/test/node/reducers/geojson-stroke-color-config.spec.js b/test/node/reducers/geojson-stroke-color-config.spec.jsindex c9e79f7f7..8fb38f3f6 100644--- a/test/node/reducers/geojson-stroke-color-config.spec.js+++ b/test/node/reducers/geojson-stroke-color-config.spec.js@@ -40,11 +40,72 @@ describe('geojson strokeColor programmatic config', () => {let fields;let rows;let dataset;+ let datasetWithRealColor;beforeAll(async () => {({fields, rows} = processGeojson(cloneDeep(geojsonData)));dataset = new KeplerTable({info: {id: 'traffic-data'}});await dataset.importData({data: {fields, rows}});++ const geojsonWithRealColor = cloneDeep(geojsonData);+ geojsonWithRealColor.features.forEach((feature, index) => {+ feature.properties.score = index + 0.5;+ });+ const realColorData = processGeojson(geojsonWithRealColor);+ datasetWithRealColor = new KeplerTable({info: {id: 'real-color-data'}});+ await datasetWithRealColor.importData({data: realColorData});+ });++ test('validateLayerWithData preserves an explicit saved fill field', () => {+ const savedConfig = {+ id: 'saved-fill-field',+ type: 'geojson',+ config: {+ dataId: 'real-color-data',+ label: 'Saved Fill Field',+ columns: {geojson: '_geojson'},+ isVisible: true,+ visConfig: {stroked: true, filled: true}+ },+ visualChannels: {+ colorField: {name: 'score', type: 'real'},+ colorScale: 'quantile'+ }+ };++ const layer = validateLayerWithData(+ datasetWithRealColor,+ savedConfig,+ INITIAL_VIS_STATE.layerClasses+ );+ expect(layer.config.colorField?.name).toBe('score');+ layer.updateLayerDomain({'real-color-data': datasetWithRealColor});+ expect(layer.config.colorDomain).toEqual([0.5, 1.5, 2.5, 3.5, 4.5]);+ });++ test('validateLayerWithData preserves an explicit null fill field', () => {+ const savedConfig = {+ id: 'saved-null-fill-field',+ type: 'geojson',+ config: {+ dataId: 'real-color-data',+ label: 'Saved Null Fill Field',+ columns: {geojson: '_geojson'},+ isVisible: true,+ visConfig: {stroked: true, filled: true}+ },+ visualChannels: {+ colorField: null,+ colorScale: 'quantile'+ }+ };++ const layer = validateLayerWithData(+ datasetWithRealColor,+ savedConfig,+ INITIAL_VIS_STATE.layerClasses+ );+ expect(layer.config.colorField).toBeNull();});test('validateLayerWithData binds strokeColorField nested inside config', () => {diff --git a/test/node/utils/layer-utils-test.js b/test/node/utils/layer-utils-test.jsindex 7b66a3f5e..a7877dd23 100644--- a/test/node/utils/layer-utils-test.js+++ b/test/node/utils/layer-utils-test.js@@ -18,7 +18,7 @@ import {createNewDataEntryMock} from 'test/helpers/table-utils';import tripGeojson, {timeStampDomain, tripBounds} from 'test/fixtures/trip-geojson';import {geoJsonWithStyle} from 'test/fixtures/geojson';import {KeplerTable, findPointFieldPairs} from '@kepler.gl/table';-import {ArrowDataContainer, createDataContainer} from '@kepler.gl/utils';+import {ArrowDataContainer, createDataContainer, findDefaultColorField} from '@kepler.gl/utils';import * as arrow from 'apache-arrow';test('layerUtils -> getGeojsonLayerMetaFromArrow uses worker-prepared data', t => {@@ -673,11 +673,13 @@ test('layerUtils -> findDefaultLayer:GeojsonLayer', t => {t.end();});-test('layerUtils -> findDefaultLayer:GeojsonLayer.wkt', t => {+test('layerUtils -> findDefaultLayer:GeojsonLayer.wkt', async t => {const {fields, rows} = processCsvData(wktCsv);const dataId = '0dj3h';const label = 'some geometry file';+ const dataset = new KeplerTable({info: {id: dataId, label}});+ await dataset.importData({data: {fields, rows}});const expected1 = new GeojsonLayer({dataId: '0dj3h',@@ -707,13 +709,13 @@ test('layerUtils -> findDefaultLayer:GeojsonLayer.wkt', t => {stroked: true,strokeColor: strokeColor2});+ const defaultColorField = findDefaultColorField(dataset);+ expected1.updateLayerConfig({colorField: defaultColorField});+ expected1.updateLayerVisualChannel(dataset, 'color');+ expected2.updateLayerConfig({colorField: defaultColorField});+ expected2.updateLayerVisualChannel(dataset, 'color');- const dataContainer = createDataContainer(rows, {fields});-- const geojsonLayers = findDefaultLayer(- {fields, id: dataId, label, fieldPairs: [], dataContainer},- LayerClasses- );+ const geojsonLayers = findDefaultLayer(dataset, LayerClasses);cmpLayers(t, [expected1, expected2], geojsonLayers);t.end();