Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: sort Trip timestamps chronologically and accept reserved IDs (#3934)
Igor Dykhta784e8e0f2dba
diff --git a/src/layers/src/geojson-layer/geojson-utils.spec.ts b/src/layers/src/geojson-layer/geojson-utils.spec.tsindex ef24ba5c0..a34c27352 100644--- a/src/layers/src/geojson-layer/geojson-utils.spec.ts+++ b/src/layers/src/geojson-layer/geojson-utils.spec.ts@@ -7,7 +7,7 @@ import {GeoJsonLayer as DeckGLGeoJsonLayer} from '@deck.gl/layers';import {GEOARROW_EXTENSIONS} from '@kepler.gl/constants';import {GEOARROW_METADATA_KEY} from '@kepler.gl/constants';import {KeplerTable, copyTableAndUpdate} from '@kepler.gl/table';-import {ArrowDataContainer} from '@kepler.gl/utils';+import {ArrowDataContainer, createDataContainer} from '@kepler.gl/utils';import {processFileData} from '@kepler.gl/processors';import * as arrow from 'apache-arrow';import GeoJsonLayer from './geojson-layer';@@ -22,7 +22,8 @@ import {getCentroidFromGeometry,parseGeoJsonRawFeature,getGeojsonFeatureTypes,- getGeojsonBounds+ getGeojsonBounds,+ groupColumnsAsGeoJson} from './geojson-utils';describe('GeoJSON coordinates', () => {@@ -576,3 +577,33 @@ describe('featureToHoverOutline', () => {expect(featureToHoverOutline(line)).toBe(line);});});++describe('table-column GeoJSON grouping', () => {+ const columns = {+ id: {value: 'id', fieldIdx: 0},+ lat: {value: 'lat', fieldIdx: 1},+ lng: {value: 'lng', fieldIdx: 2},+ sortBy: {value: 'sequence', fieldIdx: 3, optional: true}+ };++ test.each([+ {values: ['12/31/2023', '01/01/2024'], expectedOrder: [1, 0]},+ {values: [2, 10], expectedOrder: [0, 1]},+ {values: [0, null], expectedOrder: [0, 1]}+ ])('retains generic sort semantics for $values', ({values, expectedOrder}) => {+ const rows = values.map((value, index) => ['id', index, index, value]);+ const features = groupColumnsAsGeoJson(createDataContainer(rows), columns, 'sortBy');+ expect(features[0].properties?.values).toEqual(expectedOrder.map(index => rows[index]));+ });++ test('keeps input order without a sort column and handles empty data', () => {+ const rows = [+ ['id', 2, 2],+ ['id', 1, 1]+ ];+ const unsortedColumns = {...columns, sortBy: {value: null, fieldIdx: -1, optional: true}};+ const features = groupColumnsAsGeoJson(createDataContainer(rows), unsortedColumns, 'sortBy');+ expect(features[0].properties?.values).toEqual(rows);+ expect(groupColumnsAsGeoJson(createDataContainer([]), unsortedColumns, 'sortBy')).toEqual([]);+ });+});diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.tsindex e2a5b82e4..142803a94 100644--- a/src/layers/src/geojson-layer/geojson-utils.ts+++ b/src/layers/src/geojson-layer/geojson-utils.ts@@ -12,7 +12,8 @@ import {parseSync} from '@loaders.gl/core';import {WKTLoader} from '@loaders.gl/wkt';import {convertWKBToGeometry, convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';import {BinaryFeatureCollection} from '@loaders.gl/schema';-import {DataContainerInterface, getSampleData} from '@kepler.gl/utils';+import {DataContainerInterface, getSampleData, timeToUnixMilli} from '@kepler.gl/utils';+import {containValidTime} from '@kepler.gl/common-utils';import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {LayerColumns, ProtoDatasetField} from '@kepler.gl/types';import {KeplerTable} from '@kepler.gl/table';@@ -624,9 +625,11 @@ type CoordsType = [number, number, number, number | null] & {export function groupColumnsAsGeoJson(dataContainer: DataContainerInterface,columns: LayerColumns,- sortByColumn = 'timestamp'+ sortByColumn = 'timestamp',+ // Optional output for table-mode Trip animation, aligned with the returned features.+ dataToTimeStamp?: number[][]): Feature[] {- const groupedById: {[key: string]: CoordsType[]} = {};+ const groupedById: {[key: string]: CoordsType[]} = Object.create(null);const sortByFieldIdx = columns[sortByColumn].fieldIdx;const sortByRequired = !columns[sortByColumn].optional;for (let index = 0; index < dataContainer.numRows(); index++) {@@ -661,17 +664,39 @@ export function groupColumnsAsGeoJson(}}- const result: Feature[] = Object.entries(groupedById).map(- // eslint-disable-next-line @typescript-eslint/no-unused-vars- ([_, items]: [string, CoordsType[]], index) => ({+ const groups = Object.values(groupedById);+ // Match Trip's timestamp parser: infer one format from the first nonempty trip.+ const sampleTrip = sortByColumn === 'timestamp' && groups.find(items => items.length);+ const timeFormat = sampleTrip+ ? containValidTime(sampleTrip.map(item => item[3]) as any)?.format+ : null;++ const result: Feature[] = groups.map((items, index) => {+ if (sortByFieldIdx > -1) {+ if (!timeFormat || items.every(item => typeof item[3] === 'number')) {+ // Numeric timestamps already sort chronologically, without parsing or a sort cache.+ items.sort((a, b) => ascending(a[3] as any, b[3] as any));+ if (timeFormat && dataToTimeStamp) {+ dataToTimeStamp[index] = items.map(item => timeToUnixMilli(item[3] as number, timeFormat)!);+ }+ } else {+ // Parse once, then retain the sorted timestamps for Trip animation.+ const sorted = items+ .map(item => ({item, time: timeToUnixMilli(item[3] as number, timeFormat)!}))+ .sort((a, b) => ascending(a.time, b.time));+ sorted.forEach(({item}, i) => {+ items[i] = item;+ });+ if (dataToTimeStamp) {+ dataToTimeStamp[index] = sorted.map(({time}) => time);+ }+ }+ }+ return {type: 'Feature' as const,geometry: {type: 'LineString' as const,- // Sort by columns if has sortByField- // TODO: items are expected in Position[] format?- coordinates: (sortByFieldIdx > -1- ? items.sort((a, b) => ascending(a[3] as any, b[3] as any))- : items) as Position[]+ coordinates: items as Position[]},properties: {index,@@ -679,8 +704,8 @@ export function groupColumnsAsGeoJson(// Note: this can cause row materialization in case of non-row based containersvalues: items.map(item => dataContainer.rowAsArray(item.datumIndex))}- })- );+ };+ });return result;}diff --git a/src/layers/src/trip-layer/trip-layer-color.spec.ts b/src/layers/src/trip-layer/trip-layer-color.spec.tsindex cf360ac20..a16653c18 100644--- a/src/layers/src/trip-layer/trip-layer-color.spec.ts+++ b/src/layers/src/trip-layer/trip-layer-color.spec.ts@@ -4,6 +4,12 @@import TripLayer from './trip-layer';import KeplerTable from '@kepler.gl/table';import {ALL_FIELD_TYPES, DEFAULT_TEXT_LABEL, SCALE_TYPES} from '@kepler.gl/constants';+import {timeToUnixMilli} from '@kepler.gl/utils';++jest.mock('@kepler.gl/utils', () => {+ const actual = jest.requireActual('@kepler.gl/utils');+ return {...actual, timeToUnixMilli: jest.fn(actual.timeToUnixMilli)};+});/*** Build a small taxi-trip table: 3 trips (2 points each) across 3 companies.@@ -11,11 +17,11 @@ import {ALL_FIELD_TYPES, DEFAULT_TEXT_LABEL, SCALE_TYPES} from '@kepler.gl/const* trip occupies rows 0-1, so reading the color field by feature index would* return the first trip's company for the first two features.*/-async function makeTripTable() {+async function makeTripTable(rows?: any[][]) {const table = new KeplerTable({color: [255, 0, 0]} as any);await table.importData({data: {- rows: [+ rows: rows ?? [// trip 1 (Uber): 2 points['t1', 37.77, -122.42, '2024-01-01 00:00:00', 'Uber'],['t1', 37.78, -122.41, '2024-01-01 00:00:10', 'Uber'],@@ -146,3 +152,147 @@ describe('TripLayer text label collision', () => {}});});++describe('TripLayer TABLE mode grouping', () => {+ test.each([+ {+ name: 'month/day dates crossing a year',+ times: ['01/01/2024 00:00:00', '12/31/2023 23:59:50'],+ expected: [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)]+ },+ {+ name: 'ISO dates with timezone offsets',+ times: ['2024-01-01T00:00:00+00:00', '2024-01-01T00:59:50+01:00'],+ expected: [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)]+ },+ {+ name: 'numeric Unix milliseconds',+ times: [1704067200000, 1704067190000],+ expected: [1704067190000, 1704067200000]+ },+ {+ name: 'numeric Unix seconds',+ times: [1704067200, 1704067190],+ expected: [1704067190000, 1704067200000]+ }+ ])('orders $name chronologically while retaining row alignment', async ({times, expected}) => {+ const rows = [+ ['t1', 20, 10, times[0], 'Uber'],+ ['t1', 21, 11, times[1], 'Lyft'],+ ['t1', 22, 12, times[0], 'Waymo']+ ];+ const table = await makeTripTable(rows);+ const layer = makeTripLayer(table);+ jest.mocked(timeToUnixMilli).mockClear();+ layer.updateLayerMeta(table);+ const feature = layer.dataToFeature[0] as any;++ expect(timeToUnixMilli).toHaveBeenCalledTimes(rows.length);+ expect(layer.dataToTimeStamp[0]).toEqual([expected[0], expected[1], expected[1]]);+ expect(layer.config.animation.domain).toEqual(expected);+ expect(feature.geometry.coordinates.map(coord => coord.slice(0, 2))).toEqual([+ [11, 21],+ [10, 20],+ [12, 22]+ ]);+ expect(feature.geometry.coordinates.map(coord => coord[3])).toEqual([+ times[1],+ times[0],+ times[0]+ ]);+ expect(feature.properties.values).toEqual([rows[1], rows[0], rows[2]]);+ expect(feature.geometry.coordinates.map(coord => coord.datumIndex)).toEqual([1, 0, 2]);+ });++ test('aligns timestamps with empty and nonempty trips and refreshes them with the data', async () => {+ const rows = [+ ['empty', null, 10, '01/01/2024 00:00:00', 'Uber'],+ ['t1', 20, 10, '01/01/2024 00:00:00', 'Uber'],+ ['t1', 21, 11, '12/31/2023 23:59:50', 'Lyft'],+ ['t2', 22, 12, '01/01/2024 00:00:10', 'Waymo']+ ];+ const table = await makeTripTable(rows);+ const layer = makeTripLayer(table);+ layer.updateLayerMeta(table);++ expect(layer.dataToTimeStamp).toEqual([+ [],+ [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)],+ [Date.UTC(2024, 0, 1, 0, 0, 10)]+ ]);++ const replacement = await makeTripTable([+ ['t1', 20, 10, '01/02/2024 00:00:00', 'Uber'],+ ['t1', 21, 11, '01/01/2024 23:59:50', 'Lyft']+ ]);+ layer.updateLayerMeta(replacement);+ expect(layer.dataToTimeStamp).toEqual([+ [Date.UTC(2024, 0, 1, 23, 59, 50), Date.UTC(2024, 0, 2)]+ ]);+ expect(layer.config.animation.domain).toEqual([+ Date.UTC(2024, 0, 1, 23, 59, 50),+ Date.UTC(2024, 0, 2)+ ]);+ });++ test.each([{rows: []}, {rows: [['t1', 20, 10, 'not a timestamp', 'Uber']]}])(+ 'keeps animation disabled when timestamps cannot be inferred ($rows)',+ async ({rows}) => {+ const table = await makeTripTable(rows);+ const layer = makeTripLayer(table);+ layer.updateLayerMeta(table);++ expect(layer.dataToTimeStamp).toEqual([]);+ expect(layer.config.animation.domain).toBeNull();+ }+ );++ test.each(['constructor', '__proto__', 'toString'])(+ 'groups the valid trip ID %s without consulting inherited object properties',+ async id => {+ const rows = [+ [id, 20, 10, '2024-01-01 00:00:10', 'Uber'],+ ['ordinary', 40, 30, '2024-01-01 00:00:00', 'Lyft'],+ [id, 21, 11, '2024-01-01 00:00:00', 'Uber']+ ];+ const table = await makeTripTable(rows);+ const layer = makeTripLayer(table);+ layer.updateLayerMeta(table);++ expect(layer.dataToFeature).toHaveLength(2);+ expect(layer.dataToFeature.map((feature: any) => feature.properties.values)).toEqual([+ [rows[2], rows[0]],+ [rows[1]]+ ]);+ }+ );++ test('preserves ID coercion and ordering, skips missing IDs and required timestamps', async () => {+ const time = '2024-01-01 00:00:00';+ const rows = [+ ['10', 20, 10, time, 'Uber'],+ [2, 20, 11, time, 'Uber'],+ ['2', 20, 12, time, 'Uber'],+ ['', 20, 13, time, 'Uber'],+ ['ordinary', 20, 14, time, 'Uber'],+ [null, 20, 15, time, 'Uber'],+ [undefined, 20, 16, time, 'Uber'],+ ['ordinary', 20, 17, null, 'Uber'],+ ['ordinary', 20, 18, undefined, 'Uber'],+ ['ordinary', 20, 19, '', 'Uber'],+ ['ordinary', 20, 20, 0, 'Uber'],+ ['ordinary', null, 21, time, 'Uber'],+ ['ordinary', 20, null, time, 'Uber']+ ];+ const table = await makeTripTable(rows);+ const layer = makeTripLayer(table);+ layer.updateLayerMeta(table);++ expect(layer.dataToFeature.map((feature: any) => feature.properties.values)).toEqual([+ [rows[1], rows[2]],+ [rows[0]],+ [rows[3]],+ [rows[4]]+ ]);+ });+});diff --git a/src/layers/src/trip-layer/trip-layer.ts b/src/layers/src/trip-layer/trip-layer.tsindex 896a7b5c5..f4c02e339 100644--- a/src/layers/src/trip-layer/trip-layer.ts+++ b/src/layers/src/trip-layer/trip-layer.ts@@ -37,7 +37,11 @@ import {applyFiltersToTableColumns} from '../geojson-layer/geojson-utils';-import {isTripGeoJsonField, parseTripGeoJsonTimestamp} from './trip-utils';+import {+ getAnimationDomainFromTimestamps,+ isTripGeoJsonField,+ parseTripGeoJsonTimestamp+} from './trip-utils';import TripInfoModalFactory from './trip-info-modal';import {bisectRight} from 'd3-array';import {@@ -714,6 +718,7 @@ export default class TripLayer extends Layer {updateLayerMeta(dataset: KeplerTable) {const {dataContainer} = dataset;let getFeature;+ let timestamps: ReturnType<typeof parseTripGeoJsonTimestamp>;if (this.config.columnMode === COLUMN_MODE_GEOJSON) {getFeature = this.getPositionAccessor(dataContainer);if (getFeature === this.meta.getFeature) {@@ -721,12 +726,25 @@ export default class TripLayer extends Layer {return;}this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature);+ timestamps = parseTripGeoJsonTimestamp(this.dataToFeature);} else {this.dataContainer = dataContainer;- this.dataToFeature = groupColumnsAsGeoJson(dataContainer, this.config.columns, 'timestamp');+ const dataToTimeStamp: number[][] = [];+ this.dataToFeature = groupColumnsAsGeoJson(+ dataContainer,+ this.config.columns,+ 'timestamp',+ dataToTimeStamp+ );+ timestamps = {+ dataToTimeStamp,+ animationDomain: dataToTimeStamp.length+ ? getAnimationDomainFromTimestamps(dataToTimeStamp)+ : null+ };}- const {dataToTimeStamp, animationDomain} = parseTripGeoJsonTimestamp(this.dataToFeature);+ const {dataToTimeStamp, animationDomain} = timestamps;this.dataToTimeStamp = dataToTimeStamp;this.updateAnimationDomain(animationDomain);