Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: sort Trip timestamps chronologically and accept reserved IDs (#3934)

Igor Dykhta784e8e0f2dba
diff --git a/src/layers/src/geojson-layer/geojson-utils.spec.ts b/src/layers/src/geojson-layer/geojson-utils.spec.ts
index ef24ba5c0..a34c27352 100644
--- a/src/layers/src/geojson-layer/geojson-utils.spec.ts
+++ b/src/layers/src/geojson-layer/geojson-utils.spec.ts
@@ -7,7 +7,7 @@ import {GeoJsonLayer as DeckGLGeoJsonLayer} from '@deck.gl/layers';
import {GEOARROW_EXTENSIONS} from '@kepler.gl/constants';
import {GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
import {KeplerTable, copyTableAndUpdate} from '@kepler.gl/table';
-import {ArrowDataContainer} from '@kepler.gl/utils';
+import {ArrowDataContainer, createDataContainer} from '@kepler.gl/utils';
import {processFileData} from '@kepler.gl/processors';
import * as arrow from 'apache-arrow';
import GeoJsonLayer from './geojson-layer';
@@ -22,7 +22,8 @@ import {
getCentroidFromGeometry,
parseGeoJsonRawFeature,
getGeojsonFeatureTypes,
- getGeojsonBounds
+ getGeojsonBounds,
+ groupColumnsAsGeoJson
} from './geojson-utils';
describe('GeoJSON coordinates', () => {
@@ -576,3 +577,33 @@ describe('featureToHoverOutline', () => {
expect(featureToHoverOutline(line)).toBe(line);
});
});
+
+describe('table-column GeoJSON grouping', () => {
+ const columns = {
+ id: {value: 'id', fieldIdx: 0},
+ lat: {value: 'lat', fieldIdx: 1},
+ lng: {value: 'lng', fieldIdx: 2},
+ sortBy: {value: 'sequence', fieldIdx: 3, optional: true}
+ };
+
+ test.each([
+ {values: ['12/31/2023', '01/01/2024'], expectedOrder: [1, 0]},
+ {values: [2, 10], expectedOrder: [0, 1]},
+ {values: [0, null], expectedOrder: [0, 1]}
+ ])('retains generic sort semantics for $values', ({values, expectedOrder}) => {
+ const rows = values.map((value, index) => ['id', index, index, value]);
+ const features = groupColumnsAsGeoJson(createDataContainer(rows), columns, 'sortBy');
+ expect(features[0].properties?.values).toEqual(expectedOrder.map(index => rows[index]));
+ });
+
+ test('keeps input order without a sort column and handles empty data', () => {
+ const rows = [
+ ['id', 2, 2],
+ ['id', 1, 1]
+ ];
+ const unsortedColumns = {...columns, sortBy: {value: null, fieldIdx: -1, optional: true}};
+ const features = groupColumnsAsGeoJson(createDataContainer(rows), unsortedColumns, 'sortBy');
+ expect(features[0].properties?.values).toEqual(rows);
+ expect(groupColumnsAsGeoJson(createDataContainer([]), unsortedColumns, 'sortBy')).toEqual([]);
+ });
+});
diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts
index e2a5b82e4..142803a94 100644
--- a/src/layers/src/geojson-layer/geojson-utils.ts
+++ b/src/layers/src/geojson-layer/geojson-utils.ts
@@ -12,7 +12,8 @@ import {parseSync} from '@loaders.gl/core';
import {WKTLoader} from '@loaders.gl/wkt';
import {convertWKBToGeometry, convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';
import {BinaryFeatureCollection} from '@loaders.gl/schema';
-import {DataContainerInterface, getSampleData} from '@kepler.gl/utils';
+import {DataContainerInterface, getSampleData, timeToUnixMilli} from '@kepler.gl/utils';
+import {containValidTime} from '@kepler.gl/common-utils';
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {LayerColumns, ProtoDatasetField} from '@kepler.gl/types';
import {KeplerTable} from '@kepler.gl/table';
@@ -624,9 +625,11 @@ type CoordsType = [number, number, number, number | null] & {
export function groupColumnsAsGeoJson(
dataContainer: DataContainerInterface,
columns: LayerColumns,
- sortByColumn = 'timestamp'
+ sortByColumn = 'timestamp',
+ // Optional output for table-mode Trip animation, aligned with the returned features.
+ dataToTimeStamp?: number[][]
): Feature[] {
- const groupedById: {[key: string]: CoordsType[]} = {};
+ const groupedById: {[key: string]: CoordsType[]} = Object.create(null);
const sortByFieldIdx = columns[sortByColumn].fieldIdx;
const sortByRequired = !columns[sortByColumn].optional;
for (let index = 0; index < dataContainer.numRows(); index++) {
@@ -661,17 +664,39 @@ export function groupColumnsAsGeoJson(
}
}
- const result: Feature[] = Object.entries(groupedById).map(
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- ([_, items]: [string, CoordsType[]], index) => ({
+ const groups = Object.values(groupedById);
+ // Match Trip's timestamp parser: infer one format from the first nonempty trip.
+ const sampleTrip = sortByColumn === 'timestamp' && groups.find(items => items.length);
+ const timeFormat = sampleTrip
+ ? containValidTime(sampleTrip.map(item => item[3]) as any)?.format
+ : null;
+
+ const result: Feature[] = groups.map((items, index) => {
+ if (sortByFieldIdx > -1) {
+ if (!timeFormat || items.every(item => typeof item[3] === 'number')) {
+ // Numeric timestamps already sort chronologically, without parsing or a sort cache.
+ items.sort((a, b) => ascending(a[3] as any, b[3] as any));
+ if (timeFormat && dataToTimeStamp) {
+ dataToTimeStamp[index] = items.map(item => timeToUnixMilli(item[3] as number, timeFormat)!);
+ }
+ } else {
+ // Parse once, then retain the sorted timestamps for Trip animation.
+ const sorted = items
+ .map(item => ({item, time: timeToUnixMilli(item[3] as number, timeFormat)!}))
+ .sort((a, b) => ascending(a.time, b.time));
+ sorted.forEach(({item}, i) => {
+ items[i] = item;
+ });
+ if (dataToTimeStamp) {
+ dataToTimeStamp[index] = sorted.map(({time}) => time);
+ }
+ }
+ }
+ return {
type: 'Feature' as const,
geometry: {
type: 'LineString' as const,
- // Sort by columns if has sortByField
- // TODO: items are expected in Position[] format?
- coordinates: (sortByFieldIdx > -1
- ? items.sort((a, b) => ascending(a[3] as any, b[3] as any))
- : items) as Position[]
+ coordinates: items as Position[]
},
properties: {
index,
@@ -679,8 +704,8 @@ export function groupColumnsAsGeoJson(
// Note: this can cause row materialization in case of non-row based containers
values: items.map(item => dataContainer.rowAsArray(item.datumIndex))
}
- })
- );
+ };
+ });
return result;
}
diff --git a/src/layers/src/trip-layer/trip-layer-color.spec.ts b/src/layers/src/trip-layer/trip-layer-color.spec.ts
index cf360ac20..a16653c18 100644
--- a/src/layers/src/trip-layer/trip-layer-color.spec.ts
+++ b/src/layers/src/trip-layer/trip-layer-color.spec.ts
@@ -4,6 +4,12 @@
import TripLayer from './trip-layer';
import KeplerTable from '@kepler.gl/table';
import {ALL_FIELD_TYPES, DEFAULT_TEXT_LABEL, SCALE_TYPES} from '@kepler.gl/constants';
+import {timeToUnixMilli} from '@kepler.gl/utils';
+
+jest.mock('@kepler.gl/utils', () => {
+ const actual = jest.requireActual('@kepler.gl/utils');
+ return {...actual, timeToUnixMilli: jest.fn(actual.timeToUnixMilli)};
+});
/**
* Build a small taxi-trip table: 3 trips (2 points each) across 3 companies.
@@ -11,11 +17,11 @@ import {ALL_FIELD_TYPES, DEFAULT_TEXT_LABEL, SCALE_TYPES} from '@kepler.gl/const
* trip occupies rows 0-1, so reading the color field by feature index would
* return the first trip's company for the first two features.
*/
-async function makeTripTable() {
+async function makeTripTable(rows?: any[][]) {
const table = new KeplerTable({color: [255, 0, 0]} as any);
await table.importData({
data: {
- rows: [
+ rows: rows ?? [
// trip 1 (Uber): 2 points
['t1', 37.77, -122.42, '2024-01-01 00:00:00', 'Uber'],
['t1', 37.78, -122.41, '2024-01-01 00:00:10', 'Uber'],
@@ -146,3 +152,147 @@ describe('TripLayer text label collision', () => {
}
});
});
+
+describe('TripLayer TABLE mode grouping', () => {
+ test.each([
+ {
+ name: 'month/day dates crossing a year',
+ times: ['01/01/2024 00:00:00', '12/31/2023 23:59:50'],
+ expected: [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)]
+ },
+ {
+ name: 'ISO dates with timezone offsets',
+ times: ['2024-01-01T00:00:00+00:00', '2024-01-01T00:59:50+01:00'],
+ expected: [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)]
+ },
+ {
+ name: 'numeric Unix milliseconds',
+ times: [1704067200000, 1704067190000],
+ expected: [1704067190000, 1704067200000]
+ },
+ {
+ name: 'numeric Unix seconds',
+ times: [1704067200, 1704067190],
+ expected: [1704067190000, 1704067200000]
+ }
+ ])('orders $name chronologically while retaining row alignment', async ({times, expected}) => {
+ const rows = [
+ ['t1', 20, 10, times[0], 'Uber'],
+ ['t1', 21, 11, times[1], 'Lyft'],
+ ['t1', 22, 12, times[0], 'Waymo']
+ ];
+ const table = await makeTripTable(rows);
+ const layer = makeTripLayer(table);
+ jest.mocked(timeToUnixMilli).mockClear();
+ layer.updateLayerMeta(table);
+ const feature = layer.dataToFeature[0] as any;
+
+ expect(timeToUnixMilli).toHaveBeenCalledTimes(rows.length);
+ expect(layer.dataToTimeStamp[0]).toEqual([expected[0], expected[1], expected[1]]);
+ expect(layer.config.animation.domain).toEqual(expected);
+ expect(feature.geometry.coordinates.map(coord => coord.slice(0, 2))).toEqual([
+ [11, 21],
+ [10, 20],
+ [12, 22]
+ ]);
+ expect(feature.geometry.coordinates.map(coord => coord[3])).toEqual([
+ times[1],
+ times[0],
+ times[0]
+ ]);
+ expect(feature.properties.values).toEqual([rows[1], rows[0], rows[2]]);
+ expect(feature.geometry.coordinates.map(coord => coord.datumIndex)).toEqual([1, 0, 2]);
+ });
+
+ test('aligns timestamps with empty and nonempty trips and refreshes them with the data', async () => {
+ const rows = [
+ ['empty', null, 10, '01/01/2024 00:00:00', 'Uber'],
+ ['t1', 20, 10, '01/01/2024 00:00:00', 'Uber'],
+ ['t1', 21, 11, '12/31/2023 23:59:50', 'Lyft'],
+ ['t2', 22, 12, '01/01/2024 00:00:10', 'Waymo']
+ ];
+ const table = await makeTripTable(rows);
+ const layer = makeTripLayer(table);
+ layer.updateLayerMeta(table);
+
+ expect(layer.dataToTimeStamp).toEqual([
+ [],
+ [Date.UTC(2023, 11, 31, 23, 59, 50), Date.UTC(2024, 0, 1)],
+ [Date.UTC(2024, 0, 1, 0, 0, 10)]
+ ]);
+
+ const replacement = await makeTripTable([
+ ['t1', 20, 10, '01/02/2024 00:00:00', 'Uber'],
+ ['t1', 21, 11, '01/01/2024 23:59:50', 'Lyft']
+ ]);
+ layer.updateLayerMeta(replacement);
+ expect(layer.dataToTimeStamp).toEqual([
+ [Date.UTC(2024, 0, 1, 23, 59, 50), Date.UTC(2024, 0, 2)]
+ ]);
+ expect(layer.config.animation.domain).toEqual([
+ Date.UTC(2024, 0, 1, 23, 59, 50),
+ Date.UTC(2024, 0, 2)
+ ]);
+ });
+
+ test.each([{rows: []}, {rows: [['t1', 20, 10, 'not a timestamp', 'Uber']]}])(
+ 'keeps animation disabled when timestamps cannot be inferred ($rows)',
+ async ({rows}) => {
+ const table = await makeTripTable(rows);
+ const layer = makeTripLayer(table);
+ layer.updateLayerMeta(table);
+
+ expect(layer.dataToTimeStamp).toEqual([]);
+ expect(layer.config.animation.domain).toBeNull();
+ }
+ );
+
+ test.each(['constructor', '__proto__', 'toString'])(
+ 'groups the valid trip ID %s without consulting inherited object properties',
+ async id => {
+ const rows = [
+ [id, 20, 10, '2024-01-01 00:00:10', 'Uber'],
+ ['ordinary', 40, 30, '2024-01-01 00:00:00', 'Lyft'],
+ [id, 21, 11, '2024-01-01 00:00:00', 'Uber']
+ ];
+ const table = await makeTripTable(rows);
+ const layer = makeTripLayer(table);
+ layer.updateLayerMeta(table);
+
+ expect(layer.dataToFeature).toHaveLength(2);
+ expect(layer.dataToFeature.map((feature: any) => feature.properties.values)).toEqual([
+ [rows[2], rows[0]],
+ [rows[1]]
+ ]);
+ }
+ );
+
+ test('preserves ID coercion and ordering, skips missing IDs and required timestamps', async () => {
+ const time = '2024-01-01 00:00:00';
+ const rows = [
+ ['10', 20, 10, time, 'Uber'],
+ [2, 20, 11, time, 'Uber'],
+ ['2', 20, 12, time, 'Uber'],
+ ['', 20, 13, time, 'Uber'],
+ ['ordinary', 20, 14, time, 'Uber'],
+ [null, 20, 15, time, 'Uber'],
+ [undefined, 20, 16, time, 'Uber'],
+ ['ordinary', 20, 17, null, 'Uber'],
+ ['ordinary', 20, 18, undefined, 'Uber'],
+ ['ordinary', 20, 19, '', 'Uber'],
+ ['ordinary', 20, 20, 0, 'Uber'],
+ ['ordinary', null, 21, time, 'Uber'],
+ ['ordinary', 20, null, time, 'Uber']
+ ];
+ const table = await makeTripTable(rows);
+ const layer = makeTripLayer(table);
+ layer.updateLayerMeta(table);
+
+ expect(layer.dataToFeature.map((feature: any) => feature.properties.values)).toEqual([
+ [rows[1], rows[2]],
+ [rows[0]],
+ [rows[3]],
+ [rows[4]]
+ ]);
+ });
+});
diff --git a/src/layers/src/trip-layer/trip-layer.ts b/src/layers/src/trip-layer/trip-layer.ts
index 896a7b5c5..f4c02e339 100644
--- a/src/layers/src/trip-layer/trip-layer.ts
+++ b/src/layers/src/trip-layer/trip-layer.ts
@@ -37,7 +37,11 @@ import {
applyFiltersToTableColumns
} from '../geojson-layer/geojson-utils';
-import {isTripGeoJsonField, parseTripGeoJsonTimestamp} from './trip-utils';
+import {
+ getAnimationDomainFromTimestamps,
+ isTripGeoJsonField,
+ parseTripGeoJsonTimestamp
+} from './trip-utils';
import TripInfoModalFactory from './trip-info-modal';
import {bisectRight} from 'd3-array';
import {
@@ -714,6 +718,7 @@ export default class TripLayer extends Layer {
updateLayerMeta(dataset: KeplerTable) {
const {dataContainer} = dataset;
let getFeature;
+ let timestamps: ReturnType<typeof parseTripGeoJsonTimestamp>;
if (this.config.columnMode === COLUMN_MODE_GEOJSON) {
getFeature = this.getPositionAccessor(dataContainer);
if (getFeature === this.meta.getFeature) {
@@ -721,12 +726,25 @@ export default class TripLayer extends Layer {
return;
}
this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature);
+ timestamps = parseTripGeoJsonTimestamp(this.dataToFeature);
} else {
this.dataContainer = dataContainer;
- this.dataToFeature = groupColumnsAsGeoJson(dataContainer, this.config.columns, 'timestamp');
+ const dataToTimeStamp: number[][] = [];
+ this.dataToFeature = groupColumnsAsGeoJson(
+ dataContainer,
+ this.config.columns,
+ 'timestamp',
+ dataToTimeStamp
+ );
+ timestamps = {
+ dataToTimeStamp,
+ animationDomain: dataToTimeStamp.length
+ ? getAnimationDomainFromTimestamps(dataToTimeStamp)
+ : null
+ };
}
- const {dataToTimeStamp, animationDomain} = parseTripGeoJsonTimestamp(this.dataToFeature);
+ const {dataToTimeStamp, animationDomain} = timestamps;
this.dataToTimeStamp = dataToTimeStamp;
this.updateAnimationDomain(animationDomain);