Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(table): refresh cached Arrow timestamps (#3886)

Igor Dykhta24b285efcfd2
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index 1e7917acc..627fd461c 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -315,6 +315,15 @@ class KeplerTable<F extends Field = Field> {
});
}
+ // Column snapshots can reorder or replace rows without changing the schema,
+ // invalidating row-indexed timestamp mappings. Collect only already-cached
+ // field names so unused columns stay lazy and refresh scans stay limited.
+ const timestampFieldsToRebuild = isRowSnapshot
+ ? []
+ : this.fields
+ .filter(f => f.type === ALL_FIELD_TYPES.timestamp && f.filterProps)
+ .map(f => f.name);
+
if (isRowSnapshot) {
this.fields = this.fields.map((f, i) => {
const {filterProps: _filterProps, ...rest} = f;
@@ -324,6 +333,17 @@ class KeplerTable<F extends Field = Field> {
};
}) as F[];
this.gpuFilter = getGpuFilterProps([], this.id, this.fields, undefined);
+ } else if (timestampFieldsToRebuild.length) {
+ // getColumnFilterProps returns existing filterProps without recomputing.
+ // Remove the stale cache so it reads the new snapshot, while preserving
+ // the GPU filter configuration and channels across column updates.
+ this.fields = this.fields.map(f => {
+ if (f.type !== ALL_FIELD_TYPES.timestamp || !f.filterProps) {
+ return f;
+ }
+ const {filterProps: _filterProps, ...rest} = f;
+ return rest;
+ }) as F[];
}
this.allIndexes = this.dataContainer.getPlainIndex();
@@ -336,6 +356,11 @@ class KeplerTable<F extends Field = Field> {
this.changedFilters = undefined;
this.dataRevision += 1;
+ // Rebuild after row indexes and filter results are reset: CPU/GPU consumers
+ // read mappedValue directly. Caching the new mapping here avoids rescanning
+ // timestamps on later filter interactions.
+ timestampFieldsToRebuild.forEach(name => this.getColumnFilterProps(name));
+
return this;
}
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index dff297524..f86e201bf 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -10287,6 +10287,86 @@ test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapsh
t.end();
});
+test('VisStateUpdater -> timestamp refresh updates point filtering and export', async t => {
+ const firstTime = Date.UTC(2024, 1, 1);
+ const lastTime = firstTime + 86400000;
+ const snapshot = (values, rowBacked) => {
+ const source = new arrow.Table({
+ time: arrow.vectorFromArray(
+ values.map(value => new Date(value)),
+ new arrow.TimestampMillisecond('UTC')
+ ),
+ lat: arrow.vectorFromArray(
+ values.map(() => 37.77),
+ new arrow.Float64()
+ ),
+ lng: arrow.vectorFromArray(
+ values.map(() => -122.42),
+ new arrow.Float64()
+ )
+ });
+ const data = processArrowBatches(arrow.tableFromIPC(arrow.tableToIPC(source, 'file')).batches);
+ return rowBacked
+ ? {fields: data.fields, rows: values.map(value => [value, 37.77, -122.42])}
+ : data;
+ };
+
+ for (const rowBacked of [false, true]) {
+ for (const gpu of [false, true]) {
+ drainTasksForTesting();
+ const dataId = 'remote-timestamps';
+ const datasets = await createNewDataEntryMock({
+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},
+ data: snapshot([firstTime, lastTime], rowBacked),
+ metadata: {source: 'https://example.com/time.arrow', sourceFormat: 'arrow'}
+ });
+ const {props} = PointLayer.findDefaultLayerProps(datasets[dataId]);
+ const point = new PointLayer({id: 'timestamp-point', dataId, ...props[0]});
+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};
+ const initial = calculateLayerData(point, state);
+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};
+ state = reducer(state, VisStateActions.addFilter(dataId));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'time'));
+ state = reducer(
+ state,
+ VisStateActions.setFilter(0, 'value', [firstTime, firstTime + 3600000])
+ );
+ state = reducer(state, VisStateActions.setFilter(0, 'gpu', gpu));
+ const selection = state.filters[0];
+
+ for (const values of [[lastTime, firstTime], [], [lastTime, firstTime]]) {
+ state = reducer(state, VisStateActions.refreshDataset(dataId));
+ state = reducer(state, VisStateActions.refreshDatasetProgress(dataId, 50));
+ const [task] = drainTasksForTesting();
+ state = reducer(
+ state,
+ succeedTaskInTest(task, {data: snapshot(values, rowBacked), notModified: false})
+ );
+ const table = state.datasets[dataId];
+ const expected = values.length ? [1] : [];
+ table.filterTableCPU(state.filters, state.layers);
+ t.deepEqual(table.filteredIdxCPU, expected, 'export uses refreshed timestamp values');
+ const renderIndexes = rowBacked
+ ? state.layerData[0].data.map(datum => datum.index)
+ : table.allIndexes;
+ const visible = renderIndexes.filter(index => {
+ if (!state.layerData[0].getFiltered({index})) return false;
+ const value = state.layerData[0].getFilterValue({index})[0];
+ const [min, max] = table.gpuFilter.filterRange[0];
+ return value >= min && value <= max;
+ });
+ t.deepEqual(visible, expected, 'point rendering uses refreshed timestamp values');
+ t.equal(state.layers[0].isValid, true, 'empty snapshots keep the point layer valid');
+ t.deepEqual(state.filters[0].value, selection.value, 'refresh retains the selected range');
+ t.equal(state.filters[0].id, selection.id, 'refresh retains filter identity');
+ t.equal(state.filters[0].gpu, gpu, 'refresh retains GPU mode');
+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');
+ }
+ }
+ }
+ t.end();
+});
+
test('VisStateUpdater -> refreshDataset rebuilds filter domains and preserves selections', async t => {
drainTasksForTesting();
const dataId = 'remote-filter';
diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.js
index 47d7d0f76..8fb7fd0c0 100644
--- a/test/node/utils/kepler-table-test.js
+++ b/test/node/utils/kepler-table-test.js
@@ -7,7 +7,7 @@ import testData, {numericRangesCsv, testFields} from 'test/fixtures/test-csv-dat
import {preciseRound, getFilterFunction} from '@kepler.gl/utils';
import {findPointFieldPairs, KeplerTable, createNewDataEntry} from '@kepler.gl/table';
-import {processCsvData} from '@kepler.gl/processors';
+import {processArrowBatches, processCsvData} from '@kepler.gl/processors';
import {ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
@@ -510,6 +510,116 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>
t.end();
});
+test('KeplerTable -> Arrow snapshots rebuild cached timestamps and reuse them between filters', async t => {
+ const firstTime = Date.UTC(2024, 1, 1);
+ const lastTime = firstTime + 86400000;
+ const types = [
+ arrow.TimestampSecond,
+ arrow.TimestampMillisecond,
+ arrow.TimestampMicrosecond,
+ arrow.TimestampNanosecond
+ ];
+
+ for (const Timestamp of types) {
+ for (const [shape, timezone] of [
+ [null, null],
+ ['table', 'UTC'],
+ ['duckdb', 'America/New_York']
+ ]) {
+ const snapshot = values => {
+ const vector = offset =>
+ arrow.vectorFromArray(
+ values.map(value => (value === null ? null : new Date(value + offset))),
+ new Timestamp(timezone)
+ );
+ const source = new arrow.Table({
+ time: vector(0),
+ later: vector(3600000),
+ unused: vector(0)
+ });
+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));
+ const data = processArrowBatches(ipc.batches);
+ return shape === 'table' ? {...data, arrowTable: ipc} : data;
+ };
+
+ for (const gpu of [false, true]) {
+ const table = new KeplerTable({info: {id: 'timestamps'}, color: [0, 0, 0]});
+ await table.importData({data: snapshot([firstTime, lastTime, null])});
+ const filters = ['time', 'later'].map((name, index) => ({
+ ...table.getColumnFilterProps(name),
+ id: name,
+ dataId: [table.id],
+ name: [name],
+ fieldIdx: [index],
+ value: [firstTime + index * 3600000, firstTime + index * 3600000],
+ enabled: true,
+ gpu,
+ gpuChannel: [index]
+ }));
+ table.filterTable(filters, []);
+ table.filterTableCPU(filters, []);
+ t.deepEqual(table.filteredIdxCPU, [0], 'initial timestamp selection');
+
+ const reads = [0, 0, 0];
+ table.fields.forEach((field, index) => {
+ const accessor = field.valueAccessor;
+ field.valueAccessor = datum => {
+ reads[index] += 1;
+ return accessor(datum);
+ };
+ });
+ for (const values of [
+ [lastTime, firstTime, null],
+ [],
+ [null, lastTime, firstTime],
+ [firstTime, lastTime, null, firstTime]
+ ]) {
+ const data = snapshot(values);
+ const oldGpuFilter = table.gpuFilter;
+ reads.fill(0);
+ await table.update(shape === 'duckdb' ? {...data, fields: []} : data);
+ t.equal(table.gpuFilter, oldGpuFilter, 'snapshot retains GPU configuration');
+ t.deepEqual(
+ reads,
+ [values.length, values.length, 0],
+ 'only cached timestamps are scanned once'
+ );
+ t.notOk(table.fields[2].filterProps, 'unused timestamp stays lazy');
+ t.deepEqual(
+ table.fields[0].filterProps.mappedValue,
+ values,
+ 'mapping follows row order and nulls'
+ );
+ const mapped = table.fields[0].filterProps.mappedValue;
+ table.getColumnFilterProps('time');
+ table.filterTable(filters, []);
+ table.filterTableCPU(filters, []);
+ const expected = values.flatMap((value, index) => (value === firstTime ? [index] : []));
+ t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export filters use current timestamps');
+ const getValue = table.gpuFilter.filterValueAccessor(table.dataContainer)();
+ const visible = table.filteredIndex.filter(index =>
+ getValue({index}).every((value, channel) => {
+ const [min, max] = table.gpuFilter.filterRange[channel];
+ return value >= min && value <= max;
+ })
+ );
+ t.deepEqual(visible, expected, 'render filter attributes select the current rows');
+ reads.fill(0);
+ table.filterTable(filters, []);
+ table.filterTableCPU(filters, []);
+ t.equal(
+ table.fields[0].filterProps.mappedValue,
+ mapped,
+ 'filter interactions reuse the mapping'
+ );
+ t.deepEqual(reads, [0, 0, 0], 'unchanged filters do not rescan timestamps');
+ }
+ }
+ }
+ }
+ t.end();
+});
+
test('KeplerTable -> update DuckDB cols payload keeps fields and gpuFilter', async t => {
const first = arrow.tableFromArrays({
lat: [1, 2],