Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(table): refresh cached Arrow timestamps (#3886)
Igor Dykhta24b285efcfd2
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex 1e7917acc..627fd461c 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -315,6 +315,15 @@ class KeplerTable<F extends Field = Field> {});}+ // Column snapshots can reorder or replace rows without changing the schema,+ // invalidating row-indexed timestamp mappings. Collect only already-cached+ // field names so unused columns stay lazy and refresh scans stay limited.+ const timestampFieldsToRebuild = isRowSnapshot+ ? []+ : this.fields+ .filter(f => f.type === ALL_FIELD_TYPES.timestamp && f.filterProps)+ .map(f => f.name);+if (isRowSnapshot) {this.fields = this.fields.map((f, i) => {const {filterProps: _filterProps, ...rest} = f;@@ -324,6 +333,17 @@ class KeplerTable<F extends Field = Field> {};}) as F[];this.gpuFilter = getGpuFilterProps([], this.id, this.fields, undefined);+ } else if (timestampFieldsToRebuild.length) {+ // getColumnFilterProps returns existing filterProps without recomputing.+ // Remove the stale cache so it reads the new snapshot, while preserving+ // the GPU filter configuration and channels across column updates.+ this.fields = this.fields.map(f => {+ if (f.type !== ALL_FIELD_TYPES.timestamp || !f.filterProps) {+ return f;+ }+ const {filterProps: _filterProps, ...rest} = f;+ return rest;+ }) as F[];}this.allIndexes = this.dataContainer.getPlainIndex();@@ -336,6 +356,11 @@ class KeplerTable<F extends Field = Field> {this.changedFilters = undefined;this.dataRevision += 1;+ // Rebuild after row indexes and filter results are reset: CPU/GPU consumers+ // read mappedValue directly. Caching the new mapping here avoids rescanning+ // timestamps on later filter interactions.+ timestampFieldsToRebuild.forEach(name => this.getColumnFilterProps(name));+return this;}diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex dff297524..f86e201bf 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -10287,6 +10287,86 @@ test('VisStateUpdater -> refreshDataset reapplies active filters to Arrow snapsht.end();});+test('VisStateUpdater -> timestamp refresh updates point filtering and export', async t => {+ const firstTime = Date.UTC(2024, 1, 1);+ const lastTime = firstTime + 86400000;+ const snapshot = (values, rowBacked) => {+ const source = new arrow.Table({+ time: arrow.vectorFromArray(+ values.map(value => new Date(value)),+ new arrow.TimestampMillisecond('UTC')+ ),+ lat: arrow.vectorFromArray(+ values.map(() => 37.77),+ new arrow.Float64()+ ),+ lng: arrow.vectorFromArray(+ values.map(() => -122.42),+ new arrow.Float64()+ )+ });+ const data = processArrowBatches(arrow.tableFromIPC(arrow.tableToIPC(source, 'file')).batches);+ return rowBacked+ ? {fields: data.fields, rows: values.map(value => [value, 37.77, -122.42])}+ : data;+ };++ for (const rowBacked of [false, true]) {+ for (const gpu of [false, true]) {+ drainTasksForTesting();+ const dataId = 'remote-timestamps';+ const datasets = await createNewDataEntryMock({+ info: {id: dataId, type: DatasetType.EXTERNALLY_HOSTED},+ data: snapshot([firstTime, lastTime], rowBacked),+ metadata: {source: 'https://example.com/time.arrow', sourceFormat: 'arrow'}+ });+ const {props} = PointLayer.findDefaultLayerProps(datasets[dataId]);+ const point = new PointLayer({id: 'timestamp-point', dataId, ...props[0]});+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};+ const initial = calculateLayerData(point, state);+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};+ state = reducer(state, VisStateActions.addFilter(dataId));+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'time'));+ state = reducer(+ state,+ VisStateActions.setFilter(0, 'value', [firstTime, firstTime + 3600000])+ );+ state = reducer(state, VisStateActions.setFilter(0, 'gpu', gpu));+ const selection = state.filters[0];++ for (const values of [[lastTime, firstTime], [], [lastTime, firstTime]]) {+ state = reducer(state, VisStateActions.refreshDataset(dataId));+ state = reducer(state, VisStateActions.refreshDatasetProgress(dataId, 50));+ const [task] = drainTasksForTesting();+ state = reducer(+ state,+ succeedTaskInTest(task, {data: snapshot(values, rowBacked), notModified: false})+ );+ const table = state.datasets[dataId];+ const expected = values.length ? [1] : [];+ table.filterTableCPU(state.filters, state.layers);+ t.deepEqual(table.filteredIdxCPU, expected, 'export uses refreshed timestamp values');+ const renderIndexes = rowBacked+ ? state.layerData[0].data.map(datum => datum.index)+ : table.allIndexes;+ const visible = renderIndexes.filter(index => {+ if (!state.layerData[0].getFiltered({index})) return false;+ const value = state.layerData[0].getFilterValue({index})[0];+ const [min, max] = table.gpuFilter.filterRange[0];+ return value >= min && value <= max;+ });+ t.deepEqual(visible, expected, 'point rendering uses refreshed timestamp values');+ t.equal(state.layers[0].isValid, true, 'empty snapshots keep the point layer valid');+ t.deepEqual(state.filters[0].value, selection.value, 'refresh retains the selected range');+ t.equal(state.filters[0].id, selection.id, 'refresh retains filter identity');+ t.equal(state.filters[0].gpu, gpu, 'refresh retains GPU mode');+ t.equal(state.filters[0].enabled, true, 'refresh keeps the filter enabled');+ }+ }+ }+ t.end();+});+test('VisStateUpdater -> refreshDataset rebuilds filter domains and preserves selections', async t => {drainTasksForTesting();const dataId = 'remote-filter';diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.jsindex 47d7d0f76..8fb7fd0c0 100644--- a/test/node/utils/kepler-table-test.js+++ b/test/node/utils/kepler-table-test.js@@ -7,7 +7,7 @@ import testData, {numericRangesCsv, testFields} from 'test/fixtures/test-csv-datimport {preciseRound, getFilterFunction} from '@kepler.gl/utils';import {findPointFieldPairs, KeplerTable, createNewDataEntry} from '@kepler.gl/table';-import {processCsvData} from '@kepler.gl/processors';+import {processArrowBatches, processCsvData} from '@kepler.gl/processors';import {ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';import * as arrow from 'apache-arrow';@@ -510,6 +510,116 @@ test('KeplerTable -> update with arrow cols keeps gpu filter state', async t =>t.end();});+test('KeplerTable -> Arrow snapshots rebuild cached timestamps and reuse them between filters', async t => {+ const firstTime = Date.UTC(2024, 1, 1);+ const lastTime = firstTime + 86400000;+ const types = [+ arrow.TimestampSecond,+ arrow.TimestampMillisecond,+ arrow.TimestampMicrosecond,+ arrow.TimestampNanosecond+ ];++ for (const Timestamp of types) {+ for (const [shape, timezone] of [+ [null, null],+ ['table', 'UTC'],+ ['duckdb', 'America/New_York']+ ]) {+ const snapshot = values => {+ const vector = offset =>+ arrow.vectorFromArray(+ values.map(value => (value === null ? null : new Date(value + offset))),+ new Timestamp(timezone)+ );+ const source = new arrow.Table({+ time: vector(0),+ later: vector(3600000),+ unused: vector(0)+ });+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));+ const data = processArrowBatches(ipc.batches);+ return shape === 'table' ? {...data, arrowTable: ipc} : data;+ };++ for (const gpu of [false, true]) {+ const table = new KeplerTable({info: {id: 'timestamps'}, color: [0, 0, 0]});+ await table.importData({data: snapshot([firstTime, lastTime, null])});+ const filters = ['time', 'later'].map((name, index) => ({+ ...table.getColumnFilterProps(name),+ id: name,+ dataId: [table.id],+ name: [name],+ fieldIdx: [index],+ value: [firstTime + index * 3600000, firstTime + index * 3600000],+ enabled: true,+ gpu,+ gpuChannel: [index]+ }));+ table.filterTable(filters, []);+ table.filterTableCPU(filters, []);+ t.deepEqual(table.filteredIdxCPU, [0], 'initial timestamp selection');++ const reads = [0, 0, 0];+ table.fields.forEach((field, index) => {+ const accessor = field.valueAccessor;+ field.valueAccessor = datum => {+ reads[index] += 1;+ return accessor(datum);+ };+ });+ for (const values of [+ [lastTime, firstTime, null],+ [],+ [null, lastTime, firstTime],+ [firstTime, lastTime, null, firstTime]+ ]) {+ const data = snapshot(values);+ const oldGpuFilter = table.gpuFilter;+ reads.fill(0);+ await table.update(shape === 'duckdb' ? {...data, fields: []} : data);+ t.equal(table.gpuFilter, oldGpuFilter, 'snapshot retains GPU configuration');+ t.deepEqual(+ reads,+ [values.length, values.length, 0],+ 'only cached timestamps are scanned once'+ );+ t.notOk(table.fields[2].filterProps, 'unused timestamp stays lazy');+ t.deepEqual(+ table.fields[0].filterProps.mappedValue,+ values,+ 'mapping follows row order and nulls'+ );+ const mapped = table.fields[0].filterProps.mappedValue;+ table.getColumnFilterProps('time');+ table.filterTable(filters, []);+ table.filterTableCPU(filters, []);+ const expected = values.flatMap((value, index) => (value === firstTime ? [index] : []));+ t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export filters use current timestamps');+ const getValue = table.gpuFilter.filterValueAccessor(table.dataContainer)();+ const visible = table.filteredIndex.filter(index =>+ getValue({index}).every((value, channel) => {+ const [min, max] = table.gpuFilter.filterRange[channel];+ return value >= min && value <= max;+ })+ );+ t.deepEqual(visible, expected, 'render filter attributes select the current rows');+ reads.fill(0);+ table.filterTable(filters, []);+ table.filterTableCPU(filters, []);+ t.equal(+ table.fields[0].filterProps.mappedValue,+ mapped,+ 'filter interactions reuse the mapping'+ );+ t.deepEqual(reads, [0, 0, 0], 'unchanged filters do not rescan timestamps');+ }+ }+ }+ }+ t.end();+});+test('KeplerTable -> update DuckDB cols payload keeps fields and gpuFilter', async t => {const first = arrow.tableFromArrays({lat: [1, 2],