Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: bound timeline bins when refreshed domains grow (#3887)
Igor Dykhtaaafa90d4f675
diff --git a/src/utils/src/plot.ts b/src/utils/src/plot.tsindex b0396b48e..9b78dd8d9 100644--- a/src/utils/src/plot.ts+++ b/src/utils/src/plot.ts@@ -10,8 +10,12 @@ import {ticks} from 'd3-array';import isEqual from 'es-toolkit/compat/isEqual';-import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';-import moment from 'moment';+import {+ getFilterMappedValue,+ getInitialInterval,+ getIntervalByTicks,+ intervalToFunction+} from './time';import {Bin,TimeBins,@@ -34,7 +38,7 @@ import {notNullorUndefined, toArray} from '@kepler.gl/common-utils';import {ANIMATION_WINDOW,BINS,- durationDay,+ BINS_LARGE,FILTER_TYPES,TIME_AGGREGATION,AGGREGATION_TYPES,@@ -273,6 +277,8 @@ export function histogramFromDomain(}));}+const timeBinDomains = new WeakMap<TimeBins, number[]>();+/*** @param filter* @param datasets@@ -284,7 +290,10 @@ export function getTimeBins(interval: PlotType['interval'],plotBaseIndexByDataId?: Record<string, number[]>): TimeBins {- let bins = filter.timeBins || {};+ interval = getTimeBinConfig(interval, filter.domain).interval;+ const cachedDomain = filter.timeBins && timeBinDomains.get(filter.timeBins);+ // Preserve caller-supplied caches; invalidate a domain change once it is known.+ let bins = cachedDomain && !isEqual(cachedDomain, filter.domain) ? {} : filter.timeBins || {};filter.dataId.forEach(dataId => {// reuse bins if filterData did not change@@ -310,6 +319,7 @@ export function getTimeBins(};});+ timeBinDomains.set(bins, Array.isArray(filter.domain) ? filter.domain.slice(0, 2) : []);return bins;}@@ -336,24 +346,122 @@ export function binByTime(indexes, dataset, interval, filter) {}export function getBinThresholds(interval: string, domain: number[]): number[] {- const timeInterval = intervalToFunction(interval);- const [t0, t1] = domain;+ return getTimeBinConfig(interval, domain).thresholds.slice();+}- if (!timeInterval) {- // if time interval is not defined- // this should not happen- return [t0, t0 + durationDay];- }- const floor = timeInterval.floor(t0).getTime();- const ceiling = timeInterval.ceil(t1).getTime();- const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());- const lastStep = binThresholds[binThresholds.length - 1];- if (lastStep === t1) {- // when last step equal to domain max, add one more step- binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());+const MAX_TIME_BINS = 2048;+// Weak keys keep validation local to live domains without retaining old filters.+const timeBinConfigs = new WeakMap<+ number[],+ {requestedInterval: string; domain: number[]; result: {interval: string; thresholds: number[]}}+>();++/** Resolve the interval and its complete domain together, never truncating a plot. */+function getTimeBinConfig(interval: string, domain: number[]) {+ if (+ !Array.isArray(domain) ||+ domain.length !== 2 ||+ !domain.every(value => Number.isFinite(value) && Number.isFinite(new Date(value).getTime())) ||+ domain[1] < domain[0]+ ) {+ return {interval, thresholds: [] as number[]};}+ const cached = timeBinConfigs.get(domain);+ if (+ cached &&+ isEqual(cached.domain, domain) &&+ (cached.requestedInterval === interval || cached.result.interval === interval)+ ) {+ return cached.result;+ }++ const build = (id: string): number[] | null => {+ const match = /^(\d+)-(millisecond|second|minute|hour|day|week|month|year)$/.exec(id);+ const step = Number(match?.[1]);+ if (!match || !Number.isSafeInteger(step) || step <= 0) return null;+ let timeInterval = intervalToFunction(id);+ if (!timeInterval) return null;+ // Filtered D3 intervals can loop internally after Date overflow. Near the+ // Date limits, walk at most one field cycle using the unfiltered interval.+ const period = {second: 60, minute: 60, hour: 24, day: 31, month: 12}[match[2]];+ if (step > 1 && period) {+ const base = intervalToFunction(`1-${match[2]}`);+ if (+ !Number.isFinite(Number(base.offset(domain[0], -period))) ||+ !Number.isFinite(Number(base.offset(domain[1], period)))+ ) {+ const field = {+ second: (date: Date) => date.getUTCSeconds(),+ minute: (date: Date) => date.getUTCMinutes(),+ hour: (date: Date) => date.getUTCHours(),+ day: (date: Date) => date.getUTCDate() - 1,+ month: (date: Date) => date.getUTCMonth()+ }[match[2]];+ if (!field) return null;+ const walk = (value: number, direction: number, includeCurrent = false) => {+ let date = base.floor(value);+ for (let index = 0; index < period; index++) {+ if (!includeCurrent || index) date = base.offset(date, direction);+ if (!Number.isFinite(Number(date))) return date;+ if (field(date) % step === 0) return date;+ }+ return new Date(NaN);+ };+ timeInterval = {+ floor: value => walk(value, -1, true),+ offset: value => walk(value, 1)+ };+ }+ }+ if (match[2] === 'week' && step > 1) {+ // D3's filtered week.offset walks every intervening week. Use the same+ // epoch alignment without an unbounded scan for large configured steps.+ const week = intervalToFunction('1-week');+ timeInterval = {+ floor(value) {+ const floor = week.floor(value);+ const remainder = ((week.count(0, floor) % step) + step) % step;+ return week.offset(floor, -remainder);+ },+ offset: value => week.offset(value, step)+ };+ }+ let boundary = Number(timeInterval.floor(domain[0]));+ const thresholds: number[] = [];+ // Include the first boundary strictly after max, even when max is aligned.+ // This preserves the existing inclusive maximum bin and calendar padding.+ for (let index = 0; index <= MAX_TIME_BINS; index++) {+ if (!Number.isFinite(boundary)) return null;+ thresholds.push(boundary);+ if (boundary > domain[1]) return thresholds;+ const next = Number(timeInterval.floor(timeInterval.offset(boundary)));+ if (!(next > boundary)) return null;+ boundary = next;+ }+ return null;+ };- return binThresholds;+ const requestedInterval = interval;+ let thresholds = build(interval);+ // Domain-only fallback avoids copying/sorting every timestamp on refresh.+ // Retry fewer ticks when the nearest interval still exceeds the cap.+ for (let target = BINS_LARGE; !thresholds && target >= 1; target /= 10) {+ interval = getIntervalByTicks(target, domain[0], domain[1]);+ thresholds = build(interval);+ }+ const result = {interval, thresholds: thresholds || []};+ timeBinConfigs.set(domain, {requestedInterval, domain: domain.slice(), result});+ return result;+}++function normalizeTimePlotType(+ plotType: TimeRangeFilter['plotType'],+ domain: number[]+): TimeRangeFilter['plotType'] {+ const {interval} = getTimeBinConfig(plotType.interval, domain);+ return interval === plotType.interval+ ? plotType+ : {...plotType, interval, defaultTimeFormat: getDefaultTimeFormat(interval)};}const MAX_NUMERIC_BINS = 2048;@@ -1217,9 +1325,13 @@ export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWi} = filter;const isTimeFilter = filter.type === FILTER_TYPES.timeRange;- const interval = isTimeFilter- ? plotType.interval || getInitialInterval(filter, state.datasets)- : undefined;+ const timePlotType = isTimeFilter+ ? normalizeTimePlotType(+ {...plotType, interval: plotType.interval || getInitialInterval(filter, state.datasets)},+ filter.domain+ )+ : plotType;+ const interval = isTimeFilter ? timePlotType.interval : undefined;const bins = isTimeFilter? getTimeBins(filter, state.datasets, interval, plotBaseIndexByDataId): getRangeFilterBins(filter, state.datasets, BINS, plotBaseIndexByDataId);@@ -1267,7 +1379,7 @@ export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWi...filter,...(isTimeFilter? {- plotType: {...filter.plotType, interval},+ plotType: timePlotType,timeBins: bins}: {bins}),@@ -1313,6 +1425,11 @@ export function updateTimeFilterPlotType(if (typeof nextPlotType !== 'object' || !nextPlotType.aggregation || !nextPlotType.interval) {nextPlotType = getDefaultPlotType(filter, datasets);}+ const normalizedPlotType = normalizeTimePlotType(nextPlotType, filter.domain);+ if (normalizedPlotType.interval !== nextPlotType.interval) {+ nextFilter = {...filter, timeBins: undefined, lineChart: undefined};+ }+ nextPlotType = normalizedPlotType;if (filter.dataId.length > 1) {nextPlotType = {diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 6c6002ebb..dff297524 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -46,6 +46,7 @@ import {getAnimatableVisibleLayers,getBinThresholds,getFilterBinThresholds,+ getDefaultTimeFormat,getNumericPlotInterval,intervalBinFromMarks,getDefaultFilter,@@ -56,6 +57,7 @@ import {} from '@kepler.gl/utils';import {ALL_FIELD_TYPES,+ TIME_INTERVALS,DatasetType,EDITOR_MODES,LAYER_VIS_CONFIGS,@@ -10345,6 +10347,150 @@ test('VisStateUpdater -> refreshDataset rebuilds filter domains and preserves set.end();});+test('VisStateUpdater -> refreshDataset bounds timeline bins without changing the selection', async t => {+ const start = Date.UTC(2020, 0, 1);+ const end = start + 86400000;+ // Protect this regression even when run against unfixed code: fail before+ // D3 can allocate an oversized range, never reproduce the original OOM.+ const every = TIME_INTERVALS.millisecond.every;+ const stub = sinon.stub(TIME_INTERVALS.millisecond, 'every').callsFake(step => {+ const interval = every(step);+ return {+ ...interval,+ range(from, to) {+ const requested = Math.ceil((to - from) / step);+ if (requested > 2049)+ throw new Error(`Intercepted unsafe allocation of ${requested} Dates`);+ return interval.range(from, to);+ }+ };+ });+ try {+ for (const mode of ['auto', 'manual', 'safe', 'end-field']) {+ const dataId = `timeline-${mode}`;+ const datasets = await createNewDataEntryMock({+ info: {id: dataId},+ data: processCsvData(`start_time,end_time,value+2020-01-01T00:00:00.000Z,2020-01-01T00:00:00.001Z,10+2020-01-01T00:00:00.001Z,,20`)+ });+ let state = {...INITIAL_VIS_STATE, datasets};+ state = reducer(state, VisStateActions.addFilter(dataId));+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'start_time'));+ if (mode === 'end-field')+ state = reducer(state, VisStateActions.setFilter(0, 'endName', 'end_time'));+ if (mode === 'manual' || mode === 'safe') {+ state = reducer(+ state,+ VisStateActions.setFilterPlot(0, {+ plotType: {interval: mode === 'safe' ? '1-hour' : '1-millisecond'}+ })+ );+ }+ state = reducer(state, VisStateActions.setFilter(0, 'gpu', false));+ state = reducer(state, VisStateActions.setFilter(0, 'value', [start, start + 1]));+ const yAxis = state.datasets[dataId].fields.find(field => field.name === 'value');+ state = reducer(state, VisStateActions.setFilterPlot(0, {yAxis}));+ const previous = state.filters[0];+ const data = processCsvData(`start_time,end_time,value+2020-01-01T00:00:00.000Z,2020-01-01T00:00:00.001Z,30+2020-01-02T00:00:00.000Z,,40`);+ state = reducer(+ state,+ VisStateActions.refreshDatasetSuccess(dataId, {data, notModified: false})+ );+ const filter = state.filters[0];+ const effectiveInterval = mode === 'safe' ? '1-hour' : '15-minute';+ t.equal(+ filter.plotType.interval,+ effectiveInterval,+ `${mode}: should preserve safe intervals and coarsen unsafe ones`+ );+ t.deepEqual(filter.domain, [start, end], `${mode}: should retain the full refreshed domain`);+ t.deepEqual(filter.value, previous.value, `${mode}: should preserve the selected range`);+ t.equal(filter.id, previous.id, `${mode}: should preserve filter identity`);+ t.equal(filter.gpu, false, `${mode}: should preserve CPU filtering`);+ t.deepEqual(+ state.datasets[dataId].filteredIndex,+ [0],+ `${mode}: should apply the unchanged selection`+ );+ t.equal(+ filter.plotType.aggregation,+ previous.plotType.aggregation,+ `${mode}: should preserve aggregation`+ );+ t.equal(filter.yAxis.name, 'value', `${mode}: should preserve the Y field`);+ if (mode !== 'safe') {+ t.equal(+ filter.plotType.defaultTimeFormat,+ getDefaultTimeFormat(effectiveInterval),+ `${mode}: should update automatic labels`+ );+ t.notOk(+ filter.timeBins[dataId]['1-millisecond'],+ `${mode}: should remove the obsolete cache`+ );+ }+ const marks = getFilterBinThresholds(filter);+ const bins = filter.timeBins[dataId][effectiveInterval];+ t.ok(marks.length <= 2049, `${mode}: should bound playback marks`);+ t.ok(+ bins.every(bin => marks.includes(bin.x0) && marks.includes(bin.x1)),+ `${mode}: should align histogram and playback boundaries`+ );+ t.equal(+ filter.lineChart.interval,+ effectiveInterval,+ `${mode}: should use the effective interval for the line chart`+ );+ t.deepEqual(+ filter.lineChart.yDomain,+ [30, 40],+ `${mode}: should include both refreshed rows in chart aggregation`+ );+ // A saved/API interval that bypasses the UI must be normalized too.+ state = reducer(+ state,+ VisStateActions.setFilterPlot(0, {plotType: {interval: '1-millisecond'}})+ );+ t.equal(+ state.filters[0].plotType.interval,+ '15-minute',+ `${mode}: should bound an unsafe API interval`+ );+ t.deepEqual(+ state.filters[0].value,+ previous.value,+ `${mode}: API coarsening should not move the selection`+ );+ const configured = state.filters[0];+ state = reducer(+ state,+ VisStateActions.applyFilterConfig(configured.id, {+ ...configured,+ plotType: {...configured.plotType, interval: '1-millisecond'}+ })+ );+ t.equal(+ state.filters[0].plotType.interval,+ '15-minute',+ `${mode}: should bound an unsafe configured interval`+ );+ t.deepEqual(+ state.filters[0].value,+ previous.value,+ `${mode}: configured coarsening should preserve selection`+ );+ }+ } catch (error) {+ t.fail(error.message);+ } finally {+ stub.restore();+ }+ t.end();+});+test('VisStateUpdater -> refreshDataset rebuilds synced and categorical domains in place', async t => {drainTasksForTesting();const remote = await createNewDataEntryMock({diff --git a/test/node/utils/plot-test.js b/test/node/utils/plot-test.jsindex e72c33b60..5baf0a513 100644--- a/test/node/utils/plot-test.js+++ b/test/node/utils/plot-test.js@@ -2,7 +2,11 @@// Copyright contributors to the kepler.gl projectimport test from 'tape';+import sinon from 'sinon';+import {TIME_INTERVALS} from '@kepler.gl/constants';import {+ getBinThresholds,+ getTimeBins,histogramFromThreshold,histogramFromValues,histogramFromTimeIntervals,@@ -24,6 +28,231 @@ import {const values1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];+test('Utils -> time thresholds bound allocation before expanding a millisecond domain', t => {+ const every = TIME_INTERVALS.millisecond.every;+ const stub = sinon.stub(TIME_INTERVALS.millisecond, 'every').callsFake(step => {+ const interval = every(step);+ return {+ ...interval,+ range(start, stop) {+ const requested = Math.ceil((stop - start) / step);+ if (requested > 2049) {+ throw new Error(`Intercepted unsafe allocation of ${requested} Dates`);+ }+ return interval.range(start, stop);+ }+ };+ });+ try {+ const start = Date.UTC(2020, 0, 1);+ const end = start + 86400000;+ const thresholds = getBinThresholds('1-millisecond', [start, end]);+ t.ok(thresholds.length <= 2049, 'should bound the number of bins including endpoint padding');+ t.equal(thresholds[0], start, 'should keep the full domain start');+ t.ok(thresholds[thresholds.length - 1] > end, 'should include the maximum in a complete bin');+ t.equal(+ histogramFromThreshold(thresholds, [start, end]).reduce((sum, bin) => sum + bin.count, 0),+ 2,+ 'should retain both endpoint rows instead of truncating the domain'+ );+ } catch (error) {+ t.fail(error.message);+ } finally {+ stub.restore();+ }+ t.end();+});++test('Utils -> time thresholds preserve safe calendar intervals and reject invalid domains', t => {+ t.equal(+ getBinThresholds('1-millisecond', [0, 2047]).length,+ 2049,+ 'should keep exactly 2048 bins'+ );+ for (const end of [2048, 3000]) {+ const thresholds = getBinThresholds('1-millisecond', [0, end]);+ t.ok(thresholds.length > 1 && thresholds.length <= 2049, 'should coarsen beyond the cap');+ t.equal(thresholds[0], 0, 'should retain the domain start');+ t.ok(thresholds[thresholds.length - 1] > end, 'should include the domain maximum');+ t.equal(+ histogramFromThreshold(thresholds, [0, end]).reduce((sum, bin) => sum + bin.count, 0),+ 2,+ 'should retain both endpoint rows after fallback'+ );+ }+ const dates = [Date.UTC(2020, 0, 1), Date.UTC(2020, 1, 1), Date.UTC(2020, 2, 1)];+ t.deepEqual(+ getBinThresholds('1-month', [dates[0], dates[1] + 1]),+ dates,+ 'should preserve leap-year calendar boundaries'+ );+ t.deepEqual(+ getBinThresholds('1-month', [dates[0], dates[1]]),+ dates,+ 'should retain inclusive maximum padding'+ );+ for (const interval of [+ '5-second',+ '15-minute',+ '3-hour',+ '2-day',+ '2-week',+ '3-month',+ '5-year'+ ]) {+ const [step, unitName] = interval.split('-');+ const unit = TIME_INTERVALS[unitName].every(Number(step));+ const start = Date.UTC(2020, 0, 2, 1, 2, 3);+ const end = Number(unit.offset(start, 3));+ const expected = unit.range(unit.floor(start), Number(unit.ceil(end)) + 1).map(Number);+ if (expected[expected.length - 1] === end) expected.push(Number(unit.offset(end)));+ t.deepEqual(+ getBinThresholds(interval, [start, end]),+ expected,+ `${interval} should keep calendar alignment`+ );+ }+ t.ok(+ getBinThresholds('1000000-week', [0, 86400000]).length <= 2049,+ 'should bound large calendar steps'+ );+ for (const domain of [[], [NaN, 1], [0, Infinity], [2, 1], [0, 9e15]]) {+ t.deepEqual(+ getBinThresholds('1-millisecond', domain),+ [],+ 'should reject an invalid time domain'+ );+ }+ for (const interval of [undefined, '', '0-day', '-1-day', 'no-such-unit']) {+ const thresholds = getBinThresholds(interval, [0, 86400000]);+ t.ok(+ thresholds.length > 1 && thresholds.length <= 2049,+ 'should safely replace an invalid interval'+ );+ t.ok(thresholds[thresholds.length - 1] >= 86400000, 'should cover the full domain');+ }+ t.end();+});++test('Utils -> time thresholds stop before calendar padding overflows Date', t => {+ const setMonth = Date.prototype.setUTCMonth;+ let operations = 0;+ const stub = sinon.stub(Date.prototype, 'setUTCMonth').callsFake(function (...args) {+ if (++operations > 100) throw new Error('Intercepted unbounded calendar stepping');+ return setMonth.apply(this, args);+ });+ try {+ const july = Date.UTC(275760, 6, 1);+ const september = Date.UTC(275760, 8, 1);+ t.deepEqual(+ getBinThresholds('2-month', [july, july + 86400000]),+ [july, september],+ 'should keep finite calendar boundaries near the Date limit'+ );+ t.deepEqual(+ getBinThresholds('2-month', [8640000000000000 - 86400000, 8640000000000000]),+ [],+ 'should reject unrepresentable upper padding'+ );+ const minimum = -8640000000000000;+ const thresholds = getBinThresholds('2-month', [minimum, minimum + 86400000]);+ t.equal(+ thresholds[0],+ minimum,+ 'should replace unrepresentable lower padding with a finite interval'+ );+ t.ok(+ thresholds.length <= 2049 && thresholds[thresholds.length - 1] > minimum + 86400000,+ 'should bound the fallback and cover the entire domain'+ );+ } catch (error) {+ t.fail(error.message);+ } finally {+ stub.restore();+ }+ t.end();+});++test('Utils -> time bins reuse validated boundaries and invalidate changed domains', t => {+ const mappedValue = [0, 30000, 60000];+ const dataset = {+ ...mockGpuPlotDataset(+ mappedValue.map(value => [value]),+ []+ ),+ getColumnField: () => ({filterProps: {mappedValue}})+ };+ const filter = {+ type: 'timeRange',+ dataId: ['dsA'],+ name: ['time'],+ domain: [0, 30000 * 2047]+ };+ const datasets = {dsA: dataset};+ filter.timeBins = getTimeBins(filter, datasets, '30-second');+ const cachedBins = filter.timeBins.dsA['30-second'];+ const marks = getBinThresholds('30-second', filter.domain);+ const every = sinon.spy(TIME_INTERVALS.second, 'every');+ try {+ t.equal(+ getTimeBins(filter, datasets, '30-second'),+ filter.timeBins,+ 'should reuse cached bins'+ );+ t.deepEqual(getBinThresholds('30-second', filter.domain), marks, 'should reuse matching marks');+ t.notOk(every.called, 'a validated cache hit should not traverse calendar intervals again');+ marks.length = 0;+ t.equal(+ getBinThresholds('30-second', filter.domain).length,+ 2049,+ 'caller changes to returned marks must not corrupt cached boundaries'+ );++ const minuteBins = getTimeBins(filter, datasets, '1-minute').dsA['1-minute'];+ t.notEqual(minuteBins, cachedBins, 'a changed interval should create different bins');+ t.deepEqual(+ minuteBins.map(bin => bin.count),+ [2, 1],+ 'should aggregate with the new interval'+ );++ filter.domain[0] = 30000;+ const changed = getTimeBins(filter, datasets, '30-second');+ t.notEqual(changed.dsA['30-second'], cachedBins, 'should invalidate an in-place domain change');+ t.deepEqual(+ changed.dsA['30-second'].flatMap(bin => bin.indexes),+ [1, 2],+ 'should exclude rows outside the new domain'+ );+ const replaced = getTimeBins({...filter, domain: [60000, 90000]}, datasets, '30-second');+ t.deepEqual(+ replaced.dsA['30-second'].flatMap(bin => bin.indexes),+ [2],+ 'should rebuild bins when the domain array is replaced'+ );++ const growing = {...filter, domain: [0, 1]};+ growing.timeBins = getTimeBins(growing, datasets, '1-millisecond');+ growing.domain[1] = 86400000;+ const coarsened = getTimeBins(growing, datasets, '1-millisecond');+ t.notOk(coarsened.dsA['1-millisecond'], 'must not reuse unsafe bins after domain growth');+ t.ok(coarsened.dsA['15-minute'], 'should cache under the effective safe interval');+ t.equal(+ coarsened.dsA['15-minute'].reduce((sum, bin) => sum + bin.count, 0),+ 3,+ 'should retain every row after domain growth'+ );+ t.deepEqual(+ getTimeBins({...filter, domain: {}}, datasets, '30-second').dsA['30-second'],+ [],+ 'should reject a non-array domain without throwing while caching metadata'+ );+ } finally {+ every.restore();+ }+ t.end();+});+test('Utils -> getNumericBinThresholds uses numeric units', t => {t.deepEqual(getNumericBinThresholds([-1.5, 1], 0.5),