Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat: progress bar when loading remote datasets from a saved map (#3764)
Igor Dykhtaac36f69ffe3b
diff --git a/examples/demo-app-sqlrooms/src/store.js b/examples/demo-app-sqlrooms/src/store.jsindex bf1414730..47a06e12c 100644--- a/examples/demo-app-sqlrooms/src/store.js+++ b/examples/demo-app-sqlrooms/src/store.js@@ -22,6 +22,7 @@ const LEVEL1_ACTIONS = new Set(['@@kepler.gl/MOUSE_MOVE','@@kepler.gl/LAYER_HOVER','@@kepler.gl/SET_LOADING_INDICATOR',+ '@@kepler.gl/SET_LOADING_PROGRESS','@@openassistant/SET_MAP_BOUNDARY']);diff --git a/examples/demo-app/src/store.js b/examples/demo-app/src/store.jsindex bf1414730..47a06e12c 100644--- a/examples/demo-app/src/store.js+++ b/examples/demo-app/src/store.js@@ -22,6 +22,7 @@ const LEVEL1_ACTIONS = new Set(['@@kepler.gl/MOUSE_MOVE','@@kepler.gl/LAYER_HOVER','@@kepler.gl/SET_LOADING_INDICATOR',+ '@@kepler.gl/SET_LOADING_PROGRESS','@@openassistant/SET_MAP_BOUNDARY']);diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex 5155da3ad..7cb54c062 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -187,6 +187,7 @@ export const ActionTypes = {SYNC_TIME_FILTER_TIMELINE_MODE: `${ACTION_PREFIX}SYNC_TIME_FILTER_TIMELINE_MODE`,TOGGLE_PANEL_LIST_VIEW: `${ACTION_PREFIX}TOGGLE_PANEL_LIST_VIEW`,SET_LOADING_INDICATOR: `${ACTION_PREFIX}SET_LOADING_INDICATOR`,+ SET_LOADING_PROGRESS: `${ACTION_PREFIX}SET_LOADING_PROGRESS`,// uiState > export imageSET_EXPORT_IMAGE_SETTING: `${ACTION_PREFIX}SET_EXPORT_IMAGE_SETTING`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex e06ea6a34..6f16d0935 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -2376,6 +2376,8 @@ export function setTimeFilterSyncTimelineMode({export type CreateNewDatasetSuccessPayload = {results: (PromiseFulfilledResult<KeplerTable> | PromiseRejectedResult)[];addToMapOptions: AddDataToMapPayload['options'];+ /** Dataset ids whose hydrate download progress should be cleared. */+ progressIds?: string[];};/**@@ -2404,6 +2406,28 @@ export const setLoadingIndicator = createAction<SetLoadingIndicatorPayload>(ActionTypes.SET_LOADING_INDICATOR);+export type SetLoadingProgressUpdaterAction = {+ id: string;+ percent: number;+};++/**+ * Update download progress for an in-flight dataset load shown on the map+ * loading indicator (0–100).+ * @memberof visStateActions+ * @returns action+ */+export function setLoadingProgress(+ id: string,+ percent: number+): Merge<SetLoadingProgressUpdaterAction, {type: typeof ActionTypes.SET_LOADING_PROGRESS}> {+ return {+ type: ActionTypes.SET_LOADING_PROGRESS,+ id,+ percent+ };+}+/*** This declaration is needed to group actions in docs*/diff --git a/src/components/src/loading-indicator.tsx b/src/components/src/loading-indicator.tsxindex 02a05b9eb..b73a8404f 100644--- a/src/components/src/loading-indicator.tsx+++ b/src/components/src/loading-indicator.tsx@@ -2,7 +2,7 @@// Copyright contributors to the kepler.gl projectimport React, {PropsWithChildren, useRef, useEffect} from 'react';-import styled, {withTheme, keyframes} from 'styled-components';+import styled, {withTheme, keyframes, css} from 'styled-components';import {getNumRasterTilesBeingLoaded, getNumVectorTilesBeingLoaded} from '@kepler.gl/layers';@@ -12,21 +12,49 @@ type StyledContainerProps = {$bottomOffset: number;};-const spin = keyframes`- 0% { transform: rotate(0deg); }- 100% { transform: rotate(360deg); }+const indeterminateSlide = keyframes`+ 0% { transform: translateX(-100%); }+ 100% { transform: translateX(350%); }`;-const Spinner = styled.div`- display: inline-block;- width: 14px;- height: 14px;- margin-right: 8px;- border: 2px solid ${props => props.theme.textColorHl};- border-top-color: transparent;- border-radius: 50%;- animation: ${spin} 0.8s linear infinite;- vertical-align: middle;+const StyledLabel = styled.div`+ color: ${props => props.theme.textColorHl};+ line-height: 1.2;+`;++const StyledHint = styled.div`+ margin-top: 3px;+ color: ${props => props.theme.textColor};+ font-size: 11px;+ font-weight: 400;+ line-height: 1.3;+`;++const StyledTrack = styled.div`+ margin-top: 4px;+ height: 4px;+ width: 100%;+ background-color: color-mix(in srgb, ${props => props.theme.activeColor} 20%, transparent);+ border-radius: 4px;+ overflow: hidden;+`;++const barFill = css`+ height: 100%;+ border-radius: 4px;+ background-color: ${props => props.theme.activeColor};+`;++const StyledBar = styled.div<{$percent: number}>`+ ${barFill};+ width: ${props => props.$percent}%;+ transition: width 0.2s ease-out;+`;++const StyledIndeterminateBar = styled.div`+ ${barFill};+ width: 32%;+ animation: ${indeterminateSlide} 1.2s ease-in-out infinite;`;export const StyledContainer = styled.div<StyledContainerProps>`@@ -34,17 +62,17 @@ export const StyledContainer = styled.div<StyledContainerProps>`left: ${props => props.$left}px;bottom: ${props => props.theme.sidePanel.margin.left + props.$bottomOffset}px;z-index: 1;+ min-width: 220px;+ max-width: 280px;+ padding: 10px 12px;color: ${props => props.theme.textColor};opacity: ${props => (props.$isVisible ? 1 : 0)};transition: opacity 0.5s ease-in-out;background-color: ${props => props.theme.sidePanelBg};- padding: 8px 12px;- font-size: 13px;+ font-size: 12px;font-weight: 500;- box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3);- backdrop-filter: blur(4px);- display: flex;- align-items: center;+ box-shadow: 0 2px 10px rgba(0, 0, 0, 0.35);+ backdrop-filter: blur(8px);pointer-events: none;`;@@ -54,6 +82,10 @@ type LoadingIndicatorProps = {sidePanelWidth?: number;hasAttributionLogos?: boolean;hasMapScale?: boolean;+ /** Combined download progress 0–100. Omit or 0 for an indeterminate bar. */+ percent?: number;+ /** Number of remote files currently being fetched from a saved config. */+ remoteDatasetCount?: number;};/** Extra adjustment for the loading indicator when side panel is visible */@@ -61,12 +93,29 @@ const LEFT_POSITION_ADJUSTMENT = 3;/** Height of the map scale bar, used to stack the loading indicator above it */const MAP_SCALE_HEIGHT = 24;+/**+ * Average per-dataset download percents. Returns undefined when every value is+ * 0 so the indicator stays an indeterminate bar (no Content-Length).+ */+export function aggregateLoadingPercent(progress?: Record<string, number>): number | undefined {+ if (!progress) {+ return undefined;+ }+ const values = Object.values(progress);+ if (!values.length || values.every(value => value <= 0)) {+ return undefined;+ }+ return Math.round(values.reduce((sum, value) => sum + value, 0) / values.length);+}+const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({isVisible,activeSidePanel,sidePanelWidth,hasAttributionLogos,hasMapScale,+ percent,+ remoteDatasetCount = 0,theme}) => {const left =@@ -78,37 +127,67 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({const numRasterTilesInProgress = getNumRasterTilesBeingLoaded();const numVectorTilesInProgress = getNumVectorTilesBeingLoaded();- let extraMessage = '';+ const remoteMessage =+ remoteDatasetCount > 0+ ? remoteDatasetCount === 1+ ? 'Remote dataset is being loaded'+ : `${remoteDatasetCount} remote datasets are being loaded`+ : '';++ let tileMessage = '';if (numRasterTilesInProgress > 0 && numVectorTilesInProgress > 0) {- // Both types loading: show combined countconst totalTiles = numRasterTilesInProgress + numVectorTilesInProgress;- extraMessage = `${totalTiles} tile${totalTiles === 1 ? ' is' : 's are'} being loaded`;+ tileMessage = `${totalTiles} tile${totalTiles === 1 ? ' is' : 's are'} being loaded`;} else if (numRasterTilesInProgress > 0) {- // Only raster tiles loading- extraMessage = `${numRasterTilesInProgress} raster tile${+ tileMessage = `${numRasterTilesInProgress} raster tile${numRasterTilesInProgress === 1 ? ' is' : 's are'} being loaded`;} else if (numVectorTilesInProgress > 0) {- // Only vector tiles loading- extraMessage = `${numVectorTilesInProgress} vector tile${+ tileMessage = `${numVectorTilesInProgress} vector tile${numVectorTilesInProgress === 1 ? ' is' : 's are'} being loaded`;}- // Preserve the last message during fade-out+ const extraMessage = [remoteMessage, tileMessage].filter(Boolean).join('\n');++ const hasProgress = typeof percent === 'number' && percent > 0;++ // Preserve the last message / percent during fade-outconst lastMessageRef = useRef(extraMessage);+ const lastPercentRef = useRef<number | undefined>(undefined);useEffect(() => {if (isVisible && extraMessage) {lastMessageRef.current = extraMessage;}- }, [isVisible, extraMessage]);+ if (isVisible && hasProgress) {+ lastPercentRef.current = percent;+ }+ }, [isVisible, extraMessage, hasProgress, percent]);const displayMessage = isVisible ? extraMessage : lastMessageRef.current;+ const displayPercent = isVisible ? (hasProgress ? percent : undefined) : lastPercentRef.current;+ const determinatePercent =+ typeof displayPercent === 'number' && displayPercent > 0 ? displayPercent : null;return (<StyledContainer $isVisible={isVisible} $left={left} $bottomOffset={bottomOffset}>- <Spinner />- <span>{`Loading... ${displayMessage}`}</span>+ <StyledLabel>Loading...</StyledLabel>+ {displayMessage+ ? displayMessage.split('\n').map(line => <StyledHint key={line}>{line}</StyledHint>)+ : null}+ <StyledTrack+ role="progressbar"+ aria-valuemin={0}+ aria-valuemax={100}+ aria-valuenow={determinatePercent === null ? undefined : Math.round(determinatePercent)}+ aria-label="Loading"+ >+ {determinatePercent === null ? (+ <StyledIndeterminateBar />+ ) : (+ <StyledBar $percent={determinatePercent} />+ )}+ </StyledTrack></StyledContainer>);};diff --git a/src/components/src/map-container.tsx b/src/components/src/map-container.tsxindex 81a484997..2ba4b8d87 100644--- a/src/components/src/map-container.tsx+++ b/src/components/src/map-container.tsx@@ -112,7 +112,7 @@ import {} from '@kepler.gl/reducers';import {VisState} from '@kepler.gl/schemas';-import LoadingIndicator from './loading-indicator';+import LoadingIndicator, {aggregateLoadingPercent} from './loading-indicator';// Debounce the propagation of viewport change and mouse moves to redux store.// This is to avoid too many renders of other components when the map is@@ -1579,6 +1579,8 @@ export default function MapContainerFactory(sidePanelWidth={sidePanelWidth}hasAttributionLogos={attributionLogos.length > 0}hasMapScale={getApplicationConfig().enableMapScale}+ percent={aggregateLoadingPercent(visState.loadingProgress)}+ remoteDatasetCount={Object.keys(visState.loadingProgress || {}).length}/>) : null}{this.props.primary ? (diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 5471d3eba..93957ffc3 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -51,7 +51,8 @@ import {loadColumnStatsError,refreshDatasetSuccess,refreshDatasetError,- refreshDatasetProgress+ refreshDatasetProgress,+ setLoadingProgress} from '@kepler.gl/actions';// Utils@@ -399,6 +400,7 @@ export const INITIAL_VIS_STATE: VisState = {fileLoadingProgress: {},// for loading datasetsloadingIndicatorValue: 0,+ loadingProgress: {},loaders: [],loadOptions: {},@@ -3232,7 +3234,13 @@ function needsExternallyHostedHydrate(dataset: ProtoDataset): boolean {);}-async function hydrateExternallyHostedProtoDataset(dataset: ProtoDataset): Promise<ProtoDataset> {+async function hydrateExternallyHostedProtoDataset({+ arg: dataset,+ onProgress+}: {+ arg: ProtoDataset;+ onProgress: (progress: {loaded: number; total?: number; percent: number}) => void;+}): Promise<ProtoDataset> {if (!needsExternallyHostedHydrate(dataset)) {return dataset;}@@ -3240,7 +3248,8 @@ async function hydrateExternallyHostedProtoDataset(dataset: ProtoDataset): Promisource: dataset.metadata?.source as string,format: getRemoteSourceFormat(dataset.metadata),size: typeof dataset.metadata?.size === 'number' ? dataset.metadata.size : undefined,- bypassCache: getDatasetRefreshIntervalMs(dataset.metadata) > 0+ bypassCache: getDatasetRefreshIntervalMs(dataset.metadata) > 0,+ onProgress});if (!loaded.data) {throw new Error(`No data loaded from ${dataset.metadata?.source}`);@@ -3259,7 +3268,7 @@ async function hydrateExternallyHostedProtoDataset(dataset: ProtoDataset): Promi};}-const HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK = Task.fromPromise(+const HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK = Task.fromPromiseWithProgress(hydrateExternallyHostedProtoDataset,'HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK');@@ -3268,11 +3277,26 @@ const FAIL_CREATE_DATASET_TASK = Task.fromPromise(async (message: string) => {throw new Error(message);}, 'FAIL_CREATE_DATASET_TASK');+function getDatasetProgressId(dataset: ProtoDataset, index = 0): string {+ if (typeof dataset.info?.id === 'string' && dataset.info.id) {+ return dataset.info.id;+ }+ if (typeof dataset.metadata?.source === 'string' && dataset.metadata.source) {+ return dataset.metadata.source;+ }+ return `dataset-${index}`;+}+function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {if (!needsExternallyHostedHydrate(dataset)) {return createNewDataEntry(dataset, datasets);}- return HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK(dataset).chain(hydrated => {+ const progressId = getDatasetProgressId(dataset);+ return HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK({+ arg: dataset,+ onProgress: progress =>+ setLoadingProgress(progressId, Math.round((progress?.percent ?? 0) * 100))+ }).chain(hydrated => {const task = createNewDataEntry(hydrated, datasets);return (task ||@@ -3281,6 +3305,24 @@ function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {});}+function clearLoadingProgress(state: VisState, ids?: string[]): VisState {+ const current = state.loadingProgress || {};+ if (!Object.keys(current).length) {+ return state;+ }+ if ((state.loadingIndicatorValue || 0) <= 0) {+ return {...state, loadingProgress: {}};+ }+ if (!ids?.length) {+ return state;+ }+ const next = {...current};+ ids.forEach(id => {+ delete next[id];+ });+ return {...state, loadingProgress: next};+}+function patchDatasetMetadata(dataset: Datasets[string], patch: Record<string, unknown>) {return copyTableAndUpdate(dataset, {metadata: {@@ -3715,9 +3757,14 @@ export const updateVisDataUpdater = (const createDatasetTasks: TaskDescriptor[] = [];const notificationTasks: TaskDescriptor[] = [];+ const hydrateProgress: Record<string, number> = {};datasets.forEach(({info = {}, ...rest}, datasetIndex) => {- const task = createNewDataEntryTask({info, ...rest}, state.datasets);+ const proto = {info, ...rest};+ if (needsExternallyHostedHydrate(proto)) {+ hydrateProgress[getDatasetProgressId(proto, datasetIndex)] = 0;+ }+ const task = createNewDataEntryTask(proto, state.datasets);if (task) {createDatasetTasks.push(task);} else {@@ -3734,14 +3781,24 @@ export const updateVisDataUpdater = (}});+ const progressIds = Object.keys(hydrateProgress);const datasetsAllSettledTask = createDatasetTasks.length? Task.allSettled(createDatasetTasks).map(results =>- createNewDatasetSuccess({results, addToMapOptions: options})+ createNewDatasetSuccess({results, addToMapOptions: options, progressIds})): null;if (datasetsAllSettledTask) {updatedState = setLoadingIndicatorUpdater(updatedState, payload_({change: 1, type: ''}));+ if (progressIds.length) {+ updatedState = {+ ...updatedState,+ loadingProgress: {+ ...updatedState.loadingProgress,+ ...hydrateProgress+ }+ };+ }}return withTask(updatedState, [@@ -3754,7 +3811,7 @@ export const createNewDatasetSuccessUpdater = (state: VisState,action: PayloadAction<CreateNewDatasetSuccessPayload>): VisState => {- const {results, addToMapOptions} = action.payload;+ const {results, addToMapOptions, progressIds} = action.payload;const notificationTasks: TaskDescriptor[] = [];const newDataEntries = results.reduce((accu, result, idx) => {@@ -3800,7 +3857,10 @@ export const createNewDatasetSuccessUpdater = (});return withTask(- setLoadingIndicatorUpdater(updatedState, payload_({change: -1})),+ clearLoadingProgress(+ setLoadingIndicatorUpdater(updatedState, payload_({change: -1})),+ progressIds+ ),notificationTasks);};@@ -5814,12 +5874,43 @@ export const setLoadingIndicatorUpdater = (loadingIndicatorValue = 0;}+ const nextValue = Math.max(loadingIndicatorValue + change, 0);return {...state,- loadingIndicatorValue: Math.max(loadingIndicatorValue + change, 0)+ loadingIndicatorValue: nextValue,+ ...(nextValue === 0 && Object.keys(state.loadingProgress || {}).length+ ? {loadingProgress: {}}+ : {})};};+/**+ * Record download progress for an in-flight remote dataset hydrate.+ * @memberof visStateUpdaters+ * @public+ */+export function setLoadingProgressUpdater(+ state: VisState,+ {id, percent}: VisStateActions.SetLoadingProgressUpdaterAction+): VisState {+ if ((state.loadingIndicatorValue || 0) <= 0) {+ return state;+ }+ const nextPercent = Number.isFinite(percent)+ ? Math.max(0, Math.min(100, Math.round(percent)))+ : 0;+ if (state.loadingProgress?.[id] === nextPercent) {+ return state;+ }+ return {+ ...state,+ loadingProgress: {+ ...state.loadingProgress,+ [id]: nextPercent+ }+ };+}+function adjustAnimationConfigWithFilter<S extends VisState>(state: S, filterIdx: number): S {const filter = state.filters[filterIdx];if ((filter as TimeRangeFilter).syncedWithLayerTimeline) {diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex 8475633fb..f397c9d7e 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -222,7 +222,9 @@ const actionHandler = {[ActionTypes.CREATE_NEW_DATASET_SUCCESS]: visStateUpdaters.createNewDatasetSuccessUpdater,- [ActionTypes.SET_LOADING_INDICATOR]: visStateUpdaters.setLoadingIndicatorUpdater+ [ActionTypes.SET_LOADING_INDICATOR]: visStateUpdaters.setLoadingIndicatorUpdater,++ [ActionTypes.SET_LOADING_PROGRESS]: visStateUpdaters.setLoadingProgressUpdater};// construct vis-state reducerdiff --git a/src/schemas/src/vis-state-schema.ts b/src/schemas/src/vis-state-schema.tsindex be48612bc..4c20d8a73 100644--- a/src/schemas/src/vis-state-schema.ts+++ b/src/schemas/src/vis-state-schema.ts@@ -91,6 +91,8 @@ export interface VisState {fileLoading: FileLoading | false;fileLoadingProgress: FileLoadingProgress;loadingIndicatorValue: number;+ /** Per-dataset download progress (0–100) while hydrating remote files. */+ loadingProgress: Record<string, number>;loaders: Loader[];loadOptions: object;initialState?: Partial<VisState>;diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.tsindex 8b5f3a5c6..5d7e6b01e 100644--- a/src/utils/src/application-config.ts+++ b/src/utils/src/application-config.ts@@ -56,7 +56,8 @@ export type BaseMapLibraryConfig = {** - `0` — log every action (no filtering)* - `1` — suppress the highest-frequency UI noise:- * MOUSE_MOVE, LAYER_HOVER, SET_MAP_BOUNDARY, SET_LOADING_INDICATOR+ * MOUSE_MOVE, LAYER_HOVER, SET_MAP_BOUNDARY, SET_LOADING_INDICATOR,+ * SET_LOADING_PROGRESS* - `2` — suppress everything in level 1 plus map/layer update chatter:* LOAD_MAP_STYLES, UPDATE_MAP, LAYER_VISUAL_CHANGE,* ON_MAP_CLICK, FILTER_CHANGE, MAP_LOAD_STARTEDdiff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 88894d7bf..27f5822db 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -8428,6 +8428,37 @@ test('VisStateUpdater -> refreshDatasetProgress', async t => {t.end();});+test('VisStateUpdater -> hydrate remote dataset loading progress', t => {+ drainTasksForTesting();+ const proto = {+ info: {id: 'remote-1', type: DatasetType.EXTERNALLY_HOSTED, label: 'quakes.csv'},+ data: {fields: [], rows: []},+ metadata: {source: 'https://example.com/quakes.csv', sourceFormat: 'csv'}+ };++ const loadingState = reducer(INITIAL_VIS_STATE, VisStateActions.updateVisData([proto]));+ t.equal(loadingState.loadingIndicatorValue, 1, 'should show the map loading indicator');+ t.equal(loadingState.loadingProgress['remote-1'], 0, 'should seed hydrate progress');++ const progressed = reducer(loadingState, VisStateActions.setLoadingProgress('remote-1', 42));+ t.equal(progressed.loadingProgress['remote-1'], 42, 'should store download percent');++ const same = reducer(progressed, VisStateActions.setLoadingProgress('remote-1', 42));+ t.equal(same, progressed, 'should skip redundant progress updates');++ const idle = reducer(INITIAL_VIS_STATE, VisStateActions.setLoadingProgress('remote-1', 50));+ t.equal(idle, INITIAL_VIS_STATE, 'should ignore progress when nothing is loading');++ const [task] = drainTasksForTesting();+ t.ok(task, 'should schedule a hydrate/create task');++ const cleared = reducer(progressed, VisStateActions.setLoadingIndicator({change: -1}));+ t.equal(cleared.loadingIndicatorValue, 0, 'should hide the loading indicator');+ t.deepEqual(cleared.loadingProgress, {}, 'should clear hydrate progress');++ t.end();+});+test('VisStateUpdater -> refreshDataset 304 and error', async t => {drainTasksForTesting();const initialData = processCsvData('lat,lng\n1,2');