Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: show feedback while large dataset processing blocks the UI (#3790)

Igor Dykhta7f26460a7337
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts
index 499c899db..5a0ccbd3c 100644
--- a/src/actions/src/vis-state-actions.ts
+++ b/src/actions/src/vis-state-actions.ts
@@ -2648,6 +2648,8 @@ export const setLoadingIndicator = createAction<SetLoadingIndicatorPayload>(
export type SetLoadingProgressUpdaterAction = {
id: string;
percent: number;
+ /** Download is finished and the file is being parsed into a dataset. */
+ phase?: 'processing';
};
/**
@@ -2658,12 +2660,14 @@ export type SetLoadingProgressUpdaterAction = {
*/
export function setLoadingProgress(
id: string,
- percent: number
+ percent: number,
+ phase?: 'processing'
): Merge<SetLoadingProgressUpdaterAction, {type: typeof ActionTypes.SET_LOADING_PROGRESS}> {
return {
type: ActionTypes.SET_LOADING_PROGRESS,
id,
- percent
+ percent,
+ ...(phase ? {phase} : {})
};
}
diff --git a/src/components/src/common/file-uploader/upload-file-list.tsx b/src/components/src/common/file-uploader/upload-file-list.tsx
index 03e4ee845..ea0c699b0 100644
--- a/src/components/src/common/file-uploader/upload-file-list.tsx
+++ b/src/components/src/common/file-uploader/upload-file-list.tsx
@@ -2,7 +2,7 @@
// Copyright contributors to the kepler.gl project
import React, {FC} from 'react';
-import styled from 'styled-components';
+import styled, {keyframes} from 'styled-components';
import {media} from '@kepler.gl/styles';
import Checkbox from '../checkbox';
@@ -99,7 +99,8 @@ const CardBody = styled.div`
display: flex;
align-items: center;
gap: 8px;
- padding: 8px 12px 8px 8px;
+ /* Icon is 36px. A centered 16px control then sits 18px from the top and bottom. */
+ padding: 8px 18px 8px 8px;
`;
const IconWrap = styled.div`
@@ -122,11 +123,33 @@ const CardName = styled.div`
const CardCheck = styled.div`
flex-shrink: 0;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: ${props => props.theme.checkboxWidth}px;
+ height: ${props => props.theme.checkboxHeight}px;
+
+ .kg-checkbox {
+ margin: 0;
+ min-height: 0;
+ width: ${props => props.theme.checkboxWidth}px;
+ height: ${props => props.theme.checkboxHeight}px;
+ }
- .kg-checkbox,
.kg-checkbox__label {
- margin-left: 0;
- margin-bottom: 0;
+ display: block;
+ box-sizing: border-box;
+ width: ${props => props.theme.checkboxWidth}px;
+ height: ${props => props.theme.checkboxHeight}px;
+ margin: 0;
+ padding: 0 0 0 ${props => props.theme.checkboxWidth}px;
+ line-height: ${props => props.theme.checkboxHeight}px;
+ }
+
+ .kg-checkbox__label::before {
+ box-sizing: border-box;
+ width: ${props => props.theme.checkboxWidth}px;
+ height: ${props => props.theme.checkboxHeight}px;
}
`;
@@ -155,6 +178,29 @@ const CardStatus = styled.div<{$isError?: boolean}>`
color: ${props => (props.$isError ? props.theme.errorColor : props.theme.subtextColorLT)};
`;
+// Transform-only so the compositor can keep it moving while parsing blocks the main thread.
+const spin = keyframes`
+ to {
+ transform: rotate(360deg);
+ }
+`;
+
+const StatusSpinner = styled.span.attrs({
+ className: 'upload-file-list__spinner',
+ 'aria-hidden': true
+})`
+ display: block;
+ flex-shrink: 0;
+ width: ${props => props.theme.checkboxWidth}px;
+ height: ${props => props.theme.checkboxHeight}px;
+ box-sizing: border-box;
+ border-radius: 50%;
+ border: 2px solid ${props => props.theme.borderColorLT};
+ border-top-color: ${props => props.theme.primaryBtnBgd};
+ will-change: transform;
+ animation: ${spin} 0.7s linear infinite;
+`;
+
const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {
if (!items.length) {
return null;
@@ -178,7 +224,11 @@ const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {
{item.status}
</CardStatus>
</CardText>
- {item.selectable ? (
+ {!item.isSuccess && !item.isError ? (
+ <CardCheck>
+ <StatusSpinner />
+ </CardCheck>
+ ) : item.selectable ? (
<CardCheck title={item.selectionLabel} onClick={event => event.stopPropagation()}>
<Checkbox
id={`upload-file-${index}`}
diff --git a/src/components/src/common/item-selector/item-selector.tsx b/src/components/src/common/item-selector/item-selector.tsx
index 9af5eeae7..0e77d72b2 100644
--- a/src/components/src/common/item-selector/item-selector.tsx
+++ b/src/components/src/common/item-selector/item-selector.tsx
@@ -4,7 +4,7 @@
import React, {Component, createRef, ComponentType, MouseEventHandler, RefObject} from 'react';
import classnames from 'classnames';
import uniqBy from 'es-toolkit/compat/uniqBy';
-import styled, {IStyledComponent} from 'styled-components';
+import styled, {IStyledComponent, keyframes} from 'styled-components';
import Accessor from './accessor';
import ChickletedInput from './chickleted-input';
@@ -33,6 +33,51 @@ const DropdownWrapper: IStyledComponent<'web', DropdownWrapperProps> = styled.di
width: ${props => props.width}px;
`;
+const DropdownFrame = styled.div<{$preparing?: boolean}>`
+ position: relative;
+
+ ${props =>
+ props.$preparing
+ ? `
+ .typeahead {
+ pointer-events: none;
+ }
+ `
+ : ''}
+`;
+
+const PrepareOverlay = styled.div.attrs({
+ className: 'item-selector__prepare'
+})`
+ position: absolute;
+ inset: 0;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ background: rgba(0, 0, 0, 0.45);
+`;
+
+const prepareSpin = keyframes`
+ to {
+ transform: rotate(360deg);
+ }
+`;
+
+const PrepareSpinner = styled.span.attrs({
+ className: 'item-selector__prepare-spinner',
+ 'aria-hidden': true
+})`
+ display: block;
+ width: 24px;
+ height: 24px;
+ box-sizing: border-box;
+ border-radius: 50%;
+ border: 2px solid ${props => props.theme.subtextColor};
+ border-top-color: ${props => props.theme.activeColor};
+ will-change: transform;
+ animation: ${prepareSpin} 0.7s linear infinite;
+`;
+
export type ItemSelectorProps<Option> = {
selectedItems?: ReadonlyArray<Option> | string | number | boolean | object | null;
options: ReadonlyArray<Option>;
@@ -55,6 +100,11 @@ export type ItemSelectorProps<Option> = {
onBlur?: () => void;
placeholder?: string;
closeOnSelect?: boolean;
+ /**
+ * Paint a spinner over the open list before `onChange`. Used when the
+ * change parses a large dataset and would freeze the open selector.
+ */
+ deferOnChange?: boolean;
typeaheadPlaceholder?: string;
DropDownWrapperComponent?: ComponentType<any> | null;
DropdownHeaderComponent?: ComponentType<any> | null;
@@ -67,7 +117,10 @@ export type ItemSelectorProps<Option> = {
showDropdownOnMount?: boolean;
};
-class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
+class ItemSelectorUnmemoized extends Component<
+ ItemSelectorProps<any>,
+ {showTypeahead: boolean; dimensions?: any; preparing: boolean}
+> {
static defaultProps = {
multiSelect: true,
placeholder: 'placeholder.enterValue',
@@ -82,11 +135,16 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
state = {
showTypeahead: false,
+ preparing: false,
dimensions: {
width: 200
}
};
+ _unmounted = false;
+ _preparingSelection = false;
+ _prepareFrames: number[] = [];
+
componentDidMount() {
if (this.props.showDropdownOnMount) {
this.setState({showTypeahead: true});
@@ -98,6 +156,8 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
}
componentWillUnmount() {
+ this._unmounted = true;
+ this._prepareFrames.forEach(id => window.cancelAnimationFrame(id));
if (this.root.current instanceof HTMLElement) {
unobserveDimensions(this.root.current);
}
@@ -106,6 +166,9 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
root: RefObject<HTMLDivElement | null> = createRef();
handleClickOutside = () => {
+ if (this._preparingSelection) {
+ return;
+ }
this._hideTypeahead();
};
@@ -151,23 +214,45 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
};
_selectItem = item => {
+ if (this._preparingSelection) {
+ return;
+ }
const getValue = Accessor.generateOptionToStringFor(
this.props.getOptionValue || this.props.displayOption
);
const previousSelected = toArray(this.props.selectedItems);
+ const nextValue = this.props.multiSelect
+ ? uniqBy(previousSelected.concat(toArray(item)), getValue)
+ : getValue(item);
+
+ const apply = () => {
+ this.props.onChange(nextValue);
+ if (this._unmounted) {
+ return;
+ }
+ this._preparingSelection = false;
+ if (this.props.closeOnSelect) {
+ this.setState({showTypeahead: false, preparing: false});
+ this._onBlur();
+ } else {
+ this.setState({preparing: false});
+ }
+ };
- if (this.props.multiSelect) {
- const items = uniqBy(previousSelected.concat(toArray(item)), getValue);
- this.props.onChange(items);
- } else {
- this.props.onChange(getValue(item));
+ if (this.props.deferOnChange) {
+ // Keep the open list on screen, faded, until the dataset parse starts.
+ this._preparingSelection = true;
+ this.setState({preparing: true});
+ const first = window.requestAnimationFrame(() => {
+ const second = window.requestAnimationFrame(apply);
+ this._prepareFrames.push(second);
+ });
+ this._prepareFrames.push(first);
+ return;
}
- if (this.props.closeOnSelect) {
- this.setState({showTypeahead: false});
- this._onBlur();
- }
+ apply();
};
_onErase: MouseEventHandler = e => {
@@ -194,35 +279,44 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {
const DropDownWrapperComponent = this.props
.DropDownWrapperComponent as React.ComponentType<any>;
+ const {preparing} = this.state;
+
return (
<Portaled left={0} top={0} isOpened={this.state.showTypeahead} onClose={this._hideTypeahead}>
<DropDownWrapperComponent placement={placement} width={dimensions?.width}>
- <Typeahead
- customClasses={{
- results: 'list-selector',
- input: 'typeahead__input',
- listItem: 'list__item',
- listAnchor: 'list__item__anchor'
- }}
- options={this.props.options}
- filterOption={this.props.filterOption}
- fixedOptions={this.props.fixedOptions}
- placeholder={
- this.props.typeaheadPlaceholder || intl
- ? intl.formatMessage({id: 'placeholder.search'})
- : 'Search'
- }
- onOptionSelected={this._selectItem}
- customListComponent={this.props.DropDownRenderComponent}
- customListHeaderComponent={this.props.DropdownHeaderComponent}
- customListItemComponent={this.props.DropDownLineItemRenderComponent}
- displayOption={Accessor.generateOptionToStringFor(this.props.displayOption)}
- searchable={this.props.searchable}
- searchOptions={this.props.searchOptions}
- showOptionsWhenEmpty
- selectedItems={toArray(this.props.selectedItems)}
- light={this.props.inputTheme === 'light'}
- />
+ <DropdownFrame $preparing={preparing}>
+ <Typeahead
+ customClasses={{
+ results: 'list-selector',
+ input: 'typeahead__input',
+ listItem: 'list__item',
+ listAnchor: 'list__item__anchor'
+ }}
+ options={this.props.options}
+ filterOption={this.props.filterOption}
+ fixedOptions={this.props.fixedOptions}
+ placeholder={
+ this.props.typeaheadPlaceholder || intl
+ ? intl.formatMessage({id: 'placeholder.search'})
+ : 'Search'
+ }
+ onOptionSelected={this._selectItem}
+ customListComponent={this.props.DropDownRenderComponent}
+ customListHeaderComponent={this.props.DropdownHeaderComponent}
+ customListItemComponent={this.props.DropDownLineItemRenderComponent}
+ displayOption={Accessor.generateOptionToStringFor(this.props.displayOption)}
+ searchable={this.props.searchable}
+ searchOptions={this.props.searchOptions}
+ showOptionsWhenEmpty
+ selectedItems={toArray(this.props.selectedItems)}
+ light={this.props.inputTheme === 'light'}
+ />
+ {preparing ? (
+ <PrepareOverlay>
+ <PrepareSpinner />
+ </PrepareOverlay>
+ ) : null}
+ </DropdownFrame>
</DropDownWrapperComponent>
</Portaled>
);
diff --git a/src/components/src/loading-indicator.tsx b/src/components/src/loading-indicator.tsx
index b73a8404f..5edca7b50 100644
--- a/src/components/src/loading-indicator.tsx
+++ b/src/components/src/loading-indicator.tsx
@@ -86,6 +86,8 @@ type LoadingIndicatorProps = {
percent?: number;
/** Number of remote files currently being fetched from a saved config. */
remoteDatasetCount?: number;
+ /** Remote files whose download finished and that are now being parsed. */
+ processingDatasetCount?: number;
};
/** Extra adjustment for the loading indicator when side panel is visible */
@@ -116,6 +118,7 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({
hasMapScale,
percent,
remoteDatasetCount = 0,
+ processingDatasetCount = 0,
theme
}) => {
const left =
@@ -127,12 +130,21 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({
const numRasterTilesInProgress = getNumRasterTilesBeingLoaded();
const numVectorTilesInProgress = getNumVectorTilesBeingLoaded();
- const remoteMessage =
- remoteDatasetCount > 0
- ? remoteDatasetCount === 1
- ? 'Remote dataset is being loaded'
- : `${remoteDatasetCount} remote datasets are being loaded`
- : '';
+ const downloadingCount = Math.max(remoteDatasetCount - processingDatasetCount, 0);
+ const remoteMessage = [
+ downloadingCount === 1
+ ? 'Remote dataset is being loaded'
+ : downloadingCount > 1
+ ? `${downloadingCount} remote datasets are being loaded`
+ : '',
+ processingDatasetCount === 1
+ ? 'Processing remote dataset'
+ : processingDatasetCount > 1
+ ? `Processing ${processingDatasetCount} remote datasets`
+ : ''
+ ]
+ .filter(Boolean)
+ .join('\n');
let tileMessage = '';
if (numRasterTilesInProgress > 0 && numVectorTilesInProgress > 0) {
@@ -150,7 +162,8 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({
const extraMessage = [remoteMessage, tileMessage].filter(Boolean).join('\n');
- const hasProgress = typeof percent === 'number' && percent > 0;
+ const processingOnly = processingDatasetCount > 0 && downloadingCount === 0;
+ const hasProgress = !processingOnly && typeof percent === 'number' && percent > 0;
// Preserve the last message / percent during fade-out
const lastMessageRef = useRef(extraMessage);
diff --git a/src/components/src/map-container.tsx b/src/components/src/map-container.tsx
index d37834252..f80a0aaf9 100644
--- a/src/components/src/map-container.tsx
+++ b/src/components/src/map-container.tsx
@@ -1776,8 +1776,15 @@ export default function MapContainerFactory(
sidePanelWidth={sidePanelWidth}
hasAttributionLogos={attributionLogos.length > 0}
hasMapScale={getApplicationConfig().enableMapScale}
- percent={aggregateLoadingPercent(visState.loadingProgress)}
+ percent={aggregateLoadingPercent(
+ Object.fromEntries(
+ Object.entries(visState.loadingProgress || {}).filter(
+ ([id]) => !visState.loadingProcessing?.[id]
+ )
+ )
+ )}
remoteDatasetCount={Object.keys(visState.loadingProgress || {}).length}
+ processingDatasetCount={Object.keys(visState.loadingProcessing || {}).length}
/>
) : null}
{this.props.primary ? (
diff --git a/src/components/src/modals/load-data-modal.tsx b/src/components/src/modals/load-data-modal.tsx
index 90115dab3..c762cc2d5 100644
--- a/src/components/src/modals/load-data-modal.tsx
+++ b/src/components/src/modals/load-data-modal.tsx
@@ -1,8 +1,8 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {useCallback, useState} from 'react';
-import styled from 'styled-components';
+import React, {useCallback, useEffect, useRef, useState} from 'react';
+import styled, {keyframes} from 'styled-components';
import get from 'es-toolkit/compat/get';
import {IntlShape, useIntl} from 'react-intl';
@@ -85,6 +85,53 @@ const AddDataActions = styled.div`
flex-shrink: 0;
`;
+const Dimmed = styled.div<{$dimmed?: boolean}>`
+ opacity: ${props => (props.$dimmed ? 0.4 : 1)};
+ pointer-events: ${props => (props.$dimmed ? 'none' : 'auto')};
+`;
+
+const ModalMain = styled(Dimmed)`
+ display: flex;
+ flex-direction: column;
+ flex: 1;
+ min-height: 0;
+`;
+
+const spin = keyframes`
+ to {
+ transform: rotate(360deg);
+ }
+`;
+
+const ProcessingSpinner = styled.span.attrs({
+ className: 'add-data-bar__spinner',
+ 'aria-hidden': true
+})`
+ display: block;
+ flex-shrink: 0;
+ width: 16px;
+ height: 16px;
+ box-sizing: border-box;
+ border-radius: 50%;
+ border: 2px solid ${props => props.theme.borderColorLT};
+ border-top-color: ${props => props.theme.primaryBtnBgd};
+ will-change: transform;
+ animation: ${spin} 0.7s linear infinite;
+`;
+
+const ProcessingStatus = styled.div.attrs({
+ className: 'add-data-bar__processing',
+ role: 'status'
+})`
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ margin-left: auto;
+ flex-shrink: 0;
+ width: 32px;
+ height: 32px;
+`;
+
const noop = () => {
return;
};
@@ -250,6 +297,14 @@ export function LoadDataModalFactory(
? loadingMethods.filter(method => method.id === LOADING_METHODS.upload)
: loadingMethods;
const [autoCreateLayers, setAutoCreateLayers] = useState(true);
+ const [preparingLayers, setPreparingLayers] = useState(false);
+ const prepareFrames = useRef<number[]>([]);
+ useEffect(() => {
+ const frames = prepareFrames.current;
+ return () => {
+ frames.forEach(id => window.cancelAnimationFrame(id));
+ };
+ }, []);
const [deleteOriginalDataset, setDeleteOriginalDataset] = useState(true);
const [deselectedDatasets, setDeselectedDatasets] = useState<Record<string, boolean>>({});
const onToggleDataset = useCallback((key: string) => {
@@ -290,28 +345,45 @@ export function LoadDataModalFactory(
? Boolean(tilesetDraft?.canAdd) && !tilesetDraft?.loading
: Boolean(datasetsToAdd.length) && !fileLoading;
const onAddData = useCallback(() => {
- if (isTileset) {
- if (tilesetDraft?.canAdd && tilesetDraft.dataset) {
- handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);
- }
+ if (!canAdd || preparingLayers) {
return;
}
- if (datasetsToAdd.length && !fileLoading) {
+ const commit = () => {
+ if (isTileset) {
+ if (tilesetDraft?.canAdd && tilesetDraft.dataset) {
+ handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);
+ }
+ return;
+ }
onConfirmAddData(
isReplace
? {autoCreateLayers: false, datasets: datasetsToAdd, deleteOriginalDataset}
: {autoCreateLayers, datasets: datasetsToAdd}
);
+ };
+ // Layer creation blocks the main thread. Paint this status first so the
+ // spinner can keep moving on the compositor during that work.
+ // Replace never creates layers, so it commits immediately.
+ if (autoCreateLayers && !isReplace) {
+ setPreparingLayers(true);
+ const first = window.requestAnimationFrame(() => {
+ const second = window.requestAnimationFrame(commit);
+ prepareFrames.current.push(second);
+ });
+ prepareFrames.current.push(first);
+ return;
}
+ commit();
}, [
autoCreateLayers,
+ canAdd,
datasetsToAdd,
deleteOriginalDataset,
- fileLoading,
handleTilesetAdded,
isReplace,
isTileset,
onConfirmAddData,
+ preparingLayers,
tilesetDraft
]);
const uploadError = !fileLoading && isUpload ? progressErrors(fileLoadingProgress)[0] : '';
@@ -339,66 +411,83 @@ export function LoadDataModalFactory(
return (
<StyledLoadDataModal $withFooter={showAddBar}>
- <ModalTabs
- currentMethod={currentMethod?.id}
- loadingMethods={availableMethods}
- toggleMethod={selectMethod}
- />
- {isCloudMapLoading ? (
- <LoadingDialog size={64} />
- ) : (
- ElementType && <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />
- )}
+ <ModalMain $dimmed={preparingLayers}>
+ <ModalTabs
+ currentMethod={currentMethod?.id}
+ loadingMethods={availableMethods}
+ toggleMethod={selectMethod}
+ />
+ {isCloudMapLoading ? (
+ <LoadingDialog size={64} />
+ ) : (
+ ElementType && (
+ <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />
+ )
+ )}
+ </ModalMain>
{showAddBar ? (
<AddDataBar>
- {isReplace ? (
- <RemoveOriginalRow>
- <Checkbox
- id="delete-original-dataset"
- type="checkbox"
- label={intl.formatMessage(
- {id: 'modal.replaceDataset.removeOriginal'},
- {datasetName: replaceDatasetLabel || replaceDatasetId || ''}
- )}
- checked={deleteOriginalDataset}
- onChange={() => setDeleteOriginalDataset(value => !value)}
- />
- <TippyTooltip
- placement="top"
- isLightTheme
- render={() => (
- <div>{intl.formatMessage({id: 'modal.replaceDataset.removeOriginalHint'})}</div>
- )}
- >
- <InfoButton
- aria-label={intl.formatMessage({
- id: 'modal.replaceDataset.removeOriginalHint'
- })}
+ <Dimmed $dimmed={preparingLayers}>
+ {isReplace ? (
+ <RemoveOriginalRow>
+ <Checkbox
+ id="delete-original-dataset"
+ type="checkbox"
+ label={intl.formatMessage(
+ {id: 'modal.replaceDataset.removeOriginal'},
+ {datasetName: replaceDatasetLabel || replaceDatasetId || ''}
+ )}
+ checked={deleteOriginalDataset}
+ onChange={() => setDeleteOriginalDataset(value => !value)}
+ />
+ <TippyTooltip
+ placement="top"
+ isLightTheme
+ render={() => (
+ <div>
+ {intl.formatMessage({id: 'modal.replaceDataset.removeOriginalHint'})}
+ </div>
+ )}
>
- <Docs height="16px" />
- </InfoButton>
- </TippyTooltip>
- </RemoveOriginalRow>
- ) : (
- <AutoCreateLayersCheckbox
- checked={autoCreateLayers}
- onToggle={onToggleAutoCreateLayers}
- />
- )}
+ <InfoButton
+ aria-label={intl.formatMessage({
+ id: 'modal.replaceDataset.removeOriginalHint'
+ })}
+ >
+ <Docs height="16px" />
+ </InfoButton>
+ </TippyTooltip>
+ </RemoveOriginalRow>
+ ) : (
+ <AutoCreateLayersCheckbox
+ checked={autoCreateLayers}
+ disabled={preparingLayers}
+ onToggle={onToggleAutoCreateLayers}
+ />
+ )}
+ </Dimmed>
<StagedLabel />
{uploadError || tilesetError ? (
<FooterError>{uploadError || tilesetError}</FooterError>
) : null}
- <AddDataActions>
- <Button type="button" link onClick={onClose}>
- {intl.formatMessage({id: 'modal.button.defaultCancel'})}
- </Button>
- <Button type="button" cta disabled={!canAdd} onClick={onAddData}>
- {intl.formatMessage({
- id: isReplace ? 'modal.button.replace' : 'layerManager.addData'
- })}
- </Button>
- </AddDataActions>
+ {preparingLayers ? (
+ <ProcessingStatus
+ aria-label={intl.formatMessage({id: 'modal.loadData.processingLayers'})}
+ >
+ <ProcessingSpinner />
+ </ProcessingStatus>
+ ) : (
+ <AddDataActions>
+ <Button type="button" link onClick={onClose}>
+ {intl.formatMessage({id: 'modal.button.defaultCancel'})}
+ </Button>
+ <Button type="button" cta disabled={!canAdd} onClick={onAddData}>
+ {intl.formatMessage({
+ id: isReplace ? 'modal.button.replace' : 'layerManager.addData'
+ })}
+ </Button>
+ </AddDataActions>
+ )}
</AddDataBar>
) : null}
</StyledLoadDataModal>
diff --git a/src/components/src/side-panel/layer-panel/layer-type-selector.tsx b/src/components/src/side-panel/layer-panel/layer-type-selector.tsx
index ce3c4928a..56731234d 100644
--- a/src/components/src/side-panel/layer-panel/layer-type-selector.tsx
+++ b/src/components/src/side-panel/layer-panel/layer-type-selector.tsx
@@ -69,6 +69,7 @@ function LayerTypeSelectorFactory(
multiSelect={false}
disabled={disabled}
placeholder="placeholder.selectType"
+ deferOnChange
onChange={onSelect}
getOptionValue={getOptionValue}
filterOption="label"
diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.ts
index 4b1de6448..2083ad12a 100644
--- a/src/localization/src/translations/en.ts
+++ b/src/localization/src/translations/en.ts
@@ -827,6 +827,7 @@ export default {
autoCreateLayers: 'Auto create layers',
autoCreateLayersInfo:
'kepler.gl will auto detect layers if the column names in your file follow certain naming conventions.',
+ processingLayers: 'Processing data for layers',
readyToAdd: 'Ready to add'
},
tripInfo: {
diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.ts
index 728238376..1b71f9de2 100644
--- a/src/processors/src/remote-file.ts
+++ b/src/processors/src/remote-file.ts
@@ -29,9 +29,17 @@ export type KeplerRemoteFile = File & {
keplerLastModified?: string;
};
+export type RemoteLoadProgress = {
+ loaded: number;
+ total?: number;
+ percent: number;
+ /** Set once the bytes are in memory and parsing is about to block the main thread. */
+ phase?: 'processing';
+};
+
export type FetchRemoteFileOptions = {
format?: string | null;
- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void;
+ onProgress?: (progress: RemoteLoadProgress) => void;
etag?: string;
lastModified?: string;
/** Bypass the HTTP cache. Used for poll/reload, not the first remote URL load. */
@@ -229,7 +237,7 @@ export function remoteDatasetFromUrl(url: string, format?: string | null): FileC
export async function fetchRemoteFileAsKeplerFile(
url: string,
format?: string | null,
- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void
+ onProgress?: (progress: RemoteLoadProgress) => void
): Promise<KeplerRemoteFile> {
const result = await fetchRemoteFile(url, {format, onProgress});
if (!result.file) {
@@ -238,9 +246,48 @@ export async function fetchRemoteFileAsKeplerFile(
return result.file;
}
+/** Two frames are enough for the processing indicator to paint. */
+const PAINT_WAIT_MS = 100;
+
+function waitForNextPaint(): Promise<void> {
+ // Hidden tabs suspend requestAnimationFrame. A download that finishes in the
+ // background would otherwise wait until the user comes back, and there is
+ // nothing to paint there.
+ if (
+ typeof requestAnimationFrame !== 'function' ||
+ (typeof document !== 'undefined' && document.hidden)
+ ) {
+ return Promise.resolve();
+ }
+ return new Promise(resolve => {
+ let settled = false;
+ const frames: number[] = [];
+ const finish = () => {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ clearTimeout(timer);
+ if (typeof cancelAnimationFrame === 'function') {
+ frames.forEach(id => cancelAnimationFrame(id));
+ }
+ resolve();
+ };
+ // The tab can be hidden after this wait starts. Don't sit on frames that will never run.
+ const timer = setTimeout(finish, PAINT_WAIT_MS);
+ frames.push(
+ requestAnimationFrame(() => {
+ if (!settled) {
+ frames.push(requestAnimationFrame(finish));
+ }
+ })
+ );
+ });
+}
+
async function readResponseBlob(
response: Response,
- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void
+ onProgress?: (progress: RemoteLoadProgress) => void
): Promise<Blob> {
const contentLength = Number(response.headers.get('content-length'));
const total = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : undefined;
@@ -315,7 +362,7 @@ export async function loadExternallyHostedDataset(metadata: {
etag?: string;
lastModified?: string;
bypassCache?: boolean;
- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void;
+ onProgress?: (progress: RemoteLoadProgress) => void;
}): Promise<LoadExternallyHostedDatasetResult> {
const {source, format, etag, lastModified, size, bypassCache, onProgress} = metadata;
const fetched = await fetchRemoteFile(source, {
@@ -335,6 +382,15 @@ export async function loadExternallyHostedDataset(metadata: {
};
}
+ onProgress?.({
+ loaded: fetched.size ?? fetched.file.size,
+ ...(typeof fetched.size === 'number' ? {total: fetched.size} : {}),
+ percent: 1,
+ phase: 'processing'
+ });
+ // Let the loading indicator paint a processing state before parse blocks the tab.
+ await waitForNextPaint();
+
const batches = await readFileInBatches({
file: fetched.file,
fileCache: [],
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 0c9d7b500..41102d2f5 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -442,6 +442,7 @@ export const INITIAL_VIS_STATE: VisState = {
// for loading datasets
loadingIndicatorValue: 0,
loadingProgress: {},
+ loadingProcessing: {},
loaders: [],
loadOptions: {},
@@ -3364,7 +3365,11 @@ function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {
return HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK({
arg: dataset,
onProgress: progress =>
- setLoadingProgress(progressId, Math.round((progress?.percent ?? 0) * 100))
+ setLoadingProgress(
+ progressId,
+ Math.round((progress?.percent ?? 0) * 100),
+ progress?.phase === 'processing' ? 'processing' : undefined
+ )
}).chain(hydrated => {
const task = createNewDataEntry(hydrated, datasets);
return (
@@ -3376,20 +3381,23 @@ function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {
function clearLoadingProgress(state: VisState, ids?: string[]): VisState {
const current = state.loadingProgress || {};
- if (!Object.keys(current).length) {
+ const processing = state.loadingProcessing || {};
+ if (!Object.keys(current).length && !Object.keys(processing).length) {
return state;
}
if ((state.loadingIndicatorValue || 0) <= 0) {
- return {...state, loadingProgress: {}};
+ return {...state, loadingProgress: {}, loadingProcessing: {}};
}
if (!ids?.length) {
return state;
}
const next = {...current};
+ const nextProcessing = {...processing};
ids.forEach(id => {
delete next[id];
+ delete nextProcessing[id];
});
- return {...state, loadingProgress: next};
+ return {...state, loadingProgress: next, loadingProcessing: nextProcessing};
}
function patchDatasetMetadata(dataset: Datasets[string], patch: Record<string, unknown>) {
@@ -6114,8 +6122,10 @@ export const setLoadingIndicatorUpdater = (
return {
...state,
loadingIndicatorValue: nextValue,
- ...(nextValue === 0 && Object.keys(state.loadingProgress || {}).length
- ? {loadingProgress: {}}
+ ...(nextValue === 0 &&
+ (Object.keys(state.loadingProgress || {}).length ||
+ Object.keys(state.loadingProcessing || {}).length)
+ ? {loadingProgress: {}, loadingProcessing: {}}
: {})
};
};
@@ -6127,7 +6137,7 @@ export const setLoadingIndicatorUpdater = (
*/
export function setLoadingProgressUpdater(
state: VisState,
- {id, percent}: VisStateActions.SetLoadingProgressUpdaterAction
+ {id, percent, phase}: VisStateActions.SetLoadingProgressUpdaterAction
): VisState {
if ((state.loadingIndicatorValue || 0) <= 0) {
return state;
@@ -6135,7 +6145,11 @@ export function setLoadingProgressUpdater(
const nextPercent = Number.isFinite(percent)
? Math.max(0, Math.min(100, Math.round(percent)))
: 0;
- if (state.loadingProgress?.[id] === nextPercent) {
+ const processing = phase === 'processing';
+ if (
+ state.loadingProgress?.[id] === nextPercent &&
+ Boolean(state.loadingProcessing?.[id]) === processing
+ ) {
return state;
}
return {
@@ -6143,7 +6157,15 @@ export function setLoadingProgressUpdater(
loadingProgress: {
...state.loadingProgress,
[id]: nextPercent
- }
+ },
+ ...(processing
+ ? {
+ loadingProcessing: {
+ ...state.loadingProcessing,
+ [id]: true
+ }
+ }
+ : {})
};
}
diff --git a/src/schemas/src/vis-state-schema.ts b/src/schemas/src/vis-state-schema.ts
index 748859767..786cc0529 100644
--- a/src/schemas/src/vis-state-schema.ts
+++ b/src/schemas/src/vis-state-schema.ts
@@ -99,6 +99,8 @@ export interface VisState {
loadingIndicatorValue: number;
/** Per-dataset download progress (0–100) while hydrating remote files. */
loadingProgress: Record<string, number>;
+ /** Remote datasets whose download finished and whose parse is still running. */
+ loadingProcessing?: Record<string, boolean>;
loaders: Loader[];
loadOptions: object;
initialState?: Partial<VisState>;
diff --git a/test/browser/components/common/file-uploader-test.js b/test/browser/components/common/file-uploader-test.js
index ec3c41848..81d49cd47 100644
--- a/test/browser/components/common/file-uploader-test.js
+++ b/test/browser/components/common/file-uploader-test.js
@@ -102,6 +102,15 @@ test('Components -> FileUpload.onDrop -> render loading msg', t => {
const uploadMsg = wrapper.find('.file-upload-progress__message').at(0).html();
t.comment(uploadMsg);
t.ok(uploadMsg.includes('tst-file.csv'), 'should render upload file msg');
+ t.ok(
+ uploadMsg.includes('upload-file-list__spinner'),
+ 'should show a spinner in place of the checkbox while the file is still loading'
+ );
+ t.equal(
+ wrapper.find('.upload-file-list').find('Checkbox').length,
+ 0,
+ 'should hide the include checkbox until parsing finishes'
+ );
t.equal(
wrapper.find('.upload-file-list').hostNodes().length,
1,
diff --git a/test/browser/components/common/item-selector-test.js b/test/browser/components/common/item-selector-test.js
index d5e1ba9d7..d46fecf12 100644
--- a/test/browser/components/common/item-selector-test.js
+++ b/test/browser/components/common/item-selector-test.js
@@ -54,6 +54,60 @@ test('Components -> ItemSelector.render', t => {
t.end();
});
+test('Components -> ItemSelector.deferOnChange', t => {
+ const onChange = sinon.spy();
+ const wrapper = mountWithTheme(
+ <IntlWrapper>
+ <ItemSelector
+ selectedItems="normal"
+ options={['additive', 'normal', 'subtractive']}
+ multiSelect={false}
+ searchable={false}
+ deferOnChange
+ onChange={onChange}
+ />
+ </IntlWrapper>
+ );
+
+ const frames = [];
+ const prevFrame = window.requestAnimationFrame;
+ window.requestAnimationFrame = cb => {
+ frames.push(cb);
+ return frames.length;
+ };
+ try {
+ wrapper.find('.item-selector__dropdown').at(0).simulate('click');
+ wrapper.find(DropdownList).at(0).find('.list__item').at(0).simulate('click');
+
+ t.equal(onChange.called, false, 'should wait to apply the selection until the list can paint');
+ t.equal(wrapper.find(Typeahead).length, 1, 'should keep the list open');
+ t.equal(
+ wrapper.find('.item-selector__prepare-spinner').hostNodes().length,
+ 1,
+ 'should show a spinner over the open list'
+ );
+
+ act(() => {
+ while (frames.length) {
+ frames.shift()();
+ }
+ });
+ } finally {
+ window.requestAnimationFrame = prevFrame;
+ }
+ wrapper.update();
+
+ t.deepEqual(onChange.args[0], ['additive'], 'should apply the selected value');
+ t.equal(wrapper.find(Typeahead).length, 0, 'should close the list');
+ t.equal(
+ wrapper.find('.item-selector__prepare-spinner').hostNodes().length,
+ 0,
+ 'should remove the spinner'
+ );
+
+ t.end();
+});
+
test('Components -> ItemSelector.render over 100 options', t => {
let wrapper;
const onChange = sinon.spy();
diff --git a/test/browser/components/modals/load-data-modal-test.js b/test/browser/components/modals/load-data-modal-test.js
index bc83091ed..6f8bafe87 100644
--- a/test/browser/components/modals/load-data-modal-test.js
+++ b/test/browser/components/modals/load-data-modal-test.js
@@ -156,6 +156,61 @@ test('Components -> LoadDataModal -> auto create layers', t => {
t.end();
});
+test('Components -> LoadDataModal -> processing layers', t => {
+ const onConfirmAddData = sinon.spy();
+ const wrapper = mountWithTheme(
+ <IntlWrapper>
+ <LoadDataModal
+ onConfirmAddData={onConfirmAddData}
+ stagedToAdd={[{info: {label: 'points.csv'}}]}
+ />
+ </IntlWrapper>
+ );
+
+ const frames = [];
+ const prevFrame = window.requestAnimationFrame;
+ window.requestAnimationFrame = cb => {
+ frames.push(cb);
+ return frames.length;
+ };
+ try {
+ wrapper.find('.add-data-bar button').at(1).simulate('click');
+
+ t.equal(
+ onConfirmAddData.called,
+ false,
+ 'layer prep waits until the processing state can paint'
+ );
+ const processing = wrapper.find('.add-data-bar__processing').hostNodes();
+ t.equal(processing.length, 1, 'should replace the footer actions');
+ t.equal(processing.text(), '', 'should show only the spinner');
+ t.equal(
+ processing.find('.add-data-bar__spinner').hostNodes().length,
+ 1,
+ 'should show a spinner in place of the footer actions'
+ );
+ t.equal(
+ wrapper.find('.add-data-bar button').hostNodes().length,
+ 0,
+ 'should hide Cancel and Add Data while layers are prepared'
+ );
+
+ while (frames.length) {
+ frames.shift()();
+ }
+ } finally {
+ window.requestAnimationFrame = prevFrame;
+ }
+
+ t.deepEqual(
+ onConfirmAddData.args[0][0],
+ {autoCreateLayers: true, datasets: [{info: {label: 'points.csv'}}]},
+ 'Add Data commits the staged dataset after the processing state can paint'
+ );
+
+ t.end();
+});
+
test('Components -> LoadDataModal -> replace dataset', t => {
const onConfirmAddData = sinon.spy();
const wrapper = mountWithTheme(
@@ -238,7 +293,20 @@ test('Components -> LoadDataModal -> one unchecked dataset stays out', t => {
);
checks().at(1).simulate('change');
- wrapper.find('.add-data-bar button').at(1).simulate('click');
+ const frames = [];
+ const prevFrame = window.requestAnimationFrame;
+ window.requestAnimationFrame = cb => {
+ frames.push(cb);
+ return frames.length;
+ };
+ try {
+ wrapper.find('.add-data-bar button').at(1).simulate('click');
+ while (frames.length) {
+ frames.shift()();
+ }
+ } finally {
+ window.requestAnimationFrame = prevFrame;
+ }
t.deepEqual(
onConfirmAddData.args[0][0],
{autoCreateLayers: true, datasets: [points]},
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index a22e17560..01e13703f 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -8829,6 +8829,13 @@ test('VisStateUpdater -> hydrate remote dataset loading progress', t => {
const progressed = reducer(loadingState, VisStateActions.setLoadingProgress('remote-1', 42));
t.equal(progressed.loadingProgress['remote-1'], 42, 'should store download percent');
+ const processing = reducer(
+ progressed,
+ VisStateActions.setLoadingProgress('remote-1', 100, 'processing')
+ );
+ t.equal(processing.loadingProcessing['remote-1'], true, 'should mark the dataset as processing');
+ t.equal(processing.loadingIndicatorValue, 1, 'should keep the indicator up while processing');
+
const same = reducer(progressed, VisStateActions.setLoadingProgress('remote-1', 42));
t.equal(same, progressed, 'should skip redundant progress updates');
@@ -8838,9 +8845,10 @@ test('VisStateUpdater -> hydrate remote dataset loading progress', t => {
const [task] = drainTasksForTesting();
t.ok(task, 'should schedule a hydrate/create task');
- const cleared = reducer(progressed, VisStateActions.setLoadingIndicator({change: -1}));
+ const cleared = reducer(processing, VisStateActions.setLoadingIndicator({change: -1}));
t.equal(cleared.loadingIndicatorValue, 0, 'should hide the loading indicator');
t.deepEqual(cleared.loadingProgress, {}, 'should clear hydrate progress');
+ t.deepEqual(cleared.loadingProcessing, {}, 'should clear the processing flag');
t.end();
});