Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: show feedback while large dataset processing blocks the UI (#3790)
Igor Dykhta7f26460a7337
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex 499c899db..5a0ccbd3c 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -2648,6 +2648,8 @@ export const setLoadingIndicator = createAction<SetLoadingIndicatorPayload>(export type SetLoadingProgressUpdaterAction = {id: string;percent: number;+ /** Download is finished and the file is being parsed into a dataset. */+ phase?: 'processing';};/**@@ -2658,12 +2660,14 @@ export type SetLoadingProgressUpdaterAction = {*/export function setLoadingProgress(id: string,- percent: number+ percent: number,+ phase?: 'processing'): Merge<SetLoadingProgressUpdaterAction, {type: typeof ActionTypes.SET_LOADING_PROGRESS}> {return {type: ActionTypes.SET_LOADING_PROGRESS,id,- percent+ percent,+ ...(phase ? {phase} : {})};}diff --git a/src/components/src/common/file-uploader/upload-file-list.tsx b/src/components/src/common/file-uploader/upload-file-list.tsxindex 03e4ee845..ea0c699b0 100644--- a/src/components/src/common/file-uploader/upload-file-list.tsx+++ b/src/components/src/common/file-uploader/upload-file-list.tsx@@ -2,7 +2,7 @@// Copyright contributors to the kepler.gl projectimport React, {FC} from 'react';-import styled from 'styled-components';+import styled, {keyframes} from 'styled-components';import {media} from '@kepler.gl/styles';import Checkbox from '../checkbox';@@ -99,7 +99,8 @@ const CardBody = styled.div`display: flex;align-items: center;gap: 8px;- padding: 8px 12px 8px 8px;+ /* Icon is 36px. A centered 16px control then sits 18px from the top and bottom. */+ padding: 8px 18px 8px 8px;`;const IconWrap = styled.div`@@ -122,11 +123,33 @@ const CardName = styled.div`const CardCheck = styled.div`flex-shrink: 0;+ display: flex;+ align-items: center;+ justify-content: center;+ width: ${props => props.theme.checkboxWidth}px;+ height: ${props => props.theme.checkboxHeight}px;++ .kg-checkbox {+ margin: 0;+ min-height: 0;+ width: ${props => props.theme.checkboxWidth}px;+ height: ${props => props.theme.checkboxHeight}px;+ }- .kg-checkbox,.kg-checkbox__label {- margin-left: 0;- margin-bottom: 0;+ display: block;+ box-sizing: border-box;+ width: ${props => props.theme.checkboxWidth}px;+ height: ${props => props.theme.checkboxHeight}px;+ margin: 0;+ padding: 0 0 0 ${props => props.theme.checkboxWidth}px;+ line-height: ${props => props.theme.checkboxHeight}px;+ }++ .kg-checkbox__label::before {+ box-sizing: border-box;+ width: ${props => props.theme.checkboxWidth}px;+ height: ${props => props.theme.checkboxHeight}px;}`;@@ -155,6 +178,29 @@ const CardStatus = styled.div<{$isError?: boolean}>`color: ${props => (props.$isError ? props.theme.errorColor : props.theme.subtextColorLT)};`;+// Transform-only so the compositor can keep it moving while parsing blocks the main thread.+const spin = keyframes`+ to {+ transform: rotate(360deg);+ }+`;++const StatusSpinner = styled.span.attrs({+ className: 'upload-file-list__spinner',+ 'aria-hidden': true+})`+ display: block;+ flex-shrink: 0;+ width: ${props => props.theme.checkboxWidth}px;+ height: ${props => props.theme.checkboxHeight}px;+ box-sizing: border-box;+ border-radius: 50%;+ border: 2px solid ${props => props.theme.borderColorLT};+ border-top-color: ${props => props.theme.primaryBtnBgd};+ will-change: transform;+ animation: ${spin} 0.7s linear infinite;+`;+const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {if (!items.length) {return null;@@ -178,7 +224,11 @@ const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {{item.status}</CardStatus></CardText>- {item.selectable ? (+ {!item.isSuccess && !item.isError ? (+ <CardCheck>+ <StatusSpinner />+ </CardCheck>+ ) : item.selectable ? (<CardCheck title={item.selectionLabel} onClick={event => event.stopPropagation()}><Checkboxid={`upload-file-${index}`}diff --git a/src/components/src/common/item-selector/item-selector.tsx b/src/components/src/common/item-selector/item-selector.tsxindex 9af5eeae7..0e77d72b2 100644--- a/src/components/src/common/item-selector/item-selector.tsx+++ b/src/components/src/common/item-selector/item-selector.tsx@@ -4,7 +4,7 @@import React, {Component, createRef, ComponentType, MouseEventHandler, RefObject} from 'react';import classnames from 'classnames';import uniqBy from 'es-toolkit/compat/uniqBy';-import styled, {IStyledComponent} from 'styled-components';+import styled, {IStyledComponent, keyframes} from 'styled-components';import Accessor from './accessor';import ChickletedInput from './chickleted-input';@@ -33,6 +33,51 @@ const DropdownWrapper: IStyledComponent<'web', DropdownWrapperProps> = styled.diwidth: ${props => props.width}px;`;+const DropdownFrame = styled.div<{$preparing?: boolean}>`+ position: relative;++ ${props =>+ props.$preparing+ ? `+ .typeahead {+ pointer-events: none;+ }+ `+ : ''}+`;++const PrepareOverlay = styled.div.attrs({+ className: 'item-selector__prepare'+})`+ position: absolute;+ inset: 0;+ display: flex;+ align-items: center;+ justify-content: center;+ background: rgba(0, 0, 0, 0.45);+`;++const prepareSpin = keyframes`+ to {+ transform: rotate(360deg);+ }+`;++const PrepareSpinner = styled.span.attrs({+ className: 'item-selector__prepare-spinner',+ 'aria-hidden': true+})`+ display: block;+ width: 24px;+ height: 24px;+ box-sizing: border-box;+ border-radius: 50%;+ border: 2px solid ${props => props.theme.subtextColor};+ border-top-color: ${props => props.theme.activeColor};+ will-change: transform;+ animation: ${prepareSpin} 0.7s linear infinite;+`;+export type ItemSelectorProps<Option> = {selectedItems?: ReadonlyArray<Option> | string | number | boolean | object | null;options: ReadonlyArray<Option>;@@ -55,6 +100,11 @@ export type ItemSelectorProps<Option> = {onBlur?: () => void;placeholder?: string;closeOnSelect?: boolean;+ /**+ * Paint a spinner over the open list before `onChange`. Used when the+ * change parses a large dataset and would freeze the open selector.+ */+ deferOnChange?: boolean;typeaheadPlaceholder?: string;DropDownWrapperComponent?: ComponentType<any> | null;DropdownHeaderComponent?: ComponentType<any> | null;@@ -67,7 +117,10 @@ export type ItemSelectorProps<Option> = {showDropdownOnMount?: boolean;};-class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {+class ItemSelectorUnmemoized extends Component<+ ItemSelectorProps<any>,+ {showTypeahead: boolean; dimensions?: any; preparing: boolean}+> {static defaultProps = {multiSelect: true,placeholder: 'placeholder.enterValue',@@ -82,11 +135,16 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {state = {showTypeahead: false,+ preparing: false,dimensions: {width: 200}};+ _unmounted = false;+ _preparingSelection = false;+ _prepareFrames: number[] = [];+componentDidMount() {if (this.props.showDropdownOnMount) {this.setState({showTypeahead: true});@@ -98,6 +156,8 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {}componentWillUnmount() {+ this._unmounted = true;+ this._prepareFrames.forEach(id => window.cancelAnimationFrame(id));if (this.root.current instanceof HTMLElement) {unobserveDimensions(this.root.current);}@@ -106,6 +166,9 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {root: RefObject<HTMLDivElement | null> = createRef();handleClickOutside = () => {+ if (this._preparingSelection) {+ return;+ }this._hideTypeahead();};@@ -151,23 +214,45 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {};_selectItem = item => {+ if (this._preparingSelection) {+ return;+ }const getValue = Accessor.generateOptionToStringFor(this.props.getOptionValue || this.props.displayOption);const previousSelected = toArray(this.props.selectedItems);+ const nextValue = this.props.multiSelect+ ? uniqBy(previousSelected.concat(toArray(item)), getValue)+ : getValue(item);++ const apply = () => {+ this.props.onChange(nextValue);+ if (this._unmounted) {+ return;+ }+ this._preparingSelection = false;+ if (this.props.closeOnSelect) {+ this.setState({showTypeahead: false, preparing: false});+ this._onBlur();+ } else {+ this.setState({preparing: false});+ }+ };- if (this.props.multiSelect) {- const items = uniqBy(previousSelected.concat(toArray(item)), getValue);- this.props.onChange(items);- } else {- this.props.onChange(getValue(item));+ if (this.props.deferOnChange) {+ // Keep the open list on screen, faded, until the dataset parse starts.+ this._preparingSelection = true;+ this.setState({preparing: true});+ const first = window.requestAnimationFrame(() => {+ const second = window.requestAnimationFrame(apply);+ this._prepareFrames.push(second);+ });+ this._prepareFrames.push(first);+ return;}- if (this.props.closeOnSelect) {- this.setState({showTypeahead: false});- this._onBlur();- }+ apply();};_onErase: MouseEventHandler = e => {@@ -194,35 +279,44 @@ class ItemSelectorUnmemoized extends Component<ItemSelectorProps<any>> {const DropDownWrapperComponent = this.props.DropDownWrapperComponent as React.ComponentType<any>;+ const {preparing} = this.state;+return (<Portaled left={0} top={0} isOpened={this.state.showTypeahead} onClose={this._hideTypeahead}><DropDownWrapperComponent placement={placement} width={dimensions?.width}>- <Typeahead- customClasses={{- results: 'list-selector',- input: 'typeahead__input',- listItem: 'list__item',- listAnchor: 'list__item__anchor'- }}- options={this.props.options}- filterOption={this.props.filterOption}- fixedOptions={this.props.fixedOptions}- placeholder={- this.props.typeaheadPlaceholder || intl- ? intl.formatMessage({id: 'placeholder.search'})- : 'Search'- }- onOptionSelected={this._selectItem}- customListComponent={this.props.DropDownRenderComponent}- customListHeaderComponent={this.props.DropdownHeaderComponent}- customListItemComponent={this.props.DropDownLineItemRenderComponent}- displayOption={Accessor.generateOptionToStringFor(this.props.displayOption)}- searchable={this.props.searchable}- searchOptions={this.props.searchOptions}- showOptionsWhenEmpty- selectedItems={toArray(this.props.selectedItems)}- light={this.props.inputTheme === 'light'}- />+ <DropdownFrame $preparing={preparing}>+ <Typeahead+ customClasses={{+ results: 'list-selector',+ input: 'typeahead__input',+ listItem: 'list__item',+ listAnchor: 'list__item__anchor'+ }}+ options={this.props.options}+ filterOption={this.props.filterOption}+ fixedOptions={this.props.fixedOptions}+ placeholder={+ this.props.typeaheadPlaceholder || intl+ ? intl.formatMessage({id: 'placeholder.search'})+ : 'Search'+ }+ onOptionSelected={this._selectItem}+ customListComponent={this.props.DropDownRenderComponent}+ customListHeaderComponent={this.props.DropdownHeaderComponent}+ customListItemComponent={this.props.DropDownLineItemRenderComponent}+ displayOption={Accessor.generateOptionToStringFor(this.props.displayOption)}+ searchable={this.props.searchable}+ searchOptions={this.props.searchOptions}+ showOptionsWhenEmpty+ selectedItems={toArray(this.props.selectedItems)}+ light={this.props.inputTheme === 'light'}+ />+ {preparing ? (+ <PrepareOverlay>+ <PrepareSpinner />+ </PrepareOverlay>+ ) : null}+ </DropdownFrame></DropDownWrapperComponent></Portaled>);diff --git a/src/components/src/loading-indicator.tsx b/src/components/src/loading-indicator.tsxindex b73a8404f..5edca7b50 100644--- a/src/components/src/loading-indicator.tsx+++ b/src/components/src/loading-indicator.tsx@@ -86,6 +86,8 @@ type LoadingIndicatorProps = {percent?: number;/** Number of remote files currently being fetched from a saved config. */remoteDatasetCount?: number;+ /** Remote files whose download finished and that are now being parsed. */+ processingDatasetCount?: number;};/** Extra adjustment for the loading indicator when side panel is visible */@@ -116,6 +118,7 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({hasMapScale,percent,remoteDatasetCount = 0,+ processingDatasetCount = 0,theme}) => {const left =@@ -127,12 +130,21 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({const numRasterTilesInProgress = getNumRasterTilesBeingLoaded();const numVectorTilesInProgress = getNumVectorTilesBeingLoaded();- const remoteMessage =- remoteDatasetCount > 0- ? remoteDatasetCount === 1- ? 'Remote dataset is being loaded'- : `${remoteDatasetCount} remote datasets are being loaded`- : '';+ const downloadingCount = Math.max(remoteDatasetCount - processingDatasetCount, 0);+ const remoteMessage = [+ downloadingCount === 1+ ? 'Remote dataset is being loaded'+ : downloadingCount > 1+ ? `${downloadingCount} remote datasets are being loaded`+ : '',+ processingDatasetCount === 1+ ? 'Processing remote dataset'+ : processingDatasetCount > 1+ ? `Processing ${processingDatasetCount} remote datasets`+ : ''+ ]+ .filter(Boolean)+ .join('\n');let tileMessage = '';if (numRasterTilesInProgress > 0 && numVectorTilesInProgress > 0) {@@ -150,7 +162,8 @@ const LoadingIndicator: React.FC<LoadingIndicatorProps & {theme: any}> = ({const extraMessage = [remoteMessage, tileMessage].filter(Boolean).join('\n');- const hasProgress = typeof percent === 'number' && percent > 0;+ const processingOnly = processingDatasetCount > 0 && downloadingCount === 0;+ const hasProgress = !processingOnly && typeof percent === 'number' && percent > 0;// Preserve the last message / percent during fade-outconst lastMessageRef = useRef(extraMessage);diff --git a/src/components/src/map-container.tsx b/src/components/src/map-container.tsxindex d37834252..f80a0aaf9 100644--- a/src/components/src/map-container.tsx+++ b/src/components/src/map-container.tsx@@ -1776,8 +1776,15 @@ export default function MapContainerFactory(sidePanelWidth={sidePanelWidth}hasAttributionLogos={attributionLogos.length > 0}hasMapScale={getApplicationConfig().enableMapScale}- percent={aggregateLoadingPercent(visState.loadingProgress)}+ percent={aggregateLoadingPercent(+ Object.fromEntries(+ Object.entries(visState.loadingProgress || {}).filter(+ ([id]) => !visState.loadingProcessing?.[id]+ )+ )+ )}remoteDatasetCount={Object.keys(visState.loadingProgress || {}).length}+ processingDatasetCount={Object.keys(visState.loadingProcessing || {}).length}/>) : null}{this.props.primary ? (diff --git a/src/components/src/modals/load-data-modal.tsx b/src/components/src/modals/load-data-modal.tsxindex 90115dab3..c762cc2d5 100644--- a/src/components/src/modals/load-data-modal.tsx+++ b/src/components/src/modals/load-data-modal.tsx@@ -1,8 +1,8 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {useCallback, useState} from 'react';-import styled from 'styled-components';+import React, {useCallback, useEffect, useRef, useState} from 'react';+import styled, {keyframes} from 'styled-components';import get from 'es-toolkit/compat/get';import {IntlShape, useIntl} from 'react-intl';@@ -85,6 +85,53 @@ const AddDataActions = styled.div`flex-shrink: 0;`;+const Dimmed = styled.div<{$dimmed?: boolean}>`+ opacity: ${props => (props.$dimmed ? 0.4 : 1)};+ pointer-events: ${props => (props.$dimmed ? 'none' : 'auto')};+`;++const ModalMain = styled(Dimmed)`+ display: flex;+ flex-direction: column;+ flex: 1;+ min-height: 0;+`;++const spin = keyframes`+ to {+ transform: rotate(360deg);+ }+`;++const ProcessingSpinner = styled.span.attrs({+ className: 'add-data-bar__spinner',+ 'aria-hidden': true+})`+ display: block;+ flex-shrink: 0;+ width: 16px;+ height: 16px;+ box-sizing: border-box;+ border-radius: 50%;+ border: 2px solid ${props => props.theme.borderColorLT};+ border-top-color: ${props => props.theme.primaryBtnBgd};+ will-change: transform;+ animation: ${spin} 0.7s linear infinite;+`;++const ProcessingStatus = styled.div.attrs({+ className: 'add-data-bar__processing',+ role: 'status'+})`+ display: flex;+ align-items: center;+ justify-content: center;+ margin-left: auto;+ flex-shrink: 0;+ width: 32px;+ height: 32px;+`;+const noop = () => {return;};@@ -250,6 +297,14 @@ export function LoadDataModalFactory(? loadingMethods.filter(method => method.id === LOADING_METHODS.upload): loadingMethods;const [autoCreateLayers, setAutoCreateLayers] = useState(true);+ const [preparingLayers, setPreparingLayers] = useState(false);+ const prepareFrames = useRef<number[]>([]);+ useEffect(() => {+ const frames = prepareFrames.current;+ return () => {+ frames.forEach(id => window.cancelAnimationFrame(id));+ };+ }, []);const [deleteOriginalDataset, setDeleteOriginalDataset] = useState(true);const [deselectedDatasets, setDeselectedDatasets] = useState<Record<string, boolean>>({});const onToggleDataset = useCallback((key: string) => {@@ -290,28 +345,45 @@ export function LoadDataModalFactory(? Boolean(tilesetDraft?.canAdd) && !tilesetDraft?.loading: Boolean(datasetsToAdd.length) && !fileLoading;const onAddData = useCallback(() => {- if (isTileset) {- if (tilesetDraft?.canAdd && tilesetDraft.dataset) {- handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);- }+ if (!canAdd || preparingLayers) {return;}- if (datasetsToAdd.length && !fileLoading) {+ const commit = () => {+ if (isTileset) {+ if (tilesetDraft?.canAdd && tilesetDraft.dataset) {+ handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);+ }+ return;+ }onConfirmAddData(isReplace? {autoCreateLayers: false, datasets: datasetsToAdd, deleteOriginalDataset}: {autoCreateLayers, datasets: datasetsToAdd});+ };+ // Layer creation blocks the main thread. Paint this status first so the+ // spinner can keep moving on the compositor during that work.+ // Replace never creates layers, so it commits immediately.+ if (autoCreateLayers && !isReplace) {+ setPreparingLayers(true);+ const first = window.requestAnimationFrame(() => {+ const second = window.requestAnimationFrame(commit);+ prepareFrames.current.push(second);+ });+ prepareFrames.current.push(first);+ return;}+ commit();}, [autoCreateLayers,+ canAdd,datasetsToAdd,deleteOriginalDataset,- fileLoading,handleTilesetAdded,isReplace,isTileset,onConfirmAddData,+ preparingLayers,tilesetDraft]);const uploadError = !fileLoading && isUpload ? progressErrors(fileLoadingProgress)[0] : '';@@ -339,66 +411,83 @@ export function LoadDataModalFactory(return (<StyledLoadDataModal $withFooter={showAddBar}>- <ModalTabs- currentMethod={currentMethod?.id}- loadingMethods={availableMethods}- toggleMethod={selectMethod}- />- {isCloudMapLoading ? (- <LoadingDialog size={64} />- ) : (- ElementType && <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />- )}+ <ModalMain $dimmed={preparingLayers}>+ <ModalTabs+ currentMethod={currentMethod?.id}+ loadingMethods={availableMethods}+ toggleMethod={selectMethod}+ />+ {isCloudMapLoading ? (+ <LoadingDialog size={64} />+ ) : (+ ElementType && (+ <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />+ )+ )}+ </ModalMain>{showAddBar ? (<AddDataBar>- {isReplace ? (- <RemoveOriginalRow>- <Checkbox- id="delete-original-dataset"- type="checkbox"- label={intl.formatMessage(- {id: 'modal.replaceDataset.removeOriginal'},- {datasetName: replaceDatasetLabel || replaceDatasetId || ''}- )}- checked={deleteOriginalDataset}- onChange={() => setDeleteOriginalDataset(value => !value)}- />- <TippyTooltip- placement="top"- isLightTheme- render={() => (- <div>{intl.formatMessage({id: 'modal.replaceDataset.removeOriginalHint'})}</div>- )}- >- <InfoButton- aria-label={intl.formatMessage({- id: 'modal.replaceDataset.removeOriginalHint'- })}+ <Dimmed $dimmed={preparingLayers}>+ {isReplace ? (+ <RemoveOriginalRow>+ <Checkbox+ id="delete-original-dataset"+ type="checkbox"+ label={intl.formatMessage(+ {id: 'modal.replaceDataset.removeOriginal'},+ {datasetName: replaceDatasetLabel || replaceDatasetId || ''}+ )}+ checked={deleteOriginalDataset}+ onChange={() => setDeleteOriginalDataset(value => !value)}+ />+ <TippyTooltip+ placement="top"+ isLightTheme+ render={() => (+ <div>+ {intl.formatMessage({id: 'modal.replaceDataset.removeOriginalHint'})}+ </div>+ )}>- <Docs height="16px" />- </InfoButton>- </TippyTooltip>- </RemoveOriginalRow>- ) : (- <AutoCreateLayersCheckbox- checked={autoCreateLayers}- onToggle={onToggleAutoCreateLayers}- />- )}+ <InfoButton+ aria-label={intl.formatMessage({+ id: 'modal.replaceDataset.removeOriginalHint'+ })}+ >+ <Docs height="16px" />+ </InfoButton>+ </TippyTooltip>+ </RemoveOriginalRow>+ ) : (+ <AutoCreateLayersCheckbox+ checked={autoCreateLayers}+ disabled={preparingLayers}+ onToggle={onToggleAutoCreateLayers}+ />+ )}+ </Dimmed><StagedLabel />{uploadError || tilesetError ? (<FooterError>{uploadError || tilesetError}</FooterError>) : null}- <AddDataActions>- <Button type="button" link onClick={onClose}>- {intl.formatMessage({id: 'modal.button.defaultCancel'})}- </Button>- <Button type="button" cta disabled={!canAdd} onClick={onAddData}>- {intl.formatMessage({- id: isReplace ? 'modal.button.replace' : 'layerManager.addData'- })}- </Button>- </AddDataActions>+ {preparingLayers ? (+ <ProcessingStatus+ aria-label={intl.formatMessage({id: 'modal.loadData.processingLayers'})}+ >+ <ProcessingSpinner />+ </ProcessingStatus>+ ) : (+ <AddDataActions>+ <Button type="button" link onClick={onClose}>+ {intl.formatMessage({id: 'modal.button.defaultCancel'})}+ </Button>+ <Button type="button" cta disabled={!canAdd} onClick={onAddData}>+ {intl.formatMessage({+ id: isReplace ? 'modal.button.replace' : 'layerManager.addData'+ })}+ </Button>+ </AddDataActions>+ )}</AddDataBar>) : null}</StyledLoadDataModal>diff --git a/src/components/src/side-panel/layer-panel/layer-type-selector.tsx b/src/components/src/side-panel/layer-panel/layer-type-selector.tsxindex ce3c4928a..56731234d 100644--- a/src/components/src/side-panel/layer-panel/layer-type-selector.tsx+++ b/src/components/src/side-panel/layer-panel/layer-type-selector.tsx@@ -69,6 +69,7 @@ function LayerTypeSelectorFactory(multiSelect={false}disabled={disabled}placeholder="placeholder.selectType"+ deferOnChangeonChange={onSelect}getOptionValue={getOptionValue}filterOption="label"diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.tsindex 4b1de6448..2083ad12a 100644--- a/src/localization/src/translations/en.ts+++ b/src/localization/src/translations/en.ts@@ -827,6 +827,7 @@ export default {autoCreateLayers: 'Auto create layers',autoCreateLayersInfo:'kepler.gl will auto detect layers if the column names in your file follow certain naming conventions.',+ processingLayers: 'Processing data for layers',readyToAdd: 'Ready to add'},tripInfo: {diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.tsindex 728238376..1b71f9de2 100644--- a/src/processors/src/remote-file.ts+++ b/src/processors/src/remote-file.ts@@ -29,9 +29,17 @@ export type KeplerRemoteFile = File & {keplerLastModified?: string;};+export type RemoteLoadProgress = {+ loaded: number;+ total?: number;+ percent: number;+ /** Set once the bytes are in memory and parsing is about to block the main thread. */+ phase?: 'processing';+};+export type FetchRemoteFileOptions = {format?: string | null;- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void;+ onProgress?: (progress: RemoteLoadProgress) => void;etag?: string;lastModified?: string;/** Bypass the HTTP cache. Used for poll/reload, not the first remote URL load. */@@ -229,7 +237,7 @@ export function remoteDatasetFromUrl(url: string, format?: string | null): FileCexport async function fetchRemoteFileAsKeplerFile(url: string,format?: string | null,- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void+ onProgress?: (progress: RemoteLoadProgress) => void): Promise<KeplerRemoteFile> {const result = await fetchRemoteFile(url, {format, onProgress});if (!result.file) {@@ -238,9 +246,48 @@ export async function fetchRemoteFileAsKeplerFile(return result.file;}+/** Two frames are enough for the processing indicator to paint. */+const PAINT_WAIT_MS = 100;++function waitForNextPaint(): Promise<void> {+ // Hidden tabs suspend requestAnimationFrame. A download that finishes in the+ // background would otherwise wait until the user comes back, and there is+ // nothing to paint there.+ if (+ typeof requestAnimationFrame !== 'function' ||+ (typeof document !== 'undefined' && document.hidden)+ ) {+ return Promise.resolve();+ }+ return new Promise(resolve => {+ let settled = false;+ const frames: number[] = [];+ const finish = () => {+ if (settled) {+ return;+ }+ settled = true;+ clearTimeout(timer);+ if (typeof cancelAnimationFrame === 'function') {+ frames.forEach(id => cancelAnimationFrame(id));+ }+ resolve();+ };+ // The tab can be hidden after this wait starts. Don't sit on frames that will never run.+ const timer = setTimeout(finish, PAINT_WAIT_MS);+ frames.push(+ requestAnimationFrame(() => {+ if (!settled) {+ frames.push(requestAnimationFrame(finish));+ }+ })+ );+ });+}+async function readResponseBlob(response: Response,- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void+ onProgress?: (progress: RemoteLoadProgress) => void): Promise<Blob> {const contentLength = Number(response.headers.get('content-length'));const total = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : undefined;@@ -315,7 +362,7 @@ export async function loadExternallyHostedDataset(metadata: {etag?: string;lastModified?: string;bypassCache?: boolean;- onProgress?: (progress: {loaded: number; total?: number; percent: number}) => void;+ onProgress?: (progress: RemoteLoadProgress) => void;}): Promise<LoadExternallyHostedDatasetResult> {const {source, format, etag, lastModified, size, bypassCache, onProgress} = metadata;const fetched = await fetchRemoteFile(source, {@@ -335,6 +382,15 @@ export async function loadExternallyHostedDataset(metadata: {};}+ onProgress?.({+ loaded: fetched.size ?? fetched.file.size,+ ...(typeof fetched.size === 'number' ? {total: fetched.size} : {}),+ percent: 1,+ phase: 'processing'+ });+ // Let the loading indicator paint a processing state before parse blocks the tab.+ await waitForNextPaint();+const batches = await readFileInBatches({file: fetched.file,fileCache: [],diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 0c9d7b500..41102d2f5 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -442,6 +442,7 @@ export const INITIAL_VIS_STATE: VisState = {// for loading datasetsloadingIndicatorValue: 0,loadingProgress: {},+ loadingProcessing: {},loaders: [],loadOptions: {},@@ -3364,7 +3365,11 @@ function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {return HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK({arg: dataset,onProgress: progress =>- setLoadingProgress(progressId, Math.round((progress?.percent ?? 0) * 100))+ setLoadingProgress(+ progressId,+ Math.round((progress?.percent ?? 0) * 100),+ progress?.phase === 'processing' ? 'processing' : undefined+ )}).chain(hydrated => {const task = createNewDataEntry(hydrated, datasets);return (@@ -3376,20 +3381,23 @@ function createNewDataEntryTask(dataset: ProtoDataset, datasets: Datasets) {function clearLoadingProgress(state: VisState, ids?: string[]): VisState {const current = state.loadingProgress || {};- if (!Object.keys(current).length) {+ const processing = state.loadingProcessing || {};+ if (!Object.keys(current).length && !Object.keys(processing).length) {return state;}if ((state.loadingIndicatorValue || 0) <= 0) {- return {...state, loadingProgress: {}};+ return {...state, loadingProgress: {}, loadingProcessing: {}};}if (!ids?.length) {return state;}const next = {...current};+ const nextProcessing = {...processing};ids.forEach(id => {delete next[id];+ delete nextProcessing[id];});- return {...state, loadingProgress: next};+ return {...state, loadingProgress: next, loadingProcessing: nextProcessing};}function patchDatasetMetadata(dataset: Datasets[string], patch: Record<string, unknown>) {@@ -6114,8 +6122,10 @@ export const setLoadingIndicatorUpdater = (return {...state,loadingIndicatorValue: nextValue,- ...(nextValue === 0 && Object.keys(state.loadingProgress || {}).length- ? {loadingProgress: {}}+ ...(nextValue === 0 &&+ (Object.keys(state.loadingProgress || {}).length ||+ Object.keys(state.loadingProcessing || {}).length)+ ? {loadingProgress: {}, loadingProcessing: {}}: {})};};@@ -6127,7 +6137,7 @@ export const setLoadingIndicatorUpdater = (*/export function setLoadingProgressUpdater(state: VisState,- {id, percent}: VisStateActions.SetLoadingProgressUpdaterAction+ {id, percent, phase}: VisStateActions.SetLoadingProgressUpdaterAction): VisState {if ((state.loadingIndicatorValue || 0) <= 0) {return state;@@ -6135,7 +6145,11 @@ export function setLoadingProgressUpdater(const nextPercent = Number.isFinite(percent)? Math.max(0, Math.min(100, Math.round(percent))): 0;- if (state.loadingProgress?.[id] === nextPercent) {+ const processing = phase === 'processing';+ if (+ state.loadingProgress?.[id] === nextPercent &&+ Boolean(state.loadingProcessing?.[id]) === processing+ ) {return state;}return {@@ -6143,7 +6157,15 @@ export function setLoadingProgressUpdater(loadingProgress: {...state.loadingProgress,[id]: nextPercent- }+ },+ ...(processing+ ? {+ loadingProcessing: {+ ...state.loadingProcessing,+ [id]: true+ }+ }+ : {})};}diff --git a/src/schemas/src/vis-state-schema.ts b/src/schemas/src/vis-state-schema.tsindex 748859767..786cc0529 100644--- a/src/schemas/src/vis-state-schema.ts+++ b/src/schemas/src/vis-state-schema.ts@@ -99,6 +99,8 @@ export interface VisState {loadingIndicatorValue: number;/** Per-dataset download progress (0–100) while hydrating remote files. */loadingProgress: Record<string, number>;+ /** Remote datasets whose download finished and whose parse is still running. */+ loadingProcessing?: Record<string, boolean>;loaders: Loader[];loadOptions: object;initialState?: Partial<VisState>;diff --git a/test/browser/components/common/file-uploader-test.js b/test/browser/components/common/file-uploader-test.jsindex ec3c41848..81d49cd47 100644--- a/test/browser/components/common/file-uploader-test.js+++ b/test/browser/components/common/file-uploader-test.js@@ -102,6 +102,15 @@ test('Components -> FileUpload.onDrop -> render loading msg', t => {const uploadMsg = wrapper.find('.file-upload-progress__message').at(0).html();t.comment(uploadMsg);t.ok(uploadMsg.includes('tst-file.csv'), 'should render upload file msg');+ t.ok(+ uploadMsg.includes('upload-file-list__spinner'),+ 'should show a spinner in place of the checkbox while the file is still loading'+ );+ t.equal(+ wrapper.find('.upload-file-list').find('Checkbox').length,+ 0,+ 'should hide the include checkbox until parsing finishes'+ );t.equal(wrapper.find('.upload-file-list').hostNodes().length,1,diff --git a/test/browser/components/common/item-selector-test.js b/test/browser/components/common/item-selector-test.jsindex d5e1ba9d7..d46fecf12 100644--- a/test/browser/components/common/item-selector-test.js+++ b/test/browser/components/common/item-selector-test.js@@ -54,6 +54,60 @@ test('Components -> ItemSelector.render', t => {t.end();});+test('Components -> ItemSelector.deferOnChange', t => {+ const onChange = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <ItemSelector+ selectedItems="normal"+ options={['additive', 'normal', 'subtractive']}+ multiSelect={false}+ searchable={false}+ deferOnChange+ onChange={onChange}+ />+ </IntlWrapper>+ );++ const frames = [];+ const prevFrame = window.requestAnimationFrame;+ window.requestAnimationFrame = cb => {+ frames.push(cb);+ return frames.length;+ };+ try {+ wrapper.find('.item-selector__dropdown').at(0).simulate('click');+ wrapper.find(DropdownList).at(0).find('.list__item').at(0).simulate('click');++ t.equal(onChange.called, false, 'should wait to apply the selection until the list can paint');+ t.equal(wrapper.find(Typeahead).length, 1, 'should keep the list open');+ t.equal(+ wrapper.find('.item-selector__prepare-spinner').hostNodes().length,+ 1,+ 'should show a spinner over the open list'+ );++ act(() => {+ while (frames.length) {+ frames.shift()();+ }+ });+ } finally {+ window.requestAnimationFrame = prevFrame;+ }+ wrapper.update();++ t.deepEqual(onChange.args[0], ['additive'], 'should apply the selected value');+ t.equal(wrapper.find(Typeahead).length, 0, 'should close the list');+ t.equal(+ wrapper.find('.item-selector__prepare-spinner').hostNodes().length,+ 0,+ 'should remove the spinner'+ );++ t.end();+});+test('Components -> ItemSelector.render over 100 options', t => {let wrapper;const onChange = sinon.spy();diff --git a/test/browser/components/modals/load-data-modal-test.js b/test/browser/components/modals/load-data-modal-test.jsindex bc83091ed..6f8bafe87 100644--- a/test/browser/components/modals/load-data-modal-test.js+++ b/test/browser/components/modals/load-data-modal-test.js@@ -156,6 +156,61 @@ test('Components -> LoadDataModal -> auto create layers', t => {t.end();});+test('Components -> LoadDataModal -> processing layers', t => {+ const onConfirmAddData = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <LoadDataModal+ onConfirmAddData={onConfirmAddData}+ stagedToAdd={[{info: {label: 'points.csv'}}]}+ />+ </IntlWrapper>+ );++ const frames = [];+ const prevFrame = window.requestAnimationFrame;+ window.requestAnimationFrame = cb => {+ frames.push(cb);+ return frames.length;+ };+ try {+ wrapper.find('.add-data-bar button').at(1).simulate('click');++ t.equal(+ onConfirmAddData.called,+ false,+ 'layer prep waits until the processing state can paint'+ );+ const processing = wrapper.find('.add-data-bar__processing').hostNodes();+ t.equal(processing.length, 1, 'should replace the footer actions');+ t.equal(processing.text(), '', 'should show only the spinner');+ t.equal(+ processing.find('.add-data-bar__spinner').hostNodes().length,+ 1,+ 'should show a spinner in place of the footer actions'+ );+ t.equal(+ wrapper.find('.add-data-bar button').hostNodes().length,+ 0,+ 'should hide Cancel and Add Data while layers are prepared'+ );++ while (frames.length) {+ frames.shift()();+ }+ } finally {+ window.requestAnimationFrame = prevFrame;+ }++ t.deepEqual(+ onConfirmAddData.args[0][0],+ {autoCreateLayers: true, datasets: [{info: {label: 'points.csv'}}]},+ 'Add Data commits the staged dataset after the processing state can paint'+ );++ t.end();+});+test('Components -> LoadDataModal -> replace dataset', t => {const onConfirmAddData = sinon.spy();const wrapper = mountWithTheme(@@ -238,7 +293,20 @@ test('Components -> LoadDataModal -> one unchecked dataset stays out', t => {);checks().at(1).simulate('change');- wrapper.find('.add-data-bar button').at(1).simulate('click');+ const frames = [];+ const prevFrame = window.requestAnimationFrame;+ window.requestAnimationFrame = cb => {+ frames.push(cb);+ return frames.length;+ };+ try {+ wrapper.find('.add-data-bar button').at(1).simulate('click');+ while (frames.length) {+ frames.shift()();+ }+ } finally {+ window.requestAnimationFrame = prevFrame;+ }t.deepEqual(onConfirmAddData.args[0][0],{autoCreateLayers: true, datasets: [points]},diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex a22e17560..01e13703f 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -8829,6 +8829,13 @@ test('VisStateUpdater -> hydrate remote dataset loading progress', t => {const progressed = reducer(loadingState, VisStateActions.setLoadingProgress('remote-1', 42));t.equal(progressed.loadingProgress['remote-1'], 42, 'should store download percent');+ const processing = reducer(+ progressed,+ VisStateActions.setLoadingProgress('remote-1', 100, 'processing')+ );+ t.equal(processing.loadingProcessing['remote-1'], true, 'should mark the dataset as processing');+ t.equal(processing.loadingIndicatorValue, 1, 'should keep the indicator up while processing');+const same = reducer(progressed, VisStateActions.setLoadingProgress('remote-1', 42));t.equal(same, progressed, 'should skip redundant progress updates');@@ -8838,9 +8845,10 @@ test('VisStateUpdater -> hydrate remote dataset loading progress', t => {const [task] = drainTasksForTesting();t.ok(task, 'should schedule a hydrate/create task');- const cleared = reducer(progressed, VisStateActions.setLoadingIndicator({change: -1}));+ const cleared = reducer(processing, VisStateActions.setLoadingIndicator({change: -1}));t.equal(cleared.loadingIndicatorValue, 0, 'should hide the loading indicator');t.deepEqual(cleared.loadingProgress, {}, 'should clear hydrate progress');+ t.deepEqual(cleared.loadingProcessing, {}, 'should clear the processing flag');t.end();});