Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat: confirm datasets in the Add Data dialog before adding them (#3770)
Igor Dykhta5eeca8f84b23
diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.mdindex 4748eee06..813d4635d 100644--- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md+++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md@@ -64,7 +64,7 @@ Because CSV file content is uploaded as strings, kepler.gl will attempt to detec#### 2. Layer detection based on column names-kepler.gl will auto detect layer, if the column names follows certain naming convention. kepler.gl creates a point layer if your CSV has columns that are named `<name>_lat` and `<name>_lng` or `<name>_latitude` and `<name>_longitude`, or `<name>_lat` and `<name>_lon`.+kepler.gl will auto detect layer, if the column names follows certain naming convention. kepler.gl creates a point layer if your CSV has columns that are named `<name>_lat` and `<name>_lng` or `<name>_latitude` and `<name>_longitude`, or `<name>_lat` and `<name>_lon`. Uncheck **Auto create layers**, then click **Add Data** to load a file or tileset without creating layers.| layer | auto create layer from column names|---|---diff --git a/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js b/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.jsindex 084fb14fc..cfab4fd7f 100644--- a/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js+++ b/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js@@ -120,7 +120,6 @@ class LoadRemoteMap extends Component {<FormattedMessage id={'loadRemoteMap.examples'} /><ul><li>https://your.map.url/map.json</li>- <li>http://your.map.url/data.csv</li></ul></StyledInputLabel><StyledInputLabel>diff --git a/examples/demo-app-sqlrooms/src/constants/localization.js b/examples/demo-app-sqlrooms/src/constants/localization.jsindex 9c9bcd4ca..ad5c0f6eb 100644--- a/examples/demo-app-sqlrooms/src/constants/localization.js+++ b/examples/demo-app-sqlrooms/src/constants/localization.js@@ -10,7 +10,7 @@ const en = {'sampleDataViewer.rowCount': ' {rowCount} rows','loadRemoteMap.description': 'Load your map using your custom URL','loadRemoteMap.message':- 'You can use the following formats: CSV | JSON | Kepler.gl config json. Make sure the url contains the file extension.',+ 'Use a Kepler.gl config json. Make sure the url contains the file extension.','loadRemoteMap.examples': 'Examples:','loadRemoteMap.cors':'* CORS policy must be defined on your custom url domain in order to be accessible. For more info ',@@ -31,7 +31,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} riviä','loadRemoteMap.description': 'Lataa karttasi käyttämällä omaa urlia','loadRemoteMap.message':- 'Voit käyttää formaatteja: CSV | JSON | Kepler.gl asetus-json. Varmista, että url sisältää tiedostopäätteen nimen.',+ 'Käytä Kepler.gl asetus-jsonia. Varmista, että url sisältää tiedostopäätteen nimen.','loadRemoteMap.examples': 'Esimerkkejä:','loadRemoteMap.cors':'* CORS-käytäntö pitää olla määriteltynä urlin domainissa, jotta aineiston voi ladata.',@@ -45,7 +45,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} files','loadRemoteMap.description': 'Carrega el teu mapa amb la teva URL personalitzada','loadRemoteMap.message':- "Pots emprar els següents formats: CSV | JSON | Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",+ "Empra un Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",'loadRemoteMap.examples': 'Exemples:','loadRemoteMap.cors':'* La política CORS s’ha de definir al teu domini per tal que sigui accessible. Per a més informació ',@@ -59,7 +59,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} files','loadRemoteMap.description': 'Carga tu mapa con tu enlace personalizado','loadRemoteMap.message':- 'Puedes usar los siguientes formatos: CSV | JSON | Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',+ 'Usa un Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.','loadRemoteMap.examples': 'Ejemplos:','loadRemoteMap.cors':'* La política CORS debe ser definida en tu dominio para que sea accessible. Para más información ',@@ -72,8 +72,7 @@ export const messages = {'sampleMapsTab.trySampleData': '尝试样本数据','sampleDataViewer.rowCount': ' {rowCount} 行','loadRemoteMap.description': '使用自定义 URL 加载地图',- 'loadRemoteMap.message':- '您可以使用以下格式:CSV | JSON | Kepler.gl 配置 json。 确保 url 包含文件扩展名。',+ 'loadRemoteMap.message': '请使用 Kepler.gl 配置 json。确保 url 包含文件扩展名。','loadRemoteMap.examples': '示例:','loadRemoteMap.cors': '* 必须在您的自定义 url 域上定义 CORS 策略才能访问。欲了解更多信息','loadRemoteMap.clickHere': '点击此处',diff --git a/examples/demo-app/src/components/load-data-modal/load-remote-map.js b/examples/demo-app/src/components/load-data-modal/load-remote-map.jsindex 084fb14fc..cfab4fd7f 100644--- a/examples/demo-app/src/components/load-data-modal/load-remote-map.js+++ b/examples/demo-app/src/components/load-data-modal/load-remote-map.js@@ -120,7 +120,6 @@ class LoadRemoteMap extends Component {<FormattedMessage id={'loadRemoteMap.examples'} /><ul><li>https://your.map.url/map.json</li>- <li>http://your.map.url/data.csv</li></ul></StyledInputLabel><StyledInputLabel>diff --git a/examples/demo-app/src/constants/localization.js b/examples/demo-app/src/constants/localization.jsindex 9c9bcd4ca..ad5c0f6eb 100644--- a/examples/demo-app/src/constants/localization.js+++ b/examples/demo-app/src/constants/localization.js@@ -10,7 +10,7 @@ const en = {'sampleDataViewer.rowCount': ' {rowCount} rows','loadRemoteMap.description': 'Load your map using your custom URL','loadRemoteMap.message':- 'You can use the following formats: CSV | JSON | Kepler.gl config json. Make sure the url contains the file extension.',+ 'Use a Kepler.gl config json. Make sure the url contains the file extension.','loadRemoteMap.examples': 'Examples:','loadRemoteMap.cors':'* CORS policy must be defined on your custom url domain in order to be accessible. For more info ',@@ -31,7 +31,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} riviä','loadRemoteMap.description': 'Lataa karttasi käyttämällä omaa urlia','loadRemoteMap.message':- 'Voit käyttää formaatteja: CSV | JSON | Kepler.gl asetus-json. Varmista, että url sisältää tiedostopäätteen nimen.',+ 'Käytä Kepler.gl asetus-jsonia. Varmista, että url sisältää tiedostopäätteen nimen.','loadRemoteMap.examples': 'Esimerkkejä:','loadRemoteMap.cors':'* CORS-käytäntö pitää olla määriteltynä urlin domainissa, jotta aineiston voi ladata.',@@ -45,7 +45,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} files','loadRemoteMap.description': 'Carrega el teu mapa amb la teva URL personalitzada','loadRemoteMap.message':- "Pots emprar els següents formats: CSV | JSON | Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",+ "Empra un Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",'loadRemoteMap.examples': 'Exemples:','loadRemoteMap.cors':'* La política CORS s’ha de definir al teu domini per tal que sigui accessible. Per a més informació ',@@ -59,7 +59,7 @@ export const messages = {'sampleDataViewer.rowCount': ' {rowCount} files','loadRemoteMap.description': 'Carga tu mapa con tu enlace personalizado','loadRemoteMap.message':- 'Puedes usar los siguientes formatos: CSV | JSON | Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',+ 'Usa un Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.','loadRemoteMap.examples': 'Ejemplos:','loadRemoteMap.cors':'* La política CORS debe ser definida en tu dominio para que sea accessible. Para más información ',@@ -72,8 +72,7 @@ export const messages = {'sampleMapsTab.trySampleData': '尝试样本数据','sampleDataViewer.rowCount': ' {rowCount} 行','loadRemoteMap.description': '使用自定义 URL 加载地图',- 'loadRemoteMap.message':- '您可以使用以下格式:CSV | JSON | Kepler.gl 配置 json。 确保 url 包含文件扩展名。',+ 'loadRemoteMap.message': '请使用 Kepler.gl 配置 json。确保 url 包含文件扩展名。','loadRemoteMap.examples': '示例:','loadRemoteMap.cors': '* 必须在您的自定义 url 域上定义 CORS 策略才能访问。欲了解更多信息','loadRemoteMap.clickHere': '点击此处',diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex 09e45ab50..0e2cf65c2 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -102,6 +102,9 @@ export const ActionTypes = {LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,+ STAGE_LOADED_FILES: `${ACTION_PREFIX}STAGE_LOADED_FILES`,+ APPEND_STAGED_LOADED_FILES: `${ACTION_PREFIX}APPEND_STAGED_LOADED_FILES`,+ CLEAR_STAGED_LOADED_FILES: `${ACTION_PREFIX}CLEAR_STAGED_LOADED_FILES`,LAYER_COLOR_UI_CHANGE: `${ACTION_PREFIX}LAYER_COLOR_UI_CHANGE`,TOGGLE_FEATURE_LAYER: `${ACTION_PREFIX}TOGGLE_FEATURE_LAYER`,APPLY_CPU_FILTER: `${ACTION_PREFIX}APPLY_CPU_FILTER`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex bfdd4bba5..f24896a39 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -7,6 +7,7 @@ import {FileCacheItem} from '@kepler.gl/processors';import {Layer, LayerBaseConfig} from '@kepler.gl/layers';import {GroupByOp, JoinOp, KeplerTable} from '@kepler.gl/table';import {+ AddDataToMapOptions,AddDataToMapPayload,ViewportPadding,ValueOf,@@ -1828,25 +1829,36 @@ export function setMapInfo(};}+export type LoadFilesOptions = AddDataToMapOptions & {+ /** Parse the files, then wait for an explicit add instead of calling `addDataToMap`. */+ deferAddToMap?: boolean;+};+export type LoadFilesUpdaterAction = {files: File[];- onFinish?(result: any): any;+ onFinish?(result: any, options?: LoadFilesOptions): any;+ /** Forwarded to `addDataToMap` when loading finishes. Omitted options keep their defaults. */+ options?: LoadFilesOptions;};/*** Trigger file loading dispatch `addDataToMap` if succeed, or `loadFilesErr` if failed* @memberof visStateActions* @param files array of fileblob+ * @param onFinish called with the processed file cache. Defaults to `loadFilesSuccess`+ * @param options forwarded to `addDataToMap`. `autoCreateLayers` defaults to true when omitted* @returns action* @public*/export function loadFiles(files: File[],- onFinish?: (result: any) => any+ onFinish?: (result: any, options?: LoadFilesOptions) => any,+ options?: LoadFilesOptions): Merge<LoadFilesUpdaterAction, {type: typeof ActionTypes.LOAD_FILES}> {return {type: ActionTypes.LOAD_FILES,files,- onFinish+ onFinish,+ ...(options ? {options} : {})};}@@ -1856,30 +1868,89 @@ export function loadFiles(* @returns action* @public*/-export function loadNextFile(): {type: typeof ActionTypes.LOAD_NEXT_FILE} {+export function loadNextFile(loadId?: number): {+ type: typeof ActionTypes.LOAD_NEXT_FILE;+ loadId?: number;+} {return {- type: ActionTypes.LOAD_NEXT_FILE+ type: ActionTypes.LOAD_NEXT_FILE,+ ...(loadId == null ? {} : {loadId})};}export type loadFilesSuccessUpdaterAction = {result: FileCacheItem[];+ options?: AddDataToMapOptions;+ /** Set when a file-load pause finishes, so a canceled load can be ignored. */+ loadId?: number;};/*** called when all files are processed and loaded* @memberof visStateActions* @param result+ * @param options forwarded to `addDataToMap`* @returns action*/export function loadFilesSuccess(- result: FileCacheItem[]+ result: FileCacheItem[],+ options?: AddDataToMapOptions): Merge<loadFilesSuccessUpdaterAction, {type: typeof ActionTypes.LOAD_FILES_SUCCESS}> {return {type: ActionTypes.LOAD_FILES_SUCCESS,+ result,+ ...(options ? {options} : {})+ };+}++export type StageLoadedFilesUpdaterAction = {+ result: FileCacheItem[];+ /** Set when a file-load pause finishes, so a canceled load can be ignored. */+ loadId?: number;+};+/**+ * Store processed files without adding them to the map or closing the modal.+ * @memberof visStateActions+ * @param result processed file cache+ * @returns action+ * @public+ */+export function stageLoadedFiles(+ result: FileCacheItem[]+): Merge<StageLoadedFilesUpdaterAction, {type: typeof ActionTypes.STAGE_LOADED_FILES}> {+ return {+ type: ActionTypes.STAGE_LOADED_FILES,+ result+ };+}++/**+ * Add already-parsed or remote datasets to the Add Data list without starting a file load.+ * @memberof visStateActions+ * @param result datasets to keep until Add Data+ * @returns action+ * @public+ */+export function appendStagedLoadedFiles(+ result: FileCacheItem[]+): Merge<StageLoadedFilesUpdaterAction, {type: typeof ActionTypes.APPEND_STAGED_LOADED_FILES}> {+ return {+ type: ActionTypes.APPEND_STAGED_LOADED_FILES,result};}+/**+ * Drop files staged for the Add Data button and stop an in-progress modal load.+ * @memberof visStateActions+ * @returns action+ * @public+ */+export function clearStagedLoadedFiles(): {type: typeof ActionTypes.CLEAR_STAGED_LOADED_FILES} {+ return {+ type: ActionTypes.CLEAR_STAGED_LOADED_FILES+ };+}+export type LoadFileStepSuccessAction = {fileName: string;fileCache: FileCacheItem[];@@ -2179,6 +2250,8 @@ export type NextFileBatchUpdaterAction = {progress?: any;accumulated?: any;onFinish: (result: any) => any;+ /** Carried across batches so progressive loads keep the original add-data options. */+ options?: LoadFilesOptions;};};/**diff --git a/src/components/src/common/file-uploader/file-upload.tsx b/src/components/src/common/file-uploader/file-upload.tsxindex 43cc9abe9..9de154355 100644--- a/src/components/src/common/file-uploader/file-upload.tsx+++ b/src/components/src/common/file-uploader/file-upload.tsx@@ -6,25 +6,19 @@ import styled from 'styled-components';import {injectIntl, WrappedComponentProps} from 'react-intl';import UploadButton from './upload-button';import {DragNDrop, FileType} from '../icons';-import FileUploadProgress from './file-upload-progress';import FileDrop from './file-drop';+import UploadFileList, {UploadFileListItem} from './upload-file-list';import {FileLoading, FileLoadingProgress} from '@kepler.gl/types';import {GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';import {FormattedMessage} from '@kepler.gl/localization';-import {- fetchRemoteFileAsKeplerFile,- getAcceptedRemoteFileFormats,- getFileNameForRemoteUrl,- isRemoteDatasetUrl-} from '@kepler.gl/processors';+import {getAcceptedRemoteFileFormats, isRemoteDatasetUrl} from '@kepler.gl/processors';import {media} from '@kepler.gl/styles';-import {getApplicationConfig} from '@kepler.gl/utils';+import {getApplicationConfig, getError} from '@kepler.gl/utils';import Markdown from 'markdown-to-jsx';import {Button, InputLight} from '../styled-components';import LinkRenderer from '../link-renderer';-const fileIconColor = '#D3D8E0';const StyledUploadMessage = styled.div`@@ -59,7 +53,9 @@ const StyledFileDrop = styled.div<StyledFileDropProps>`border-color: ${props =>props.$dragOver ? props.theme.textColorLT : props.theme.subtextColorLT};text-align: center;- width: 100%;+ flex: 1;+ min-width: 0;+ width: auto;min-height: 360px;padding: 24px 12px 16px;display: flex;@@ -120,6 +116,17 @@ const StyledFileUpload = styled.div`}`;+const UploadColumns = styled.div`+ display: flex;+ align-items: stretch;+ gap: 16px;+ width: 100%;++ ${media.portable`+ flex-direction: column;+ `}+`;+const StyledMessage = styled.div`display: flex;justify-content: center;@@ -188,42 +195,6 @@ const StyledRemoteFetchButton = styled(Button)`flex-shrink: 0;`;-const REMOTE_DOWNLOAD_SHARE = 0.85;--function scaleFileProgress(- progress: FileLoadingProgress,- start: number,- end: number-): FileLoadingProgress {- return Object.keys(progress).reduce<FileLoadingProgress>((accu, key) => {- const item = progress[key];- accu[key] = {- ...item,- percent: start + (item.percent || 0) * (end - start)- };- return accu;- }, {});-}--function getCombinedLoadProgress({- fileLoading,- fileLoadingProgress,- remoteProgress-}: {- fileLoading: FileLoading | false;- fileLoadingProgress: FileLoadingProgress;- remoteProgress: FileLoadingProgress;-}): FileLoadingProgress {- const isRemoteFetch = Object.keys(remoteProgress).length > 0;- if (!isRemoteFetch) {- return fileLoadingProgress;- }- if (fileLoading) {- return scaleFileProgress(fileLoadingProgress, REMOTE_DOWNLOAD_SHARE, 1);- }- return scaleFileProgress(remoteProgress, 0, REMOTE_DOWNLOAD_SHARE);-}-type FileUploadProps = {onFileUpload: (files: File[]) => void;fileLoading: FileLoading | false;@@ -237,6 +208,16 @@ type FileUploadProps = {displayedFileExtensions?: string[];/** Set to true if app wants to do its own file filtering */disableExtensionFilter?: boolean;+ /** Parsed files held until Add Data, used if this uploader remounts. */+ stagedToAdd?: Array<{+ info?: {label?: string; format?: string};+ metadata?: {source?: string};+ }> | null;+ /** Stage a remote URL without downloading it. Add Data runs the remote load. */+ onAddRemoteDataset?: (remote: {url: string; format?: string}) => void;+ /** Keys of datasets the user unchecked. Omitted keys stay selected. */+ deselectedDatasets?: Record<string, boolean>;+ onToggleDataset?: (key: string) => void;} & WrappedComponentProps;type FileUploadState = {@@ -247,10 +228,66 @@ type FileUploadState = {remoteUrl: string;remoteFormat: string;remoteError: {message: string} | null;- remoteLoading: boolean;- remoteProgress: FileLoadingProgress;+ deselectedDatasets: Record<string, boolean>;};+function formatFileSize(size?: number): string {+ if (!Number.isFinite(size)) {+ return '';+ }+ const bytes = size as number;+ if (bytes < 1024) {+ return `${bytes} B`;+ }+ if (bytes < 1024 * 1024) {+ return `${(bytes / 1024).toFixed(1)} KB`;+ }+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;+}++function extensionOf(name: string): string {+ const part = name.includes('.') ? name.split('.').pop() : '';+ return part && part !== name ? part : '';+}++function fileListStatus(+ intl,+ file: File,+ progress: FileLoadingProgress[string] | undefined,+ loading: boolean+): Pick<UploadFileListItem, 'status' | 'percent' | 'isError' | 'isSuccess'> {+ if (progress?.error) {+ return {+ status: getError(progress.error) || intl.formatMessage({id: 'fileUploader.unableToLoad'}),+ percent: progress.percent || 0,+ isError: true,+ isSuccess: false+ };+ }+ if (!loading) {+ const size = formatFileSize(file.size);+ return {+ status: size+ ? intl.formatMessage({id: 'fileUploader.readyToAdd'}, {size})+ : intl.formatMessage({id: 'fileUploader.readyToAddNoSize'}),+ percent: 1,+ isError: false,+ isSuccess: true+ };+ }+ const size = formatFileSize(file.size);+ const loaded = formatFileSize((progress?.percent || 0) * (file.size || 0));+ return {+ status:+ size && loaded+ ? intl.formatMessage({id: 'fileUploader.loadingProgress'}, {loaded, total: size})+ : progress?.message || intl.formatMessage({id: 'fileUploader.loading'}),+ percent: progress?.percent || 0,+ isError: false,+ isSuccess: false+ };+}+function FileUploadFactory() {/** @augments {Component<FileUploadProps>} */class FileUpload extends Component<FileUploadProps, FileUploadState> {@@ -262,23 +299,14 @@ function FileUploadFactory() {remoteUrl: '',remoteFormat: 'auto',remoteError: null,- remoteLoading: false,- remoteProgress: {}+ deselectedDatasets: {}};static getDerivedStateFromProps(props, state) {- if (state.fileLoading && props.fileLoading === false && state.files.length) {- return {- files: [],- fileLoading: props.fileLoading,- remoteLoading: false,- remoteProgress: {}- };+ if (state.fileLoading !== props.fileLoading) {+ return {fileLoading: props.fileLoading};}- return {- fileLoading: props.fileLoading,- ...(props.fileLoading ? {remoteLoading: false} : {})- };+ return null;}frame = createRef<HTMLDivElement>();@@ -311,10 +339,13 @@ function FileUploadFactory() {}}- const nextState = {files: filesToLoad, errorFiles, dragOver: false};-- this.setState(nextState, () =>- nextState.files.length ? this.props.onFileUpload(nextState.files) : null+ this.setState(+ prev => ({+ files: [...prev.files, ...filesToLoad],+ errorFiles: [...prev.errorFiles, ...errorFiles],+ dragOver: false+ }),+ () => (filesToLoad.length ? this.props.onFileUpload(filesToLoad) : null));};@@ -341,12 +372,27 @@ function FileUploadFactory() {}};- _handleLoadRemoteFile = async () => {+ _toggleDataset = (key: string) => {+ if (!key) {+ return;+ }+ if (this.props.onToggleDataset) {+ this.props.onToggleDataset(key);+ return;+ }+ this.setState(prev => ({+ deselectedDatasets: {+ ...prev.deselectedDatasets,+ [key]: !prev.deselectedDatasets[key]+ }+ }));+ };++ _handleLoadRemoteFile = () => {const {remoteUrl, remoteFormat, remoteError} = this.state;- const {intl} = this.props;const showFormatSelector = getApplicationConfig().enableRemoteFileFormatSelector;const format = showFormatSelector && remoteFormat !== 'auto' ? remoteFormat : undefined;- if (!remoteUrl || remoteError || this.state.remoteLoading) {+ if (!remoteUrl || remoteError) {if (!remoteUrl) {this.setState({remoteError: {message: 'Incorrect URL'}});}@@ -357,74 +403,32 @@ function FileUploadFactory() {return;}- const fileName = getFileNameForRemoteUrl(remoteUrl, format);- const downloading = intl.formatMessage({id: 'fileUploader.downloading'});- this.setState({- remoteLoading: true,- remoteError: null,- remoteProgress: {- [fileName]: {- fileName,- percent: 0,- message: downloading,- error: null- }- }- });-- try {- let lastUpdate = 0;- const file = await fetchRemoteFileAsKeplerFile(remoteUrl, format, ({percent}) => {- const now = Date.now();- if (percent < 1 && now - lastUpdate < 100) {- return;- }- lastUpdate = now;- this.setState(prev => {- const name = Object.keys(prev.remoteProgress)[0] || fileName;- return {- remoteProgress: {- [name]: {- fileName: name,- percent,- message: downloading,- error: null- }- }- };- });- });- this.setState(- {- files: [file],- errorFiles: [],- dragOver: false- },- () => this.props.onFileUpload([file])- );- } catch (error) {- this.setState({- remoteLoading: false,- remoteProgress: {},- remoteError: {- message: error instanceof Error ? error.message : `Failed to load ${remoteUrl}`- }- });- }+ this.props.onAddRemoteDataset?.({url: remoteUrl, ...(format ? {format} : {})});+ this.setState({remoteUrl: '', remoteError: null});};render() {+ const {dragOver, files, errorFiles, remoteUrl, remoteFormat, remoteError} = this.state;const {- dragOver,- files,- errorFiles,- remoteUrl,- remoteFormat,- remoteError,- remoteLoading,- remoteProgress- } = this.state;- const {fileLoading, fileLoadingProgress, theme, intl} = this.props;+ fileLoading,+ fileLoadingProgress,+ intl,+ stagedToAdd,+ deselectedDatasets: deselectedFromProps+ } = this.props;+ const deselected = deselectedFromProps || this.state.deselectedDatasets;+ const selectionFor = (+ item: Omit<UploadFileListItem, 'id' | 'selectable' | 'selected' | 'onToggle'>,+ key: string,+ selectable: boolean+ ): UploadFileListItem => ({+ ...item,+ id: key,+ selectable,+ selected: selectable && !deselected[key],+ selectionLabel: intl.formatMessage({id: 'fileUploader.includeDataset'}, {name: item.name}),+ onToggle: selectable ? () => this._toggleDataset(key) : undefined+ });const {fileExtensions = [], displayedFileExtensions} = this.props;const iconExtensions = displayedFileExtensions?.length? displayedFileExtensions@@ -433,6 +437,73 @@ function FileUploadFactory() {const fileUploadInfoText = `${intl.formatMessage({id: 'fileUploader.configUploadMessage'})}(${GUIDES_FILE_FORMAT_DOC}).`;+ const progress = fileLoadingProgress || {};+ const uploadItems: UploadFileListItem[] = [];+ const seen = new Set<string>();+ const localNames = new Set(files.map(file => file.name));+ (stagedToAdd || []).forEach(item => {+ const name = item?.info?.label;+ const source = item?.metadata?.source;+ if (!name || seen.has(source || name)) {+ return;+ }+ if (!source && localNames.has(name)) {+ return;+ }+ const key = source || name;+ seen.add(key);+ uploadItems.push(+ selectionFor(+ {+ name,+ ext: extensionOf(name) || 'url',+ status: source || intl.formatMessage({id: 'fileUploader.readyToAddNoSize'}),+ percent: 1,+ isSuccess: true+ },+ key,+ true+ )+ );+ });+ files.forEach(file => {+ if (seen.has(file.name)) {+ return;+ }+ seen.add(file.name);+ const fileProgress = progress[file.name];+ const status = fileListStatus(+ intl,+ file,+ fileProgress,+ Boolean(fileLoading && fileProgress)+ );+ uploadItems.push(+ selectionFor(+ {+ name: file.name,+ ext: extensionOf(file.name),+ ...status+ },+ file.name,+ !status.isError+ )+ );+ });+ errorFiles.forEach(name => {+ if (seen.has(name)) {+ return;+ }+ seen.add(name);+ uploadItems.push({+ name,+ ext: extensionOf(name),+ status: intl.formatMessage({id: 'fileUploader.unsupported'}),+ percent: 0,+ isError: true,+ isSuccess: false+ });+ });return (<StyledFileUpload className="file-uploader" ref={this.frame}>{FileDrop ? (@@ -456,110 +527,92 @@ function FileUploadFactory() {{fileUploadInfoText}</Markdown></StyledUploadMessage>- <StyledFileDrop $dragOver={dragOver}>- <StyledDropBody>- <StyledFileTypeFow className="file-type-row">- {iconExtensions.map(ext => (- <FileType key={ext} ext={ext} height="50px" fontSize="9px" />- ))}- </StyledFileTypeFow>- {fileLoading || remoteLoading ? (- <FileUploadProgress- fileLoadingProgress={getCombinedLoadProgress({- fileLoading,- fileLoadingProgress,- remoteProgress- })}- theme={theme}- />- ) : (- <>- <StyledDragNDropIcon- style={{opacity: dragOver ? 0.5 : 1}}- className="file-upload-display-message"+ <UploadColumns>+ <StyledFileDrop $dragOver={dragOver}>+ <StyledDropBody>+ <StyledFileTypeFow className="file-type-row">+ {iconExtensions.map(ext => (+ <FileType key={ext} ext={ext} height="50px" fontSize="9px" />+ ))}+ </StyledFileTypeFow>+ <StyledDragNDropIcon+ style={{opacity: dragOver ? 0.5 : 1}}+ className="file-upload-display-message"+ >+ <DragNDrop height="36px" />+ </StyledDragNDropIcon>+ <StyledDragFileWrapper>+ <StyledActionLine>+ <FormattedMessage+ id={'fileUploader.dropMessage'}+ values={{+ browse: (+ <UploadButton key="browse" onUpload={this._handleFileInput}>+ {intl.formatMessage({id: 'fileUploader.browseFiles'})}+ </UploadButton>+ )+ }}+ />+ </StyledActionLine>+ <StyledRemoteUrlForm+ className="file-uploader__remote-url"+ onClick={event => event.stopPropagation()}>- <DragNDrop height="36px" />- {errorFiles.length ? (- <WarningMsg>- <FormattedMessage- id={'fileUploader.fileNotSupported'}- values={{errorFiles: errorFiles.join(', ')}}- />- </WarningMsg>- ) : null}- </StyledDragNDropIcon>- {!files.length ? (- <StyledDragFileWrapper>- <StyledActionLine>- <FormattedMessage- id={'fileUploader.dropMessage'}- values={{- browse: (- <UploadButton key="browse" onUpload={this._handleFileInput}>- {intl.formatMessage({id: 'fileUploader.browseFiles'})}- </UploadButton>- )- }}- />- </StyledActionLine>- <StyledRemoteUrlForm- className="file-uploader__remote-url"- onClick={event => event.stopPropagation()}+ <StyledRemoteUrlRow>+ <StyledRemoteUrlInput+ type="url"+ value={remoteUrl}+ aria-label={intl.formatMessage({+ id: 'fileUploader.urlPlaceholder'+ })}+ placeholder={intl.formatMessage({+ id: 'fileUploader.urlPlaceholder'+ })}+ onChange={this._onRemoteUrlChange}+ onKeyDown={this._onRemoteUrlKeyDown}+ />+ {showFormatSelector ? (+ <StyledRemoteFormatSelect+ className="file-uploader__remote-format"+ aria-label={intl.formatMessage({id: 'fileUploader.format'})}+ value={remoteFormat}+ onChange={this._onRemoteFormatChange}+ >+ {getAcceptedRemoteFileFormats().map(format => (+ <option key={format} value={format}>+ {format === 'auto'+ ? intl.formatMessage({id: 'fileUploader.formatAuto'})+ : format.toUpperCase()}+ </option>+ ))}+ </StyledRemoteFormatSelect>+ ) : null}+ <StyledRemoteFetchButton+ type="button"+ className="file-uploader__remote-add"+ cta+ small+ disabled={!remoteUrl}+ onClick={this._handleLoadRemoteFile}>- <StyledRemoteUrlRow>- <StyledRemoteUrlInput- type="url"- value={remoteUrl}- aria-label={intl.formatMessage({- id: 'fileUploader.urlPlaceholder'- })}- placeholder={intl.formatMessage({- id: 'fileUploader.urlPlaceholder'- })}- onChange={this._onRemoteUrlChange}- onKeyDown={this._onRemoteUrlKeyDown}- disabled={remoteLoading}- />- {showFormatSelector ? (- <StyledRemoteFormatSelect- className="file-uploader__remote-format"- aria-label={intl.formatMessage({id: 'fileUploader.format'})}- value={remoteFormat}- onChange={this._onRemoteFormatChange}- disabled={remoteLoading}- >- {getAcceptedRemoteFileFormats().map(format => (- <option key={format} value={format}>- {format === 'auto'- ? intl.formatMessage({id: 'fileUploader.formatAuto'})- : format.toUpperCase()}- </option>- ))}- </StyledRemoteFormatSelect>- ) : null}- <StyledRemoteFetchButton- type="button"- cta- small- disabled={!remoteUrl || remoteLoading}- onClick={this._handleLoadRemoteFile}- >- <FormattedMessage id={'fileUploader.fetch'} />- </StyledRemoteFetchButton>- </StyledRemoteUrlRow>- {remoteError ? <WarningMsg>{remoteError.message}</WarningMsg> : null}- </StyledRemoteUrlForm>- </StyledDragFileWrapper>- ) : null}- </>- )}- </StyledDropBody>- {fileLoading || remoteLoading ? null : (+ <FormattedMessage id={'fileUploader.addUrl'} />+ </StyledRemoteFetchButton>+ </StyledRemoteUrlRow>+ {remoteError ? <WarningMsg>{remoteError.message}</WarningMsg> : null}+ </StyledRemoteUrlForm>+ </StyledDragFileWrapper>+ </StyledDropBody><StyledDisclaimer><FormattedMessage id={'fileUploader.disclaimer'} /></StyledDisclaimer>- )}- </StyledFileDrop>+ </StyledFileDrop>+ <UploadFileList+ title={+ uploadItems.length ? intl.formatMessage({id: 'fileUploader.toAdd'}) : undefined+ }+ items={uploadItems}+ />+ </UploadColumns></FileDrop>) : null}</StyledFileUpload>diff --git a/src/components/src/common/file-uploader/upload-file-list.tsx b/src/components/src/common/file-uploader/upload-file-list.tsxnew file mode 100644index 000000000..03e4ee845--- /dev/null+++ b/src/components/src/common/file-uploader/upload-file-list.tsx@@ -0,0 +1,201 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React, {FC} from 'react';+import styled from 'styled-components';++import {media} from '@kepler.gl/styles';+import Checkbox from '../checkbox';+import {FileType} from '../icons';++export type StagedDatasetRef = {+ info?: {id?: string; label?: string};+ metadata?: {source?: string};+};++/** Key shared by a card and the staged dataset it will add. Remote URLs use the URL. */+export function datasetSelectionKey(item?: StagedDatasetRef | null): string {+ return item?.metadata?.source || item?.info?.label || item?.info?.id || '';+}++export function selectedStagedDatasets<T extends StagedDatasetRef>(+ staged: T[] | null | undefined,+ deselected?: Record<string, boolean>+): T[] {+ return (staged || []).filter(item => {+ const key = datasetSelectionKey(item);+ return Boolean(key) && !deselected?.[key];+ });+}++export type UploadFileListItem = {+ id?: string;+ name: string;+ ext: string;+ status: string;+ percent: number;+ isError?: boolean;+ isSuccess?: boolean;+ selectable?: boolean;+ selected?: boolean;+ selectionLabel?: string;+ onToggle?: () => void;+};++type UploadFileListProps = {+ title?: string;+ items: UploadFileListItem[];+};++const List = styled.div.attrs({+ className: 'upload-file-list'+})`+ width: 320px;+ flex-shrink: 0;+ max-height: 420px;+ overflow-y: auto;++ ${media.portable`+ width: 100%;+ max-height: 240px;+ `}+`;++const ListTitle = styled.div`+ font-size: 13px;+ color: ${props => props.theme.textColorLT};+ margin-bottom: 8px;+`;++const Card = styled.div.attrs({+ className: 'file-upload-progress__message'+})<{$isError?: boolean}>`+ position: relative;+ overflow: hidden;+ background: #fff;+ color: ${props => (props.$isError ? props.theme.errorColor : props.theme.textColorLT)};+ border: 1px solid+ ${props => (props.$isError ? props.theme.errorColor : props.theme.selectBorderColorLT)};+ border-radius: 4px;+ box-shadow: 0 0 5px -2px rgba(0, 0, 0, 0.1);+ margin-bottom: 12px;++ &:last-child {+ margin-bottom: 0;+ }+`;++const Progress = styled.div<{$percent: number}>`+ position: absolute;+ left: 0;+ top: 0;+ bottom: 0;+ width: ${props => props.$percent}%;+ background: rgba(36, 115, 189, 0.12);+`;++const CardBody = styled.div`+ position: relative;+ display: flex;+ align-items: center;+ gap: 8px;+ padding: 8px 12px 8px 8px;+`;++const IconWrap = styled.div`+ flex-shrink: 0;+ color: ${props => props.theme.subtextColorLT};+`;++const CardText = styled.div`+ min-width: 0;+ flex: 1;+`;++const CardName = styled.div`+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+ font-size: 12px;+ font-weight: 500;+`;++const CardCheck = styled.div`+ flex-shrink: 0;++ .kg-checkbox,+ .kg-checkbox__label {+ margin-left: 0;+ margin-bottom: 0;+ }+`;++/** Names the checkbox for assistive tech without taking space in the card. */+const CheckLabel = styled.span.attrs({+ className: 'upload-file-list__check-label'+})`+ position: absolute;+ width: 1px;+ height: 1px;+ padding: 0;+ margin: -1px;+ overflow: hidden;+ clip: rect(0, 0, 0, 0);+ white-space: nowrap;+ border: 0;+`;++const CardStatus = styled.div<{$isError?: boolean}>`+ margin-top: 2px;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+ font-size: 11px;+ line-height: 14px;+ color: ${props => (props.$isError ? props.theme.errorColor : props.theme.subtextColorLT)};+`;++const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {+ if (!items.length) {+ return null;+ }++ return (+ <List>+ {title ? <ListTitle>{title}</ListTitle> : null}+ {items.map((item, index) => (+ <Card key={`${item.id || item.name}-${item.status}`} $isError={item.isError}>+ {!item.isSuccess && !item.isError ? (+ <Progress $percent={Math.round(item.percent * 100)} />+ ) : null}+ <CardBody>+ <IconWrap>+ <FileType ext={item.ext} height="36px" fontSize="8px" />+ </IconWrap>+ <CardText>+ <CardName title={item.name}>{item.name}</CardName>+ <CardStatus $isError={item.isError} title={item.status}>+ {item.status}+ </CardStatus>+ </CardText>+ {item.selectable ? (+ <CardCheck title={item.selectionLabel} onClick={event => event.stopPropagation()}>+ <Checkbox+ id={`upload-file-${index}`}+ type="checkbox"+ label={+ item.selectionLabel ? <CheckLabel>{item.selectionLabel}</CheckLabel> : undefined+ }+ checked={Boolean(item.selected)}+ onChange={() => item.onToggle?.()}+ />+ </CardCheck>+ ) : null}+ </CardBody>+ </Card>+ ))}+ </List>+ );+};++export default UploadFileList;diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex 3bdd3f6fc..ebea2de4c 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -15,6 +15,7 @@ import {MapStyle,ProviderState} from '@kepler.gl/reducers';+import {remoteDatasetFromUrl} from '@kepler.gl/processors';import {exportHtml, exportMap, exportJson, exportImage} from '@kepler.gl/utils';import ModalDialogFactory from './modals/modal-dialog';@@ -85,6 +86,11 @@ const smallModalCss = css`const LoadDataModalStyle = css`top: 60px;+ padding-bottom: 0;++ ${media.portable`+ padding-bottom: 0;+ `}`;const ExportVideoModalStyle = css`@@ -166,6 +172,9 @@ export default function ModalContainerFactory(};_closeModal = () => {+ if (this.props.uiState.currentModal === ADD_DATA_ID) {+ this.props.visStateActions.clearStagedLoadedFiles();+ }this.props.uiStateActions.toggleModal(null);};@@ -180,12 +189,32 @@ export default function ModalContainerFactory(};_onFileUpload = fileList => {- this.props.visStateActions.loadFiles(fileList);+ this.props.visStateActions.loadFiles(fileList, VisStateActions.stageLoadedFiles, {+ deferAddToMap: true+ });+ };++ _onAddRemoteDataset = ({url, format}: {url: string; format?: string}) => {+ this.props.visStateActions.appendStagedLoadedFiles([remoteDatasetFromUrl(url, format)]);+ };++ _onConfirmAddData = ({+ autoCreateLayers,+ datasets+ }: {autoCreateLayers?: boolean; datasets?: any[]} = {}) => {+ const staged = datasets || this.props.visState.stagedToAdd;+ if (!staged?.length) {+ return;+ }+ this.props.visStateActions.loadFilesSuccess(staged, {+ autoCreateLayers: autoCreateLayers !== false+ });};_onTilesetAdded = (tileset: {name: string; type: string; metadata: Record<string, any>},- processedMetadata?: Record<string, any>+ processedMetadata?: Record<string, any>,+ options?: {autoCreateLayers?: boolean}) => {this.props.visStateActions.updateVisData({@@ -207,7 +236,7 @@ export default function ModalContainerFactory(disableDataOperation: true},{- autoCreateLayers: true,+ autoCreateLayers: options?.autoCreateLayers !== false,centerMap: true});@@ -373,6 +402,9 @@ export default function ModalContainerFactory({...providerState}onClose={this._closeModal}onFileUpload={this._onFileUpload}+ onAddRemoteDataset={this._onAddRemoteDataset}+ onConfirmAddData={this._onConfirmAddData}+ stagedToAdd={visState.stagedToAdd}onTilesetAdded={this._onTilesetAdded}onLoadCloudMap={this._onLoadCloudMap}loadFiles={uiState.loadFiles}diff --git a/src/components/src/modals/auto-create-layers-checkbox.tsx b/src/components/src/modals/auto-create-layers-checkbox.tsxnew file mode 100644index 000000000..73d2c6fcb--- /dev/null+++ b/src/components/src/modals/auto-create-layers-checkbox.tsx@@ -0,0 +1,75 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React, {FC} from 'react';+import {useIntl} from 'react-intl';+import styled from 'styled-components';++import Checkbox from '../common/checkbox';+import {Docs} from '../common/icons';+import TippyTooltip from '../common/tippy-tooltip';++type AutoCreateLayersCheckboxProps = {+ checked: boolean;+ onToggle: () => void;+ disabled?: boolean;+};++const AutoCreateLayersRow = styled.div.attrs({+ className: 'auto-create-layers'+})`+ display: flex;+ align-items: center;++ .kg-checkbox {+ margin-left: 0;+ }++ .kg-checkbox__label {+ margin-bottom: 0;+ margin-left: 0;+ color: ${props => props.theme.textColorLT};+ }+`;++const InfoButton = styled.span`+ display: inline-flex;+ align-items: center;+ margin-left: 6px;+ color: ${props => props.theme.subtextColorLT};+ cursor: help;++ &:hover {+ color: ${props => props.theme.textColorLT};+ }+`;++const AutoCreateLayersCheckbox: FC<AutoCreateLayersCheckboxProps> = ({+ checked,+ onToggle,+ disabled = false+}) => {+ const intl = useIntl();+ const label = intl.formatMessage({id: 'modal.loadData.autoCreateLayers'});+ const info = intl.formatMessage({id: 'modal.loadData.autoCreateLayersInfo'});++ return (+ <AutoCreateLayersRow>+ <Checkbox+ id="auto-create-layers"+ type="checkbox"+ label={label}+ checked={checked}+ disabled={disabled}+ onChange={onToggle}+ />+ <TippyTooltip placement="top" isLightTheme render={() => <div>{info}</div>}>+ <InfoButton aria-label={info}>+ <Docs height="16px" />+ </InfoButton>+ </TippyTooltip>+ </AutoCreateLayersRow>+ );+};++export default AutoCreateLayersCheckbox;diff --git a/src/components/src/modals/load-data-modal.tsx b/src/components/src/modals/load-data-modal.tsxindex ea5032974..2e487e211 100644--- a/src/components/src/modals/load-data-modal.tsx+++ b/src/components/src/modals/load-data-modal.tsx@@ -1,27 +1,85 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {useState} from 'react';+import React, {useCallback, useState} from 'react';import styled from 'styled-components';import get from 'es-toolkit/compat/get';import {IntlShape, useIntl} from 'react-intl';+import {Button} from '../common';import FileUploadFactory from '../common/file-uploader/file-upload';+import {selectedStagedDatasets} from '../common/file-uploader/upload-file-list';import LoadStorageMapFactory from './load-storage-map';import LoadTilesetFactory from './tilesets-modals/load-tileset';import ModalTabsFactory from './modal-tabs';import LoadingDialog from './loading-dialog';+import AutoCreateLayersCheckbox from './auto-create-layers-checkbox';import {LOADING_METHODS} from '@kepler.gl/constants';+import {media} from '@kepler.gl/styles';import {FileLoading, FileLoadingProgress, LoadFiles} from '@kepler.gl/types';const StyledLoadDataModal = styled.div.attrs({className: 'load-data-modal'-})`- padding: ${props => props.theme.modalPadding};+})<{$withFooter?: boolean}>`+ padding: 10px 0 ${props => (props.$withFooter ? 0 : '50px')};min-height: 360px;display: flex;flex-direction: column;++ ${props =>+ props.$withFooter+ ? ''+ : media.portable`+ padding-bottom: 34px;+ `}+`;++const AddDataBar = styled.div.attrs({+ className: 'add-data-bar'+})`+ display: flex;+ align-items: center;+ gap: 16px;+ box-sizing: border-box;+ /* Pull the rule out to the dialog edges. Side padding matches the modal. */+ margin: 16px -72px 0;+ padding: 16px 72px;+ border-top: 1px solid #d8d8d8;++ ${media.portable`+ margin-left: -36px;+ margin-right: -36px;+ padding-left: 36px;+ padding-right: 36px;+ `}+`;++const StagedLabel = styled.div`+ min-width: 0;+ flex: 1;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+ color: ${props => props.theme.subtextColorLT};+ font-size: 12px;+`;++const FooterError = styled.div`+ color: ${props => props.theme.negativeBtnColor};+ font-size: 12px;+ max-width: 220px;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+`;++const AddDataActions = styled.div`+ display: flex;+ align-items: center;+ gap: 12px;+ margin-left: auto;+ flex-shrink: 0;`;const noop = () => {@@ -36,13 +94,46 @@ export interface LoadingMethod {tabElementType?: React.ComponentType<{onClick: React.MouseEventHandler; intl: IntlShape}>;}+export type LoadDataOptions = {+ autoCreateLayers?: boolean;+ /** Staged datasets the user left checked. */+ datasets?: Array<{info?: {label?: string}; metadata?: {source?: string}}>;+};++type TilesetDraft = {+ canAdd: boolean;+ loading?: boolean;+ error?: string | null;+ dataset?: {name: string; type: string; metadata: Record<string, any>};+ metadata?: Record<string, any>;+};++function progressErrors(progress: FileLoadingProgress = {}): string[] {+ return Object.values(progress)+ .map(item => {+ const error = item?.error;+ if (!error) {+ return '';+ }+ return typeof error === 'string' ? error : error.message || '';+ })+ .filter(Boolean);+}+type LoadDataModalProps = {// call backsonFileUpload: (files: File[]) => void;+ onAddRemoteDataset?: (remote: {url: string; format?: string}) => void;+ onConfirmAddData?: (options: LoadDataOptions) => void;+ stagedToAdd?: Array<{+ info?: {id?: string; label?: string};+ metadata?: {source?: string};+ }> | null;onLoadCloudMap: (provider: any, vis: any) => void;onTilesetAdded: (tileset: {name: string; type: string; metadata: Record<string, any>},- processedMetadata?: Record<string, any>+ processedMetadata?: Record<string, any>,+ options?: LoadDataOptions) => void;fileLoading: FileLoading | false;loadingMethods?: LoadingMethod[];@@ -96,37 +187,123 @@ export function LoadDataModalFactory(defaultLoadingMethods: LoadDataModalProps['loadingMethods'];} = ({onFileUpload = noop,+ onAddRemoteDataset = noop,onTilesetAdded = noop,+ onConfirmAddData = noop,+ onClose = noop,fileLoading = false,+ fileLoadingProgress = {},+ stagedToAdd = null,loadingMethods = defaultLoadingMethods,isCloudMapLoading,...restProps}) => {const intl = useIntl();+ const [autoCreateLayers, setAutoCreateLayers] = useState(true);+ const [deselectedDatasets, setDeselectedDatasets] = useState<Record<string, boolean>>({});+ const onToggleDataset = useCallback((key: string) => {+ setDeselectedDatasets(prev => ({...prev, [key]: !prev[key]}));+ }, []);+ const [tilesetDraft, setTilesetDraft] = useState<TilesetDraft | null>(null);+ const onToggleAutoCreateLayers = useCallback(() => {+ setAutoCreateLayers(value => !value);+ }, []);+ const handleFileUpload = useCallback(+ (files: File[]) => {+ onFileUpload(files);+ },+ [onFileUpload]+ );+ const handleTilesetAdded = useCallback(+ (+ tileset: {name: string; type: string; metadata: Record<string, any>},+ processedMetadata?: Record<string, any>+ ) => {+ onTilesetAdded(tileset, processedMetadata, {autoCreateLayers});+ },+ [onTilesetAdded, autoCreateLayers]+ );+ const onTilesetDraftChange = useCallback((draft: TilesetDraft) => {+ setTilesetDraft(draft);+ }, []);+ const [currentMethod, toggleMethod] = useState(getDefaultMethod(loadingMethods));+ const selectMethod = useCallback((method: LoadingMethod) => {+ setTilesetDraft(null);+ toggleMethod(method);+ }, []);+ const isUpload = currentMethod?.id === LOADING_METHODS.upload;+ const isTileset = currentMethod?.id === LOADING_METHODS.tileset;+ const showAddBar = isUpload || isTileset;+ const datasetsToAdd = selectedStagedDatasets(stagedToAdd, deselectedDatasets);+ const canAdd = isTileset+ ? Boolean(tilesetDraft?.canAdd) && !tilesetDraft?.loading+ : Boolean(datasetsToAdd.length) && !fileLoading;+ const onAddData = useCallback(() => {+ if (isTileset) {+ if (tilesetDraft?.canAdd && tilesetDraft.dataset) {+ handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);+ }+ return;+ }+ if (datasetsToAdd.length && !fileLoading) {+ onConfirmAddData({autoCreateLayers, datasets: datasetsToAdd});+ }+ }, [+ autoCreateLayers,+ datasetsToAdd,+ fileLoading,+ handleTilesetAdded,+ isTileset,+ onConfirmAddData,+ tilesetDraft+ ]);+ const uploadError = !fileLoading && isUpload ? progressErrors(fileLoadingProgress)[0] : '';+ const tilesetError = isTileset ? tilesetDraft?.error : '';+const currentModalProps = {...restProps,- onFileUpload,- onTilesetAdded,+ onFileUpload: handleFileUpload,+ onTilesetAdded: handleTilesetAdded,fileLoading,- isCloudMapLoading+ fileLoadingProgress,+ isCloudMapLoading,+ ...(isUpload ? {stagedToAdd, onAddRemoteDataset, deselectedDatasets, onToggleDataset} : {}),+ ...(isTileset ? {confirmInParent: true, onTilesetDraftChange} : {})};- // const {loadingMethods, isCloudMapLoading} = props;- const [currentMethod, toggleMethod] = useState(getDefaultMethod(loadingMethods));-const ElementType = currentMethod?.elementType;return (- <StyledLoadDataModal>+ <StyledLoadDataModal $withFooter={showAddBar}><ModalTabscurrentMethod={currentMethod?.id}loadingMethods={loadingMethods}- toggleMethod={toggleMethod}+ toggleMethod={selectMethod}/>{isCloudMapLoading ? (<LoadingDialog size={64} />) : (ElementType && <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />)}+ {showAddBar ? (+ <AddDataBar>+ <AutoCreateLayersCheckbox+ checked={autoCreateLayers}+ onToggle={onToggleAutoCreateLayers}+ />+ <StagedLabel />+ {uploadError || tilesetError ? (+ <FooterError>{uploadError || tilesetError}</FooterError>+ ) : null}+ <AddDataActions>+ <Button type="button" link onClick={onClose}>+ {intl.formatMessage({id: 'modal.button.defaultCancel'})}+ </Button>+ <Button type="button" cta disabled={!canAdd} onClick={onAddData}>+ {intl.formatMessage({id: 'layerManager.addData'})}+ </Button>+ </AddDataActions>+ </AddDataBar>+ ) : null}</StyledLoadDataModal>);};diff --git a/src/components/src/modals/tilesets-modals/load-data-footer.tsx b/src/components/src/modals/tilesets-modals/load-data-footer.tsxindex 981f87b04..32d65d7dc 100644--- a/src/components/src/modals/tilesets-modals/load-data-footer.tsx+++ b/src/components/src/modals/tilesets-modals/load-data-footer.tsx@@ -55,6 +55,11 @@ type ThemeProps = {theme: any;};+const ButtonRow = styled.div`+ display: flex;+ align-items: center;+`;+const LoadDataFooter = ({disabled,intl,@@ -67,7 +72,7 @@ const LoadDataFooter = ({}: LoadDataFooterProps & WrappedComponentProps & ThemeProps) => {return (<LoadDataFooterContainer>- <div>+ <ButtonRow>{prependText}<AddDataButtondisabled={disabled}@@ -86,7 +91,7 @@ const LoadDataFooter = ({})}</AddDataButton>{errorText && <ErrorContainer>{errorText}</ErrorContainer>}- </div>+ </ButtonRow></LoadDataFooterContainer>);};diff --git a/src/components/src/modals/tilesets-modals/load-tileset.tsx b/src/components/src/modals/tilesets-modals/load-tileset.tsxindex 4744db5a9..b3629d25a 100644--- a/src/components/src/modals/tilesets-modals/load-tileset.tsx+++ b/src/components/src/modals/tilesets-modals/load-tileset.tsx@@ -1,7 +1,7 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {useCallback, useMemo, useState} from 'react';+import React, {useCallback, useEffect, useMemo, useState} from 'react';import {IntlShape} from 'react-intl';import JSONPretty from 'react-json-pretty';import {AutoSizer} from 'react-virtualized';@@ -100,6 +100,15 @@ type LoadTilesetTabProps = {isAddingDatasets: boolean;onTilesetAdded: (tilesetInfo: any, metadata?: any) => void;intl: IntlShape;+ /** The parent modal owns Cancel / Add Data, so this tab only reports readiness. */+ confirmInParent?: boolean;+ onTilesetDraftChange?: (draft: {+ canAdd: boolean;+ loading?: boolean;+ error?: string | null;+ dataset?: any;+ metadata?: any;+ }) => void;};const TILE_TYPES = [@@ -140,7 +149,12 @@ function isReady(response) {}function LoadTilesetTabFactory() {- const LoadTilesetTab: React.FC<LoadTilesetTabProps> = ({onTilesetAdded, isAddingDatasets}) => {+ const LoadTilesetTab: React.FC<LoadTilesetTabProps> = ({+ onTilesetAdded,+ isAddingDatasets,+ confirmInParent = false,+ onTilesetDraftChange+ }) => {const [typeIndex, setTypeIndex] = useState<number>(0);const [response, setResponse] = useState<MetaResponse>({});@@ -156,6 +170,16 @@ function LoadTilesetTabFactory() {}}, [onTilesetAdded, response]);+ useEffect(() => {+ onTilesetDraftChange?.({+ canAdd: isReady(response),+ loading: Boolean(response.loading),+ error: error ? getError(error) : null,+ dataset: response.dataset,+ metadata: response.metadata+ });+ }, [error, onTilesetDraftChange, response]);+// temp patch to hide raster tile layer while in developmentconst enableRasterTileLayer = getApplicationConfig().enableRasterTileLayer;const enableWMSLayer = getApplicationConfig().enableWMSLayer;@@ -215,13 +239,15 @@ function LoadTilesetTabFactory() {) : null}</MetaContainer></Container>- <LoadDataFooter- disabled={Boolean(error) || !isReady(response)}- isLoading={loading || isAddingDatasets}- onConfirm={createTileDataset}- confirmText="tilesetSetup.addTilesetText"- errorText={error && getError(error)}- />+ {confirmInParent ? null : (+ <LoadDataFooter+ disabled={Boolean(error) || !isReady(response)}+ isLoading={loading || isAddingDatasets}+ onConfirm={createTileDataset}+ confirmText="tilesetSetup.addTilesetText"+ errorText={error && getError(error)}+ />+ )}</LoadTilesetTabContainer>);};diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.tsindex 4be42b7e6..fbf8f90bf 100644--- a/src/localization/src/translations/en.ts+++ b/src/localization/src/translations/en.ts@@ -804,7 +804,11 @@ export default {loadData: {upload: 'Load Files',tileset: 'Tileset',- storage: 'Load from Storage'+ storage: 'Load from Storage',+ autoCreateLayers: 'Auto create layers',+ autoCreateLayersInfo:+ 'kepler.gl will auto detect layers if the column names in your file follow certain naming conventions.',+ readyToAdd: 'Ready to add'},tripInfo: {title: 'Create trips from GeoJson',@@ -1030,8 +1034,17 @@ ${'```'}fileNotSupported: 'File {errorFiles} is not supported.',or: 'or',dropMessage: 'Drag & Drop Your File(s) Here, {browse}, or',+ toAdd: 'To add to map',+ includeDataset: 'Include {name}',+ readyToAdd: '{size} - Ready to add file to map',+ readyToAddNoSize: 'Ready to add file to map',+ loadingProgress: 'Loading ({loaded} / {total})',+ loading: 'Loading...',+ unableToLoad: 'Unable to load file',+ unsupported: 'Unsupported file type or format',urlPlaceholder: 'Enter your dataset URL',fetch: 'Fetch',+ addUrl: 'Add',format: 'Format',formatAuto: 'Auto',cors: 'The URL must allow [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS).'diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex c80f59dc4..08853b069 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -12,7 +12,12 @@ import {} from '@kepler.gl/utils';import {generateHashId} from '@kepler.gl/common-utils';import {DATASET_FORMATS, DatasetType, REMOTE_FILE_EXTENSIONS} from '@kepler.gl/constants';-import {AddDataToMapPayload, LoadedMap, ProcessorResult} from '@kepler.gl/types';+import {+ AddDataToMapOptions,+ AddDataToMapPayload,+ LoadedMap,+ ProcessorResult+} from '@kepler.gl/types';import {KeplerTable} from '@kepler.gl/table';import type {Feature, FeatureCollection} from 'geojson';@@ -261,7 +266,10 @@ export async function* makeProgressIterator(}// eslint-disable-next-line complexity-function getPersistedRemoteFormat(keplerFormat?: string, fileName?: string): string | undefined {+export function getPersistedRemoteFormat(+ keplerFormat?: string,+ fileName?: string+): string | undefined {if (keplerFormat &&keplerFormat !== 'auto' &&@@ -527,7 +535,10 @@ export async function processFileData({}}-export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPayload[] {+export function filesToDataPayload(+ fileCache: FileCacheItem[],+ options?: AddDataToMapOptions+): AddDataToMapPayload[] {// seperate out files which could be a single datasets. or a keplergl map jsonconst collection = fileCache.reduce<{datasets: FileCacheItem[];@@ -540,7 +551,8 @@ export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPaylaccu.keplerMaps.push({...data,options: {- centerMap: !(data.config && data.config.mapState)+ centerMap: !(data.config && data.config.mapState),+ ...(options || {})}});} else if (DATASET_FORMATS[info?.format]) {@@ -562,5 +574,8 @@ export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPayl// add kepler map first with config// add datasets later in one add data call- return collection.keplerMaps.concat({datasets: collection.datasets});+ return collection.keplerMaps.concat({+ datasets: collection.datasets,+ ...(options && collection.datasets.length ? {options} : {})+ });}diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.tsindex aa0a678c3..728238376 100644--- a/src/processors/src/remote-file.ts+++ b/src/processors/src/remote-file.ts@@ -3,14 +3,23 @@import {parseUri} from '@kepler.gl/common-utils';import {+ DATASET_FORMATS,+ DatasetType,MIME_TO_REMOTE_FILE_EXTENSION,REMOTE_FILE_EXTENSIONS,REMOTE_FILE_MIME_TYPES,RemoteFileFormat} from '@kepler.gl/constants';import {ProcessorResult} from '@kepler.gl/types';+import {generateHashIdFromString} from '@kepler.gl/utils';-import {processFileData, ProcessFileDataContent, readFileInBatches} from './file-handler';+import {+ getPersistedRemoteFormat,+ processFileData,+ ProcessFileDataContent,+ readFileInBatches+} from './file-handler';+import {FileCacheItem} from './types';export type KeplerRemoteFile = File & {keplerSourceUrl?: string;@@ -184,6 +193,34 @@ export async function fetchRemoteFile(};}+/**+ * A remote URL to hold until Add Data. Rows stay empty so adding it runs the+ * externally hosted download instead of parsing a file in the modal.+ */+export function remoteDatasetFromUrl(url: string, format?: string | null): FileCacheItem {+ if (!isRemoteDatasetUrl(url)) {+ throw new Error('Remote dataset URL must use http or https');+ }+ const label = getFileNameForRemoteUrl(url, format);+ const sourceFormat = getPersistedRemoteFormat(+ format && format !== 'auto' ? format : undefined,+ label+ );+ return {+ data: {fields: [], rows: []},+ info: {+ id: generateHashIdFromString(url),+ label,+ format: DATASET_FORMATS.row,+ type: DatasetType.EXTERNALLY_HOSTED+ },+ metadata: {+ source: url,+ ...(sourceFormat ? {sourceFormat} : {})+ }+ };+}+/*** Fetch a remote dataset URL and wrap it as a File the existing loadFiles pipeline can parse.* Sets File.type from an explicit format or the response Content-Type so extensionlessdiff --git a/src/reducers/src/combined-updaters.ts b/src/reducers/src/combined-updaters.tsindex a4e3a403d..839676609 100644--- a/src/reducers/src/combined-updaters.ts+++ b/src/reducers/src/combined-updaters.ts@@ -32,6 +32,7 @@ import {MapStyle} from './map-style-updaters';import {ProviderState} from './provider-state-updaters';import {loadFilesSuccessUpdaterAction,+ StageLoadedFilesUpdaterAction,MapStyleChangeUpdaterAction,LayerTypeChangeUpdaterAction,ToggleSplitMapUpdaterAction,@@ -46,7 +47,8 @@ import {BASE_MAP_COLOR_MODES,OVERLAY_BLENDINGS,NO_MAP_ID,- MapSplitMode+ MapSplitMode,+ ADD_DATA_ID} from '@kepler.gl/constants';import {getBasemapColorsForStyle, DEFAULT_BASEMAP_COLOR} from '@kepler.gl/deckgl-layers';@@ -235,13 +237,17 @@ export const loadFilesSuccessUpdater = (state: KeplerGlState,action: loadFilesSuccessUpdaterAction): KeplerGlState => {+ if (isStaleFileLoadFinish(state.visState.fileLoading, action.loadId)) {+ return state;+ }// still more to load- const payloads = filesToDataPayload(action.result);+ const payloads = filesToDataPayload(action.result, action.options);const nextState = compose_([pick_('visState')(merge_({fileLoading: false,- fileLoadingProgress: {}+ fileLoadingProgress: {},+ stagedToAdd: null}))])(state);@@ -252,6 +258,83 @@ export const loadFilesSuccessUpdater = (return stateWithData as KeplerGlState;};+function keptLoadErrors(progress: Record<string, {error?: unknown}> = {}) {+ return Object.fromEntries(Object.entries(progress).filter(([, value]) => value?.error));+}++/** A finish from the 200ms pause is stale once that load was canceled or replaced. */+function isStaleFileLoadFinish(fileLoading: {loadId?: number} | false, loadId?: number): boolean {+ if (loadId == null) {+ return false;+ }+ return !fileLoading || fileLoading.loadId !== loadId;+}++/**+ * Finish a deferred file load: keep the parsed cache, clear the loading flag,+ * and leave the Add Data modal open until the user confirms.+ * @memberof combinedUpdaters+ * @public+ */+export const stageLoadedFilesUpdater = (+ state: KeplerGlState,+ action: StageLoadedFilesUpdaterAction+): KeplerGlState => {+ if (isStaleFileLoadFinish(state.visState.fileLoading, action.loadId)) {+ return state;+ }+ const modalOpen = state.uiState.currentModal === ADD_DATA_ID;+ const result = action.result || [];+ const previous =+ modalOpen && Array.isArray(state.visState.stagedToAdd) ? state.visState.stagedToAdd : [];+ const stagedToAdd = [...previous, ...result];++ return compose_<KeplerGlState>([+ pick_('visState')(+ merge_({+ fileLoading: false,+ fileLoadingProgress: modalOpen ? keptLoadErrors(state.visState.fileLoadingProgress) : {},+ stagedToAdd: modalOpen && stagedToAdd.length ? stagedToAdd : null+ })+ ),+ pick_('uiState')(apply_(uiStateLoadFilesSuccessUpdater, payload_(null)))+ ])(state);+};++function stagedItemKey(item): string {+ return item?.metadata?.source || item?.info?.id || '';+}++/**+ * Add datasets to the Add Data list without interrupting a file that is still parsing.+ * @memberof combinedUpdaters+ * @public+ */+export const appendStagedLoadedFilesUpdater = (+ state: KeplerGlState,+ action: StageLoadedFilesUpdaterAction+): KeplerGlState => {+ if (state.uiState.currentModal !== ADD_DATA_ID) {+ return state;+ }+ const previous = Array.isArray(state.visState.stagedToAdd) ? state.visState.stagedToAdd : [];+ const seen = new Set(previous.map(stagedItemKey).filter(Boolean));+ const next = (action.result || []).filter(item => {+ const key = stagedItemKey(item);+ if (!key || seen.has(key)) {+ return false;+ }+ seen.add(key);+ return true;+ });+ if (!next.length) {+ return state;+ }+ return compose_<KeplerGlState>([+ pick_('visState')(merge_({stagedToAdd: [...previous, ...next]}))+ ])(state);+};+export const addDataToMapComposed = addDataToMapUpdater;/**diff --git a/src/reducers/src/composers.ts b/src/reducers/src/composers.tsindex 40f53676e..a60722398 100644--- a/src/reducers/src/composers.ts+++ b/src/reducers/src/composers.ts@@ -14,6 +14,8 @@ const actionHandler = {[ActionTypes.SET_MAP_VIEW_MODE]: combinedUpdaters.combinedSetMapViewModeUpdater,[ActionTypes.LAYER_TYPE_CHANGE]: combinedUpdaters.combinedLayerTypeChangeUpdater,[ActionTypes.LOAD_FILES_SUCCESS]: combinedUpdaters.loadFilesSuccessUpdater,+ [ActionTypes.STAGE_LOADED_FILES]: combinedUpdaters.stageLoadedFilesUpdater,+ [ActionTypes.APPEND_STAGED_LOADED_FILES]: combinedUpdaters.appendStagedLoadedFilesUpdater,[ActionTypes.TOGGLE_SPLIT_MAP]: combinedUpdaters.toggleSplitMapUpdater,[ActionTypes.SET_MAP_SPLIT_MODE]: combinedUpdaters.setMapSplitModeUpdater,[ActionTypes.REPLACE_DATA_IN_MAP]: combinedUpdaters.replaceDataInMapUpdaterdiff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex f41370154..541965302 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -423,6 +423,7 @@ export const INITIAL_VIS_STATE: VisState = {fileLoading: false,fileLoadingProgress: {},+ stagedToAdd: null,// for loading datasetsloadingIndicatorValue: 0,loadingProgress: {},@@ -3786,18 +3787,20 @@ export const updateVisDataUpdater = (return updatedState;}- const createDatasetTasks: TaskDescriptor[] = [];+ const readyTasks: TaskDescriptor[] = [];+ const remoteTasks: TaskDescriptor[] = [];const notificationTasks: TaskDescriptor[] = [];const hydrateProgress: Record<string, number> = {};datasets.forEach(({info = {}, ...rest}, datasetIndex) => {const proto = {info, ...rest};- if (needsExternallyHostedHydrate(proto)) {+ const remote = needsExternallyHostedHydrate(proto);+ if (remote) {hydrateProgress[getDatasetProgressId(proto, datasetIndex)] = 0;}const task = createNewDataEntryTask(proto, state.datasets);if (task) {- createDatasetTasks.push(task);+ (remote ? remoteTasks : readyTasks).push(task);} else {notificationTasks.push(ACTION_TASK_ADD_NOTIFICATION().map(() =>@@ -3812,15 +3815,22 @@ export const updateVisDataUpdater = (}});+ // Parsed datasets are committed on their own. A remote download in the same+ // add must not hold them until the URL finishes.const progressIds = Object.keys(hydrateProgress);- const datasetsAllSettledTask = createDatasetTasks.length- ? Task.allSettled(createDatasetTasks).map(results =>- createNewDatasetSuccess({results, addToMapOptions: options, progressIds})- )- : null;+ const batches: Array<{tasks: TaskDescriptor[]; progressIds: string[]}> = [];+ if (readyTasks.length) {+ batches.push({tasks: readyTasks, progressIds: []});+ }+ if (remoteTasks.length) {+ batches.push({tasks: remoteTasks, progressIds});+ }- if (datasetsAllSettledTask) {- updatedState = setLoadingIndicatorUpdater(updatedState, payload_({change: 1, type: ''}));+ if (batches.length) {+ updatedState = setLoadingIndicatorUpdater(+ updatedState,+ payload_({change: batches.length, type: ''})+ );if (progressIds.length) {updatedState = {...updatedState,@@ -3833,7 +3843,15 @@ export const updateVisDataUpdater = (}return withTask(updatedState, [- ...(datasetsAllSettledTask ? [datasetsAllSettledTask] : []),+ ...batches.map(batch =>+ Task.allSettled(batch.tasks).map(results =>+ createNewDatasetSuccess({+ results,+ addToMapOptions: options,+ progressIds: batch.progressIds+ })+ )+ ),...notificationTasks]);};@@ -4425,38 +4443,114 @@ export function closeSpecificMapAtIndex<S extends VisState>(* @memberof visStateUpdaters* @public*/+let nextFileLoadId = 1;++function createFileLoadId(): number {+ const loadId = nextFileLoadId;+ nextFileLoadId += 1;+ return loadId;+}++function finishFileLoad(+ onFinish: (payload: any, options?: VisStateActions.LoadFilesOptions) => any,+ fileCache: any[],+ options: VisStateActions.LoadFilesOptions | undefined,+ loadId: number+) {+ const action = options ? onFinish(fileCache, options) : onFinish(fileCache);+ return {...action, loadId};+}+export const loadFilesUpdater = (state: VisState,action: VisStateActions.LoadFilesUpdaterAction): VisState => {- const {files, onFinish = loadFilesSuccess} = action;+ const {files, onFinish = loadFilesSuccess, options} = action;if (!files.length) {return state;}const companionFiles = Array.from(files);const filesToLoad = getFilesToParse(companionFiles);++ // Another drop while a deferred parse is running must join that load.+ // Replacing `fileLoading` makes the in-flight file finish against the new+ // cache and drop the files that were already being parsed.+ if (state.fileLoading && state.fileLoading.options?.deferAddToMap && options?.deferAddToMap) {+ return appendDeferredFileLoad(state, {filesToLoad, companionFiles});+ }+if (!filesToLoad.length) {return state;}+ // A deferred load is another batch for the same Add Data click. Keep files+ // already staged, and keep their error cards, instead of starting over.+ const previousErrors =+ options?.deferAddToMap && state.fileLoadingProgress+ ? Object.fromEntries(+ Object.entries(state.fileLoadingProgress).filter(([, value]) => value?.error)+ )+ : {};+const fileLoadingProgress = filesToLoad.reduce((accu, f, i) => merge_(initialFileLoadingProgress(f, i))(accu),- {}+ previousErrors);const fileLoading = {+ loadId: createFileLoadId(),fileCache: [],filesToLoad,companionFiles,- onFinish+ onFinish,+ ...(options ? {options} : {})};- const nextState = merge_({fileLoadingProgress, fileLoading})(state);+ const nextState = merge_({+ fileLoadingProgress,+ fileLoading,+ ...(options?.deferAddToMap ? {} : {stagedToAdd: null})+ })(state);return loadNextFileUpdater(nextState);};+/**+ * Add a drop to the deferred load that is already parsing.+ * The current file keeps its cache; the new files run after it.+ */+function appendDeferredFileLoad(+ state: VisState,+ {+ filesToLoad,+ companionFiles+ }: {+ filesToLoad: File[];+ companionFiles: File[];+ }+): VisState {+ const loading = state.fileLoading;+ if (!loading || (!filesToLoad.length && !companionFiles.length)) {+ return state;+ }++ const fileLoadingProgress = filesToLoad.reduce(+ (accu, file, index) => merge_(initialFileLoadingProgress(file, index))(accu),+ state.fileLoadingProgress+ );++ return {+ ...state,+ fileLoadingProgress,+ fileLoading: {+ ...loading,+ filesToLoad: [...Array.from(loading.filesToLoad), ...filesToLoad],+ companionFiles: [...(loading.companionFiles || []), ...companionFiles]+ }+ };+}+/*** Sucessfully loaded one file, move on to the next one* @memberof visStateUpdaters@@ -4470,7 +4564,7 @@ export function loadFileStepSuccessUpdater(return state;}const {fileName, fileCache} = action;- const {filesToLoad, onFinish} = state.fileLoading;+ const loadId = state.fileLoading.loadId;const stateWithProgress = updateFileLoadingProgressUpdater(state, {fileName,progress: {percent: 1, message: 'Done'}@@ -4479,9 +4573,11 @@ export function loadFileStepSuccessUpdater(// save processed file to fileCacheconst stateWithCache = pick_('fileLoading')(merge_({fileCache}))(stateWithProgress);+ // Decide after the pause, and only for this load. A drop during these+ // 200ms joins the queue; cancel or a newer load makes this continuation stale.return withTask(stateWithCache,- DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))+ DELAY_TASK(200).map(() => loadNextFile(loadId)));}@@ -4492,11 +4588,18 @@ export function loadFileStepSuccessUpdater(* @memberof visStateUpdaters* @public*/-export function loadNextFileUpdater(state: VisState): VisState {- if (!state.fileLoading) {+export function loadNextFileUpdater(state: VisState, action?: {loadId?: number}): VisState {+ const loadId = action?.loadId;+ if (!state.fileLoading || (loadId != null && state.fileLoading.loadId !== loadId)) {return state;}- const {filesToLoad} = state.fileLoading;+ const {filesToLoad, onFinish, fileCache, options, loadId: currentLoadId} = state.fileLoading;+ if (!filesToLoad.length) {+ return withTask(+ state,+ ACTION_TASK().map(() => finishFileLoad(onFinish, fileCache, options, currentLoadId))+ );+ }const [file, ...remainingFilesToLoad] = filesToLoad;// save filesToLoad to state@@ -4515,7 +4618,8 @@ export function loadNextFileUpdater(state: VisState): VisState {nextState.fileLoading && nextState.fileLoading.fileCache,loaders,loadOptions,- nextState.fileLoading ? nextState.fileLoading.companionFiles : undefined+ nextState.fileLoading ? nextState.fileLoading.companionFiles : undefined,+ options));}@@ -4525,7 +4629,8 @@ export function makeLoadFileTask(fileCache,loaders: Loader[] = [],loadOptions = {},- companionFiles?: File[]+ companionFiles?: File[],+ addDataOptions?: VisStateActions.LoadFilesOptions) {return LOAD_FILE_TASK({file, fileCache, loaders, loadOptions, companionFiles}).bimap(// prettier ignore@@ -4538,7 +4643,8 @@ export function makeLoadFileTask(processFileContent({content: result,fileCache- })+ }),+ ...(addDataOptions ? {options: addDataOptions} : {})}),// error@@ -4592,7 +4698,7 @@ export function parseProgress(prevProgress = {}, progress) {export const nextFileBatchUpdater = (state: VisState,{- payload: {gen, fileName, progress, accumulated, onFinish}+ payload: {gen, fileName, progress, accumulated, onFinish, options}}: VisStateActions.NextFileBatchUpdaterAction): VisState => {const stateWithProgress = updateFileLoadingProgressUpdater(state, {@@ -4600,8 +4706,11 @@ export const nextFileBatchUpdater = (progress: parseProgress(state.fileLoadingProgress[fileName], progress)});+ const deferAddToMap = Boolean(options?.deferAddToMap);+return withTask(stateWithProgress, [...(getApplicationConfig().useArrowProgressiveLoading &&+ !deferAddToMap &&fileName.endsWith('arrow') &&accumulated?.data?.length > 0? [@@ -4611,7 +4720,7 @@ export const nextFileBatchUpdater = (content: {...accumulated, skipArrowCompact: true},fileCache: []}).bimap(- result => loadFilesSuccess(result),+ result => (options ? loadFilesSuccess(result, options) : loadFilesSuccess(result)),err => loadFilesErr(fileName, err))]@@ -4625,7 +4734,8 @@ export const nextFileBatchUpdater = (fileName,progress: value.progress,accumulated: value,- onFinish+ onFinish,+ ...(options ? {options} : {})});},err => loadFilesErr(fileName, err)@@ -4638,6 +4748,18 @@ export const nextFileBatchUpdater = (* @memberof visStateUpdaters* @public*/+/**+ * Drop files waiting on the Add Data button and abort an in-progress modal load.+ * @memberof visStateUpdaters+ * @public+ */+export const clearStagedLoadedFilesUpdater = (state: VisState): VisState => ({+ ...state,+ fileLoading: false,+ fileLoadingProgress: {},+ stagedToAdd: null+});+export const loadFilesErrUpdater = (state: VisState,{error, fileName}: VisStateActions.LoadFilesErrUpdaterAction@@ -4647,17 +4769,17 @@ export const loadFilesErrUpdater = (if (!state.fileLoading) {return state;}- const {filesToLoad, onFinish, fileCache} = state.fileLoading;+ const loadId = state.fileLoading.loadId;const nextState = updateFileLoadingProgressUpdater(state, {fileName,progress: {error}});- // kick off next file or finish+ // Same pause as a successful file. Re-check this load's queue when it ends.return withTask(nextState,- DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))+ DELAY_TASK(200).map(() => loadNextFile(loadId)));};diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex 07b5dabdf..863900b8b 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -54,6 +54,8 @@ const actionHandler = {[ActionTypes.LOAD_FILES_ERR]: visStateUpdaters.loadFilesErrUpdater,+ [ActionTypes.CLEAR_STAGED_LOADED_FILES]: visStateUpdaters.clearStagedLoadedFilesUpdater,+[ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,[ActionTypes.LOAD_FILE_STEP_SUCCESS]: visStateUpdaters.loadFileStepSuccessUpdater,diff --git a/src/schemas/src/vis-state-schema.ts b/src/schemas/src/vis-state-schema.tsindex 682c5f0e1..c749d7a75 100644--- a/src/schemas/src/vis-state-schema.ts+++ b/src/schemas/src/vis-state-schema.ts@@ -92,6 +92,8 @@ export interface VisState {splitMapsToBeMerged: SplitMap[];fileLoading: FileLoading | false;fileLoadingProgress: FileLoadingProgress;+ /** Parsed files held until the Add Data button commits them. */+ stagedToAdd: any[] | null;loadingIndicatorValue: number;/** Per-dataset download progress (0–100) while hydrating remote files. */loadingProgress: Record<string, number>;diff --git a/src/types/reducers.d.ts b/src/types/reducers.d.tsindex 6dc4873b8..40d2650fb 100644--- a/src/types/reducers.d.ts+++ b/src/types/reducers.d.ts@@ -422,11 +422,21 @@ export type MapInfo = {title: string;description: string;};+/** Options captured when a file load starts and forwarded to `addDataToMap`. */+export type FileLoadingOptions = {+ autoCreateLayers?: boolean;+ /** Parse the files, then wait for an explicit add instead of calling `addDataToMap`. */+ deferAddToMap?: boolean;+};+export type FileLoading = {+ /** Identifies this load so a delayed finish from an older one can be ignored. */+ loadId: number;filesToLoad: FileList | File[];companionFiles?: File[];- onFinish: (payload: any) => any;+ onFinish: (payload: any, options?: FileLoadingOptions) => any;fileCache: any[];+ options?: FileLoadingOptions;};export type FileLoadingProgress = {[key: string]: {diff --git a/test/browser/components/common/file-uploader-test.js b/test/browser/components/common/file-uploader-test.jsindex 0a758b0c2..e1169e6cf 100644--- a/test/browser/components/common/file-uploader-test.js+++ b/test/browser/components/common/file-uploader-test.js@@ -6,7 +6,7 @@ import test from 'tape';import sinon from 'sinon';import {IntlWrapper, mountWithTheme} from 'test/helpers/component-utils';-import {FileUpload, WarningMsg, FileDrop, UploadButton} from '@kepler.gl/components';+import {FileUpload, FileDrop, UploadButton} from '@kepler.gl/components';import {initApplicationConfig} from '@kepler.gl/utils';test('Components -> FileUploader.render', t => {@@ -101,7 +101,109 @@ test('Components -> FileUpload.onDrop -> render loading msg', t => {const uploadMsg = wrapper.find('.file-upload-progress__message').at(0).html();t.comment(uploadMsg);- t.ok(uploadMsg.includes('tst-file.csv', 'should render upload file msg'));+ t.ok(uploadMsg.includes('tst-file.csv'), 'should render upload file msg');+ t.equal(+ wrapper.find('.upload-file-list').hostNodes().length,+ 1,+ 'should list the file beside the drop zone'+ );++ t.end();+});++test('Components -> FileUpload.onDrop keeps earlier files', t => {+ const first = [{type: 'text/csv', name: 'first.csv', size: 10}];+ const second = [{type: 'text/csv', name: 'second.csv', size: 20}];+ const onFileUpload = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <FileUpload+ onFileUpload={onFileUpload}+ fileExtensions={['csv']}+ stagedToAdd={[{info: {label: 'kept.csv', format: 'csv'}}]}+ />+ </IntlWrapper>+ );+ const drop = files => {+ wrapper+ .find('.file-uploader__file-drop')+ .at(0)+ .simulate('drop', {+ stopPropagation: () => {},+ dataTransfer: {types: ['Files'], files}+ });+ };++ drop(first);+ drop(second);++ const files = wrapper.find(FileUpload).children().first().state().files;+ t.deepEqual(+ files.map(file => file.name),+ ['first.csv', 'second.csv'],+ 'should append dropped files'+ );+ t.deepEqual(onFileUpload.lastCall.args[0], second, 'should upload only the new files');+ const listText = wrapper.find('.upload-file-list').text();+ t.ok(listText.includes('kept.csv'), 'should keep a file staged before this uploader mounted');+ t.ok(listText.includes('first.csv'), 'should keep the first dropped file');+ t.ok(listText.includes('second.csv'), 'should show the latest dropped file');+ const fileChecks = wrapper.find('.upload-file-list').find('Checkbox');+ t.equal(fileChecks.length, 3, 'each dataset card has a checkbox');+ t.equal(fileChecks.at(0).props().checked, true, 'datasets start selected');+ t.equal(+ fileChecks.at(0).find('.upload-file-list__check-label').text(),+ 'Include kept.csv',+ 'checkbox names the dataset it includes'+ );+ fileChecks.at(0).simulate('change');+ t.equal(+ wrapper.find('.upload-file-list').find('Checkbox').at(0).props().checked,+ false,+ 'should let the user deselect a dataset'+ );++ t.end();+});++test('Components -> FileUpload remote URL is staged without downloading', t => {+ const onFileUpload = sinon.spy();+ const onAddRemoteDataset = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <FileUpload+ onFileUpload={onFileUpload}+ onAddRemoteDataset={onAddRemoteDataset}+ fileExtensions={['csv']}+ stagedToAdd={[+ {+ info: {label: 'quakes.csv', format: 'row'},+ metadata: {source: 'https://example.com/quakes.csv'}+ }+ ]}+ />+ </IntlWrapper>+ );++ const listText = wrapper.find('.upload-file-list').text();+ t.ok(listText.includes('quakes.csv'), 'should list the remote dataset');+ t.ok(listText.includes('https://example.com/quakes.csv'), 'should show the remote url');++ const urlInput = wrapper.find('.file-uploader__remote-url input').hostNodes().first();+ urlInput.simulate('change', {target: {value: 'https://example.com/cities.csv'}});+ wrapper.find('.file-uploader__remote-add').hostNodes().first().simulate('click');++ t.ok(onFileUpload.notCalled, 'should not download the url through the file loader');+ t.deepEqual(+ onAddRemoteDataset.lastCall.args[0],+ {url: 'https://example.com/cities.csv'},+ 'should stage the url for Add Data'+ );+ t.equal(+ wrapper.find(FileUpload).children().first().state().remoteUrl,+ '',+ 'should clear the url field after adding it'+ );t.end();});@@ -131,7 +233,15 @@ test('Components -> FileUpload.onDrop -> render error msg', t => {FileDropDiv.simulate('drop', mockEvent);t.ok(onFileUpload.notCalled, 'onFileUpload should not get called');- t.ok(wrapper.find(WarningMsg), 'should render WarningMsg');+ t.ok(+ wrapper.find('.upload-file-list').text().includes('tst-file.png'),+ 'should list the unsupported file'+ );+ t.equal(+ wrapper.find('.upload-file-list').find('Checkbox').length,+ 0,+ 'unsupported files cannot be selected'+ );const errorFiles = wrapper.find(FileUpload).children().first().state().errorFiles;t.deepEqual(errorFiles, ['tst-file.png'], 'should save files to errorFiles');diff --git a/test/browser/components/modals/load-data-modal-test.js b/test/browser/components/modals/load-data-modal-test.jsindex 20aca1910..e9e3515b9 100644--- a/test/browser/components/modals/load-data-modal-test.js+++ b/test/browser/components/modals/load-data-modal-test.js@@ -3,17 +3,20 @@import React from 'react';import test from 'tape';+import sinon from 'sinon';import {IntlWrapper, mountWithTheme} from 'test/helpers/component-utils';import {STYLED_COMPONENTS_DUPLICATED_ENTRIES} from '../../../helpers/utils';import {LoadDataModalFactory,ModalTabItem,LoadStorageMapFactory,+ FileUploadFactory,appInjector} from '@kepler.gl/components';const LoadDataModal = appInjector.get(LoadDataModalFactory);const LoadStorageMap = appInjector.get(LoadStorageMapFactory);+const FileUpload = appInjector.get(FileUploadFactory);test('Components -> LoadDataModal.mount', t => {// mount@@ -68,3 +71,83 @@ test('Components -> LoadDataModal -> custom loading method', t => {t.equal(wrapper.find(MockTabComp).length, 1, 'should render MockTabComp');t.end();});++test('Components -> LoadDataModal -> auto create layers', t => {+ const onFileUpload = sinon.spy();+ const onTilesetAdded = sinon.spy();+ const onConfirmAddData = sinon.spy();+ const onClose = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <LoadDataModal+ onFileUpload={onFileUpload}+ onTilesetAdded={onTilesetAdded}+ onConfirmAddData={onConfirmAddData}+ onClose={onClose}+ stagedToAdd={[{info: {label: 'points.csv'}}]}+ />+ </IntlWrapper>+ );++ const checkbox = () => wrapper.find('.add-data-bar').find('Checkbox').first();+ t.equal(checkbox().props().checked, true, 'auto create layers is checked by default');+ t.equal(+ wrapper.find('.file-uploader .auto-create-layers').length,+ 0,+ 'checkbox sits in the footer'+ );++ checkbox().simulate('change');+ t.equal(checkbox().props().checked, false, 'should uncheck auto create layers');++ const files = [{name: 'points.csv'}];+ const uploader = wrapper.find(FileUpload);+ t.ok(uploader.length > 0, 'should find the file uploader rendered by the modal');+ uploader.first().props().onFileUpload(files);+ t.deepEqual(onFileUpload.args[0][0], files, 'should upload the selected files');+ t.equal(onFileUpload.args[0][1], undefined, 'parsing waits to apply autoCreateLayers');++ const buttons = wrapper.find('.add-data-bar button');+ buttons.at(0).simulate('click');+ t.equal(onClose.calledOnce, true, 'cancel closes the modal');++ buttons.at(1).simulate('click');+ t.deepEqual(+ onConfirmAddData.args[0][0],+ {autoCreateLayers: false, datasets: [{info: {label: 'points.csv'}}]},+ 'Add Data commits the current autoCreateLayers choice'+ );++ const fileCheckbox = wrapper.find('.upload-file-list').find('Checkbox').first();+ t.equal(fileCheckbox.props().checked, true, 'staged datasets start selected');+ fileCheckbox.simulate('change');+ t.equal(+ wrapper.find('.add-data-bar button').at(1).props().disabled,+ true,+ 'Add Data waits until a dataset is checked'+ );+ wrapper.find('.add-data-bar button').at(1).simulate('click');+ t.equal(onConfirmAddData.callCount, 1, 'an unchecked dataset is not added');++ const tabs = wrapper.find('.load-data-modal__tab__item');+ let clickedTileset = false;+ for (let i = 0; i < tabs.length; i++) {+ if (tabs.at(i).text() === 'Tileset') {+ tabs.at(i).simulate('click');+ clickedTileset = true;+ break;+ }+ }+ t.equal(clickedTileset, true, 'should find the tileset tab');+ t.equal(+ wrapper.find('.add-data-bar').find('Checkbox').first().props().checked,+ false,+ 'tileset tab keeps the unchecked choice'+ );+ t.equal(wrapper.find('.load-data-footer').length, 0, 'tileset confirm uses the shared footer');+ const tilesetButtons = wrapper.find('.add-data-bar button');+ t.equal(tilesetButtons.at(1).props().disabled, true, 'Add Data waits until the tileset is ready');+ t.equal(onTilesetAdded.called, false, 'opening the tileset tab does not add data');++ t.end();+});diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.jsindex d734fec8d..f533681b9 100644--- a/test/node/processors/file-handler-test.js+++ b/test/node/processors/file-handler-test.js@@ -7,6 +7,7 @@ import {makeProgressIterator,filesToDataPayload,processFileData,+ remoteDatasetFromUrl,processArrowBatches,getGeoJsonFromLoaderResult,readBatch@@ -167,6 +168,55 @@ test('#file-handler -> filesToDataPayload', t => {t.end();});+test('#file-handler -> filesToDataPayload autoCreateLayers', t => {+ const fileCache = [+ {+ data: {+ fields: parsedFields,+ rows: parsedRows+ },+ info: {label: 'rows-data.json', format: 'row'}+ },+ {+ data: {+ datasets: [],+ config: keplerglMap.config+ },+ info: {label: 'keplergl-map.json', format: 'keplergl'}+ }+ ];++ const result = filesToDataPayload(fileCache, {autoCreateLayers: false});++ t.deepEqual(+ result[0].options,+ {centerMap: true, autoCreateLayers: false},+ 'should keep kepler map centerMap and forward autoCreateLayers'+ );+ t.deepEqual(+ result[1].options,+ {autoCreateLayers: false},+ 'should forward autoCreateLayers for dataset files'+ );++ t.end();+});++test('#file-handler -> remoteDatasetFromUrl waits for the remote download', t => {+ const dataset = remoteDatasetFromUrl('https://example.com/quakes.csv?sv=1', 'csv');+ t.equal(dataset.info.type, 'externally-hosted', 'should be a remote dataset');+ t.equal(dataset.info.label, 'quakes.csv', 'should name the card from the url');+ t.deepEqual(dataset.data, {fields: [], rows: []}, 'should not include downloaded rows');+ t.equal(dataset.metadata.source, 'https://example.com/quakes.csv?sv=1');+ t.equal(dataset.metadata.sourceFormat, 'csv', 'should keep the selected format');++ const payload = filesToDataPayload([dataset]);+ t.equal(payload[0].datasets[0].info.type, 'externally-hosted', 'Add Data keeps the remote type');+ t.deepEqual(payload[0].datasets[0].data, {fields: [], rows: []}, 'Add Data still has no rows');++ t.end();+});+test('#file-handler -> filesToDataPayload remote metadata', t => {const fileCache = [{diff --git a/test/node/reducers/composer-state-test.js b/test/node/reducers/composer-state-test.jsindex 6fd21e2ce..b3897e096 100644--- a/test/node/reducers/composer-state-test.js+++ b/test/node/reducers/composer-state-test.js@@ -6,7 +6,7 @@import {drainTasksForTesting, succeedTaskWithValues} from '@kepler.gl/tasks';import test from 'tape';-import {registerEntry} from '@kepler.gl/actions';+import {appendStagedLoadedFiles, registerEntry, stageLoadedFiles} from '@kepler.gl/actions';import {processCsvData} from '@kepler.gl/processors';import keplerGlReducer, {addDataToMapUpdater,@@ -1178,3 +1178,106 @@ test('#composerStateReducer - replaceDataInMapUpdater: syncedTimeFilter & no matt.deepEqual(nextState.visState.splitMapsToBeMerged, [], 'should reset splitMapsToBeMerged');t.end();});++test('#composerStateReducer -> stageLoadedFilesUpdater keeps the modal open', t => {+ const base = keplerGlReducer(undefined, registerEntry({id: 'test'})).test;+ const result = [{info: {label: 'points.csv', format: 'csv'}, data: []}];+ const loadingState = {+ ...base,+ uiState: {+ ...base.uiState,+ currentModal: 'addData',+ loadFiles: {fileLoading: true}+ },+ visState: {+ ...base.visState,+ fileLoading: {filesToLoad: [], fileCache: [], onFinish: () => null},+ fileLoadingProgress: {+ a: {percent: 1, message: 'Done', fileName: 'points.csv', error: null}+ }+ }+ };++ const staged = combinedUpdaters.stageLoadedFilesUpdater(loadingState, stageLoadedFiles(result));+ t.equal(staged.uiState.currentModal, 'addData', 'should leave the Add Data modal open');+ t.equal(staged.uiState.loadFiles.fileLoading, false, 'should clear the loading flag');+ t.equal(staged.visState.fileLoading, false, 'should finish the visState load');+ t.equal(staged.visState.stagedToAdd[0].info.label, 'points.csv', 'should keep the parsed file');+ t.equal(staged.visState.datasets, base.visState.datasets, 'should not add datasets yet');++ const closed = {+ ...loadingState,+ uiState: {...loadingState.uiState, currentModal: null}+ };+ const dropped = combinedUpdaters.stageLoadedFilesUpdater(closed, stageLoadedFiles(result));+ t.equal(+ dropped.visState.stagedToAdd,+ null,+ 'should discard files when the modal is already closed'+ );++ const withPrevious = {+ ...loadingState,+ visState: {+ ...loadingState.visState,+ stagedToAdd: [{info: {label: 'points.csv', format: 'csv'}, data: []}]+ }+ };+ const appended = combinedUpdaters.stageLoadedFilesUpdater(+ withPrevious,+ stageLoadedFiles([{info: {label: 'cities.csv', format: 'csv'}, data: []}])+ );+ t.deepEqual(+ appended.visState.stagedToAdd.map(item => item.info.label),+ ['points.csv', 'cities.csv'],+ 'should append a later file instead of replacing the staged list'+ );++ const replacedLoad = {+ ...loadingState,+ visState: {+ ...loadingState.visState,+ fileLoading: {...loadingState.visState.fileLoading, loadId: 2},+ stagedToAdd: null+ }+ };+ const staleFinish = combinedUpdaters.stageLoadedFilesUpdater(replacedLoad, {+ ...stageLoadedFiles(result),+ loadId: 1+ });+ t.equal(staleFinish, replacedLoad, 'a finish from a canceled load does not replace the new one');++ const canceledLoad = {+ ...loadingState,+ visState: {...loadingState.visState, fileLoading: false, stagedToAdd: null}+ };+ const afterCancel = combinedUpdaters.stageLoadedFilesUpdater(canceledLoad, {+ ...stageLoadedFiles(result),+ loadId: 1+ });+ t.equal(afterCancel.visState.fileLoading, false, 'a canceled load stays stopped');+ t.equal(afterCancel.visState.stagedToAdd, null, 'a canceled load is not restored');++ const remote = {+ data: {fields: [], rows: []},+ info: {id: 'remote-1', label: 'quakes.csv', format: 'row', type: 'externally-hosted'},+ metadata: {source: 'https://example.com/quakes.csv', sourceFormat: 'csv'}+ };+ const withRemote = combinedUpdaters.appendStagedLoadedFilesUpdater(+ appended,+ appendStagedLoadedFiles([remote])+ );+ t.deepEqual(+ withRemote.visState.stagedToAdd.map(item => item.info.label),+ ['points.csv', 'cities.csv', 'quakes.csv'],+ 'should add a remote url without replacing staged files'+ );+ t.equal(withRemote.visState.fileLoading, false, 'should not start a file load for a remote url');+ const duplicate = combinedUpdaters.appendStagedLoadedFilesUpdater(+ withRemote,+ appendStagedLoadedFiles([remote])+ );+ t.equal(duplicate, withRemote, 'should ignore a url that is already staged');++ t.end();+});diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 43a181455..3af03dee8 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -26,7 +26,7 @@ import {calculateLayerData} from '@kepler.gl/reducers';-import {processCsvData, processGeojson} from '@kepler.gl/processors';+import {processCsvData, processGeojson, remoteDatasetFromUrl} from '@kepler.gl/processors';import {Layer, KeplerGlLayers, COLUMN_MODE_TABLE} from '@kepler.gl/layers';import {maybeToDate} from '@kepler.gl/table';import {@@ -6769,7 +6769,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {'test-file-2.csv': {percent: 0, message: '', fileName: 'test-file-2.csv', error: null}};- t.deepEqual(nextState.fileLoading, expectedFileLoading, 'should save fileLoading in state');+ t.deepEqual(+ nextState.fileLoading,+ {...expectedFileLoading, loadId: nextState.fileLoading.loadId},+ 'should save fileLoading in state'+ );+ t.equal(typeof nextState.fileLoading.loadId, 'number', 'should assign a load session');t.deepEqual(nextState.fileLoadingProgress,expectedFileLoadingProgress,@@ -6845,6 +6850,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.deepEqual(nextState3Err.fileLoading,{+ loadId: nextState.fileLoading.loadId,fileCache: [],filesToLoad: [],companionFiles: mockFiles,@@ -6930,6 +6936,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.deepEqual(resultState5.fileLoading,{+ loadId: nextState.fileLoading.loadId,fileCache: fileProcessResult,filesToLoad: [{type: 'text/csv', name: 'test-file-2.csv'}],companionFiles: mockFiles,@@ -6969,6 +6976,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.deepEqual(resultState6.fileLoading,{+ loadId: nextState.fileLoading.loadId,fileCache: fileProcessResult,filesToLoad: [],companionFiles: mockFiles,@@ -7012,6 +7020,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.deepEqual(resultState10.fileLoading,{+ loadId: nextState.fileLoading.loadId,fileCache: file2ProcessResult,filesToLoad: [],companionFiles: mockFiles,@@ -7024,7 +7033,11 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.equal(task10.type, 'DELAY_TASK', 'should return an DELAY_TASK for onFinish');// calling delayed task succeed to trigger load next file- const _resultState11 = reducer(resultState10, succeedTaskInTest(task10));+ const resultState11 = reducer(resultState10, succeedTaskInTest(task10));+ const [task11, ...more11] = drainTasksForTesting();+ t.equal(more11.length, 0, 'should create 1 task');+ t.equal(task11.type, 'ACTION_TASK', 'an empty queue finishes the load');+ reducer(resultState11, succeedTaskInTest(task11));t.ok(loadFilesSuccessSpy.calledOnce);const expectedArgs = [@@ -7038,6 +7051,207 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.end();});+test('#visStateReducer -> LOAD_FILES autoCreateLayers option', t => {+ drainTasksForTesting();++ const initialState = CloneDeep(InitialState).visState;+ const mockFiles = [{type: 'text/csv', name: 'test-file.csv'}];+ const nextState = reducer(+ initialState,+ VisStateActions.loadFiles(mockFiles, undefined, {autoCreateLayers: false})+ );++ t.equal(+ nextState.fileLoading.options.autoCreateLayers,+ false,+ 'should keep autoCreateLayers on the in-progress file load'+ );++ drainTasksForTesting();+ t.end();+});++test('#visStateReducer -> LOAD_FILES deferAddToMap', t => {+ drainTasksForTesting();++ const initialState = CloneDeep(InitialState).visState;+ const mockFiles = [{type: 'text/csv', name: 'test-file.csv'}];+ const nextState = reducer(+ initialState,+ VisStateActions.loadFiles(mockFiles, VisStateActions.stageLoadedFiles, {deferAddToMap: true})+ );++ t.equal(+ nextState.fileLoading.onFinish,+ VisStateActions.stageLoadedFiles,+ 'should stage instead of adding'+ );+ t.equal(+ nextState.fileLoading.options.deferAddToMap,+ true,+ 'should keep deferAddToMap on the load'+ );++ const alreadyStaged = [{info: {label: 'points.csv'}}];+ const withStaged = {...initialState, stagedToAdd: alreadyStaged};+ const deferred = reducer(+ withStaged,+ VisStateActions.loadFiles(mockFiles, VisStateActions.stageLoadedFiles, {deferAddToMap: true})+ );+ t.equal(deferred.stagedToAdd, alreadyStaged, 'should keep files already staged for Add Data');++ const replaced = reducer(withStaged, VisStateActions.loadFiles(mockFiles));+ t.equal(replaced.stagedToAdd, null, 'should clear staged files when adding them immediately');++ drainTasksForTesting();+ const remote = remoteDatasetFromUrl('https://example.com/quakes.csv');+ const hydrating = reducer(+ CloneDeep(InitialState).visState,+ VisStateActions.updateVisData([remote])+ );+ t.equal(+ hydrating.loadingProgress[remote.info.id],+ 0,+ 'adding a staged remote url should start the remote download'+ );+ t.equal(+ hydrating.datasets[remote.info.id],+ undefined,+ 'should not insert rows before the download'+ );+ drainTasksForTesting();++ const queued = reducer(+ nextState,+ VisStateActions.loadFiles(+ [{type: 'text/csv', name: 'extra.csv'}],+ VisStateActions.stageLoadedFiles,+ {+ deferAddToMap: true+ }+ )+ );+ t.deepEqual(+ queued.fileLoading.filesToLoad.map(file => file.name),+ ['extra.csv'],+ 'a second drop waits behind the file already parsing'+ );+ t.equal(+ queued.fileLoading.fileCache,+ nextState.fileLoading.fileCache,+ 'should keep the in-progress cache'+ );+ t.ok(queued.fileLoadingProgress['test-file.csv'], 'should keep progress for the first file');+ t.ok(queued.fileLoadingProgress['extra.csv'], 'should show progress for the queued file');++ const readyToFinish = {+ ...nextState,+ fileLoading: {+ ...nextState.fileLoading,+ filesToLoad: [],+ fileCache: [{info: {label: 'test-file.csv'}}]+ }+ };+ const droppedDuringPause = reducer(+ readyToFinish,+ VisStateActions.loadFiles(+ [{type: 'text/csv', name: 'late.csv'}],+ VisStateActions.stageLoadedFiles,+ {deferAddToMap: true}+ )+ );+ const continued = reducer(droppedDuringPause, VisStateActions.loadNextFile());+ t.equal(+ continued.fileLoadingProgress['late.csv'].message,+ 'loading...',+ 'a file dropped during the finish pause starts parsing'+ );+ t.equal(+ continued.fileLoading.fileCache,+ readyToFinish.fileLoading.fileCache,+ 'should keep the parsed files while the queued file loads'+ );++ const canceled = reducer(readyToFinish, VisStateActions.clearStagedLoadedFiles());+ drainTasksForTesting();+ const ignored = reducer(canceled, VisStateActions.loadNextFile(readyToFinish.fileLoading.loadId));+ t.equal(ignored.fileLoading, false, 'a canceled load is not resumed');+ t.equal(ignored.stagedToAdd, null, 'cancel stays discarded');+ t.equal(drainTasksForTesting().length, 0, 'a canceled pause does not stage files');++ const restarted = reducer(+ canceled,+ VisStateActions.loadFiles(+ [{type: 'text/csv', name: 'fresh.csv'}],+ VisStateActions.stageLoadedFiles,+ {deferAddToMap: true}+ )+ );+ drainTasksForTesting();+ const stale = reducer(restarted, VisStateActions.loadNextFile(readyToFinish.fileLoading.loadId));+ t.equal(stale.fileLoading, restarted.fileLoading, 'a stale pause does not replace the new load');+ t.equal(drainTasksForTesting().length, 0, 'a stale pause does not stage or parse');++ const cleared = reducer(nextState, VisStateActions.clearStagedLoadedFiles());+ t.equal(cleared.fileLoading, false, 'cancel should stop the in-progress load');+ t.equal(cleared.stagedToAdd, null, 'cancel should drop staged files');++ drainTasksForTesting();+ t.end();+});++test('#visStateReducer -> local dataset is not blocked by a remote download', t => {+ drainTasksForTesting();+ const local = {+ info: {id: 'local-points', label: 'points.csv'},+ data: processCsvData('lat,lng\n1,2')+ };+ const remote = remoteDatasetFromUrl('https://example.com/slow.csv');+ const nextState = reducer(+ CloneDeep(InitialState).visState,+ VisStateActions.updateVisData([local, remote])+ );+ const tasks = drainTasksForTesting();+ const ready = tasks.find(+ task => task.label.includes('CREATE_TABLE_TASK') && !task.label.includes('HYDRATE')+ );+ const remoteTask = tasks.find(task =>+ task.label.includes('HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK')+ );++ t.ok(ready, 'should commit the parsed dataset on its own');+ t.ok(remoteTask, 'should download the remote dataset separately');+ t.equal(+ nextState.datasets['local-points'],+ undefined,+ 'parsed rows are inserted by the ready task'+ );+ t.equal(nextState.loadingProgress[remote.info.id], 0, 'remote download stays in progress');++ ready.run(+ (effect, resolve, reject, ctx) => effect(resolve, reject, ctx),+ action => {+ const added = reducer(nextState, action);+ t.equal(+ added.datasets['local-points'].id,+ 'local-points',+ 'local dataset is added before the remote download'+ );+ t.equal(added.datasets[remote.info.id], undefined, 'remote dataset is still absent');+ t.equal(+ added.loadingIndicatorValue > 0,+ true,+ 'loading indicator stays up for the remote download'+ );+ t.end();+ },+ err => {+ t.error(err);+ t.end();+ }+ );+});+test('#visStateReducer -> setLayerAnimationTimeConfig', t => {// change Trip layer isVisibleconst nextState = reducer(