Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat: confirm datasets in the Add Data dialog before adding them (#3770)

Igor Dykhta5eeca8f84b23
diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
index 4748eee06..813d4635d 100644
--- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
+++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
@@ -64,7 +64,7 @@ Because CSV file content is uploaded as strings, kepler.gl will attempt to detec
#### 2. Layer detection based on column names
-kepler.gl will auto detect layer, if the column names follows certain naming convention. kepler.gl creates a point layer if your CSV has columns that are named `<name>_lat` and `<name>_lng` or `<name>_latitude` and `<name>_longitude`, or `<name>_lat` and `<name>_lon`.
+kepler.gl will auto detect layer, if the column names follows certain naming convention. kepler.gl creates a point layer if your CSV has columns that are named `<name>_lat` and `<name>_lng` or `<name>_latitude` and `<name>_longitude`, or `<name>_lat` and `<name>_lon`. Uncheck **Auto create layers**, then click **Add Data** to load a file or tileset without creating layers.
| layer | auto create layer from column names
|---|---
diff --git a/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js b/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js
index 084fb14fc..cfab4fd7f 100644
--- a/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js
+++ b/examples/demo-app-sqlrooms/src/components/load-data-modal/load-remote-map.js
@@ -120,7 +120,6 @@ class LoadRemoteMap extends Component {
<FormattedMessage id={'loadRemoteMap.examples'} />
<ul>
<li>https://your.map.url/map.json</li>
- <li>http://your.map.url/data.csv</li>
</ul>
</StyledInputLabel>
<StyledInputLabel>
diff --git a/examples/demo-app-sqlrooms/src/constants/localization.js b/examples/demo-app-sqlrooms/src/constants/localization.js
index 9c9bcd4ca..ad5c0f6eb 100644
--- a/examples/demo-app-sqlrooms/src/constants/localization.js
+++ b/examples/demo-app-sqlrooms/src/constants/localization.js
@@ -10,7 +10,7 @@ const en = {
'sampleDataViewer.rowCount': ' {rowCount} rows',
'loadRemoteMap.description': 'Load your map using your custom URL',
'loadRemoteMap.message':
- 'You can use the following formats: CSV | JSON | Kepler.gl config json. Make sure the url contains the file extension.',
+ 'Use a Kepler.gl config json. Make sure the url contains the file extension.',
'loadRemoteMap.examples': 'Examples:',
'loadRemoteMap.cors':
'* CORS policy must be defined on your custom url domain in order to be accessible. For more info ',
@@ -31,7 +31,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} riviä',
'loadRemoteMap.description': 'Lataa karttasi käyttämällä omaa urlia',
'loadRemoteMap.message':
- 'Voit käyttää formaatteja: CSV | JSON | Kepler.gl asetus-json. Varmista, että url sisältää tiedostopäätteen nimen.',
+ 'Käytä Kepler.gl asetus-jsonia. Varmista, että url sisältää tiedostopäätteen nimen.',
'loadRemoteMap.examples': 'Esimerkkejä:',
'loadRemoteMap.cors':
'* CORS-käytäntö pitää olla määriteltynä urlin domainissa, jotta aineiston voi ladata.',
@@ -45,7 +45,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} files',
'loadRemoteMap.description': 'Carrega el teu mapa amb la teva URL personalitzada',
'loadRemoteMap.message':
- "Pots emprar els següents formats: CSV | JSON | Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",
+ "Empra un Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",
'loadRemoteMap.examples': 'Exemples:',
'loadRemoteMap.cors':
'* La política CORS s’ha de definir al teu domini per tal que sigui accessible. Per a més informació ',
@@ -59,7 +59,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} files',
'loadRemoteMap.description': 'Carga tu mapa con tu enlace personalizado',
'loadRemoteMap.message':
- 'Puedes usar los siguientes formatos: CSV | JSON | Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',
+ 'Usa un Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',
'loadRemoteMap.examples': 'Ejemplos:',
'loadRemoteMap.cors':
'* La política CORS debe ser definida en tu dominio para que sea accessible. Para más información ',
@@ -72,8 +72,7 @@ export const messages = {
'sampleMapsTab.trySampleData': '尝试样本数据',
'sampleDataViewer.rowCount': ' {rowCount} 行',
'loadRemoteMap.description': '使用自定义 URL 加载地图',
- 'loadRemoteMap.message':
- '您可以使用以下格式:CSV | JSON | Kepler.gl 配置 json。 确保 url 包含文件扩展名。',
+ 'loadRemoteMap.message': '请使用 Kepler.gl 配置 json。确保 url 包含文件扩展名。',
'loadRemoteMap.examples': '示例:',
'loadRemoteMap.cors': '* 必须在您的自定义 url 域上定义 CORS 策略才能访问。欲了解更多信息',
'loadRemoteMap.clickHere': '点击此处',
diff --git a/examples/demo-app/src/components/load-data-modal/load-remote-map.js b/examples/demo-app/src/components/load-data-modal/load-remote-map.js
index 084fb14fc..cfab4fd7f 100644
--- a/examples/demo-app/src/components/load-data-modal/load-remote-map.js
+++ b/examples/demo-app/src/components/load-data-modal/load-remote-map.js
@@ -120,7 +120,6 @@ class LoadRemoteMap extends Component {
<FormattedMessage id={'loadRemoteMap.examples'} />
<ul>
<li>https://your.map.url/map.json</li>
- <li>http://your.map.url/data.csv</li>
</ul>
</StyledInputLabel>
<StyledInputLabel>
diff --git a/examples/demo-app/src/constants/localization.js b/examples/demo-app/src/constants/localization.js
index 9c9bcd4ca..ad5c0f6eb 100644
--- a/examples/demo-app/src/constants/localization.js
+++ b/examples/demo-app/src/constants/localization.js
@@ -10,7 +10,7 @@ const en = {
'sampleDataViewer.rowCount': ' {rowCount} rows',
'loadRemoteMap.description': 'Load your map using your custom URL',
'loadRemoteMap.message':
- 'You can use the following formats: CSV | JSON | Kepler.gl config json. Make sure the url contains the file extension.',
+ 'Use a Kepler.gl config json. Make sure the url contains the file extension.',
'loadRemoteMap.examples': 'Examples:',
'loadRemoteMap.cors':
'* CORS policy must be defined on your custom url domain in order to be accessible. For more info ',
@@ -31,7 +31,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} riviä',
'loadRemoteMap.description': 'Lataa karttasi käyttämällä omaa urlia',
'loadRemoteMap.message':
- 'Voit käyttää formaatteja: CSV | JSON | Kepler.gl asetus-json. Varmista, että url sisältää tiedostopäätteen nimen.',
+ 'Käytä Kepler.gl asetus-jsonia. Varmista, että url sisältää tiedostopäätteen nimen.',
'loadRemoteMap.examples': 'Esimerkkejä:',
'loadRemoteMap.cors':
'* CORS-käytäntö pitää olla määriteltynä urlin domainissa, jotta aineiston voi ladata.',
@@ -45,7 +45,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} files',
'loadRemoteMap.description': 'Carrega el teu mapa amb la teva URL personalitzada',
'loadRemoteMap.message':
- "Pots emprar els següents formats: CSV | JSON | Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",
+ "Empra un Kepler.gl config json. Assegura't que la URL contingui l'extensió de l'arxiu.",
'loadRemoteMap.examples': 'Exemples:',
'loadRemoteMap.cors':
'* La política CORS s’ha de definir al teu domini per tal que sigui accessible. Per a més informació ',
@@ -59,7 +59,7 @@ export const messages = {
'sampleDataViewer.rowCount': ' {rowCount} files',
'loadRemoteMap.description': 'Carga tu mapa con tu enlace personalizado',
'loadRemoteMap.message':
- 'Puedes usar los siguientes formatos: CSV | JSON | Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',
+ 'Usa un Kepler.gl config json. Asegurate que el enlace contenga la extensión del archivo.',
'loadRemoteMap.examples': 'Ejemplos:',
'loadRemoteMap.cors':
'* La política CORS debe ser definida en tu dominio para que sea accessible. Para más información ',
@@ -72,8 +72,7 @@ export const messages = {
'sampleMapsTab.trySampleData': '尝试样本数据',
'sampleDataViewer.rowCount': ' {rowCount} 行',
'loadRemoteMap.description': '使用自定义 URL 加载地图',
- 'loadRemoteMap.message':
- '您可以使用以下格式:CSV | JSON | Kepler.gl 配置 json。 确保 url 包含文件扩展名。',
+ 'loadRemoteMap.message': '请使用 Kepler.gl 配置 json。确保 url 包含文件扩展名。',
'loadRemoteMap.examples': '示例:',
'loadRemoteMap.cors': '* 必须在您的自定义 url 域上定义 CORS 策略才能访问。欲了解更多信息',
'loadRemoteMap.clickHere': '点击此处',
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.ts
index 09e45ab50..0e2cf65c2 100644
--- a/src/actions/src/action-types.ts
+++ b/src/actions/src/action-types.ts
@@ -102,6 +102,9 @@ export const ActionTypes = {
LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,
LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,
LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,
+ STAGE_LOADED_FILES: `${ACTION_PREFIX}STAGE_LOADED_FILES`,
+ APPEND_STAGED_LOADED_FILES: `${ACTION_PREFIX}APPEND_STAGED_LOADED_FILES`,
+ CLEAR_STAGED_LOADED_FILES: `${ACTION_PREFIX}CLEAR_STAGED_LOADED_FILES`,
LAYER_COLOR_UI_CHANGE: `${ACTION_PREFIX}LAYER_COLOR_UI_CHANGE`,
TOGGLE_FEATURE_LAYER: `${ACTION_PREFIX}TOGGLE_FEATURE_LAYER`,
APPLY_CPU_FILTER: `${ACTION_PREFIX}APPLY_CPU_FILTER`,
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts
index bfdd4bba5..f24896a39 100644
--- a/src/actions/src/vis-state-actions.ts
+++ b/src/actions/src/vis-state-actions.ts
@@ -7,6 +7,7 @@ import {FileCacheItem} from '@kepler.gl/processors';
import {Layer, LayerBaseConfig} from '@kepler.gl/layers';
import {GroupByOp, JoinOp, KeplerTable} from '@kepler.gl/table';
import {
+ AddDataToMapOptions,
AddDataToMapPayload,
ViewportPadding,
ValueOf,
@@ -1828,25 +1829,36 @@ export function setMapInfo(
};
}
+export type LoadFilesOptions = AddDataToMapOptions & {
+ /** Parse the files, then wait for an explicit add instead of calling `addDataToMap`. */
+ deferAddToMap?: boolean;
+};
+
export type LoadFilesUpdaterAction = {
files: File[];
- onFinish?(result: any): any;
+ onFinish?(result: any, options?: LoadFilesOptions): any;
+ /** Forwarded to `addDataToMap` when loading finishes. Omitted options keep their defaults. */
+ options?: LoadFilesOptions;
};
/**
* Trigger file loading dispatch `addDataToMap` if succeed, or `loadFilesErr` if failed
* @memberof visStateActions
* @param files array of fileblob
+ * @param onFinish called with the processed file cache. Defaults to `loadFilesSuccess`
+ * @param options forwarded to `addDataToMap`. `autoCreateLayers` defaults to true when omitted
* @returns action
* @public
*/
export function loadFiles(
files: File[],
- onFinish?: (result: any) => any
+ onFinish?: (result: any, options?: LoadFilesOptions) => any,
+ options?: LoadFilesOptions
): Merge<LoadFilesUpdaterAction, {type: typeof ActionTypes.LOAD_FILES}> {
return {
type: ActionTypes.LOAD_FILES,
files,
- onFinish
+ onFinish,
+ ...(options ? {options} : {})
};
}
@@ -1856,30 +1868,89 @@ export function loadFiles(
* @returns action
* @public
*/
-export function loadNextFile(): {type: typeof ActionTypes.LOAD_NEXT_FILE} {
+export function loadNextFile(loadId?: number): {
+ type: typeof ActionTypes.LOAD_NEXT_FILE;
+ loadId?: number;
+} {
return {
- type: ActionTypes.LOAD_NEXT_FILE
+ type: ActionTypes.LOAD_NEXT_FILE,
+ ...(loadId == null ? {} : {loadId})
};
}
export type loadFilesSuccessUpdaterAction = {
result: FileCacheItem[];
+ options?: AddDataToMapOptions;
+ /** Set when a file-load pause finishes, so a canceled load can be ignored. */
+ loadId?: number;
};
/**
* called when all files are processed and loaded
* @memberof visStateActions
* @param result
+ * @param options forwarded to `addDataToMap`
* @returns action
*/
export function loadFilesSuccess(
- result: FileCacheItem[]
+ result: FileCacheItem[],
+ options?: AddDataToMapOptions
): Merge<loadFilesSuccessUpdaterAction, {type: typeof ActionTypes.LOAD_FILES_SUCCESS}> {
return {
type: ActionTypes.LOAD_FILES_SUCCESS,
+ result,
+ ...(options ? {options} : {})
+ };
+}
+
+export type StageLoadedFilesUpdaterAction = {
+ result: FileCacheItem[];
+ /** Set when a file-load pause finishes, so a canceled load can be ignored. */
+ loadId?: number;
+};
+/**
+ * Store processed files without adding them to the map or closing the modal.
+ * @memberof visStateActions
+ * @param result processed file cache
+ * @returns action
+ * @public
+ */
+export function stageLoadedFiles(
+ result: FileCacheItem[]
+): Merge<StageLoadedFilesUpdaterAction, {type: typeof ActionTypes.STAGE_LOADED_FILES}> {
+ return {
+ type: ActionTypes.STAGE_LOADED_FILES,
+ result
+ };
+}
+
+/**
+ * Add already-parsed or remote datasets to the Add Data list without starting a file load.
+ * @memberof visStateActions
+ * @param result datasets to keep until Add Data
+ * @returns action
+ * @public
+ */
+export function appendStagedLoadedFiles(
+ result: FileCacheItem[]
+): Merge<StageLoadedFilesUpdaterAction, {type: typeof ActionTypes.APPEND_STAGED_LOADED_FILES}> {
+ return {
+ type: ActionTypes.APPEND_STAGED_LOADED_FILES,
result
};
}
+/**
+ * Drop files staged for the Add Data button and stop an in-progress modal load.
+ * @memberof visStateActions
+ * @returns action
+ * @public
+ */
+export function clearStagedLoadedFiles(): {type: typeof ActionTypes.CLEAR_STAGED_LOADED_FILES} {
+ return {
+ type: ActionTypes.CLEAR_STAGED_LOADED_FILES
+ };
+}
+
export type LoadFileStepSuccessAction = {
fileName: string;
fileCache: FileCacheItem[];
@@ -2179,6 +2250,8 @@ export type NextFileBatchUpdaterAction = {
progress?: any;
accumulated?: any;
onFinish: (result: any) => any;
+ /** Carried across batches so progressive loads keep the original add-data options. */
+ options?: LoadFilesOptions;
};
};
/**
diff --git a/src/components/src/common/file-uploader/file-upload.tsx b/src/components/src/common/file-uploader/file-upload.tsx
index 43cc9abe9..9de154355 100644
--- a/src/components/src/common/file-uploader/file-upload.tsx
+++ b/src/components/src/common/file-uploader/file-upload.tsx
@@ -6,25 +6,19 @@ import styled from 'styled-components';
import {injectIntl, WrappedComponentProps} from 'react-intl';
import UploadButton from './upload-button';
import {DragNDrop, FileType} from '../icons';
-import FileUploadProgress from './file-upload-progress';
import FileDrop from './file-drop';
+import UploadFileList, {UploadFileListItem} from './upload-file-list';
import {FileLoading, FileLoadingProgress} from '@kepler.gl/types';
import {GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';
import {FormattedMessage} from '@kepler.gl/localization';
-import {
- fetchRemoteFileAsKeplerFile,
- getAcceptedRemoteFileFormats,
- getFileNameForRemoteUrl,
- isRemoteDatasetUrl
-} from '@kepler.gl/processors';
+import {getAcceptedRemoteFileFormats, isRemoteDatasetUrl} from '@kepler.gl/processors';
import {media} from '@kepler.gl/styles';
-import {getApplicationConfig} from '@kepler.gl/utils';
+import {getApplicationConfig, getError} from '@kepler.gl/utils';
import Markdown from 'markdown-to-jsx';
import {Button, InputLight} from '../styled-components';
import LinkRenderer from '../link-renderer';
-
const fileIconColor = '#D3D8E0';
const StyledUploadMessage = styled.div`
@@ -59,7 +53,9 @@ const StyledFileDrop = styled.div<StyledFileDropProps>`
border-color: ${props =>
props.$dragOver ? props.theme.textColorLT : props.theme.subtextColorLT};
text-align: center;
- width: 100%;
+ flex: 1;
+ min-width: 0;
+ width: auto;
min-height: 360px;
padding: 24px 12px 16px;
display: flex;
@@ -120,6 +116,17 @@ const StyledFileUpload = styled.div`
}
`;
+const UploadColumns = styled.div`
+ display: flex;
+ align-items: stretch;
+ gap: 16px;
+ width: 100%;
+
+ ${media.portable`
+ flex-direction: column;
+ `}
+`;
+
const StyledMessage = styled.div`
display: flex;
justify-content: center;
@@ -188,42 +195,6 @@ const StyledRemoteFetchButton = styled(Button)`
flex-shrink: 0;
`;
-const REMOTE_DOWNLOAD_SHARE = 0.85;
-
-function scaleFileProgress(
- progress: FileLoadingProgress,
- start: number,
- end: number
-): FileLoadingProgress {
- return Object.keys(progress).reduce<FileLoadingProgress>((accu, key) => {
- const item = progress[key];
- accu[key] = {
- ...item,
- percent: start + (item.percent || 0) * (end - start)
- };
- return accu;
- }, {});
-}
-
-function getCombinedLoadProgress({
- fileLoading,
- fileLoadingProgress,
- remoteProgress
-}: {
- fileLoading: FileLoading | false;
- fileLoadingProgress: FileLoadingProgress;
- remoteProgress: FileLoadingProgress;
-}): FileLoadingProgress {
- const isRemoteFetch = Object.keys(remoteProgress).length > 0;
- if (!isRemoteFetch) {
- return fileLoadingProgress;
- }
- if (fileLoading) {
- return scaleFileProgress(fileLoadingProgress, REMOTE_DOWNLOAD_SHARE, 1);
- }
- return scaleFileProgress(remoteProgress, 0, REMOTE_DOWNLOAD_SHARE);
-}
-
type FileUploadProps = {
onFileUpload: (files: File[]) => void;
fileLoading: FileLoading | false;
@@ -237,6 +208,16 @@ type FileUploadProps = {
displayedFileExtensions?: string[];
/** Set to true if app wants to do its own file filtering */
disableExtensionFilter?: boolean;
+ /** Parsed files held until Add Data, used if this uploader remounts. */
+ stagedToAdd?: Array<{
+ info?: {label?: string; format?: string};
+ metadata?: {source?: string};
+ }> | null;
+ /** Stage a remote URL without downloading it. Add Data runs the remote load. */
+ onAddRemoteDataset?: (remote: {url: string; format?: string}) => void;
+ /** Keys of datasets the user unchecked. Omitted keys stay selected. */
+ deselectedDatasets?: Record<string, boolean>;
+ onToggleDataset?: (key: string) => void;
} & WrappedComponentProps;
type FileUploadState = {
@@ -247,10 +228,66 @@ type FileUploadState = {
remoteUrl: string;
remoteFormat: string;
remoteError: {message: string} | null;
- remoteLoading: boolean;
- remoteProgress: FileLoadingProgress;
+ deselectedDatasets: Record<string, boolean>;
};
+function formatFileSize(size?: number): string {
+ if (!Number.isFinite(size)) {
+ return '';
+ }
+ const bytes = size as number;
+ if (bytes < 1024) {
+ return `${bytes} B`;
+ }
+ if (bytes < 1024 * 1024) {
+ return `${(bytes / 1024).toFixed(1)} KB`;
+ }
+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function extensionOf(name: string): string {
+ const part = name.includes('.') ? name.split('.').pop() : '';
+ return part && part !== name ? part : '';
+}
+
+function fileListStatus(
+ intl,
+ file: File,
+ progress: FileLoadingProgress[string] | undefined,
+ loading: boolean
+): Pick<UploadFileListItem, 'status' | 'percent' | 'isError' | 'isSuccess'> {
+ if (progress?.error) {
+ return {
+ status: getError(progress.error) || intl.formatMessage({id: 'fileUploader.unableToLoad'}),
+ percent: progress.percent || 0,
+ isError: true,
+ isSuccess: false
+ };
+ }
+ if (!loading) {
+ const size = formatFileSize(file.size);
+ return {
+ status: size
+ ? intl.formatMessage({id: 'fileUploader.readyToAdd'}, {size})
+ : intl.formatMessage({id: 'fileUploader.readyToAddNoSize'}),
+ percent: 1,
+ isError: false,
+ isSuccess: true
+ };
+ }
+ const size = formatFileSize(file.size);
+ const loaded = formatFileSize((progress?.percent || 0) * (file.size || 0));
+ return {
+ status:
+ size && loaded
+ ? intl.formatMessage({id: 'fileUploader.loadingProgress'}, {loaded, total: size})
+ : progress?.message || intl.formatMessage({id: 'fileUploader.loading'}),
+ percent: progress?.percent || 0,
+ isError: false,
+ isSuccess: false
+ };
+}
+
function FileUploadFactory() {
/** @augments {Component<FileUploadProps>} */
class FileUpload extends Component<FileUploadProps, FileUploadState> {
@@ -262,23 +299,14 @@ function FileUploadFactory() {
remoteUrl: '',
remoteFormat: 'auto',
remoteError: null,
- remoteLoading: false,
- remoteProgress: {}
+ deselectedDatasets: {}
};
static getDerivedStateFromProps(props, state) {
- if (state.fileLoading && props.fileLoading === false && state.files.length) {
- return {
- files: [],
- fileLoading: props.fileLoading,
- remoteLoading: false,
- remoteProgress: {}
- };
+ if (state.fileLoading !== props.fileLoading) {
+ return {fileLoading: props.fileLoading};
}
- return {
- fileLoading: props.fileLoading,
- ...(props.fileLoading ? {remoteLoading: false} : {})
- };
+ return null;
}
frame = createRef<HTMLDivElement>();
@@ -311,10 +339,13 @@ function FileUploadFactory() {
}
}
- const nextState = {files: filesToLoad, errorFiles, dragOver: false};
-
- this.setState(nextState, () =>
- nextState.files.length ? this.props.onFileUpload(nextState.files) : null
+ this.setState(
+ prev => ({
+ files: [...prev.files, ...filesToLoad],
+ errorFiles: [...prev.errorFiles, ...errorFiles],
+ dragOver: false
+ }),
+ () => (filesToLoad.length ? this.props.onFileUpload(filesToLoad) : null)
);
};
@@ -341,12 +372,27 @@ function FileUploadFactory() {
}
};
- _handleLoadRemoteFile = async () => {
+ _toggleDataset = (key: string) => {
+ if (!key) {
+ return;
+ }
+ if (this.props.onToggleDataset) {
+ this.props.onToggleDataset(key);
+ return;
+ }
+ this.setState(prev => ({
+ deselectedDatasets: {
+ ...prev.deselectedDatasets,
+ [key]: !prev.deselectedDatasets[key]
+ }
+ }));
+ };
+
+ _handleLoadRemoteFile = () => {
const {remoteUrl, remoteFormat, remoteError} = this.state;
- const {intl} = this.props;
const showFormatSelector = getApplicationConfig().enableRemoteFileFormatSelector;
const format = showFormatSelector && remoteFormat !== 'auto' ? remoteFormat : undefined;
- if (!remoteUrl || remoteError || this.state.remoteLoading) {
+ if (!remoteUrl || remoteError) {
if (!remoteUrl) {
this.setState({remoteError: {message: 'Incorrect URL'}});
}
@@ -357,74 +403,32 @@ function FileUploadFactory() {
return;
}
- const fileName = getFileNameForRemoteUrl(remoteUrl, format);
- const downloading = intl.formatMessage({id: 'fileUploader.downloading'});
- this.setState({
- remoteLoading: true,
- remoteError: null,
- remoteProgress: {
- [fileName]: {
- fileName,
- percent: 0,
- message: downloading,
- error: null
- }
- }
- });
-
- try {
- let lastUpdate = 0;
- const file = await fetchRemoteFileAsKeplerFile(remoteUrl, format, ({percent}) => {
- const now = Date.now();
- if (percent < 1 && now - lastUpdate < 100) {
- return;
- }
- lastUpdate = now;
- this.setState(prev => {
- const name = Object.keys(prev.remoteProgress)[0] || fileName;
- return {
- remoteProgress: {
- [name]: {
- fileName: name,
- percent,
- message: downloading,
- error: null
- }
- }
- };
- });
- });
- this.setState(
- {
- files: [file],
- errorFiles: [],
- dragOver: false
- },
- () => this.props.onFileUpload([file])
- );
- } catch (error) {
- this.setState({
- remoteLoading: false,
- remoteProgress: {},
- remoteError: {
- message: error instanceof Error ? error.message : `Failed to load ${remoteUrl}`
- }
- });
- }
+ this.props.onAddRemoteDataset?.({url: remoteUrl, ...(format ? {format} : {})});
+ this.setState({remoteUrl: '', remoteError: null});
};
render() {
+ const {dragOver, files, errorFiles, remoteUrl, remoteFormat, remoteError} = this.state;
const {
- dragOver,
- files,
- errorFiles,
- remoteUrl,
- remoteFormat,
- remoteError,
- remoteLoading,
- remoteProgress
- } = this.state;
- const {fileLoading, fileLoadingProgress, theme, intl} = this.props;
+ fileLoading,
+ fileLoadingProgress,
+ intl,
+ stagedToAdd,
+ deselectedDatasets: deselectedFromProps
+ } = this.props;
+ const deselected = deselectedFromProps || this.state.deselectedDatasets;
+ const selectionFor = (
+ item: Omit<UploadFileListItem, 'id' | 'selectable' | 'selected' | 'onToggle'>,
+ key: string,
+ selectable: boolean
+ ): UploadFileListItem => ({
+ ...item,
+ id: key,
+ selectable,
+ selected: selectable && !deselected[key],
+ selectionLabel: intl.formatMessage({id: 'fileUploader.includeDataset'}, {name: item.name}),
+ onToggle: selectable ? () => this._toggleDataset(key) : undefined
+ });
const {fileExtensions = [], displayedFileExtensions} = this.props;
const iconExtensions = displayedFileExtensions?.length
? displayedFileExtensions
@@ -433,6 +437,73 @@ function FileUploadFactory() {
const fileUploadInfoText = `${intl.formatMessage({
id: 'fileUploader.configUploadMessage'
})}(${GUIDES_FILE_FORMAT_DOC}).`;
+ const progress = fileLoadingProgress || {};
+ const uploadItems: UploadFileListItem[] = [];
+ const seen = new Set<string>();
+ const localNames = new Set(files.map(file => file.name));
+ (stagedToAdd || []).forEach(item => {
+ const name = item?.info?.label;
+ const source = item?.metadata?.source;
+ if (!name || seen.has(source || name)) {
+ return;
+ }
+ if (!source && localNames.has(name)) {
+ return;
+ }
+ const key = source || name;
+ seen.add(key);
+ uploadItems.push(
+ selectionFor(
+ {
+ name,
+ ext: extensionOf(name) || 'url',
+ status: source || intl.formatMessage({id: 'fileUploader.readyToAddNoSize'}),
+ percent: 1,
+ isSuccess: true
+ },
+ key,
+ true
+ )
+ );
+ });
+ files.forEach(file => {
+ if (seen.has(file.name)) {
+ return;
+ }
+ seen.add(file.name);
+ const fileProgress = progress[file.name];
+ const status = fileListStatus(
+ intl,
+ file,
+ fileProgress,
+ Boolean(fileLoading && fileProgress)
+ );
+ uploadItems.push(
+ selectionFor(
+ {
+ name: file.name,
+ ext: extensionOf(file.name),
+ ...status
+ },
+ file.name,
+ !status.isError
+ )
+ );
+ });
+ errorFiles.forEach(name => {
+ if (seen.has(name)) {
+ return;
+ }
+ seen.add(name);
+ uploadItems.push({
+ name,
+ ext: extensionOf(name),
+ status: intl.formatMessage({id: 'fileUploader.unsupported'}),
+ percent: 0,
+ isError: true,
+ isSuccess: false
+ });
+ });
return (
<StyledFileUpload className="file-uploader" ref={this.frame}>
{FileDrop ? (
@@ -456,110 +527,92 @@ function FileUploadFactory() {
{fileUploadInfoText}
</Markdown>
</StyledUploadMessage>
- <StyledFileDrop $dragOver={dragOver}>
- <StyledDropBody>
- <StyledFileTypeFow className="file-type-row">
- {iconExtensions.map(ext => (
- <FileType key={ext} ext={ext} height="50px" fontSize="9px" />
- ))}
- </StyledFileTypeFow>
- {fileLoading || remoteLoading ? (
- <FileUploadProgress
- fileLoadingProgress={getCombinedLoadProgress({
- fileLoading,
- fileLoadingProgress,
- remoteProgress
- })}
- theme={theme}
- />
- ) : (
- <>
- <StyledDragNDropIcon
- style={{opacity: dragOver ? 0.5 : 1}}
- className="file-upload-display-message"
+ <UploadColumns>
+ <StyledFileDrop $dragOver={dragOver}>
+ <StyledDropBody>
+ <StyledFileTypeFow className="file-type-row">
+ {iconExtensions.map(ext => (
+ <FileType key={ext} ext={ext} height="50px" fontSize="9px" />
+ ))}
+ </StyledFileTypeFow>
+ <StyledDragNDropIcon
+ style={{opacity: dragOver ? 0.5 : 1}}
+ className="file-upload-display-message"
+ >
+ <DragNDrop height="36px" />
+ </StyledDragNDropIcon>
+ <StyledDragFileWrapper>
+ <StyledActionLine>
+ <FormattedMessage
+ id={'fileUploader.dropMessage'}
+ values={{
+ browse: (
+ <UploadButton key="browse" onUpload={this._handleFileInput}>
+ {intl.formatMessage({id: 'fileUploader.browseFiles'})}
+ </UploadButton>
+ )
+ }}
+ />
+ </StyledActionLine>
+ <StyledRemoteUrlForm
+ className="file-uploader__remote-url"
+ onClick={event => event.stopPropagation()}
>
- <DragNDrop height="36px" />
- {errorFiles.length ? (
- <WarningMsg>
- <FormattedMessage
- id={'fileUploader.fileNotSupported'}
- values={{errorFiles: errorFiles.join(', ')}}
- />
- </WarningMsg>
- ) : null}
- </StyledDragNDropIcon>
- {!files.length ? (
- <StyledDragFileWrapper>
- <StyledActionLine>
- <FormattedMessage
- id={'fileUploader.dropMessage'}
- values={{
- browse: (
- <UploadButton key="browse" onUpload={this._handleFileInput}>
- {intl.formatMessage({id: 'fileUploader.browseFiles'})}
- </UploadButton>
- )
- }}
- />
- </StyledActionLine>
- <StyledRemoteUrlForm
- className="file-uploader__remote-url"
- onClick={event => event.stopPropagation()}
+ <StyledRemoteUrlRow>
+ <StyledRemoteUrlInput
+ type="url"
+ value={remoteUrl}
+ aria-label={intl.formatMessage({
+ id: 'fileUploader.urlPlaceholder'
+ })}
+ placeholder={intl.formatMessage({
+ id: 'fileUploader.urlPlaceholder'
+ })}
+ onChange={this._onRemoteUrlChange}
+ onKeyDown={this._onRemoteUrlKeyDown}
+ />
+ {showFormatSelector ? (
+ <StyledRemoteFormatSelect
+ className="file-uploader__remote-format"
+ aria-label={intl.formatMessage({id: 'fileUploader.format'})}
+ value={remoteFormat}
+ onChange={this._onRemoteFormatChange}
+ >
+ {getAcceptedRemoteFileFormats().map(format => (
+ <option key={format} value={format}>
+ {format === 'auto'
+ ? intl.formatMessage({id: 'fileUploader.formatAuto'})
+ : format.toUpperCase()}
+ </option>
+ ))}
+ </StyledRemoteFormatSelect>
+ ) : null}
+ <StyledRemoteFetchButton
+ type="button"
+ className="file-uploader__remote-add"
+ cta
+ small
+ disabled={!remoteUrl}
+ onClick={this._handleLoadRemoteFile}
>
- <StyledRemoteUrlRow>
- <StyledRemoteUrlInput
- type="url"
- value={remoteUrl}
- aria-label={intl.formatMessage({
- id: 'fileUploader.urlPlaceholder'
- })}
- placeholder={intl.formatMessage({
- id: 'fileUploader.urlPlaceholder'
- })}
- onChange={this._onRemoteUrlChange}
- onKeyDown={this._onRemoteUrlKeyDown}
- disabled={remoteLoading}
- />
- {showFormatSelector ? (
- <StyledRemoteFormatSelect
- className="file-uploader__remote-format"
- aria-label={intl.formatMessage({id: 'fileUploader.format'})}
- value={remoteFormat}
- onChange={this._onRemoteFormatChange}
- disabled={remoteLoading}
- >
- {getAcceptedRemoteFileFormats().map(format => (
- <option key={format} value={format}>
- {format === 'auto'
- ? intl.formatMessage({id: 'fileUploader.formatAuto'})
- : format.toUpperCase()}
- </option>
- ))}
- </StyledRemoteFormatSelect>
- ) : null}
- <StyledRemoteFetchButton
- type="button"
- cta
- small
- disabled={!remoteUrl || remoteLoading}
- onClick={this._handleLoadRemoteFile}
- >
- <FormattedMessage id={'fileUploader.fetch'} />
- </StyledRemoteFetchButton>
- </StyledRemoteUrlRow>
- {remoteError ? <WarningMsg>{remoteError.message}</WarningMsg> : null}
- </StyledRemoteUrlForm>
- </StyledDragFileWrapper>
- ) : null}
- </>
- )}
- </StyledDropBody>
- {fileLoading || remoteLoading ? null : (
+ <FormattedMessage id={'fileUploader.addUrl'} />
+ </StyledRemoteFetchButton>
+ </StyledRemoteUrlRow>
+ {remoteError ? <WarningMsg>{remoteError.message}</WarningMsg> : null}
+ </StyledRemoteUrlForm>
+ </StyledDragFileWrapper>
+ </StyledDropBody>
<StyledDisclaimer>
<FormattedMessage id={'fileUploader.disclaimer'} />
</StyledDisclaimer>
- )}
- </StyledFileDrop>
+ </StyledFileDrop>
+ <UploadFileList
+ title={
+ uploadItems.length ? intl.formatMessage({id: 'fileUploader.toAdd'}) : undefined
+ }
+ items={uploadItems}
+ />
+ </UploadColumns>
</FileDrop>
) : null}
</StyledFileUpload>
diff --git a/src/components/src/common/file-uploader/upload-file-list.tsx b/src/components/src/common/file-uploader/upload-file-list.tsx
new file mode 100644
index 000000000..03e4ee845
--- /dev/null
+++ b/src/components/src/common/file-uploader/upload-file-list.tsx
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React, {FC} from 'react';
+import styled from 'styled-components';
+
+import {media} from '@kepler.gl/styles';
+import Checkbox from '../checkbox';
+import {FileType} from '../icons';
+
+export type StagedDatasetRef = {
+ info?: {id?: string; label?: string};
+ metadata?: {source?: string};
+};
+
+/** Key shared by a card and the staged dataset it will add. Remote URLs use the URL. */
+export function datasetSelectionKey(item?: StagedDatasetRef | null): string {
+ return item?.metadata?.source || item?.info?.label || item?.info?.id || '';
+}
+
+export function selectedStagedDatasets<T extends StagedDatasetRef>(
+ staged: T[] | null | undefined,
+ deselected?: Record<string, boolean>
+): T[] {
+ return (staged || []).filter(item => {
+ const key = datasetSelectionKey(item);
+ return Boolean(key) && !deselected?.[key];
+ });
+}
+
+export type UploadFileListItem = {
+ id?: string;
+ name: string;
+ ext: string;
+ status: string;
+ percent: number;
+ isError?: boolean;
+ isSuccess?: boolean;
+ selectable?: boolean;
+ selected?: boolean;
+ selectionLabel?: string;
+ onToggle?: () => void;
+};
+
+type UploadFileListProps = {
+ title?: string;
+ items: UploadFileListItem[];
+};
+
+const List = styled.div.attrs({
+ className: 'upload-file-list'
+})`
+ width: 320px;
+ flex-shrink: 0;
+ max-height: 420px;
+ overflow-y: auto;
+
+ ${media.portable`
+ width: 100%;
+ max-height: 240px;
+ `}
+`;
+
+const ListTitle = styled.div`
+ font-size: 13px;
+ color: ${props => props.theme.textColorLT};
+ margin-bottom: 8px;
+`;
+
+const Card = styled.div.attrs({
+ className: 'file-upload-progress__message'
+})<{$isError?: boolean}>`
+ position: relative;
+ overflow: hidden;
+ background: #fff;
+ color: ${props => (props.$isError ? props.theme.errorColor : props.theme.textColorLT)};
+ border: 1px solid
+ ${props => (props.$isError ? props.theme.errorColor : props.theme.selectBorderColorLT)};
+ border-radius: 4px;
+ box-shadow: 0 0 5px -2px rgba(0, 0, 0, 0.1);
+ margin-bottom: 12px;
+
+ &:last-child {
+ margin-bottom: 0;
+ }
+`;
+
+const Progress = styled.div<{$percent: number}>`
+ position: absolute;
+ left: 0;
+ top: 0;
+ bottom: 0;
+ width: ${props => props.$percent}%;
+ background: rgba(36, 115, 189, 0.12);
+`;
+
+const CardBody = styled.div`
+ position: relative;
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ padding: 8px 12px 8px 8px;
+`;
+
+const IconWrap = styled.div`
+ flex-shrink: 0;
+ color: ${props => props.theme.subtextColorLT};
+`;
+
+const CardText = styled.div`
+ min-width: 0;
+ flex: 1;
+`;
+
+const CardName = styled.div`
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ font-size: 12px;
+ font-weight: 500;
+`;
+
+const CardCheck = styled.div`
+ flex-shrink: 0;
+
+ .kg-checkbox,
+ .kg-checkbox__label {
+ margin-left: 0;
+ margin-bottom: 0;
+ }
+`;
+
+/** Names the checkbox for assistive tech without taking space in the card. */
+const CheckLabel = styled.span.attrs({
+ className: 'upload-file-list__check-label'
+})`
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ padding: 0;
+ margin: -1px;
+ overflow: hidden;
+ clip: rect(0, 0, 0, 0);
+ white-space: nowrap;
+ border: 0;
+`;
+
+const CardStatus = styled.div<{$isError?: boolean}>`
+ margin-top: 2px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ font-size: 11px;
+ line-height: 14px;
+ color: ${props => (props.$isError ? props.theme.errorColor : props.theme.subtextColorLT)};
+`;
+
+const UploadFileList: FC<UploadFileListProps> = ({title, items}) => {
+ if (!items.length) {
+ return null;
+ }
+
+ return (
+ <List>
+ {title ? <ListTitle>{title}</ListTitle> : null}
+ {items.map((item, index) => (
+ <Card key={`${item.id || item.name}-${item.status}`} $isError={item.isError}>
+ {!item.isSuccess && !item.isError ? (
+ <Progress $percent={Math.round(item.percent * 100)} />
+ ) : null}
+ <CardBody>
+ <IconWrap>
+ <FileType ext={item.ext} height="36px" fontSize="8px" />
+ </IconWrap>
+ <CardText>
+ <CardName title={item.name}>{item.name}</CardName>
+ <CardStatus $isError={item.isError} title={item.status}>
+ {item.status}
+ </CardStatus>
+ </CardText>
+ {item.selectable ? (
+ <CardCheck title={item.selectionLabel} onClick={event => event.stopPropagation()}>
+ <Checkbox
+ id={`upload-file-${index}`}
+ type="checkbox"
+ label={
+ item.selectionLabel ? <CheckLabel>{item.selectionLabel}</CheckLabel> : undefined
+ }
+ checked={Boolean(item.selected)}
+ onChange={() => item.onToggle?.()}
+ />
+ </CardCheck>
+ ) : null}
+ </CardBody>
+ </Card>
+ ))}
+ </List>
+ );
+};
+
+export default UploadFileList;
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index 3bdd3f6fc..ebea2de4c 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -15,6 +15,7 @@ import {
MapStyle,
ProviderState
} from '@kepler.gl/reducers';
+import {remoteDatasetFromUrl} from '@kepler.gl/processors';
import {exportHtml, exportMap, exportJson, exportImage} from '@kepler.gl/utils';
import ModalDialogFactory from './modals/modal-dialog';
@@ -85,6 +86,11 @@ const smallModalCss = css`
const LoadDataModalStyle = css`
top: 60px;
+ padding-bottom: 0;
+
+ ${media.portable`
+ padding-bottom: 0;
+ `}
`;
const ExportVideoModalStyle = css`
@@ -166,6 +172,9 @@ export default function ModalContainerFactory(
};
_closeModal = () => {
+ if (this.props.uiState.currentModal === ADD_DATA_ID) {
+ this.props.visStateActions.clearStagedLoadedFiles();
+ }
this.props.uiStateActions.toggleModal(null);
};
@@ -180,12 +189,32 @@ export default function ModalContainerFactory(
};
_onFileUpload = fileList => {
- this.props.visStateActions.loadFiles(fileList);
+ this.props.visStateActions.loadFiles(fileList, VisStateActions.stageLoadedFiles, {
+ deferAddToMap: true
+ });
+ };
+
+ _onAddRemoteDataset = ({url, format}: {url: string; format?: string}) => {
+ this.props.visStateActions.appendStagedLoadedFiles([remoteDatasetFromUrl(url, format)]);
+ };
+
+ _onConfirmAddData = ({
+ autoCreateLayers,
+ datasets
+ }: {autoCreateLayers?: boolean; datasets?: any[]} = {}) => {
+ const staged = datasets || this.props.visState.stagedToAdd;
+ if (!staged?.length) {
+ return;
+ }
+ this.props.visStateActions.loadFilesSuccess(staged, {
+ autoCreateLayers: autoCreateLayers !== false
+ });
};
_onTilesetAdded = (
tileset: {name: string; type: string; metadata: Record<string, any>},
- processedMetadata?: Record<string, any>
+ processedMetadata?: Record<string, any>,
+ options?: {autoCreateLayers?: boolean}
) => {
this.props.visStateActions.updateVisData(
{
@@ -207,7 +236,7 @@ export default function ModalContainerFactory(
disableDataOperation: true
},
{
- autoCreateLayers: true,
+ autoCreateLayers: options?.autoCreateLayers !== false,
centerMap: true
}
);
@@ -373,6 +402,9 @@ export default function ModalContainerFactory(
{...providerState}
onClose={this._closeModal}
onFileUpload={this._onFileUpload}
+ onAddRemoteDataset={this._onAddRemoteDataset}
+ onConfirmAddData={this._onConfirmAddData}
+ stagedToAdd={visState.stagedToAdd}
onTilesetAdded={this._onTilesetAdded}
onLoadCloudMap={this._onLoadCloudMap}
loadFiles={uiState.loadFiles}
diff --git a/src/components/src/modals/auto-create-layers-checkbox.tsx b/src/components/src/modals/auto-create-layers-checkbox.tsx
new file mode 100644
index 000000000..73d2c6fcb
--- /dev/null
+++ b/src/components/src/modals/auto-create-layers-checkbox.tsx
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React, {FC} from 'react';
+import {useIntl} from 'react-intl';
+import styled from 'styled-components';
+
+import Checkbox from '../common/checkbox';
+import {Docs} from '../common/icons';
+import TippyTooltip from '../common/tippy-tooltip';
+
+type AutoCreateLayersCheckboxProps = {
+ checked: boolean;
+ onToggle: () => void;
+ disabled?: boolean;
+};
+
+const AutoCreateLayersRow = styled.div.attrs({
+ className: 'auto-create-layers'
+})`
+ display: flex;
+ align-items: center;
+
+ .kg-checkbox {
+ margin-left: 0;
+ }
+
+ .kg-checkbox__label {
+ margin-bottom: 0;
+ margin-left: 0;
+ color: ${props => props.theme.textColorLT};
+ }
+`;
+
+const InfoButton = styled.span`
+ display: inline-flex;
+ align-items: center;
+ margin-left: 6px;
+ color: ${props => props.theme.subtextColorLT};
+ cursor: help;
+
+ &:hover {
+ color: ${props => props.theme.textColorLT};
+ }
+`;
+
+const AutoCreateLayersCheckbox: FC<AutoCreateLayersCheckboxProps> = ({
+ checked,
+ onToggle,
+ disabled = false
+}) => {
+ const intl = useIntl();
+ const label = intl.formatMessage({id: 'modal.loadData.autoCreateLayers'});
+ const info = intl.formatMessage({id: 'modal.loadData.autoCreateLayersInfo'});
+
+ return (
+ <AutoCreateLayersRow>
+ <Checkbox
+ id="auto-create-layers"
+ type="checkbox"
+ label={label}
+ checked={checked}
+ disabled={disabled}
+ onChange={onToggle}
+ />
+ <TippyTooltip placement="top" isLightTheme render={() => <div>{info}</div>}>
+ <InfoButton aria-label={info}>
+ <Docs height="16px" />
+ </InfoButton>
+ </TippyTooltip>
+ </AutoCreateLayersRow>
+ );
+};
+
+export default AutoCreateLayersCheckbox;
diff --git a/src/components/src/modals/load-data-modal.tsx b/src/components/src/modals/load-data-modal.tsx
index ea5032974..2e487e211 100644
--- a/src/components/src/modals/load-data-modal.tsx
+++ b/src/components/src/modals/load-data-modal.tsx
@@ -1,27 +1,85 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {useState} from 'react';
+import React, {useCallback, useState} from 'react';
import styled from 'styled-components';
import get from 'es-toolkit/compat/get';
import {IntlShape, useIntl} from 'react-intl';
+import {Button} from '../common';
import FileUploadFactory from '../common/file-uploader/file-upload';
+import {selectedStagedDatasets} from '../common/file-uploader/upload-file-list';
import LoadStorageMapFactory from './load-storage-map';
import LoadTilesetFactory from './tilesets-modals/load-tileset';
import ModalTabsFactory from './modal-tabs';
import LoadingDialog from './loading-dialog';
+import AutoCreateLayersCheckbox from './auto-create-layers-checkbox';
import {LOADING_METHODS} from '@kepler.gl/constants';
+import {media} from '@kepler.gl/styles';
import {FileLoading, FileLoadingProgress, LoadFiles} from '@kepler.gl/types';
const StyledLoadDataModal = styled.div.attrs({
className: 'load-data-modal'
-})`
- padding: ${props => props.theme.modalPadding};
+})<{$withFooter?: boolean}>`
+ padding: 10px 0 ${props => (props.$withFooter ? 0 : '50px')};
min-height: 360px;
display: flex;
flex-direction: column;
+
+ ${props =>
+ props.$withFooter
+ ? ''
+ : media.portable`
+ padding-bottom: 34px;
+ `}
+`;
+
+const AddDataBar = styled.div.attrs({
+ className: 'add-data-bar'
+})`
+ display: flex;
+ align-items: center;
+ gap: 16px;
+ box-sizing: border-box;
+ /* Pull the rule out to the dialog edges. Side padding matches the modal. */
+ margin: 16px -72px 0;
+ padding: 16px 72px;
+ border-top: 1px solid #d8d8d8;
+
+ ${media.portable`
+ margin-left: -36px;
+ margin-right: -36px;
+ padding-left: 36px;
+ padding-right: 36px;
+ `}
+`;
+
+const StagedLabel = styled.div`
+ min-width: 0;
+ flex: 1;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ color: ${props => props.theme.subtextColorLT};
+ font-size: 12px;
+`;
+
+const FooterError = styled.div`
+ color: ${props => props.theme.negativeBtnColor};
+ font-size: 12px;
+ max-width: 220px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+`;
+
+const AddDataActions = styled.div`
+ display: flex;
+ align-items: center;
+ gap: 12px;
+ margin-left: auto;
+ flex-shrink: 0;
`;
const noop = () => {
@@ -36,13 +94,46 @@ export interface LoadingMethod {
tabElementType?: React.ComponentType<{onClick: React.MouseEventHandler; intl: IntlShape}>;
}
+export type LoadDataOptions = {
+ autoCreateLayers?: boolean;
+ /** Staged datasets the user left checked. */
+ datasets?: Array<{info?: {label?: string}; metadata?: {source?: string}}>;
+};
+
+type TilesetDraft = {
+ canAdd: boolean;
+ loading?: boolean;
+ error?: string | null;
+ dataset?: {name: string; type: string; metadata: Record<string, any>};
+ metadata?: Record<string, any>;
+};
+
+function progressErrors(progress: FileLoadingProgress = {}): string[] {
+ return Object.values(progress)
+ .map(item => {
+ const error = item?.error;
+ if (!error) {
+ return '';
+ }
+ return typeof error === 'string' ? error : error.message || '';
+ })
+ .filter(Boolean);
+}
+
type LoadDataModalProps = {
// call backs
onFileUpload: (files: File[]) => void;
+ onAddRemoteDataset?: (remote: {url: string; format?: string}) => void;
+ onConfirmAddData?: (options: LoadDataOptions) => void;
+ stagedToAdd?: Array<{
+ info?: {id?: string; label?: string};
+ metadata?: {source?: string};
+ }> | null;
onLoadCloudMap: (provider: any, vis: any) => void;
onTilesetAdded: (
tileset: {name: string; type: string; metadata: Record<string, any>},
- processedMetadata?: Record<string, any>
+ processedMetadata?: Record<string, any>,
+ options?: LoadDataOptions
) => void;
fileLoading: FileLoading | false;
loadingMethods?: LoadingMethod[];
@@ -96,37 +187,123 @@ export function LoadDataModalFactory(
defaultLoadingMethods: LoadDataModalProps['loadingMethods'];
} = ({
onFileUpload = noop,
+ onAddRemoteDataset = noop,
onTilesetAdded = noop,
+ onConfirmAddData = noop,
+ onClose = noop,
fileLoading = false,
+ fileLoadingProgress = {},
+ stagedToAdd = null,
loadingMethods = defaultLoadingMethods,
isCloudMapLoading,
...restProps
}) => {
const intl = useIntl();
+ const [autoCreateLayers, setAutoCreateLayers] = useState(true);
+ const [deselectedDatasets, setDeselectedDatasets] = useState<Record<string, boolean>>({});
+ const onToggleDataset = useCallback((key: string) => {
+ setDeselectedDatasets(prev => ({...prev, [key]: !prev[key]}));
+ }, []);
+ const [tilesetDraft, setTilesetDraft] = useState<TilesetDraft | null>(null);
+ const onToggleAutoCreateLayers = useCallback(() => {
+ setAutoCreateLayers(value => !value);
+ }, []);
+ const handleFileUpload = useCallback(
+ (files: File[]) => {
+ onFileUpload(files);
+ },
+ [onFileUpload]
+ );
+ const handleTilesetAdded = useCallback(
+ (
+ tileset: {name: string; type: string; metadata: Record<string, any>},
+ processedMetadata?: Record<string, any>
+ ) => {
+ onTilesetAdded(tileset, processedMetadata, {autoCreateLayers});
+ },
+ [onTilesetAdded, autoCreateLayers]
+ );
+ const onTilesetDraftChange = useCallback((draft: TilesetDraft) => {
+ setTilesetDraft(draft);
+ }, []);
+ const [currentMethod, toggleMethod] = useState(getDefaultMethod(loadingMethods));
+ const selectMethod = useCallback((method: LoadingMethod) => {
+ setTilesetDraft(null);
+ toggleMethod(method);
+ }, []);
+ const isUpload = currentMethod?.id === LOADING_METHODS.upload;
+ const isTileset = currentMethod?.id === LOADING_METHODS.tileset;
+ const showAddBar = isUpload || isTileset;
+ const datasetsToAdd = selectedStagedDatasets(stagedToAdd, deselectedDatasets);
+ const canAdd = isTileset
+ ? Boolean(tilesetDraft?.canAdd) && !tilesetDraft?.loading
+ : Boolean(datasetsToAdd.length) && !fileLoading;
+ const onAddData = useCallback(() => {
+ if (isTileset) {
+ if (tilesetDraft?.canAdd && tilesetDraft.dataset) {
+ handleTilesetAdded(tilesetDraft.dataset, tilesetDraft.metadata);
+ }
+ return;
+ }
+ if (datasetsToAdd.length && !fileLoading) {
+ onConfirmAddData({autoCreateLayers, datasets: datasetsToAdd});
+ }
+ }, [
+ autoCreateLayers,
+ datasetsToAdd,
+ fileLoading,
+ handleTilesetAdded,
+ isTileset,
+ onConfirmAddData,
+ tilesetDraft
+ ]);
+ const uploadError = !fileLoading && isUpload ? progressErrors(fileLoadingProgress)[0] : '';
+ const tilesetError = isTileset ? tilesetDraft?.error : '';
+
const currentModalProps = {
...restProps,
- onFileUpload,
- onTilesetAdded,
+ onFileUpload: handleFileUpload,
+ onTilesetAdded: handleTilesetAdded,
fileLoading,
- isCloudMapLoading
+ fileLoadingProgress,
+ isCloudMapLoading,
+ ...(isUpload ? {stagedToAdd, onAddRemoteDataset, deselectedDatasets, onToggleDataset} : {}),
+ ...(isTileset ? {confirmInParent: true, onTilesetDraftChange} : {})
};
- // const {loadingMethods, isCloudMapLoading} = props;
- const [currentMethod, toggleMethod] = useState(getDefaultMethod(loadingMethods));
-
const ElementType = currentMethod?.elementType;
return (
- <StyledLoadDataModal>
+ <StyledLoadDataModal $withFooter={showAddBar}>
<ModalTabs
currentMethod={currentMethod?.id}
loadingMethods={loadingMethods}
- toggleMethod={toggleMethod}
+ toggleMethod={selectMethod}
/>
{isCloudMapLoading ? (
<LoadingDialog size={64} />
) : (
ElementType && <ElementType key={currentMethod?.id} intl={intl} {...currentModalProps} />
)}
+ {showAddBar ? (
+ <AddDataBar>
+ <AutoCreateLayersCheckbox
+ checked={autoCreateLayers}
+ onToggle={onToggleAutoCreateLayers}
+ />
+ <StagedLabel />
+ {uploadError || tilesetError ? (
+ <FooterError>{uploadError || tilesetError}</FooterError>
+ ) : null}
+ <AddDataActions>
+ <Button type="button" link onClick={onClose}>
+ {intl.formatMessage({id: 'modal.button.defaultCancel'})}
+ </Button>
+ <Button type="button" cta disabled={!canAdd} onClick={onAddData}>
+ {intl.formatMessage({id: 'layerManager.addData'})}
+ </Button>
+ </AddDataActions>
+ </AddDataBar>
+ ) : null}
</StyledLoadDataModal>
);
};
diff --git a/src/components/src/modals/tilesets-modals/load-data-footer.tsx b/src/components/src/modals/tilesets-modals/load-data-footer.tsx
index 981f87b04..32d65d7dc 100644
--- a/src/components/src/modals/tilesets-modals/load-data-footer.tsx
+++ b/src/components/src/modals/tilesets-modals/load-data-footer.tsx
@@ -55,6 +55,11 @@ type ThemeProps = {
theme: any;
};
+const ButtonRow = styled.div`
+ display: flex;
+ align-items: center;
+`;
+
const LoadDataFooter = ({
disabled,
intl,
@@ -67,7 +72,7 @@ const LoadDataFooter = ({
}: LoadDataFooterProps & WrappedComponentProps & ThemeProps) => {
return (
<LoadDataFooterContainer>
- <div>
+ <ButtonRow>
{prependText}
<AddDataButton
disabled={disabled}
@@ -86,7 +91,7 @@ const LoadDataFooter = ({
})}
</AddDataButton>
{errorText && <ErrorContainer>{errorText}</ErrorContainer>}
- </div>
+ </ButtonRow>
</LoadDataFooterContainer>
);
};
diff --git a/src/components/src/modals/tilesets-modals/load-tileset.tsx b/src/components/src/modals/tilesets-modals/load-tileset.tsx
index 4744db5a9..b3629d25a 100644
--- a/src/components/src/modals/tilesets-modals/load-tileset.tsx
+++ b/src/components/src/modals/tilesets-modals/load-tileset.tsx
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {useCallback, useMemo, useState} from 'react';
+import React, {useCallback, useEffect, useMemo, useState} from 'react';
import {IntlShape} from 'react-intl';
import JSONPretty from 'react-json-pretty';
import {AutoSizer} from 'react-virtualized';
@@ -100,6 +100,15 @@ type LoadTilesetTabProps = {
isAddingDatasets: boolean;
onTilesetAdded: (tilesetInfo: any, metadata?: any) => void;
intl: IntlShape;
+ /** The parent modal owns Cancel / Add Data, so this tab only reports readiness. */
+ confirmInParent?: boolean;
+ onTilesetDraftChange?: (draft: {
+ canAdd: boolean;
+ loading?: boolean;
+ error?: string | null;
+ dataset?: any;
+ metadata?: any;
+ }) => void;
};
const TILE_TYPES = [
@@ -140,7 +149,12 @@ function isReady(response) {
}
function LoadTilesetTabFactory() {
- const LoadTilesetTab: React.FC<LoadTilesetTabProps> = ({onTilesetAdded, isAddingDatasets}) => {
+ const LoadTilesetTab: React.FC<LoadTilesetTabProps> = ({
+ onTilesetAdded,
+ isAddingDatasets,
+ confirmInParent = false,
+ onTilesetDraftChange
+ }) => {
const [typeIndex, setTypeIndex] = useState<number>(0);
const [response, setResponse] = useState<MetaResponse>({});
@@ -156,6 +170,16 @@ function LoadTilesetTabFactory() {
}
}, [onTilesetAdded, response]);
+ useEffect(() => {
+ onTilesetDraftChange?.({
+ canAdd: isReady(response),
+ loading: Boolean(response.loading),
+ error: error ? getError(error) : null,
+ dataset: response.dataset,
+ metadata: response.metadata
+ });
+ }, [error, onTilesetDraftChange, response]);
+
// temp patch to hide raster tile layer while in development
const enableRasterTileLayer = getApplicationConfig().enableRasterTileLayer;
const enableWMSLayer = getApplicationConfig().enableWMSLayer;
@@ -215,13 +239,15 @@ function LoadTilesetTabFactory() {
) : null}
</MetaContainer>
</Container>
- <LoadDataFooter
- disabled={Boolean(error) || !isReady(response)}
- isLoading={loading || isAddingDatasets}
- onConfirm={createTileDataset}
- confirmText="tilesetSetup.addTilesetText"
- errorText={error && getError(error)}
- />
+ {confirmInParent ? null : (
+ <LoadDataFooter
+ disabled={Boolean(error) || !isReady(response)}
+ isLoading={loading || isAddingDatasets}
+ onConfirm={createTileDataset}
+ confirmText="tilesetSetup.addTilesetText"
+ errorText={error && getError(error)}
+ />
+ )}
</LoadTilesetTabContainer>
);
};
diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.ts
index 4be42b7e6..fbf8f90bf 100644
--- a/src/localization/src/translations/en.ts
+++ b/src/localization/src/translations/en.ts
@@ -804,7 +804,11 @@ export default {
loadData: {
upload: 'Load Files',
tileset: 'Tileset',
- storage: 'Load from Storage'
+ storage: 'Load from Storage',
+ autoCreateLayers: 'Auto create layers',
+ autoCreateLayersInfo:
+ 'kepler.gl will auto detect layers if the column names in your file follow certain naming conventions.',
+ readyToAdd: 'Ready to add'
},
tripInfo: {
title: 'Create trips from GeoJson',
@@ -1030,8 +1034,17 @@ ${'```'}
fileNotSupported: 'File {errorFiles} is not supported.',
or: 'or',
dropMessage: 'Drag & Drop Your File(s) Here, {browse}, or',
+ toAdd: 'To add to map',
+ includeDataset: 'Include {name}',
+ readyToAdd: '{size} - Ready to add file to map',
+ readyToAddNoSize: 'Ready to add file to map',
+ loadingProgress: 'Loading ({loaded} / {total})',
+ loading: 'Loading...',
+ unableToLoad: 'Unable to load file',
+ unsupported: 'Unsupported file type or format',
urlPlaceholder: 'Enter your dataset URL',
fetch: 'Fetch',
+ addUrl: 'Add',
format: 'Format',
formatAuto: 'Auto',
cors: 'The URL must allow [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS).'
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index c80f59dc4..08853b069 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -12,7 +12,12 @@ import {
} from '@kepler.gl/utils';
import {generateHashId} from '@kepler.gl/common-utils';
import {DATASET_FORMATS, DatasetType, REMOTE_FILE_EXTENSIONS} from '@kepler.gl/constants';
-import {AddDataToMapPayload, LoadedMap, ProcessorResult} from '@kepler.gl/types';
+import {
+ AddDataToMapOptions,
+ AddDataToMapPayload,
+ LoadedMap,
+ ProcessorResult
+} from '@kepler.gl/types';
import {KeplerTable} from '@kepler.gl/table';
import type {Feature, FeatureCollection} from 'geojson';
@@ -261,7 +266,10 @@ export async function* makeProgressIterator(
}
// eslint-disable-next-line complexity
-function getPersistedRemoteFormat(keplerFormat?: string, fileName?: string): string | undefined {
+export function getPersistedRemoteFormat(
+ keplerFormat?: string,
+ fileName?: string
+): string | undefined {
if (
keplerFormat &&
keplerFormat !== 'auto' &&
@@ -527,7 +535,10 @@ export async function processFileData({
}
}
-export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPayload[] {
+export function filesToDataPayload(
+ fileCache: FileCacheItem[],
+ options?: AddDataToMapOptions
+): AddDataToMapPayload[] {
// seperate out files which could be a single datasets. or a keplergl map json
const collection = fileCache.reduce<{
datasets: FileCacheItem[];
@@ -540,7 +551,8 @@ export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPayl
accu.keplerMaps.push({
...data,
options: {
- centerMap: !(data.config && data.config.mapState)
+ centerMap: !(data.config && data.config.mapState),
+ ...(options || {})
}
});
} else if (DATASET_FORMATS[info?.format]) {
@@ -562,5 +574,8 @@ export function filesToDataPayload(fileCache: FileCacheItem[]): AddDataToMapPayl
// add kepler map first with config
// add datasets later in one add data call
- return collection.keplerMaps.concat({datasets: collection.datasets});
+ return collection.keplerMaps.concat({
+ datasets: collection.datasets,
+ ...(options && collection.datasets.length ? {options} : {})
+ });
}
diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.ts
index aa0a678c3..728238376 100644
--- a/src/processors/src/remote-file.ts
+++ b/src/processors/src/remote-file.ts
@@ -3,14 +3,23 @@
import {parseUri} from '@kepler.gl/common-utils';
import {
+ DATASET_FORMATS,
+ DatasetType,
MIME_TO_REMOTE_FILE_EXTENSION,
REMOTE_FILE_EXTENSIONS,
REMOTE_FILE_MIME_TYPES,
RemoteFileFormat
} from '@kepler.gl/constants';
import {ProcessorResult} from '@kepler.gl/types';
+import {generateHashIdFromString} from '@kepler.gl/utils';
-import {processFileData, ProcessFileDataContent, readFileInBatches} from './file-handler';
+import {
+ getPersistedRemoteFormat,
+ processFileData,
+ ProcessFileDataContent,
+ readFileInBatches
+} from './file-handler';
+import {FileCacheItem} from './types';
export type KeplerRemoteFile = File & {
keplerSourceUrl?: string;
@@ -184,6 +193,34 @@ export async function fetchRemoteFile(
};
}
+/**
+ * A remote URL to hold until Add Data. Rows stay empty so adding it runs the
+ * externally hosted download instead of parsing a file in the modal.
+ */
+export function remoteDatasetFromUrl(url: string, format?: string | null): FileCacheItem {
+ if (!isRemoteDatasetUrl(url)) {
+ throw new Error('Remote dataset URL must use http or https');
+ }
+ const label = getFileNameForRemoteUrl(url, format);
+ const sourceFormat = getPersistedRemoteFormat(
+ format && format !== 'auto' ? format : undefined,
+ label
+ );
+ return {
+ data: {fields: [], rows: []},
+ info: {
+ id: generateHashIdFromString(url),
+ label,
+ format: DATASET_FORMATS.row,
+ type: DatasetType.EXTERNALLY_HOSTED
+ },
+ metadata: {
+ source: url,
+ ...(sourceFormat ? {sourceFormat} : {})
+ }
+ };
+}
+
/**
* Fetch a remote dataset URL and wrap it as a File the existing loadFiles pipeline can parse.
* Sets File.type from an explicit format or the response Content-Type so extensionless
diff --git a/src/reducers/src/combined-updaters.ts b/src/reducers/src/combined-updaters.ts
index a4e3a403d..839676609 100644
--- a/src/reducers/src/combined-updaters.ts
+++ b/src/reducers/src/combined-updaters.ts
@@ -32,6 +32,7 @@ import {MapStyle} from './map-style-updaters';
import {ProviderState} from './provider-state-updaters';
import {
loadFilesSuccessUpdaterAction,
+ StageLoadedFilesUpdaterAction,
MapStyleChangeUpdaterAction,
LayerTypeChangeUpdaterAction,
ToggleSplitMapUpdaterAction,
@@ -46,7 +47,8 @@ import {
BASE_MAP_COLOR_MODES,
OVERLAY_BLENDINGS,
NO_MAP_ID,
- MapSplitMode
+ MapSplitMode,
+ ADD_DATA_ID
} from '@kepler.gl/constants';
import {getBasemapColorsForStyle, DEFAULT_BASEMAP_COLOR} from '@kepler.gl/deckgl-layers';
@@ -235,13 +237,17 @@ export const loadFilesSuccessUpdater = (
state: KeplerGlState,
action: loadFilesSuccessUpdaterAction
): KeplerGlState => {
+ if (isStaleFileLoadFinish(state.visState.fileLoading, action.loadId)) {
+ return state;
+ }
// still more to load
- const payloads = filesToDataPayload(action.result);
+ const payloads = filesToDataPayload(action.result, action.options);
const nextState = compose_([
pick_('visState')(
merge_({
fileLoading: false,
- fileLoadingProgress: {}
+ fileLoadingProgress: {},
+ stagedToAdd: null
})
)
])(state);
@@ -252,6 +258,83 @@ export const loadFilesSuccessUpdater = (
return stateWithData as KeplerGlState;
};
+function keptLoadErrors(progress: Record<string, {error?: unknown}> = {}) {
+ return Object.fromEntries(Object.entries(progress).filter(([, value]) => value?.error));
+}
+
+/** A finish from the 200ms pause is stale once that load was canceled or replaced. */
+function isStaleFileLoadFinish(fileLoading: {loadId?: number} | false, loadId?: number): boolean {
+ if (loadId == null) {
+ return false;
+ }
+ return !fileLoading || fileLoading.loadId !== loadId;
+}
+
+/**
+ * Finish a deferred file load: keep the parsed cache, clear the loading flag,
+ * and leave the Add Data modal open until the user confirms.
+ * @memberof combinedUpdaters
+ * @public
+ */
+export const stageLoadedFilesUpdater = (
+ state: KeplerGlState,
+ action: StageLoadedFilesUpdaterAction
+): KeplerGlState => {
+ if (isStaleFileLoadFinish(state.visState.fileLoading, action.loadId)) {
+ return state;
+ }
+ const modalOpen = state.uiState.currentModal === ADD_DATA_ID;
+ const result = action.result || [];
+ const previous =
+ modalOpen && Array.isArray(state.visState.stagedToAdd) ? state.visState.stagedToAdd : [];
+ const stagedToAdd = [...previous, ...result];
+
+ return compose_<KeplerGlState>([
+ pick_('visState')(
+ merge_({
+ fileLoading: false,
+ fileLoadingProgress: modalOpen ? keptLoadErrors(state.visState.fileLoadingProgress) : {},
+ stagedToAdd: modalOpen && stagedToAdd.length ? stagedToAdd : null
+ })
+ ),
+ pick_('uiState')(apply_(uiStateLoadFilesSuccessUpdater, payload_(null)))
+ ])(state);
+};
+
+function stagedItemKey(item): string {
+ return item?.metadata?.source || item?.info?.id || '';
+}
+
+/**
+ * Add datasets to the Add Data list without interrupting a file that is still parsing.
+ * @memberof combinedUpdaters
+ * @public
+ */
+export const appendStagedLoadedFilesUpdater = (
+ state: KeplerGlState,
+ action: StageLoadedFilesUpdaterAction
+): KeplerGlState => {
+ if (state.uiState.currentModal !== ADD_DATA_ID) {
+ return state;
+ }
+ const previous = Array.isArray(state.visState.stagedToAdd) ? state.visState.stagedToAdd : [];
+ const seen = new Set(previous.map(stagedItemKey).filter(Boolean));
+ const next = (action.result || []).filter(item => {
+ const key = stagedItemKey(item);
+ if (!key || seen.has(key)) {
+ return false;
+ }
+ seen.add(key);
+ return true;
+ });
+ if (!next.length) {
+ return state;
+ }
+ return compose_<KeplerGlState>([
+ pick_('visState')(merge_({stagedToAdd: [...previous, ...next]}))
+ ])(state);
+};
+
export const addDataToMapComposed = addDataToMapUpdater;
/**
diff --git a/src/reducers/src/composers.ts b/src/reducers/src/composers.ts
index 40f53676e..a60722398 100644
--- a/src/reducers/src/composers.ts
+++ b/src/reducers/src/composers.ts
@@ -14,6 +14,8 @@ const actionHandler = {
[ActionTypes.SET_MAP_VIEW_MODE]: combinedUpdaters.combinedSetMapViewModeUpdater,
[ActionTypes.LAYER_TYPE_CHANGE]: combinedUpdaters.combinedLayerTypeChangeUpdater,
[ActionTypes.LOAD_FILES_SUCCESS]: combinedUpdaters.loadFilesSuccessUpdater,
+ [ActionTypes.STAGE_LOADED_FILES]: combinedUpdaters.stageLoadedFilesUpdater,
+ [ActionTypes.APPEND_STAGED_LOADED_FILES]: combinedUpdaters.appendStagedLoadedFilesUpdater,
[ActionTypes.TOGGLE_SPLIT_MAP]: combinedUpdaters.toggleSplitMapUpdater,
[ActionTypes.SET_MAP_SPLIT_MODE]: combinedUpdaters.setMapSplitModeUpdater,
[ActionTypes.REPLACE_DATA_IN_MAP]: combinedUpdaters.replaceDataInMapUpdater
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index f41370154..541965302 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -423,6 +423,7 @@ export const INITIAL_VIS_STATE: VisState = {
fileLoading: false,
fileLoadingProgress: {},
+ stagedToAdd: null,
// for loading datasets
loadingIndicatorValue: 0,
loadingProgress: {},
@@ -3786,18 +3787,20 @@ export const updateVisDataUpdater = (
return updatedState;
}
- const createDatasetTasks: TaskDescriptor[] = [];
+ const readyTasks: TaskDescriptor[] = [];
+ const remoteTasks: TaskDescriptor[] = [];
const notificationTasks: TaskDescriptor[] = [];
const hydrateProgress: Record<string, number> = {};
datasets.forEach(({info = {}, ...rest}, datasetIndex) => {
const proto = {info, ...rest};
- if (needsExternallyHostedHydrate(proto)) {
+ const remote = needsExternallyHostedHydrate(proto);
+ if (remote) {
hydrateProgress[getDatasetProgressId(proto, datasetIndex)] = 0;
}
const task = createNewDataEntryTask(proto, state.datasets);
if (task) {
- createDatasetTasks.push(task);
+ (remote ? remoteTasks : readyTasks).push(task);
} else {
notificationTasks.push(
ACTION_TASK_ADD_NOTIFICATION().map(() =>
@@ -3812,15 +3815,22 @@ export const updateVisDataUpdater = (
}
});
+ // Parsed datasets are committed on their own. A remote download in the same
+ // add must not hold them until the URL finishes.
const progressIds = Object.keys(hydrateProgress);
- const datasetsAllSettledTask = createDatasetTasks.length
- ? Task.allSettled(createDatasetTasks).map(results =>
- createNewDatasetSuccess({results, addToMapOptions: options, progressIds})
- )
- : null;
+ const batches: Array<{tasks: TaskDescriptor[]; progressIds: string[]}> = [];
+ if (readyTasks.length) {
+ batches.push({tasks: readyTasks, progressIds: []});
+ }
+ if (remoteTasks.length) {
+ batches.push({tasks: remoteTasks, progressIds});
+ }
- if (datasetsAllSettledTask) {
- updatedState = setLoadingIndicatorUpdater(updatedState, payload_({change: 1, type: ''}));
+ if (batches.length) {
+ updatedState = setLoadingIndicatorUpdater(
+ updatedState,
+ payload_({change: batches.length, type: ''})
+ );
if (progressIds.length) {
updatedState = {
...updatedState,
@@ -3833,7 +3843,15 @@ export const updateVisDataUpdater = (
}
return withTask(updatedState, [
- ...(datasetsAllSettledTask ? [datasetsAllSettledTask] : []),
+ ...batches.map(batch =>
+ Task.allSettled(batch.tasks).map(results =>
+ createNewDatasetSuccess({
+ results,
+ addToMapOptions: options,
+ progressIds: batch.progressIds
+ })
+ )
+ ),
...notificationTasks
]);
};
@@ -4425,38 +4443,114 @@ export function closeSpecificMapAtIndex<S extends VisState>(
* @memberof visStateUpdaters
* @public
*/
+let nextFileLoadId = 1;
+
+function createFileLoadId(): number {
+ const loadId = nextFileLoadId;
+ nextFileLoadId += 1;
+ return loadId;
+}
+
+function finishFileLoad(
+ onFinish: (payload: any, options?: VisStateActions.LoadFilesOptions) => any,
+ fileCache: any[],
+ options: VisStateActions.LoadFilesOptions | undefined,
+ loadId: number
+) {
+ const action = options ? onFinish(fileCache, options) : onFinish(fileCache);
+ return {...action, loadId};
+}
+
export const loadFilesUpdater = (
state: VisState,
action: VisStateActions.LoadFilesUpdaterAction
): VisState => {
- const {files, onFinish = loadFilesSuccess} = action;
+ const {files, onFinish = loadFilesSuccess, options} = action;
if (!files.length) {
return state;
}
const companionFiles = Array.from(files);
const filesToLoad = getFilesToParse(companionFiles);
+
+ // Another drop while a deferred parse is running must join that load.
+ // Replacing `fileLoading` makes the in-flight file finish against the new
+ // cache and drop the files that were already being parsed.
+ if (state.fileLoading && state.fileLoading.options?.deferAddToMap && options?.deferAddToMap) {
+ return appendDeferredFileLoad(state, {filesToLoad, companionFiles});
+ }
+
if (!filesToLoad.length) {
return state;
}
+ // A deferred load is another batch for the same Add Data click. Keep files
+ // already staged, and keep their error cards, instead of starting over.
+ const previousErrors =
+ options?.deferAddToMap && state.fileLoadingProgress
+ ? Object.fromEntries(
+ Object.entries(state.fileLoadingProgress).filter(([, value]) => value?.error)
+ )
+ : {};
+
const fileLoadingProgress = filesToLoad.reduce(
(accu, f, i) => merge_(initialFileLoadingProgress(f, i))(accu),
- {}
+ previousErrors
);
const fileLoading = {
+ loadId: createFileLoadId(),
fileCache: [],
filesToLoad,
companionFiles,
- onFinish
+ onFinish,
+ ...(options ? {options} : {})
};
- const nextState = merge_({fileLoadingProgress, fileLoading})(state);
+ const nextState = merge_({
+ fileLoadingProgress,
+ fileLoading,
+ ...(options?.deferAddToMap ? {} : {stagedToAdd: null})
+ })(state);
return loadNextFileUpdater(nextState);
};
+/**
+ * Add a drop to the deferred load that is already parsing.
+ * The current file keeps its cache; the new files run after it.
+ */
+function appendDeferredFileLoad(
+ state: VisState,
+ {
+ filesToLoad,
+ companionFiles
+ }: {
+ filesToLoad: File[];
+ companionFiles: File[];
+ }
+): VisState {
+ const loading = state.fileLoading;
+ if (!loading || (!filesToLoad.length && !companionFiles.length)) {
+ return state;
+ }
+
+ const fileLoadingProgress = filesToLoad.reduce(
+ (accu, file, index) => merge_(initialFileLoadingProgress(file, index))(accu),
+ state.fileLoadingProgress
+ );
+
+ return {
+ ...state,
+ fileLoadingProgress,
+ fileLoading: {
+ ...loading,
+ filesToLoad: [...Array.from(loading.filesToLoad), ...filesToLoad],
+ companionFiles: [...(loading.companionFiles || []), ...companionFiles]
+ }
+ };
+}
+
/**
* Sucessfully loaded one file, move on to the next one
* @memberof visStateUpdaters
@@ -4470,7 +4564,7 @@ export function loadFileStepSuccessUpdater(
return state;
}
const {fileName, fileCache} = action;
- const {filesToLoad, onFinish} = state.fileLoading;
+ const loadId = state.fileLoading.loadId;
const stateWithProgress = updateFileLoadingProgressUpdater(state, {
fileName,
progress: {percent: 1, message: 'Done'}
@@ -4479,9 +4573,11 @@ export function loadFileStepSuccessUpdater(
// save processed file to fileCache
const stateWithCache = pick_('fileLoading')(merge_({fileCache}))(stateWithProgress);
+ // Decide after the pause, and only for this load. A drop during these
+ // 200ms joins the queue; cancel or a newer load makes this continuation stale.
return withTask(
stateWithCache,
- DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
+ DELAY_TASK(200).map(() => loadNextFile(loadId))
);
}
@@ -4492,11 +4588,18 @@ export function loadFileStepSuccessUpdater(
* @memberof visStateUpdaters
* @public
*/
-export function loadNextFileUpdater(state: VisState): VisState {
- if (!state.fileLoading) {
+export function loadNextFileUpdater(state: VisState, action?: {loadId?: number}): VisState {
+ const loadId = action?.loadId;
+ if (!state.fileLoading || (loadId != null && state.fileLoading.loadId !== loadId)) {
return state;
}
- const {filesToLoad} = state.fileLoading;
+ const {filesToLoad, onFinish, fileCache, options, loadId: currentLoadId} = state.fileLoading;
+ if (!filesToLoad.length) {
+ return withTask(
+ state,
+ ACTION_TASK().map(() => finishFileLoad(onFinish, fileCache, options, currentLoadId))
+ );
+ }
const [file, ...remainingFilesToLoad] = filesToLoad;
// save filesToLoad to state
@@ -4515,7 +4618,8 @@ export function loadNextFileUpdater(state: VisState): VisState {
nextState.fileLoading && nextState.fileLoading.fileCache,
loaders,
loadOptions,
- nextState.fileLoading ? nextState.fileLoading.companionFiles : undefined
+ nextState.fileLoading ? nextState.fileLoading.companionFiles : undefined,
+ options
)
);
}
@@ -4525,7 +4629,8 @@ export function makeLoadFileTask(
fileCache,
loaders: Loader[] = [],
loadOptions = {},
- companionFiles?: File[]
+ companionFiles?: File[],
+ addDataOptions?: VisStateActions.LoadFilesOptions
) {
return LOAD_FILE_TASK({file, fileCache, loaders, loadOptions, companionFiles}).bimap(
// prettier ignore
@@ -4538,7 +4643,8 @@ export function makeLoadFileTask(
processFileContent({
content: result,
fileCache
- })
+ }),
+ ...(addDataOptions ? {options: addDataOptions} : {})
}),
// error
@@ -4592,7 +4698,7 @@ export function parseProgress(prevProgress = {}, progress) {
export const nextFileBatchUpdater = (
state: VisState,
{
- payload: {gen, fileName, progress, accumulated, onFinish}
+ payload: {gen, fileName, progress, accumulated, onFinish, options}
}: VisStateActions.NextFileBatchUpdaterAction
): VisState => {
const stateWithProgress = updateFileLoadingProgressUpdater(state, {
@@ -4600,8 +4706,11 @@ export const nextFileBatchUpdater = (
progress: parseProgress(state.fileLoadingProgress[fileName], progress)
});
+ const deferAddToMap = Boolean(options?.deferAddToMap);
+
return withTask(stateWithProgress, [
...(getApplicationConfig().useArrowProgressiveLoading &&
+ !deferAddToMap &&
fileName.endsWith('arrow') &&
accumulated?.data?.length > 0
? [
@@ -4611,7 +4720,7 @@ export const nextFileBatchUpdater = (
content: {...accumulated, skipArrowCompact: true},
fileCache: []
}).bimap(
- result => loadFilesSuccess(result),
+ result => (options ? loadFilesSuccess(result, options) : loadFilesSuccess(result)),
err => loadFilesErr(fileName, err)
)
]
@@ -4625,7 +4734,8 @@ export const nextFileBatchUpdater = (
fileName,
progress: value.progress,
accumulated: value,
- onFinish
+ onFinish,
+ ...(options ? {options} : {})
});
},
err => loadFilesErr(fileName, err)
@@ -4638,6 +4748,18 @@ export const nextFileBatchUpdater = (
* @memberof visStateUpdaters
* @public
*/
+/**
+ * Drop files waiting on the Add Data button and abort an in-progress modal load.
+ * @memberof visStateUpdaters
+ * @public
+ */
+export const clearStagedLoadedFilesUpdater = (state: VisState): VisState => ({
+ ...state,
+ fileLoading: false,
+ fileLoadingProgress: {},
+ stagedToAdd: null
+});
+
export const loadFilesErrUpdater = (
state: VisState,
{error, fileName}: VisStateActions.LoadFilesErrUpdaterAction
@@ -4647,17 +4769,17 @@ export const loadFilesErrUpdater = (
if (!state.fileLoading) {
return state;
}
- const {filesToLoad, onFinish, fileCache} = state.fileLoading;
+ const loadId = state.fileLoading.loadId;
const nextState = updateFileLoadingProgressUpdater(state, {
fileName,
progress: {error}
});
- // kick off next file or finish
+ // Same pause as a successful file. Re-check this load's queue when it ends.
return withTask(
nextState,
- DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
+ DELAY_TASK(200).map(() => loadNextFile(loadId))
);
};
diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.ts
index 07b5dabdf..863900b8b 100644
--- a/src/reducers/src/vis-state.ts
+++ b/src/reducers/src/vis-state.ts
@@ -54,6 +54,8 @@ const actionHandler = {
[ActionTypes.LOAD_FILES_ERR]: visStateUpdaters.loadFilesErrUpdater,
+ [ActionTypes.CLEAR_STAGED_LOADED_FILES]: visStateUpdaters.clearStagedLoadedFilesUpdater,
+
[ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,
[ActionTypes.LOAD_FILE_STEP_SUCCESS]: visStateUpdaters.loadFileStepSuccessUpdater,
diff --git a/src/schemas/src/vis-state-schema.ts b/src/schemas/src/vis-state-schema.ts
index 682c5f0e1..c749d7a75 100644
--- a/src/schemas/src/vis-state-schema.ts
+++ b/src/schemas/src/vis-state-schema.ts
@@ -92,6 +92,8 @@ export interface VisState {
splitMapsToBeMerged: SplitMap[];
fileLoading: FileLoading | false;
fileLoadingProgress: FileLoadingProgress;
+ /** Parsed files held until the Add Data button commits them. */
+ stagedToAdd: any[] | null;
loadingIndicatorValue: number;
/** Per-dataset download progress (0–100) while hydrating remote files. */
loadingProgress: Record<string, number>;
diff --git a/src/types/reducers.d.ts b/src/types/reducers.d.ts
index 6dc4873b8..40d2650fb 100644
--- a/src/types/reducers.d.ts
+++ b/src/types/reducers.d.ts
@@ -422,11 +422,21 @@ export type MapInfo = {
title: string;
description: string;
};
+/** Options captured when a file load starts and forwarded to `addDataToMap`. */
+export type FileLoadingOptions = {
+ autoCreateLayers?: boolean;
+ /** Parse the files, then wait for an explicit add instead of calling `addDataToMap`. */
+ deferAddToMap?: boolean;
+};
+
export type FileLoading = {
+ /** Identifies this load so a delayed finish from an older one can be ignored. */
+ loadId: number;
filesToLoad: FileList | File[];
companionFiles?: File[];
- onFinish: (payload: any) => any;
+ onFinish: (payload: any, options?: FileLoadingOptions) => any;
fileCache: any[];
+ options?: FileLoadingOptions;
};
export type FileLoadingProgress = {
[key: string]: {
diff --git a/test/browser/components/common/file-uploader-test.js b/test/browser/components/common/file-uploader-test.js
index 0a758b0c2..e1169e6cf 100644
--- a/test/browser/components/common/file-uploader-test.js
+++ b/test/browser/components/common/file-uploader-test.js
@@ -6,7 +6,7 @@ import test from 'tape';
import sinon from 'sinon';
import {IntlWrapper, mountWithTheme} from 'test/helpers/component-utils';
-import {FileUpload, WarningMsg, FileDrop, UploadButton} from '@kepler.gl/components';
+import {FileUpload, FileDrop, UploadButton} from '@kepler.gl/components';
import {initApplicationConfig} from '@kepler.gl/utils';
test('Components -> FileUploader.render', t => {
@@ -101,7 +101,109 @@ test('Components -> FileUpload.onDrop -> render loading msg', t => {
const uploadMsg = wrapper.find('.file-upload-progress__message').at(0).html();
t.comment(uploadMsg);
- t.ok(uploadMsg.includes('tst-file.csv', 'should render upload file msg'));
+ t.ok(uploadMsg.includes('tst-file.csv'), 'should render upload file msg');
+ t.equal(
+ wrapper.find('.upload-file-list').hostNodes().length,
+ 1,
+ 'should list the file beside the drop zone'
+ );
+
+ t.end();
+});
+
+test('Components -> FileUpload.onDrop keeps earlier files', t => {
+ const first = [{type: 'text/csv', name: 'first.csv', size: 10}];
+ const second = [{type: 'text/csv', name: 'second.csv', size: 20}];
+ const onFileUpload = sinon.spy();
+ const wrapper = mountWithTheme(
+ <IntlWrapper>
+ <FileUpload
+ onFileUpload={onFileUpload}
+ fileExtensions={['csv']}
+ stagedToAdd={[{info: {label: 'kept.csv', format: 'csv'}}]}
+ />
+ </IntlWrapper>
+ );
+ const drop = files => {
+ wrapper
+ .find('.file-uploader__file-drop')
+ .at(0)
+ .simulate('drop', {
+ stopPropagation: () => {},
+ dataTransfer: {types: ['Files'], files}
+ });
+ };
+
+ drop(first);
+ drop(second);
+
+ const files = wrapper.find(FileUpload).children().first().state().files;
+ t.deepEqual(
+ files.map(file => file.name),
+ ['first.csv', 'second.csv'],
+ 'should append dropped files'
+ );
+ t.deepEqual(onFileUpload.lastCall.args[0], second, 'should upload only the new files');
+ const listText = wrapper.find('.upload-file-list').text();
+ t.ok(listText.includes('kept.csv'), 'should keep a file staged before this uploader mounted');
+ t.ok(listText.includes('first.csv'), 'should keep the first dropped file');
+ t.ok(listText.includes('second.csv'), 'should show the latest dropped file');
+ const fileChecks = wrapper.find('.upload-file-list').find('Checkbox');
+ t.equal(fileChecks.length, 3, 'each dataset card has a checkbox');
+ t.equal(fileChecks.at(0).props().checked, true, 'datasets start selected');
+ t.equal(
+ fileChecks.at(0).find('.upload-file-list__check-label').text(),
+ 'Include kept.csv',
+ 'checkbox names the dataset it includes'
+ );
+ fileChecks.at(0).simulate('change');
+ t.equal(
+ wrapper.find('.upload-file-list').find('Checkbox').at(0).props().checked,
+ false,
+ 'should let the user deselect a dataset'
+ );
+
+ t.end();
+});
+
+test('Components -> FileUpload remote URL is staged without downloading', t => {
+ const onFileUpload = sinon.spy();
+ const onAddRemoteDataset = sinon.spy();
+ const wrapper = mountWithTheme(
+ <IntlWrapper>
+ <FileUpload
+ onFileUpload={onFileUpload}
+ onAddRemoteDataset={onAddRemoteDataset}
+ fileExtensions={['csv']}
+ stagedToAdd={[
+ {
+ info: {label: 'quakes.csv', format: 'row'},
+ metadata: {source: 'https://example.com/quakes.csv'}
+ }
+ ]}
+ />
+ </IntlWrapper>
+ );
+
+ const listText = wrapper.find('.upload-file-list').text();
+ t.ok(listText.includes('quakes.csv'), 'should list the remote dataset');
+ t.ok(listText.includes('https://example.com/quakes.csv'), 'should show the remote url');
+
+ const urlInput = wrapper.find('.file-uploader__remote-url input').hostNodes().first();
+ urlInput.simulate('change', {target: {value: 'https://example.com/cities.csv'}});
+ wrapper.find('.file-uploader__remote-add').hostNodes().first().simulate('click');
+
+ t.ok(onFileUpload.notCalled, 'should not download the url through the file loader');
+ t.deepEqual(
+ onAddRemoteDataset.lastCall.args[0],
+ {url: 'https://example.com/cities.csv'},
+ 'should stage the url for Add Data'
+ );
+ t.equal(
+ wrapper.find(FileUpload).children().first().state().remoteUrl,
+ '',
+ 'should clear the url field after adding it'
+ );
t.end();
});
@@ -131,7 +233,15 @@ test('Components -> FileUpload.onDrop -> render error msg', t => {
FileDropDiv.simulate('drop', mockEvent);
t.ok(onFileUpload.notCalled, 'onFileUpload should not get called');
- t.ok(wrapper.find(WarningMsg), 'should render WarningMsg');
+ t.ok(
+ wrapper.find('.upload-file-list').text().includes('tst-file.png'),
+ 'should list the unsupported file'
+ );
+ t.equal(
+ wrapper.find('.upload-file-list').find('Checkbox').length,
+ 0,
+ 'unsupported files cannot be selected'
+ );
const errorFiles = wrapper.find(FileUpload).children().first().state().errorFiles;
t.deepEqual(errorFiles, ['tst-file.png'], 'should save files to errorFiles');
diff --git a/test/browser/components/modals/load-data-modal-test.js b/test/browser/components/modals/load-data-modal-test.js
index 20aca1910..e9e3515b9 100644
--- a/test/browser/components/modals/load-data-modal-test.js
+++ b/test/browser/components/modals/load-data-modal-test.js
@@ -3,17 +3,20 @@
import React from 'react';
import test from 'tape';
+import sinon from 'sinon';
import {IntlWrapper, mountWithTheme} from 'test/helpers/component-utils';
import {STYLED_COMPONENTS_DUPLICATED_ENTRIES} from '../../../helpers/utils';
import {
LoadDataModalFactory,
ModalTabItem,
LoadStorageMapFactory,
+ FileUploadFactory,
appInjector
} from '@kepler.gl/components';
const LoadDataModal = appInjector.get(LoadDataModalFactory);
const LoadStorageMap = appInjector.get(LoadStorageMapFactory);
+const FileUpload = appInjector.get(FileUploadFactory);
test('Components -> LoadDataModal.mount', t => {
// mount
@@ -68,3 +71,83 @@ test('Components -> LoadDataModal -> custom loading method', t => {
t.equal(wrapper.find(MockTabComp).length, 1, 'should render MockTabComp');
t.end();
});
+
+test('Components -> LoadDataModal -> auto create layers', t => {
+ const onFileUpload = sinon.spy();
+ const onTilesetAdded = sinon.spy();
+ const onConfirmAddData = sinon.spy();
+ const onClose = sinon.spy();
+ const wrapper = mountWithTheme(
+ <IntlWrapper>
+ <LoadDataModal
+ onFileUpload={onFileUpload}
+ onTilesetAdded={onTilesetAdded}
+ onConfirmAddData={onConfirmAddData}
+ onClose={onClose}
+ stagedToAdd={[{info: {label: 'points.csv'}}]}
+ />
+ </IntlWrapper>
+ );
+
+ const checkbox = () => wrapper.find('.add-data-bar').find('Checkbox').first();
+ t.equal(checkbox().props().checked, true, 'auto create layers is checked by default');
+ t.equal(
+ wrapper.find('.file-uploader .auto-create-layers').length,
+ 0,
+ 'checkbox sits in the footer'
+ );
+
+ checkbox().simulate('change');
+ t.equal(checkbox().props().checked, false, 'should uncheck auto create layers');
+
+ const files = [{name: 'points.csv'}];
+ const uploader = wrapper.find(FileUpload);
+ t.ok(uploader.length > 0, 'should find the file uploader rendered by the modal');
+ uploader.first().props().onFileUpload(files);
+ t.deepEqual(onFileUpload.args[0][0], files, 'should upload the selected files');
+ t.equal(onFileUpload.args[0][1], undefined, 'parsing waits to apply autoCreateLayers');
+
+ const buttons = wrapper.find('.add-data-bar button');
+ buttons.at(0).simulate('click');
+ t.equal(onClose.calledOnce, true, 'cancel closes the modal');
+
+ buttons.at(1).simulate('click');
+ t.deepEqual(
+ onConfirmAddData.args[0][0],
+ {autoCreateLayers: false, datasets: [{info: {label: 'points.csv'}}]},
+ 'Add Data commits the current autoCreateLayers choice'
+ );
+
+ const fileCheckbox = wrapper.find('.upload-file-list').find('Checkbox').first();
+ t.equal(fileCheckbox.props().checked, true, 'staged datasets start selected');
+ fileCheckbox.simulate('change');
+ t.equal(
+ wrapper.find('.add-data-bar button').at(1).props().disabled,
+ true,
+ 'Add Data waits until a dataset is checked'
+ );
+ wrapper.find('.add-data-bar button').at(1).simulate('click');
+ t.equal(onConfirmAddData.callCount, 1, 'an unchecked dataset is not added');
+
+ const tabs = wrapper.find('.load-data-modal__tab__item');
+ let clickedTileset = false;
+ for (let i = 0; i < tabs.length; i++) {
+ if (tabs.at(i).text() === 'Tileset') {
+ tabs.at(i).simulate('click');
+ clickedTileset = true;
+ break;
+ }
+ }
+ t.equal(clickedTileset, true, 'should find the tileset tab');
+ t.equal(
+ wrapper.find('.add-data-bar').find('Checkbox').first().props().checked,
+ false,
+ 'tileset tab keeps the unchecked choice'
+ );
+ t.equal(wrapper.find('.load-data-footer').length, 0, 'tileset confirm uses the shared footer');
+ const tilesetButtons = wrapper.find('.add-data-bar button');
+ t.equal(tilesetButtons.at(1).props().disabled, true, 'Add Data waits until the tileset is ready');
+ t.equal(onTilesetAdded.called, false, 'opening the tileset tab does not add data');
+
+ t.end();
+});
diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.js
index d734fec8d..f533681b9 100644
--- a/test/node/processors/file-handler-test.js
+++ b/test/node/processors/file-handler-test.js
@@ -7,6 +7,7 @@ import {
makeProgressIterator,
filesToDataPayload,
processFileData,
+ remoteDatasetFromUrl,
processArrowBatches,
getGeoJsonFromLoaderResult,
readBatch
@@ -167,6 +168,55 @@ test('#file-handler -> filesToDataPayload', t => {
t.end();
});
+test('#file-handler -> filesToDataPayload autoCreateLayers', t => {
+ const fileCache = [
+ {
+ data: {
+ fields: parsedFields,
+ rows: parsedRows
+ },
+ info: {label: 'rows-data.json', format: 'row'}
+ },
+ {
+ data: {
+ datasets: [],
+ config: keplerglMap.config
+ },
+ info: {label: 'keplergl-map.json', format: 'keplergl'}
+ }
+ ];
+
+ const result = filesToDataPayload(fileCache, {autoCreateLayers: false});
+
+ t.deepEqual(
+ result[0].options,
+ {centerMap: true, autoCreateLayers: false},
+ 'should keep kepler map centerMap and forward autoCreateLayers'
+ );
+ t.deepEqual(
+ result[1].options,
+ {autoCreateLayers: false},
+ 'should forward autoCreateLayers for dataset files'
+ );
+
+ t.end();
+});
+
+test('#file-handler -> remoteDatasetFromUrl waits for the remote download', t => {
+ const dataset = remoteDatasetFromUrl('https://example.com/quakes.csv?sv=1', 'csv');
+ t.equal(dataset.info.type, 'externally-hosted', 'should be a remote dataset');
+ t.equal(dataset.info.label, 'quakes.csv', 'should name the card from the url');
+ t.deepEqual(dataset.data, {fields: [], rows: []}, 'should not include downloaded rows');
+ t.equal(dataset.metadata.source, 'https://example.com/quakes.csv?sv=1');
+ t.equal(dataset.metadata.sourceFormat, 'csv', 'should keep the selected format');
+
+ const payload = filesToDataPayload([dataset]);
+ t.equal(payload[0].datasets[0].info.type, 'externally-hosted', 'Add Data keeps the remote type');
+ t.deepEqual(payload[0].datasets[0].data, {fields: [], rows: []}, 'Add Data still has no rows');
+
+ t.end();
+});
+
test('#file-handler -> filesToDataPayload remote metadata', t => {
const fileCache = [
{
diff --git a/test/node/reducers/composer-state-test.js b/test/node/reducers/composer-state-test.js
index 6fd21e2ce..b3897e096 100644
--- a/test/node/reducers/composer-state-test.js
+++ b/test/node/reducers/composer-state-test.js
@@ -6,7 +6,7 @@
import {drainTasksForTesting, succeedTaskWithValues} from '@kepler.gl/tasks';
import test from 'tape';
-import {registerEntry} from '@kepler.gl/actions';
+import {appendStagedLoadedFiles, registerEntry, stageLoadedFiles} from '@kepler.gl/actions';
import {processCsvData} from '@kepler.gl/processors';
import keplerGlReducer, {
addDataToMapUpdater,
@@ -1178,3 +1178,106 @@ test('#composerStateReducer - replaceDataInMapUpdater: syncedTimeFilter & no mat
t.deepEqual(nextState.visState.splitMapsToBeMerged, [], 'should reset splitMapsToBeMerged');
t.end();
});
+
+test('#composerStateReducer -> stageLoadedFilesUpdater keeps the modal open', t => {
+ const base = keplerGlReducer(undefined, registerEntry({id: 'test'})).test;
+ const result = [{info: {label: 'points.csv', format: 'csv'}, data: []}];
+ const loadingState = {
+ ...base,
+ uiState: {
+ ...base.uiState,
+ currentModal: 'addData',
+ loadFiles: {fileLoading: true}
+ },
+ visState: {
+ ...base.visState,
+ fileLoading: {filesToLoad: [], fileCache: [], onFinish: () => null},
+ fileLoadingProgress: {
+ a: {percent: 1, message: 'Done', fileName: 'points.csv', error: null}
+ }
+ }
+ };
+
+ const staged = combinedUpdaters.stageLoadedFilesUpdater(loadingState, stageLoadedFiles(result));
+ t.equal(staged.uiState.currentModal, 'addData', 'should leave the Add Data modal open');
+ t.equal(staged.uiState.loadFiles.fileLoading, false, 'should clear the loading flag');
+ t.equal(staged.visState.fileLoading, false, 'should finish the visState load');
+ t.equal(staged.visState.stagedToAdd[0].info.label, 'points.csv', 'should keep the parsed file');
+ t.equal(staged.visState.datasets, base.visState.datasets, 'should not add datasets yet');
+
+ const closed = {
+ ...loadingState,
+ uiState: {...loadingState.uiState, currentModal: null}
+ };
+ const dropped = combinedUpdaters.stageLoadedFilesUpdater(closed, stageLoadedFiles(result));
+ t.equal(
+ dropped.visState.stagedToAdd,
+ null,
+ 'should discard files when the modal is already closed'
+ );
+
+ const withPrevious = {
+ ...loadingState,
+ visState: {
+ ...loadingState.visState,
+ stagedToAdd: [{info: {label: 'points.csv', format: 'csv'}, data: []}]
+ }
+ };
+ const appended = combinedUpdaters.stageLoadedFilesUpdater(
+ withPrevious,
+ stageLoadedFiles([{info: {label: 'cities.csv', format: 'csv'}, data: []}])
+ );
+ t.deepEqual(
+ appended.visState.stagedToAdd.map(item => item.info.label),
+ ['points.csv', 'cities.csv'],
+ 'should append a later file instead of replacing the staged list'
+ );
+
+ const replacedLoad = {
+ ...loadingState,
+ visState: {
+ ...loadingState.visState,
+ fileLoading: {...loadingState.visState.fileLoading, loadId: 2},
+ stagedToAdd: null
+ }
+ };
+ const staleFinish = combinedUpdaters.stageLoadedFilesUpdater(replacedLoad, {
+ ...stageLoadedFiles(result),
+ loadId: 1
+ });
+ t.equal(staleFinish, replacedLoad, 'a finish from a canceled load does not replace the new one');
+
+ const canceledLoad = {
+ ...loadingState,
+ visState: {...loadingState.visState, fileLoading: false, stagedToAdd: null}
+ };
+ const afterCancel = combinedUpdaters.stageLoadedFilesUpdater(canceledLoad, {
+ ...stageLoadedFiles(result),
+ loadId: 1
+ });
+ t.equal(afterCancel.visState.fileLoading, false, 'a canceled load stays stopped');
+ t.equal(afterCancel.visState.stagedToAdd, null, 'a canceled load is not restored');
+
+ const remote = {
+ data: {fields: [], rows: []},
+ info: {id: 'remote-1', label: 'quakes.csv', format: 'row', type: 'externally-hosted'},
+ metadata: {source: 'https://example.com/quakes.csv', sourceFormat: 'csv'}
+ };
+ const withRemote = combinedUpdaters.appendStagedLoadedFilesUpdater(
+ appended,
+ appendStagedLoadedFiles([remote])
+ );
+ t.deepEqual(
+ withRemote.visState.stagedToAdd.map(item => item.info.label),
+ ['points.csv', 'cities.csv', 'quakes.csv'],
+ 'should add a remote url without replacing staged files'
+ );
+ t.equal(withRemote.visState.fileLoading, false, 'should not start a file load for a remote url');
+ const duplicate = combinedUpdaters.appendStagedLoadedFilesUpdater(
+ withRemote,
+ appendStagedLoadedFiles([remote])
+ );
+ t.equal(duplicate, withRemote, 'should ignore a url that is already staged');
+
+ t.end();
+});
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index 43a181455..3af03dee8 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -26,7 +26,7 @@ import {
calculateLayerData
} from '@kepler.gl/reducers';
-import {processCsvData, processGeojson} from '@kepler.gl/processors';
+import {processCsvData, processGeojson, remoteDatasetFromUrl} from '@kepler.gl/processors';
import {Layer, KeplerGlLayers, COLUMN_MODE_TABLE} from '@kepler.gl/layers';
import {maybeToDate} from '@kepler.gl/table';
import {
@@ -6769,7 +6769,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {
'test-file-2.csv': {percent: 0, message: '', fileName: 'test-file-2.csv', error: null}
};
- t.deepEqual(nextState.fileLoading, expectedFileLoading, 'should save fileLoading in state');
+ t.deepEqual(
+ nextState.fileLoading,
+ {...expectedFileLoading, loadId: nextState.fileLoading.loadId},
+ 'should save fileLoading in state'
+ );
+ t.equal(typeof nextState.fileLoading.loadId, 'number', 'should assign a load session');
t.deepEqual(
nextState.fileLoadingProgress,
expectedFileLoadingProgress,
@@ -6845,6 +6850,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.deepEqual(
nextState3Err.fileLoading,
{
+ loadId: nextState.fileLoading.loadId,
fileCache: [],
filesToLoad: [],
companionFiles: mockFiles,
@@ -6930,6 +6936,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.deepEqual(
resultState5.fileLoading,
{
+ loadId: nextState.fileLoading.loadId,
fileCache: fileProcessResult,
filesToLoad: [{type: 'text/csv', name: 'test-file-2.csv'}],
companionFiles: mockFiles,
@@ -6969,6 +6976,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.deepEqual(
resultState6.fileLoading,
{
+ loadId: nextState.fileLoading.loadId,
fileCache: fileProcessResult,
filesToLoad: [],
companionFiles: mockFiles,
@@ -7012,6 +7020,7 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.deepEqual(
resultState10.fileLoading,
{
+ loadId: nextState.fileLoading.loadId,
fileCache: file2ProcessResult,
filesToLoad: [],
companionFiles: mockFiles,
@@ -7024,7 +7033,11 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.equal(task10.type, 'DELAY_TASK', 'should return an DELAY_TASK for onFinish');
// calling delayed task succeed to trigger load next file
- const _resultState11 = reducer(resultState10, succeedTaskInTest(task10));
+ const resultState11 = reducer(resultState10, succeedTaskInTest(task10));
+ const [task11, ...more11] = drainTasksForTesting();
+ t.equal(more11.length, 0, 'should create 1 task');
+ t.equal(task11.type, 'ACTION_TASK', 'an empty queue finishes the load');
+ reducer(resultState11, succeedTaskInTest(task11));
t.ok(loadFilesSuccessSpy.calledOnce);
const expectedArgs = [
@@ -7038,6 +7051,207 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.end();
});
+test('#visStateReducer -> LOAD_FILES autoCreateLayers option', t => {
+ drainTasksForTesting();
+
+ const initialState = CloneDeep(InitialState).visState;
+ const mockFiles = [{type: 'text/csv', name: 'test-file.csv'}];
+ const nextState = reducer(
+ initialState,
+ VisStateActions.loadFiles(mockFiles, undefined, {autoCreateLayers: false})
+ );
+
+ t.equal(
+ nextState.fileLoading.options.autoCreateLayers,
+ false,
+ 'should keep autoCreateLayers on the in-progress file load'
+ );
+
+ drainTasksForTesting();
+ t.end();
+});
+
+test('#visStateReducer -> LOAD_FILES deferAddToMap', t => {
+ drainTasksForTesting();
+
+ const initialState = CloneDeep(InitialState).visState;
+ const mockFiles = [{type: 'text/csv', name: 'test-file.csv'}];
+ const nextState = reducer(
+ initialState,
+ VisStateActions.loadFiles(mockFiles, VisStateActions.stageLoadedFiles, {deferAddToMap: true})
+ );
+
+ t.equal(
+ nextState.fileLoading.onFinish,
+ VisStateActions.stageLoadedFiles,
+ 'should stage instead of adding'
+ );
+ t.equal(
+ nextState.fileLoading.options.deferAddToMap,
+ true,
+ 'should keep deferAddToMap on the load'
+ );
+
+ const alreadyStaged = [{info: {label: 'points.csv'}}];
+ const withStaged = {...initialState, stagedToAdd: alreadyStaged};
+ const deferred = reducer(
+ withStaged,
+ VisStateActions.loadFiles(mockFiles, VisStateActions.stageLoadedFiles, {deferAddToMap: true})
+ );
+ t.equal(deferred.stagedToAdd, alreadyStaged, 'should keep files already staged for Add Data');
+
+ const replaced = reducer(withStaged, VisStateActions.loadFiles(mockFiles));
+ t.equal(replaced.stagedToAdd, null, 'should clear staged files when adding them immediately');
+
+ drainTasksForTesting();
+ const remote = remoteDatasetFromUrl('https://example.com/quakes.csv');
+ const hydrating = reducer(
+ CloneDeep(InitialState).visState,
+ VisStateActions.updateVisData([remote])
+ );
+ t.equal(
+ hydrating.loadingProgress[remote.info.id],
+ 0,
+ 'adding a staged remote url should start the remote download'
+ );
+ t.equal(
+ hydrating.datasets[remote.info.id],
+ undefined,
+ 'should not insert rows before the download'
+ );
+ drainTasksForTesting();
+
+ const queued = reducer(
+ nextState,
+ VisStateActions.loadFiles(
+ [{type: 'text/csv', name: 'extra.csv'}],
+ VisStateActions.stageLoadedFiles,
+ {
+ deferAddToMap: true
+ }
+ )
+ );
+ t.deepEqual(
+ queued.fileLoading.filesToLoad.map(file => file.name),
+ ['extra.csv'],
+ 'a second drop waits behind the file already parsing'
+ );
+ t.equal(
+ queued.fileLoading.fileCache,
+ nextState.fileLoading.fileCache,
+ 'should keep the in-progress cache'
+ );
+ t.ok(queued.fileLoadingProgress['test-file.csv'], 'should keep progress for the first file');
+ t.ok(queued.fileLoadingProgress['extra.csv'], 'should show progress for the queued file');
+
+ const readyToFinish = {
+ ...nextState,
+ fileLoading: {
+ ...nextState.fileLoading,
+ filesToLoad: [],
+ fileCache: [{info: {label: 'test-file.csv'}}]
+ }
+ };
+ const droppedDuringPause = reducer(
+ readyToFinish,
+ VisStateActions.loadFiles(
+ [{type: 'text/csv', name: 'late.csv'}],
+ VisStateActions.stageLoadedFiles,
+ {deferAddToMap: true}
+ )
+ );
+ const continued = reducer(droppedDuringPause, VisStateActions.loadNextFile());
+ t.equal(
+ continued.fileLoadingProgress['late.csv'].message,
+ 'loading...',
+ 'a file dropped during the finish pause starts parsing'
+ );
+ t.equal(
+ continued.fileLoading.fileCache,
+ readyToFinish.fileLoading.fileCache,
+ 'should keep the parsed files while the queued file loads'
+ );
+
+ const canceled = reducer(readyToFinish, VisStateActions.clearStagedLoadedFiles());
+ drainTasksForTesting();
+ const ignored = reducer(canceled, VisStateActions.loadNextFile(readyToFinish.fileLoading.loadId));
+ t.equal(ignored.fileLoading, false, 'a canceled load is not resumed');
+ t.equal(ignored.stagedToAdd, null, 'cancel stays discarded');
+ t.equal(drainTasksForTesting().length, 0, 'a canceled pause does not stage files');
+
+ const restarted = reducer(
+ canceled,
+ VisStateActions.loadFiles(
+ [{type: 'text/csv', name: 'fresh.csv'}],
+ VisStateActions.stageLoadedFiles,
+ {deferAddToMap: true}
+ )
+ );
+ drainTasksForTesting();
+ const stale = reducer(restarted, VisStateActions.loadNextFile(readyToFinish.fileLoading.loadId));
+ t.equal(stale.fileLoading, restarted.fileLoading, 'a stale pause does not replace the new load');
+ t.equal(drainTasksForTesting().length, 0, 'a stale pause does not stage or parse');
+
+ const cleared = reducer(nextState, VisStateActions.clearStagedLoadedFiles());
+ t.equal(cleared.fileLoading, false, 'cancel should stop the in-progress load');
+ t.equal(cleared.stagedToAdd, null, 'cancel should drop staged files');
+
+ drainTasksForTesting();
+ t.end();
+});
+
+test('#visStateReducer -> local dataset is not blocked by a remote download', t => {
+ drainTasksForTesting();
+ const local = {
+ info: {id: 'local-points', label: 'points.csv'},
+ data: processCsvData('lat,lng\n1,2')
+ };
+ const remote = remoteDatasetFromUrl('https://example.com/slow.csv');
+ const nextState = reducer(
+ CloneDeep(InitialState).visState,
+ VisStateActions.updateVisData([local, remote])
+ );
+ const tasks = drainTasksForTesting();
+ const ready = tasks.find(
+ task => task.label.includes('CREATE_TABLE_TASK') && !task.label.includes('HYDRATE')
+ );
+ const remoteTask = tasks.find(task =>
+ task.label.includes('HYDRATE_EXTERNALLY_HOSTED_DATASET_TASK')
+ );
+
+ t.ok(ready, 'should commit the parsed dataset on its own');
+ t.ok(remoteTask, 'should download the remote dataset separately');
+ t.equal(
+ nextState.datasets['local-points'],
+ undefined,
+ 'parsed rows are inserted by the ready task'
+ );
+ t.equal(nextState.loadingProgress[remote.info.id], 0, 'remote download stays in progress');
+
+ ready.run(
+ (effect, resolve, reject, ctx) => effect(resolve, reject, ctx),
+ action => {
+ const added = reducer(nextState, action);
+ t.equal(
+ added.datasets['local-points'].id,
+ 'local-points',
+ 'local dataset is added before the remote download'
+ );
+ t.equal(added.datasets[remote.info.id], undefined, 'remote dataset is still absent');
+ t.equal(
+ added.loadingIndicatorValue > 0,
+ true,
+ 'loading indicator stays up for the remote download'
+ );
+ t.end();
+ },
+ err => {
+ t.error(err);
+ t.end();
+ }
+ );
+});
+
test('#visStateReducer -> setLayerAnimationTimeConfig', t => {
// change Trip layer isVisible
const nextState = reducer(