Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Enhancement] Refactor load file tasks to better handle multiple file types (#986)

Shan He7936828bc3c5
diff --git a/src/actions/vis-state-actions.js b/src/actions/vis-state-actions.js
index 080bc556c..c0e82aa55 100644
--- a/src/actions/vis-state-actions.js
+++ b/src/actions/vis-state-actions.js
@@ -550,12 +550,32 @@ export function loadFiles(files) {
};
}
-export function loadFileSuccess({fileCache, filesToLoad, totalCount}) {
+/**
+ * called with next file to load
+ * @param {object} payload
+ * @param {Array<object>} payload.fileCache
+ * @param {Array<object>} payload.filesToLoad
+ * @param {number} payload.totalCount
+ * @param {Function} payload.onFinish - action creator to execute when all files are loaded
+ */
+export function loadNextFile({fileCache, filesToLoad, totalCount, onFinish}) {
return {
- type: ActionTypes.LOAD_FILES_SUCCESS,
+ type: ActionTypes.LOAD_NEXT_FILE,
fileCache,
filesToLoad,
- totalCount
+ totalCount,
+ onFinish
+ };
+}
+
+/**
+ * called when all files are processed and loaded
+ * @param {Array<object>} result
+ */
+export function loadFileSuccess(result) {
+ return {
+ type: ActionTypes.LOAD_FILES_SUCCESS,
+ result
};
}
diff --git a/src/constants/action-types.js b/src/constants/action-types.js
index 9e1304adb..6f3b65f53 100644
--- a/src/constants/action-types.js
+++ b/src/constants/action-types.js
@@ -92,6 +92,7 @@ const ActionTypes = keyMirror({
TOGGLE_LAYER_FOR_MAP: null,
SET_FILTER_PLOT: null,
LOAD_FILES: null,
+ LOAD_NEXT_FILE: null,
LOAD_FILES_ERR: null,
LOAD_FILES_SUCCESS: null,
LAYER_COLOR_UI_CHANGE: null,
diff --git a/src/constants/index.js b/src/constants/index.js
index 96980b4b1..8e94baf99 100644
--- a/src/constants/index.js
+++ b/src/constants/index.js
@@ -25,6 +25,7 @@ export {
AGGREGATION_TYPES,
ALL_FIELD_TYPES,
DATA_TABLE_ID,
+ DATASET_FORMATS,
DEFAULT_COLOR_RANGE,
DEFAULT_LAYER_GROUPS,
DEFAULT_MAP_STYLES,
diff --git a/src/processors/file-handler.js b/src/processors/file-handler.js
index 7593ded87..da55ce924 100644
--- a/src/processors/file-handler.js
+++ b/src/processors/file-handler.js
@@ -26,7 +26,7 @@ import {
processKeplerglJSON,
processRowObject
} from './data-processor';
-import {isPlainObject} from 'utils/utils';
+import {isPlainObject, generateHashId} from 'utils/utils';
import {DATASET_FORMATS} from 'constants/default-settings';
const FILE_HANDLERS = {
@@ -171,3 +171,38 @@ export function determineJsonProcess({dataset, format}, defaultProcessor) {
return defaultProcessor;
}
+
+export function filesToDataPayload(fileCache) {
+ // seperate out files which could be a single datasets. or a keplergl map json
+ const collection = fileCache.reduce(
+ (accu, file) => {
+ const {data, info = {}} = file;
+ const {format} = info;
+ if (format === DATASET_FORMATS.keplergl) {
+ // if file contains a single kepler map dataset & config
+ accu.keplerMaps.push({
+ ...data,
+ options: {
+ centerMap: !(data.config && data.config.mapState)
+ }
+ });
+ } else if (DATASET_FORMATS[format]) {
+ // if file contains only data
+ const newDataset = {
+ data,
+ info: {
+ id: info.id || generateHashId(4),
+ ...info
+ }
+ };
+ accu.datasets.push(newDataset);
+ }
+ return accu;
+ },
+ {datasets: [], keplerMaps: []}
+ );
+
+ // add kepler map first with config
+ // add datasets later in one add data call
+ return collection.keplerMaps.concat({datasets: collection.datasets});
+}
diff --git a/src/processors/index.js b/src/processors/index.js
index fb8ad393d..77e9316ac 100644
--- a/src/processors/index.js
+++ b/src/processors/index.js
@@ -30,6 +30,6 @@ export {
} from './data-processor';
// File Handlers
-export {getFileHandler, getFileType} from './file-handler';
+export {getFileHandler, getFileType, readFile, filesToDataPayload} from './file-handler';
export {Processors as default} from './data-processor';
diff --git a/src/reducers/combined-updaters.js b/src/reducers/combined-updaters.js
index fbfd42b1c..3a3754bbc 100644
--- a/src/reducers/combined-updaters.js
+++ b/src/reducers/combined-updaters.js
@@ -18,6 +18,27 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// to use, copy, modify, merge_, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
import {toggleModalUpdater, loadFilesSuccessUpdater} from './ui-state-updaters';
import {
updateVisDataUpdater as visStateUpdateVisDataUpdater,
@@ -28,6 +49,8 @@ import {receiveMapConfigUpdater as styleMapConfigUpdater} from './map-style-upda
import {findMapBounds} from 'utils/data-utils';
import KeplerGlSchema from 'schemas';
import {isPlainObject} from 'utils/utils';
+import {filesToDataPayload} from 'processors/file-handler';
+import Console from 'global/console';
// compose action to apply result multiple reducers, with the output of one
@@ -78,10 +101,47 @@ const combinedUpdaters = null;
export const isValidConfig = config =>
isPlainObject(config) && isPlainObject(config.config) && config.version;
+
export const defaultAddDataToMapOptions = {
centerMap: true,
keepExistingConfig: false
};
+
+const identity = state => state;
+
+/* eslint-disable-next-line no-unused-vars */
+function log(text) {
+ return value => Console.log(text, value);
+}
+
+function payload_(p) {
+ return {payload: p};
+}
+
+function apply_(updater, payload) {
+ return state => updater(state, payload);
+}
+
+function with_(fn) {
+ return state => fn(state)(state);
+}
+
+function if_(pred, fn) {
+ return pred ? fn : identity;
+}
+
+function compose_(fns) {
+ return state => fns.reduce((state2, fn) => fn(state2), state);
+}
+
+function merge_(obj) {
+ return state => ({...state, ...obj});
+}
+
+function pick_(prop) {
+ return fn => state => ({...state, [prop]: fn(state[prop])});
+}
+
/**
* Combine data and full configuration update in a single action
*
@@ -118,61 +178,58 @@ export const addDataToMapUpdater = (state, {payload}) => {
parsedConfig = KeplerGlSchema.parseSavedConfig(config);
}
const oldLayers = state.visState.layers;
+ const filterNewlyAddedLayers = layers => layers.filter(nl => !oldLayers.find(ol => ol === nl));
- // Update visState store
- let mergedState = {
- ...state,
- visState: visStateUpdateVisDataUpdater(state.visState, {
- datasets,
- options,
- config: parsedConfig
- })
- };
+ return compose_([
+ pick_('visState')(
+ apply_(visStateUpdateVisDataUpdater, {
+ datasets,
+ options,
+ config: parsedConfig
+ })
+ ),
- // merge map info
- // save title, description provider and queryParams
- if (info) {
- mergedState = {
- ...mergedState,
- visState: setMapInfoUpdater(mergedState.visState, {info})
- };
- }
+ if_(info, pick_('visState')(apply_(setMapInfoUpdater, {info}))),
- let bounds;
- if (options.centerMap) {
- // find map bounds for new layers
- const newLayers = mergedState.visState.layers.filter(nl => !oldLayers.find(ol => ol === nl));
- bounds = findMapBounds(newLayers);
- }
+ with_(({visState}) =>
+ pick_('mapState')(
+ apply_(
+ stateMapConfigUpdater,
+ payload_({
+ config: parsedConfig,
+ options,
+ bounds: options.centerMap
+ ? findMapBounds(filterNewlyAddedLayers(visState.layers))
+ : null
+ })
+ )
+ )
+ ),
- // Update mapState store
- mergedState = {
- ...mergedState,
- mapState: stateMapConfigUpdater(mergedState.mapState, {
- payload: {config: parsedConfig, options, bounds}
- })
- };
+ pick_('mapStyle')(apply_(styleMapConfigUpdater, payload_({config: parsedConfig, options}))),
- // Update mapStyle store
- mergedState = {
- ...mergedState,
- mapStyle: styleMapConfigUpdater(mergedState.mapStyle, {
- payload: {config: parsedConfig, options}
- })
- };
+ pick_('uiState')(apply_(loadFilesSuccessUpdater)),
- // Update uiState
- mergedState = {
- ...mergedState,
- uiState: {
- ...toggleModalUpdater(loadFilesSuccessUpdater(mergedState.uiState), {
- payload: null
- }),
- ...(options.hasOwnProperty('readOnly') ? {readOnly: options.readOnly} : {})
- }
- };
+ pick_('uiState')(apply_(toggleModalUpdater, payload_(null))),
+
+ pick_('uiState')(merge_(options.hasOwnProperty('readOnly') ? {readOnly: options.readOnly} : {}))
+ ])(state);
+};
+
+export const loadFileSuccessUpdater = (state, action) => {
+ // still more to load
+ const payloads = filesToDataPayload(action.result);
+ const nextState = compose_([
+ pick_('visState')(
+ merge_({
+ fileLoading: false,
+ fileLoadingProgress: 100
+ })
+ )
+ ])(state);
- return mergedState;
+ // make multiple add data to map calls
+ return compose_(payloads.map(p => apply_(addDataToMapUpdater, payload_(p))))(nextState);
};
export const addDataToMapComposed = addDataToMapUpdater;
diff --git a/src/reducers/composers.js b/src/reducers/composers.js
index cc7baf7a1..7819fb873 100644
--- a/src/reducers/composers.js
+++ b/src/reducers/composers.js
@@ -26,7 +26,8 @@ import * as combinedUpdaters from './combined-updaters';
* It is used to generate documentation
*/
const actionHandler = {
- [ActionTypes.ADD_DATA_TO_MAP]: combinedUpdaters.addDataToMapUpdater
+ [ActionTypes.ADD_DATA_TO_MAP]: combinedUpdaters.addDataToMapUpdater,
+ [ActionTypes.LOAD_FILES_SUCCESS]: combinedUpdaters.loadFileSuccessUpdater
};
export default actionHandler;
diff --git a/src/reducers/vis-state-updaters.js b/src/reducers/vis-state-updaters.js
index 0b22b8d12..0297accf0 100644
--- a/src/reducers/vis-state-updaters.js
+++ b/src/reducers/vis-state-updaters.js
@@ -28,11 +28,10 @@ import copy from 'copy-to-clipboard';
import {parseFieldValue} from 'utils/data-utils';
// Tasks
-import {LOAD_FILE_TASK, ACTION_TASK} from 'tasks/tasks';
+import {LOAD_FILE_TASK} from 'tasks/tasks';
// Actions
-import {loadFilesErr, loadFileSuccess} from 'actions/vis-state-actions';
-import {addDataToMap} from 'actions';
+import {loadFilesErr, loadNextFile, loadFileSuccess} from 'actions/vis-state-actions';
// Utils
import {getDefaultInteraction, findFieldsToShow} from 'utils/interaction-utils';
@@ -53,7 +52,7 @@ import {
} from 'utils/filter-utils';
import {setFilterGpuMode, assignGpuChannel} from 'utils/gpu-filter-utils';
import {createNewDataEntry, sortDatasetByColumn} from 'utils/dataset-utils';
-import {set, toArray, generateHashId} from 'utils/utils';
+import {set, toArray} from 'utils/utils';
import {findDefaultLayer, calculateLayerData} from 'utils/layer-utils/layer-utils';
@@ -74,7 +73,7 @@ import {
import {Layer, LayerClasses} from 'layers';
import {DEFAULT_TEXT_LABEL} from 'layers/layer-factory';
-import {EDITOR_MODES, DATASET_FORMATS, SORT_ORDER} from 'constants/default-settings';
+import {EDITOR_MODES, SORT_ORDER} from 'constants/default-settings';
// react-palm
// disable capture exception for react-palm call to withTask
@@ -1317,11 +1316,12 @@ function closeSpecificMapAtIndex(state, action) {
* @param {Object} state `visState`
* @param {Object} action action
* @param {Array<Object>} action.files array of fileblob
+ * @param {Function} action.onFinish action creator to execute after load file succeed
* @returns {Object} nextState
* @public
*/
export const loadFilesUpdater = (state, action) => {
- const {files} = action;
+ const {files, onFinish = loadFileSuccess} = action;
if (!files.length) {
return state;
}
@@ -1333,90 +1333,42 @@ export const loadFilesUpdater = (state, action) => {
fileLoading: true,
fileLoadingProgress: 0
},
- makeLoadFileTask(files.length, files, fileCache)
+ makeLoadFileTask(files.length, files, fileCache, onFinish)
);
};
-function makeLoadFileTask(totalCount, filesToLoad, fileCache) {
- const file = filesToLoad.pop();
+export function loadNextFileUpdater(state, action) {
+ const {fileCache, filesToLoad, totalCount, onFinish} = action;
+ const fileLoadingProgress = ((totalCount - filesToLoad.length) / totalCount) * 100;
+
+ return withTask(
+ {
+ ...state,
+ fileLoadingProgress
+ },
+ makeLoadFileTask(totalCount, filesToLoad, fileCache, onFinish)
+ );
+}
+
+export function makeLoadFileTask(totalCount, filesToLoad, fileCache, onFinish) {
+ const [file, ...remainingFilesToLoad] = filesToLoad;
return LOAD_FILE_TASK({file, fileCache}).bimap(
// success
result =>
- loadFileSuccess({
- fileCache: result,
- filesToLoad: [...filesToLoad],
- totalCount
- }),
+ remainingFilesToLoad.length
+ ? loadNextFile({
+ fileCache: result,
+ filesToLoad: remainingFilesToLoad,
+ totalCount,
+ onFinish
+ })
+ : onFinish(result),
// error
loadFilesErr
);
}
-export const loadFileSuccessUpdater = (state, action) => {
- const {fileCache, filesToLoad = [], totalCount} = action;
-
- // still more to load
- if (filesToLoad.length) {
- const fileLoadingProgress = ((totalCount - filesToLoad.length) / totalCount) * 100;
-
- return withTask(
- {
- ...state,
- fileLoadingProgress
- },
- makeLoadFileTask(totalCount, filesToLoad, fileCache)
- );
- }
-
- const result = fileCache.reduce(
- (accu, file) => {
- const {data, info = {}} = file;
- const {format} = info;
-
- if (format) {
- if (format !== DATASET_FORMATS.keplergl) {
- const newDataset = {
- data,
- info: {
- id: generateHashId(4),
- ...info
- }
- };
-
- accu.datasets.push(newDataset);
- return accu;
- }
-
- return {
- datasets: accu.datasets.concat(data.datasets),
- // we need to deep merge this thing unless we find a better solution
- // this case will only happen if we allow to load multiple keplergl json files
- config: {
- ...accu.config,
- ...(data.config || {})
- }
- };
- }
- return accu;
- },
- {datasets: [], config: {}}
- );
-
- const options = {
- centerMap: !(result.config && result.config.mapState)
- };
-
- return withTask(
- {
- ...state,
- fileLoading: false,
- fileLoadingProgress: 100
- },
- ACTION_TASK().map(_ => addDataToMap({...result, options}))
- );
-};
-
/**
* Trigger loading file error
* @memberof visStateUpdaters
diff --git a/src/reducers/vis-state.js b/src/reducers/vis-state.js
index 73e36b2a6..8a15eab08 100644
--- a/src/reducers/vis-state.js
+++ b/src/reducers/vis-state.js
@@ -55,7 +55,7 @@ const actionHandler = {
[ActionTypes.LOAD_FILES_ERR]: visStateUpdaters.loadFilesErrUpdater,
- [ActionTypes.LOAD_FILES_SUCCESS]: visStateUpdaters.loadFileSuccessUpdater,
+ [ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,
[ActionTypes.MAP_CLICK]: visStateUpdaters.mapClickUpdater,