Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Enhancement] Refactor load file tasks to better handle multiple file types (#986)
Shan He7936828bc3c5
diff --git a/src/actions/vis-state-actions.js b/src/actions/vis-state-actions.jsindex 080bc556c..c0e82aa55 100644--- a/src/actions/vis-state-actions.js+++ b/src/actions/vis-state-actions.js@@ -550,12 +550,32 @@ export function loadFiles(files) {};}-export function loadFileSuccess({fileCache, filesToLoad, totalCount}) {+/**+ * called with next file to load+ * @param {object} payload+ * @param {Array<object>} payload.fileCache+ * @param {Array<object>} payload.filesToLoad+ * @param {number} payload.totalCount+ * @param {Function} payload.onFinish - action creator to execute when all files are loaded+ */+export function loadNextFile({fileCache, filesToLoad, totalCount, onFinish}) {return {- type: ActionTypes.LOAD_FILES_SUCCESS,+ type: ActionTypes.LOAD_NEXT_FILE,fileCache,filesToLoad,- totalCount+ totalCount,+ onFinish+ };+}++/**+ * called when all files are processed and loaded+ * @param {Array<object>} result+ */+export function loadFileSuccess(result) {+ return {+ type: ActionTypes.LOAD_FILES_SUCCESS,+ result};}diff --git a/src/constants/action-types.js b/src/constants/action-types.jsindex 9e1304adb..6f3b65f53 100644--- a/src/constants/action-types.js+++ b/src/constants/action-types.js@@ -92,6 +92,7 @@ const ActionTypes = keyMirror({TOGGLE_LAYER_FOR_MAP: null,SET_FILTER_PLOT: null,LOAD_FILES: null,+ LOAD_NEXT_FILE: null,LOAD_FILES_ERR: null,LOAD_FILES_SUCCESS: null,LAYER_COLOR_UI_CHANGE: null,diff --git a/src/constants/index.js b/src/constants/index.jsindex 96980b4b1..8e94baf99 100644--- a/src/constants/index.js+++ b/src/constants/index.js@@ -25,6 +25,7 @@ export {AGGREGATION_TYPES,ALL_FIELD_TYPES,DATA_TABLE_ID,+ DATASET_FORMATS,DEFAULT_COLOR_RANGE,DEFAULT_LAYER_GROUPS,DEFAULT_MAP_STYLES,diff --git a/src/processors/file-handler.js b/src/processors/file-handler.jsindex 7593ded87..da55ce924 100644--- a/src/processors/file-handler.js+++ b/src/processors/file-handler.js@@ -26,7 +26,7 @@ import {processKeplerglJSON,processRowObject} from './data-processor';-import {isPlainObject} from 'utils/utils';+import {isPlainObject, generateHashId} from 'utils/utils';import {DATASET_FORMATS} from 'constants/default-settings';const FILE_HANDLERS = {@@ -171,3 +171,38 @@ export function determineJsonProcess({dataset, format}, defaultProcessor) {return defaultProcessor;}++export function filesToDataPayload(fileCache) {+ // seperate out files which could be a single datasets. or a keplergl map json+ const collection = fileCache.reduce(+ (accu, file) => {+ const {data, info = {}} = file;+ const {format} = info;+ if (format === DATASET_FORMATS.keplergl) {+ // if file contains a single kepler map dataset & config+ accu.keplerMaps.push({+ ...data,+ options: {+ centerMap: !(data.config && data.config.mapState)+ }+ });+ } else if (DATASET_FORMATS[format]) {+ // if file contains only data+ const newDataset = {+ data,+ info: {+ id: info.id || generateHashId(4),+ ...info+ }+ };+ accu.datasets.push(newDataset);+ }+ return accu;+ },+ {datasets: [], keplerMaps: []}+ );++ // add kepler map first with config+ // add datasets later in one add data call+ return collection.keplerMaps.concat({datasets: collection.datasets});+}diff --git a/src/processors/index.js b/src/processors/index.jsindex fb8ad393d..77e9316ac 100644--- a/src/processors/index.js+++ b/src/processors/index.js@@ -30,6 +30,6 @@ export {} from './data-processor';// File Handlers-export {getFileHandler, getFileType} from './file-handler';+export {getFileHandler, getFileType, readFile, filesToDataPayload} from './file-handler';export {Processors as default} from './data-processor';diff --git a/src/reducers/combined-updaters.js b/src/reducers/combined-updaters.jsindex fbfd42b1c..3a3754bbc 100644--- a/src/reducers/combined-updaters.js+++ b/src/reducers/combined-updaters.js@@ -18,6 +18,27 @@// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN// THE SOFTWARE.+// Copyright (c) 2020 Uber Technologies, Inc.+//+// Permission is hereby granted, free of charge, to any person obtaining a copy+// of this software and associated documentation files (the "Software"), to deal+// in the Software without restriction, including without limitation the rights+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+// to use, copy, modify, merge_, publish, distribute, sublicense, and/or sell+// copies of the Software, and to permit persons to whom the Software is+// furnished to do so, subject to the following conditions:+//+// The above copyright notice and this permission notice shall be included in+// all copies or substantial portions of the Software.+//+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+// THE SOFTWARE.+import {toggleModalUpdater, loadFilesSuccessUpdater} from './ui-state-updaters';import {updateVisDataUpdater as visStateUpdateVisDataUpdater,@@ -28,6 +49,8 @@ import {receiveMapConfigUpdater as styleMapConfigUpdater} from './map-style-updaimport {findMapBounds} from 'utils/data-utils';import KeplerGlSchema from 'schemas';import {isPlainObject} from 'utils/utils';+import {filesToDataPayload} from 'processors/file-handler';+import Console from 'global/console';// compose action to apply result multiple reducers, with the output of one@@ -78,10 +101,47 @@ const combinedUpdaters = null;export const isValidConfig = config =>isPlainObject(config) && isPlainObject(config.config) && config.version;+export const defaultAddDataToMapOptions = {centerMap: true,keepExistingConfig: false};++const identity = state => state;++/* eslint-disable-next-line no-unused-vars */+function log(text) {+ return value => Console.log(text, value);+}++function payload_(p) {+ return {payload: p};+}++function apply_(updater, payload) {+ return state => updater(state, payload);+}++function with_(fn) {+ return state => fn(state)(state);+}++function if_(pred, fn) {+ return pred ? fn : identity;+}++function compose_(fns) {+ return state => fns.reduce((state2, fn) => fn(state2), state);+}++function merge_(obj) {+ return state => ({...state, ...obj});+}++function pick_(prop) {+ return fn => state => ({...state, [prop]: fn(state[prop])});+}+/*** Combine data and full configuration update in a single action*@@ -118,61 +178,58 @@ export const addDataToMapUpdater = (state, {payload}) => {parsedConfig = KeplerGlSchema.parseSavedConfig(config);}const oldLayers = state.visState.layers;+ const filterNewlyAddedLayers = layers => layers.filter(nl => !oldLayers.find(ol => ol === nl));- // Update visState store- let mergedState = {- ...state,- visState: visStateUpdateVisDataUpdater(state.visState, {- datasets,- options,- config: parsedConfig- })- };+ return compose_([+ pick_('visState')(+ apply_(visStateUpdateVisDataUpdater, {+ datasets,+ options,+ config: parsedConfig+ })+ ),- // merge map info- // save title, description provider and queryParams- if (info) {- mergedState = {- ...mergedState,- visState: setMapInfoUpdater(mergedState.visState, {info})- };- }+ if_(info, pick_('visState')(apply_(setMapInfoUpdater, {info}))),- let bounds;- if (options.centerMap) {- // find map bounds for new layers- const newLayers = mergedState.visState.layers.filter(nl => !oldLayers.find(ol => ol === nl));- bounds = findMapBounds(newLayers);- }+ with_(({visState}) =>+ pick_('mapState')(+ apply_(+ stateMapConfigUpdater,+ payload_({+ config: parsedConfig,+ options,+ bounds: options.centerMap+ ? findMapBounds(filterNewlyAddedLayers(visState.layers))+ : null+ })+ )+ )+ ),- // Update mapState store- mergedState = {- ...mergedState,- mapState: stateMapConfigUpdater(mergedState.mapState, {- payload: {config: parsedConfig, options, bounds}- })- };+ pick_('mapStyle')(apply_(styleMapConfigUpdater, payload_({config: parsedConfig, options}))),- // Update mapStyle store- mergedState = {- ...mergedState,- mapStyle: styleMapConfigUpdater(mergedState.mapStyle, {- payload: {config: parsedConfig, options}- })- };+ pick_('uiState')(apply_(loadFilesSuccessUpdater)),- // Update uiState- mergedState = {- ...mergedState,- uiState: {- ...toggleModalUpdater(loadFilesSuccessUpdater(mergedState.uiState), {- payload: null- }),- ...(options.hasOwnProperty('readOnly') ? {readOnly: options.readOnly} : {})- }- };+ pick_('uiState')(apply_(toggleModalUpdater, payload_(null))),++ pick_('uiState')(merge_(options.hasOwnProperty('readOnly') ? {readOnly: options.readOnly} : {}))+ ])(state);+};++export const loadFileSuccessUpdater = (state, action) => {+ // still more to load+ const payloads = filesToDataPayload(action.result);+ const nextState = compose_([+ pick_('visState')(+ merge_({+ fileLoading: false,+ fileLoadingProgress: 100+ })+ )+ ])(state);- return mergedState;+ // make multiple add data to map calls+ return compose_(payloads.map(p => apply_(addDataToMapUpdater, payload_(p))))(nextState);};export const addDataToMapComposed = addDataToMapUpdater;diff --git a/src/reducers/composers.js b/src/reducers/composers.jsindex cc7baf7a1..7819fb873 100644--- a/src/reducers/composers.js+++ b/src/reducers/composers.js@@ -26,7 +26,8 @@ import * as combinedUpdaters from './combined-updaters';* It is used to generate documentation*/const actionHandler = {- [ActionTypes.ADD_DATA_TO_MAP]: combinedUpdaters.addDataToMapUpdater+ [ActionTypes.ADD_DATA_TO_MAP]: combinedUpdaters.addDataToMapUpdater,+ [ActionTypes.LOAD_FILES_SUCCESS]: combinedUpdaters.loadFileSuccessUpdater};export default actionHandler;diff --git a/src/reducers/vis-state-updaters.js b/src/reducers/vis-state-updaters.jsindex 0b22b8d12..0297accf0 100644--- a/src/reducers/vis-state-updaters.js+++ b/src/reducers/vis-state-updaters.js@@ -28,11 +28,10 @@ import copy from 'copy-to-clipboard';import {parseFieldValue} from 'utils/data-utils';// Tasks-import {LOAD_FILE_TASK, ACTION_TASK} from 'tasks/tasks';+import {LOAD_FILE_TASK} from 'tasks/tasks';// Actions-import {loadFilesErr, loadFileSuccess} from 'actions/vis-state-actions';-import {addDataToMap} from 'actions';+import {loadFilesErr, loadNextFile, loadFileSuccess} from 'actions/vis-state-actions';// Utilsimport {getDefaultInteraction, findFieldsToShow} from 'utils/interaction-utils';@@ -53,7 +52,7 @@ import {} from 'utils/filter-utils';import {setFilterGpuMode, assignGpuChannel} from 'utils/gpu-filter-utils';import {createNewDataEntry, sortDatasetByColumn} from 'utils/dataset-utils';-import {set, toArray, generateHashId} from 'utils/utils';+import {set, toArray} from 'utils/utils';import {findDefaultLayer, calculateLayerData} from 'utils/layer-utils/layer-utils';@@ -74,7 +73,7 @@ import {import {Layer, LayerClasses} from 'layers';import {DEFAULT_TEXT_LABEL} from 'layers/layer-factory';-import {EDITOR_MODES, DATASET_FORMATS, SORT_ORDER} from 'constants/default-settings';+import {EDITOR_MODES, SORT_ORDER} from 'constants/default-settings';// react-palm// disable capture exception for react-palm call to withTask@@ -1317,11 +1316,12 @@ function closeSpecificMapAtIndex(state, action) {* @param {Object} state `visState`* @param {Object} action action* @param {Array<Object>} action.files array of fileblob+ * @param {Function} action.onFinish action creator to execute after load file succeed* @returns {Object} nextState* @public*/export const loadFilesUpdater = (state, action) => {- const {files} = action;+ const {files, onFinish = loadFileSuccess} = action;if (!files.length) {return state;}@@ -1333,90 +1333,42 @@ export const loadFilesUpdater = (state, action) => {fileLoading: true,fileLoadingProgress: 0},- makeLoadFileTask(files.length, files, fileCache)+ makeLoadFileTask(files.length, files, fileCache, onFinish));};-function makeLoadFileTask(totalCount, filesToLoad, fileCache) {- const file = filesToLoad.pop();+export function loadNextFileUpdater(state, action) {+ const {fileCache, filesToLoad, totalCount, onFinish} = action;+ const fileLoadingProgress = ((totalCount - filesToLoad.length) / totalCount) * 100;++ return withTask(+ {+ ...state,+ fileLoadingProgress+ },+ makeLoadFileTask(totalCount, filesToLoad, fileCache, onFinish)+ );+}++export function makeLoadFileTask(totalCount, filesToLoad, fileCache, onFinish) {+ const [file, ...remainingFilesToLoad] = filesToLoad;return LOAD_FILE_TASK({file, fileCache}).bimap(// successresult =>- loadFileSuccess({- fileCache: result,- filesToLoad: [...filesToLoad],- totalCount- }),+ remainingFilesToLoad.length+ ? loadNextFile({+ fileCache: result,+ filesToLoad: remainingFilesToLoad,+ totalCount,+ onFinish+ })+ : onFinish(result),// errorloadFilesErr);}-export const loadFileSuccessUpdater = (state, action) => {- const {fileCache, filesToLoad = [], totalCount} = action;-- // still more to load- if (filesToLoad.length) {- const fileLoadingProgress = ((totalCount - filesToLoad.length) / totalCount) * 100;-- return withTask(- {- ...state,- fileLoadingProgress- },- makeLoadFileTask(totalCount, filesToLoad, fileCache)- );- }-- const result = fileCache.reduce(- (accu, file) => {- const {data, info = {}} = file;- const {format} = info;-- if (format) {- if (format !== DATASET_FORMATS.keplergl) {- const newDataset = {- data,- info: {- id: generateHashId(4),- ...info- }- };-- accu.datasets.push(newDataset);- return accu;- }-- return {- datasets: accu.datasets.concat(data.datasets),- // we need to deep merge this thing unless we find a better solution- // this case will only happen if we allow to load multiple keplergl json files- config: {- ...accu.config,- ...(data.config || {})- }- };- }- return accu;- },- {datasets: [], config: {}}- );-- const options = {- centerMap: !(result.config && result.config.mapState)- };-- return withTask(- {- ...state,- fileLoading: false,- fileLoadingProgress: 100- },- ACTION_TASK().map(_ => addDataToMap({...result, options}))- );-};-/*** Trigger loading file error* @memberof visStateUpdatersdiff --git a/src/reducers/vis-state.js b/src/reducers/vis-state.jsindex 73e36b2a6..8a15eab08 100644--- a/src/reducers/vis-state.js+++ b/src/reducers/vis-state.js@@ -55,7 +55,7 @@ const actionHandler = {[ActionTypes.LOAD_FILES_ERR]: visStateUpdaters.loadFilesErrUpdater,- [ActionTypes.LOAD_FILES_SUCCESS]: visStateUpdaters.loadFileSuccessUpdater,+ [ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,[ActionTypes.MAP_CLICK]: visStateUpdaters.mapClickUpdater,