Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Enhancement] Save merger and schema to visState (#1235)
Shan He7ff0c459bcfd
diff --git a/src/components/modal-container.js b/src/components/modal-container.jsindex 1217bfd2d..3923aca68 100644--- a/src/components/modal-container.js+++ b/src/components/modal-container.js@@ -27,7 +27,6 @@ import get from 'lodash.get';import document from 'global/document';import ModalDialogFactory from './modals/modal-dialog';-import KeplerGlSchema from 'schemas';import {exportJson, exportHtml, exportData, exportImage, exportMap} from 'utils/export-utils';import {isValidMapInfo} from 'utils/map-info-utils';@@ -397,7 +396,7 @@ export default function ModalContainerFactory(};break;case EXPORT_MAP_ID:- const keplerGlConfig = KeplerGlSchema.getConfigToSave({+ const keplerGlConfig = visState.schema.getConfigToSave({mapStyle,visState,mapState,diff --git a/src/reducers/combined-updaters.js b/src/reducers/combined-updaters.jsindex 48077b382..b5e5403c9 100644--- a/src/reducers/combined-updaters.js+++ b/src/reducers/combined-updaters.js@@ -29,7 +29,6 @@ import {import {receiveMapConfigUpdater as stateMapConfigUpdater} from './map-state-updaters';import {receiveMapConfigUpdater as styleMapConfigUpdater} from './map-style-updaters';import {findMapBounds} from 'utils/data-utils';-import KeplerGlSchema from 'schemas';import {isPlainObject} from 'utils/utils';import {filesToDataPayload} from 'processors/file-handler';import {payload_, apply_, with_, if_, compose_, merge_, pick_} from './composer-helpers';@@ -130,7 +129,7 @@ export const addDataToMapUpdater = (state, {payload}) => {if (isValidConfig(config)) {// if passed in saved config- parsedConfig = KeplerGlSchema.parseSavedConfig(config);+ parsedConfig = state.visState.schema.parseSavedConfig(config);}const oldLayers = state.visState.layers;const filterNewlyAddedLayers = layers => layers.filter(nl => !oldLayers.find(ol => ol === nl));diff --git a/src/reducers/provider-state-updaters.js b/src/reducers/provider-state-updaters.jsindex 436019f79..e922a7b96 100644--- a/src/reducers/provider-state-updaters.js+++ b/src/reducers/provider-state-updaters.js@@ -46,7 +46,6 @@ import {DATASET_FORMATS} from 'constants/default-settings';import {toArray} from 'utils/utils';-import KeplerGlSchema from 'schemas';export const INITIAL_PROVIDER_STATE = {isProviderLoading: false,@@ -266,8 +265,6 @@ function parseLoadMapResponse(response, loadParams, provider) {return {info, data};});- const parsedConfig = map.config && KeplerGlSchema.parseSavedConfig(map.config);-const info = {...map.info,provider: provider.name,@@ -276,7 +273,7 @@ function parseLoadMapResponse(response, loadParams, provider) {return {datasets: parsedDatasets,info,- ...(parsedConfig ? {config: parsedConfig} : {})+ ...(map.config ? {config: map.config} : {})};}diff --git a/src/reducers/vis-state-merger.d.ts b/src/reducers/vis-state-merger.d.tsindex e7d48e72e..e7cddb7c7 100644--- a/src/reducers/vis-state-merger.d.ts+++ b/src/reducers/vis-state-merger.d.ts@@ -24,3 +24,14 @@ export function mergeSplitMaps(): VisState;export function mergeInteractionTooltipConfig(state: VisState);++export function isValidMerger(arg: any): boolean;+export type Merger = {+ merge: (state: VisState, config: any) => VisState,+ prop: string,+ toMergeProp?: string+};++export type VisStateMergers = Merger[];++export const VIS_STATE_MERGERS: VisStateMergers;diff --git a/src/reducers/vis-state-merger.js b/src/reducers/vis-state-merger.jsindex 4bc685581..a966366d9 100644--- a/src/reducers/vis-state-merger.js+++ b/src/reducers/vis-state-merger.js@@ -22,7 +22,7 @@ import uniq from 'lodash.uniq';import pick from 'lodash.pick';import isEqual from 'lodash.isequal';import flattenDeep from 'lodash.flattendeep';-import {toArray} from 'utils/utils';+import {toArray, isObject} from 'utils/utils';import {applyFiltersToDatasets,@@ -128,7 +128,7 @@ export function mergeFilters(state, filtersToMerge) {...state,filters: updatedFilters,datasets: filtered,- filterToBeMerged: unmerged+ filterToBeMerged: [...state.filterToBeMerged, ...unmerged]};}@@ -176,7 +176,7 @@ export function mergeLayers(state, layersToMerge) {...state,layers,layerOrder,- layerToBeMerged: unmerged+ layerToBeMerged: [...state.layerToBeMerged, ...unmerged]};}@@ -274,7 +274,7 @@ export function mergeSplitMaps(state, splitMaps = []) {return {...state,splitMaps: merged,- splitMapsToBeMerged: unmerged+ splitMapsToBeMerged: [...state.splitMapsToBeMerged, ...unmerged]};}@@ -503,3 +503,16 @@ export function validateLayerWithData({fields, id: dataId}, savedLayer, layerClareturn newLayer;}++export function isValidMerger(merger) {+ return isObject(merger) && typeof merger.merge === 'function' && typeof merger.prop === 'string';+}++export const VIS_STATE_MERGERS = [+ {merge: mergeLayers, prop: 'layers', toMergeProp: 'layerToBeMerged'},+ {merge: mergeFilters, prop: 'filters', toMergeProp: 'filterToBeMerged'},+ {merge: mergeInteractions, prop: 'interactionConfig', toMergeProp: 'interactionToBeMerged'},+ {merge: mergeLayerBlending, prop: 'layerBlending'},+ {merge: mergeSplitMaps, prop: 'splitMaps', toMergeProp: 'splitMapsToBeMerged'},+ {merge: mergeAnimationConfig, prop: 'animationConfig'}+];diff --git a/src/reducers/vis-state-updaters.d.ts b/src/reducers/vis-state-updaters.d.tsindex 9c8775223..a673b4e1a 100644--- a/src/reducers/vis-state-updaters.d.ts+++ b/src/reducers/vis-state-updaters.d.ts@@ -4,6 +4,8 @@ import {ParsedConfig} from '../schemas';import * as VisStateActions from 'actions/vis-state-actions';import ActionTypes from 'constants/action-types';import {LoaderObject} from '@loaders.gl/loader-utils';+import {VisStateMergers} from './vis-state-merger';+import KeplerGLSchema from 'schemas';export type HistogramBin = {x0: number | undefined;@@ -300,12 +302,14 @@ export type VisState = {animationConfig: AnimationConfig;editor: Editor;splitMaps: SplitMap[];- splitMapsToBeMerged?: SplitMap[];+ splitMapsToBeMerged: SplitMap[];fileLoading: FileLoading | false;fileLoadingProgress: FileLoadingProgress;loaders: LoaderObject[];loadOptions: object;initialState?: Partial<VisState>;+ mergers: VisStateMergers;+ schema: KeplerGLSchema};export function addDefaultLayers(diff --git a/src/reducers/vis-state-updaters.js b/src/reducers/vis-state-updaters.jsindex 58bf891db..c7ee2c7bd 100644--- a/src/reducers/vis-state-updaters.js+++ b/src/reducers/vis-state-updaters.js@@ -59,14 +59,7 @@ import {set, toArray} from 'utils/utils';import {calculateLayerData, findDefaultLayer} from 'utils/layer-utils';-import {- mergeAnimationConfig,- mergeFilters,- mergeInteractions,- mergeLayerBlending,- mergeLayers,- mergeSplitMaps-} from './vis-state-merger';+import {isValidMerger, VIS_STATE_MERGERS} from './vis-state-merger';import {addNewLayersToSplitMap,@@ -80,6 +73,8 @@ import {EDITOR_MODES, SORT_ORDER} from 'constants/default-settings';import {pick_, merge_} from './composer-helpers';import {processFileContent} from 'actions/vis-state-actions';+import KeplerGLSchema from 'schemas';+// type imports/** @typedef {import('./vis-state-updaters').Field} Field *//** @typedef {import('./vis-state-updaters').Filter} Filter */@@ -194,7 +189,8 @@ export const INITIAL_VIS_STATE = {// }// ]],- //+ splitMapsToBeMerged: [],+// defaults layer classeslayerClasses: LayerClasses,@@ -208,7 +204,12 @@ export const INITIAL_VIS_STATE = {fileLoadingProgress: {},loaders: [],- loadOptions: {}+ loadOptions: {},++ // visStateMergers+ mergers: VIS_STATE_MERGERS,++ schema: KeplerGLSchema};/**@@ -939,25 +940,15 @@ export const receiveMapConfigUpdater = (state, {payload: {config = {}, options =return state;}- const {- filters,- layers,- interactionConfig,- layerBlending,- splitMaps,- animationConfig- } = config.visState;-const {keepExistingConfig} = options;// reset config if keepExistingConfig is falsylet mergedState = !keepExistingConfig ? resetMapConfigUpdater(state) : state;- mergedState = mergeLayers(mergedState, layers);- mergedState = mergeFilters(mergedState, filters);- mergedState = mergeInteractions(mergedState, interactionConfig);- mergedState = mergeLayerBlending(mergedState, layerBlending);- mergedState = mergeSplitMaps(mergedState, splitMaps);- mergedState = mergeAnimationConfig(mergedState, animationConfig);+ for (const merger of state.mergers) {+ if (isValidMerger(merger) && config.visState[merger.prop]) {+ mergedState = merger.merge(mergedState, config.visState[merger.prop]);+ }+ }return mergedState;};@@ -1117,23 +1108,21 @@ export const toggleLayerForMapUpdater = (state, {mapIndex, layerId}) => {* @public*//* eslint-disable max-statements */+// eslint-disable-next-line complexityexport const updateVisDataUpdater = (state, action) => {// datasets can be a single data entries or an array of multiple data entriesconst {config, options} = action;-const datasets = toArray(action.datasets);const newDataEntries = datasets.reduce(- (accu, {info = {}, data}) => ({+ (accu, {info = {}, data} = {}) => ({...accu,...(createNewDataEntry({info, data}, state.datasets) || {})}),{});- if (!Object.keys(newDataEntries).length) {- return state;- }+ const dataEmpty = Object.keys(newDataEntries).length < 1;// apply config if passed from actionconst previousState = config@@ -1142,7 +1131,7 @@ export const updateVisDataUpdater = (state, action) => {}): state;- const stateWithNewData = {+ let mergedState = {...previousState,datasets: {...previousState.datasets,@@ -1150,23 +1139,18 @@ export const updateVisDataUpdater = (state, action) => {}};- // previously saved config before data loaded- const {- filterToBeMerged = [],- layerToBeMerged = [],- interactionToBeMerged = {},- splitMapsToBeMerged = []- } = stateWithNewData;-- // We need to merge layers before filters because polygon filters requires layers to be loaded- let mergedState = mergeLayers(stateWithNewData, layerToBeMerged);-- mergedState = mergeFilters(mergedState, filterToBeMerged);-- // merge state with saved splitMaps- mergedState = mergeSplitMaps(mergedState, splitMapsToBeMerged);+ // merge state with config to be merged+ for (const merger of mergedState.mergers) {+ if (isValidMerger(merger) && merger.toMergeProp && mergedState[merger.toMergeProp]) {+ const toMerge = mergedState[merger.toMergeProp];+ mergedState[merger.toMergeProp] = INITIAL_VIS_STATE[merger.toMergeProp];+ mergedState = merger.merge(mergedState, toMerge);+ }+ }- let newLayers = mergedState.layers.filter(l => l.config.dataId in newDataEntries);+ let newLayers = !dataEmpty+ ? mergedState.layers.filter(l => l.config.dataId in newDataEntries)+ : [];if (!newLayers.length && (options || {}).autoCreateLayers !== false) {// no layer merged, find defaults@@ -1184,9 +1168,6 @@ export const updateVisDataUpdater = (state, action) => {};}- // merge state with saved interactions- mergedState = mergeInteractions(mergedState, interactionToBeMerged);-// if no tooltips merged add default tooltipsObject.keys(newDataEntries).forEach(dataId => {const tooltipFields = mergedState.interactionConfig.tooltip.config.fieldsToShow[dataId];@@ -1195,7 +1176,11 @@ export const updateVisDataUpdater = (state, action) => {}});- let updatedState = updateAllLayerDomainData(mergedState, Object.keys(newDataEntries), undefined);+ let updatedState = updateAllLayerDomainData(+ mergedState,+ dataEmpty ? Object.keys(mergedState.datasets) : Object.keys(newDataEntries),+ undefined+ );// register layer animation domain,// need to be called after layer data is calculateddiff --git a/src/schemas/index.js b/src/schemas/index.jsindex ba07a028e..c9fb063fd 100644--- a/src/schemas/index.js+++ b/src/schemas/index.js@@ -19,5 +19,15 @@// THE SOFTWARE.// Schemas-export {default, default as KeplerGlSchema} from './schema-manager';+export {+ default,+ default as KeplerGlSchema,+ reducerSchema,+ KeplerGLSchema as KeplerGLSchemaClass+} from './schema-manager';export {CURRENT_VERSION, VERSIONS} from './versions';+export {default as visStateSchema} from './vis-state-schema';+export {default as datasetSchema} from './dataset-schema';+export {default as mapStyleSchema} from './map-style-schema';+export {default as mapStateSchema} from './map-state-schema';+export {default as Schema} from './schema';diff --git a/src/schemas/schema-manager.d.ts b/src/schemas/schema-manager.d.tsindex afba58e1a..cb7eeec70 100644--- a/src/schemas/schema-manager.d.ts+++ b/src/schemas/schema-manager.d.ts@@ -5,9 +5,11 @@ import {SplitMap,AnimationConfig,VisState,- RGBColor,+ RGBColor,Merge} from 'reducers';+import {Schema} from './schema';+import {LayerTextLabel} from 'layers/layer-factory';export type SavedFilter = {@@ -203,7 +205,7 @@ export class KeplerGLSchema {validateVersion(version: any): string | null;hasDataChanged(state: any): boolean;}-+export const reducerSchema: {[key: string]: Schema};const KeplerGLSchemaManager: KeplerGLSchema;export default KeplerGLSchemaManager;diff --git a/src/schemas/schema-manager.js b/src/schemas/schema-manager.jsindex 3a63e55ff..50065136f 100644--- a/src/schemas/schema-manager.js+++ b/src/schemas/schema-manager.js@@ -28,19 +28,24 @@ import mapStateSchema from './map-state-schema';import {CURRENT_VERSION, VERSIONS} from './versions';import {isPlainObject} from 'utils/utils';-const REDUCER_SCHEMAS = {+export const reducerSchema = {visState: visStateSchema,mapState: mapStateSchema,mapStyle: mapStyleSchema};/** @type {typeof import('./schema-manager').KeplerGLSchema} */-class KeplerGLSchema {- constructor() {- this._validVersions = VERSIONS;- this._version = CURRENT_VERSION;- this._reducerSchemas = REDUCER_SCHEMAS;- this._datasetSchema = datasetSchema;+export class KeplerGLSchema {+ constructor({+ reducers = reducerSchema,+ datasets = datasetSchema,+ validVersions = VERSIONS,+ version = CURRENT_VERSION+ } = {}) {+ this._validVersions = validVersions;+ this._version = version;+ this._reducerSchemas = reducers;+ this._datasetSchema = datasets;this._datasetLastSaved = null;this._savedDataset = null;diff --git a/src/utils/export-utils.js b/src/utils/export-utils.jsindex ab1af0322..5b5e78f85 100644--- a/src/utils/export-utils.js+++ b/src/utils/export-utils.js@@ -33,8 +33,6 @@ import {formatCsv} from 'processors/data-processor';import get from 'lodash.get';import {set, generateHashId} from 'utils/utils';-import KeplerGlSchema from 'schemas';-/*** Default file names*/@@ -155,12 +153,13 @@ export function exportToJsonString(data) {export function getMapJSON(state, options = DEFAULT_EXPORT_JSON_SETTINGS) {const {hasData} = options;+ const schema = state.visState.schema;if (!hasData) {- return KeplerGlSchema.getConfigToSave(state);+ return schema.getConfigToSave(state);}- let mapToSave = KeplerGlSchema.save(state);+ let mapToSave = schema.save(state);// add file name if title is not providedconst title = get(mapToSave, ['info', 'title']);if (!title || !title.length) {diff --git a/test/browser/components/side-panel/color-selector-test.js b/test/browser/components/side-panel/color-selector-test.jsindex 802ed474f..df1727573 100644--- a/test/browser/components/side-panel/color-selector-test.js+++ b/test/browser/components/side-panel/color-selector-test.js@@ -186,7 +186,6 @@ test('Components -> ArcLayerColorSelector.render -> render single color', t => {const initialState = StateWTrips.visState;const {layers} = initialState;const arcLayer = layers.find(l => l.type === 'arc');-const updateLayerConfig = sinon.spy();const updateLayerVisConfig = sinon.spy();const updateLayerColorUI = sinon.spy();diff --git a/test/node/reducers/vis-state-merger-test.js b/test/node/reducers/vis-state-merger-test.jsindex e462a2529..873d130df 100644--- a/test/node/reducers/vis-state-merger-test.js+++ b/test/node/reducers/vis-state-merger-test.js@@ -363,6 +363,7 @@ test('VisStateMerger.v1.split -> mergeLayers -> toEmptyState', t => {// merge Stateconst mergedState = visStateReducer(oldVisState, receiveMapConfig(parsedConfig));+Object.keys(oldVisState).forEach(key => {if (key === 'layerToBeMerged') {t.deepEqual(@@ -372,7 +373,7 @@ test('VisStateMerger.v1.split -> mergeLayers -> toEmptyState', t => {);} else if (key === 'splitMaps') {t.deepEqual(mergedState.splitMaps, [], 'Should wait to merge splitMaps');- } else if (key === 'splitMaps') {+ } else if (key === 'splitMapsToBeMerged') {t.deepEqual(mergedState.splitMapsToBeMerged,expectedConfig,@@ -1263,7 +1264,8 @@ test('VisStateMerger - mergeSplitMaps -> split to split', t => {test('VisStateMerger - mergeSplitMaps', t => {const testState1 = {layers: [],- splitMaps: [{layers: {a: true}}, {layers: {a: false}}]+ splitMaps: [{layers: {a: true}}, {layers: {a: false}}],+ splitMapsToBeMerged: []};t.deepEqual(@@ -1282,7 +1284,8 @@ test('VisStateMerger - mergeSplitMaps', t => {const testState2 = {layers: [{id: 'c', config: {isVisible: true}}],- splitMaps: [{layers: {a: true}}, {layers: {a: false}}]+ splitMaps: [{layers: {a: true}}, {layers: {a: false}}],+ splitMapsToBeMerged: []};t.deepEqual(@@ -1297,7 +1300,8 @@ test('VisStateMerger - mergeSplitMaps', t => {const testState3 = {layers: [{id: 'c', config: {isVisible: true}}],- splitMaps: []+ splitMaps: [],+ splitMapsToBeMerged: []};t.deepEqual(mergeSplitMaps(testState3, testSM),@@ -1315,7 +1319,8 @@ test('VisStateMerger - mergeSplitMaps', t => {{id: 'b', config: {isVisible: false}},{id: 'c', config: {isVisible: true}}],- splitMaps: []+ splitMaps: [],+ splitMapsToBeMerged: []};t.deepEqual(mergeSplitMaps(testState4, testSM),diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 07f226d45..c00dd6013 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -868,11 +868,7 @@ test('#visStateReducer -> REMOVE_LAYER', t => {});test('#visStateReducer -> UPDATE_VIS_DATA.1 -> No data', t => {- const oldState = {- datasets: {},- layers: [{name: 'a'}, {name: 'b'}],- layerData: [{data: 1}, {data: 2}]- };+ const oldState = CloneDeep(InitialState).visState;t.deepEqual(reducer(oldState, VisStateActions.updateVisData([{info: null, data: null}])),@@ -2462,8 +2458,8 @@ test('#visStateReducer -> REMOVE_DATASET w filter and layer', t => {initialState: [],layerToBeMerged: [],filterToBeMerged: [],+ interactionToBeMerged: undefined,splitMapsToBeMerged: [],- interactionToBeMerged: [],editor: oldState.editor,mapInfo: {title: '',@@ -2472,7 +2468,9 @@ test('#visStateReducer -> REMOVE_DATASET w filter and layer', t => {fileLoading: oldState.fileLoading,fileLoadingProgress: oldState.fileLoadingProgress,loaders: oldState.loaders,- loadOptions: oldState.loadOptions+ loadOptions: oldState.loadOptions,+ mergers: oldState.mergers,+ schema: oldState.schema};const newReducer = reducer(oldState, VisStateActions.removeDataset(testCsvDataId));@@ -2701,8 +2699,8 @@ test('#visStateReducer -> SPLIT_MAP: REMOVE_DATASET', t => {initialState: [],layerToBeMerged: [],filterToBeMerged: [],+ interactionToBeMerged: undefined,splitMapsToBeMerged: [],- interactionToBeMerged: [],editor: oldState.editor,mapInfo: {title: '',@@ -2711,7 +2709,9 @@ test('#visStateReducer -> SPLIT_MAP: REMOVE_DATASET', t => {fileLoading: oldState.fileLoading,fileLoadingProgress: oldState.fileLoadingProgress,loaders: oldState.loaders,- loadOptions: oldState.loadOptions+ loadOptions: oldState.loadOptions,+ schema: oldState.schema,+ mergers: oldState.mergers};const newReducer = reducer(oldState, VisStateActions.removeDataset(testGeoJsonDataId));