Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
Add `visState.loaders` to let app inject a list of loaders.gl loaders. (#1192)
Ib Green8270f35317ff
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.jsindex c26394325..b23b5f97d 100644--- a/examples/demo-app/src/reducers/index.js+++ b/examples/demo-app/src/reducers/index.js@@ -89,6 +89,10 @@ const demoReducer = combineReducers({exportMapboxAccessToken: AUTH_TOKENS.EXPORT_MAPBOX_TOKEN}}+ },+ visState: {+ loaders: [], // Add additional loaders.gl loaders here+ loadOptions: {} // Add additional loaders.gl loader options here}}),app: appReducerdiff --git a/package.json b/package.jsonindex 2b39af748..391eea3a1 100644--- a/package.json+++ b/package.json@@ -87,6 +87,7 @@"@loaders.gl/csv": "^2.2.5","@loaders.gl/gltf": "^2.2.5","@loaders.gl/json": "^2.2.5",+ "@loaders.gl/loader-utils": "^2.2.5","@loaders.gl/polyfills": "^2.2.5","@luma.gl/constants": "^8.2.0","@luma.gl/core": "^8.2.0",diff --git a/src/actions/vis-state-actions.d.ts b/src/actions/vis-state-actions.d.tsindex 2db3777a4..0431ff048 100644--- a/src/actions/vis-state-actions.d.ts+++ b/src/actions/vis-state-actions.d.ts@@ -26,6 +26,9 @@ import {FileCacheItem} from '../processors/file-handler';import {Layer, LayerConfig, LayerVisConfig} from 'layers';import {Feature, InteractionConfig} from 'reducers/vis-state-updaters';import {ValueOf, Merge} from '../reducers/types';+// TODO - import LoaderObject type from @loaders.gl/core when supported+// TODO - import LoadOptions type from @loaders.gl/core when supported+import {LoaderObject} from '@loaders.gl/loader-utils';export type LayerConfigChangeUpdaterAction = {oldLayer: Layer;@@ -199,7 +202,7 @@ export function copyTableColumn(column: string): Merge<CopyTableColumnUpdaterAction, {type: ActionTypes.COPY_TABLE_COLUMN}>;-export type AddDaataToMapOUpdaterptions = {+export type AddDataToMapUpdaterOptions = {centrMap?: boolean;readOnly?: boolean;keepExistingConfig?: boolean;diff --git a/src/actions/vis-state-actions.js b/src/actions/vis-state-actions.jsindex 003a43a2d..79ab2496c 100644--- a/src/actions/vis-state-actions.js+++ b/src/actions/vis-state-actions.js@@ -582,10 +582,11 @@ export function setMapInfo(info) {* @returns action* @public*/-export function loadFiles(files) {+export function loadFiles(files, onFinish) {return {type: ActionTypes.LOAD_FILES,- files+ files,+ onFinish};}diff --git a/src/components/modal-container.d.ts b/src/components/modal-container.d.tsindex ae51a5525..88e300e98 100644--- a/src/components/modal-container.d.ts+++ b/src/components/modal-container.d.ts@@ -3,13 +3,13 @@ import * as VisStateActions from '../actions/vis-state-actions';import * as UIStateActions from '../actions/ui-state-actions';import * as MapStyleActions from '../actions/map-style-actions';import * as ProviderActions from '../actions/provider-actions';-import {OnErrorCallBack, OnSuccessCallBack} from '../actions/provider-actions';+import {OnErrorCallBack, OnSuccessCallBack} from 'actions/provider-actions';-import {MapState} from '../reducers/map-state-updaters';-import {MapStyle} from '../reducers/map-style-updaters';-import {UiState} from '../reducers/ui-state-updaters';-import {VisState} from '../reducers/vis-state-updaters';-import {ProviderState} from '../reducers/provider-state-updaters';+import {MapState} from 'reducers/map-state-updaters';+import {MapStyle} from 'reducers/map-style-updaters';+import {UiState} from 'reducers/ui-state-updaters';+import {VisState} from 'reducers/vis-state-updaters';+import {ProviderState} from 'reducers/provider-state-updaters';import {OnSuccessCallBack} from 'actions';export type ModalContainerProps = {@@ -32,7 +32,7 @@ export type ModalContainerProps = {onLoadCloudMapSuccess: OnSuccessCallBack,onLoadCloudMapError: OnErrorCallBack,onExportToCloudSuccess:OnSuccessCallBack,- onExportToCloudError: OnErrorCallBack,+ onExportToCloudError: OnErrorCallBack};export default function ModalContainerFactory(): React.Component<ModelContainerProps>;diff --git a/src/components/modal-container.js b/src/components/modal-container.jsindex 981e2f686..3588605fd 100644--- a/src/components/modal-container.js+++ b/src/components/modal-container.js@@ -173,8 +173,8 @@ export default function ModalContainerFactory(this._closeModal();};- _onFileUpload = blob => {- this.props.visStateActions.loadFiles(blob);+ _onFileUpload = fileList => {+ this.props.visStateActions.loadFiles(fileList);};_onExportImage = () => {diff --git a/src/processors/file-handler.d.ts b/src/processors/file-handler.d.tsindex f0aa8f572..a95e99c55 100644--- a/src/processors/file-handler.d.ts+++ b/src/processors/file-handler.d.ts@@ -31,6 +31,8 @@ export type FileCacheItem = {export function readFileInBatches(payload: {file: File;fileCache: FileCacheItem[];+ loaders: LoaderObject;+ loadOptions: object;}): AsyncGenerator;export function processFileData(payload: {diff --git a/src/processors/file-handler.js b/src/processors/file-handler.jsindex a6221ba33..c851e0efe 100644--- a/src/processors/file-handler.js+++ b/src/processors/file-handler.js@@ -132,17 +132,16 @@ export async function* readBatch(asyncIterator, fileName) {}}-export async function readFileInBatches({file, fileCache = []}) {- const batchIterator = await parseInBatches(- file,- [JSONLoader, CSVLoader],- {- csv: CSV_LOADER_OPTIONS,- json: JSON_LOADER_OPTIONS,- metadata: true- },- file.name- );+export async function readFileInBatches({file, fileCache = [], loaders = [], loadOptions = {}}) {+ loaders = [JSONLoader, CSVLoader, ...loaders];+ loadOptions = {+ csv: CSV_LOADER_OPTIONS,+ json: JSON_LOADER_OPTIONS,+ metadata: true,+ ...loadOptions+ };++ const batchIterator = await parseInBatches(file, loaders, loadOptions, file.name);const progressIterator = makeProgressIterator(batchIterator, {size: file.size});return readBatch(progressIterator, file.name);diff --git a/src/reducers/vis-state-updaters.d.ts b/src/reducers/vis-state-updaters.d.tsindex 301cc5aa0..0b443450e 100644--- a/src/reducers/vis-state-updaters.d.ts+++ b/src/reducers/vis-state-updaters.d.ts@@ -3,6 +3,7 @@ import {AddDataToMapOptions, ReceiveMapConfigPayload} from '../actions/actions';import {ParsedConfig} from '../schemas';import * as VisStateActions from 'actions/vis-state-actions';import ActionTypes from 'constants/action-types';+import {LoaderObject} from '@loaders.gl/loader-utils';export type HistogramBin = {x0: number | undefined;@@ -302,6 +303,8 @@ export type VisState = {splitMapsToBeMerged?: SplitMap[];fileLoading: FileLoading | false;fileLoadingProgress: FileLoadingProgress;+ loaders?: LoaderObject[];+ loadOptions?: object;initialState?: Partial<VisState>;};diff --git a/src/reducers/vis-state-updaters.js b/src/reducers/vis-state-updaters.jsindex afec88f90..f80409614 100644--- a/src/reducers/vis-state-updaters.js+++ b/src/reducers/vis-state-updaters.js@@ -1312,11 +1312,15 @@ export function loadNextFileUpdater(state) {progress: {percent: 0, message: 'loading...'}});- return withTask(stateWithProgress, makeLoadFileTask(file, nextState.fileLoading.fileCache));+ const {loaders, loadOptions} = state;+ return withTask(+ stateWithProgress,+ makeLoadFileTask(file, nextState.fileLoading.fileCache, loaders, loadOptions)+ );}-export function makeLoadFileTask(file, fileCache) {- return LOAD_FILE_TASK({file, fileCache}).bimap(+export function makeLoadFileTask(file, fileCache, loaders = [], loadOptions = {}) {+ return LOAD_FILE_TASK({file, fileCache, loaders, loadOptions}).bimap(// prettier ignore// successgen =>diff --git a/src/tasks/tasks.js b/src/tasks/tasks.jsindex 219648c6b..e3277c386 100644--- a/src/tasks/tasks.js+++ b/src/tasks/tasks.js@@ -23,7 +23,8 @@ import {json as requestJson} from 'd3-request';import {readFileInBatches, processFileData} from '../processors/file-handler';export const LOAD_FILE_TASK = Task.fromPromise(- ({file, fileCache}) => readFileInBatches({file, fileCache}),+ ({file, fileCache, loaders, loadOptions}) =>+ readFileInBatches({file, fileCache, loaders, loadOptions}),'LOAD_FILE_TASK');diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 3e1cc5bd6..c12c79a5d 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -4206,17 +4206,19 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.equal(more.length, 0, 'should ceate 1 task');- const exptectedTask1 = {+ const expectedTask1 = {type: 'LOAD_FILE_TASK',- payload: {file: {type: 'text/csv', name: 'test-file.csv'}, fileCache: []}+ payload: {+ file: {type: 'text/csv', name: 'test-file.csv'},+ fileCache: [],+ loaders: [],+ loadOptions: {}+ }};- t.equal(task1.type, exptectedTask1.type, 'should create LOAD_FILE_TASK task');- t.deepEqual(- task1.payload,- exptectedTask1.payload,- 'should create LOAD_FILE_TASK correct payload'- );+ t.comment(JSON.stringify(task1));+ t.equal(task1.type, expectedTask1.type, 'should create LOAD_FILE_TASK task');+ t.deepEqual(task1.payload, expectedTask1.payload, 'should create LOAD_FILE_TASK correct payload');const expectedFileLoading = {fileCache: [],@@ -4283,7 +4285,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.equal(task3Err.type, 'LOAD_FILE_TASK', 'should return an LOAD_FILE_TASK');t.deepEqual(task3Err.payload,- {file: {type: 'text/csv', name: 'test-file-2.csv'}, fileCache: []},+ {+ file: {type: 'text/csv', name: 'test-file-2.csv'},+ fileCache: [],+ loaders: [],+ loadOptions: {}+ },'should return an LOAD_FILE_TASK with 2nd file to load');const expectedErrFileLoadingProgress = {@@ -4399,7 +4406,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.equal(task6.type, 'LOAD_FILE_TASK', 'should return an LOAD_FILE_TASK');t.deepEqual(task6.payload,- {file: {type: 'text/csv', name: 'test-file-2.csv'}, fileCache: fileProcessResult},+ {+ file: {type: 'text/csv', name: 'test-file-2.csv'},+ fileCache: fileProcessResult,+ loaders: [],+ loadOptions: {}+ },'should return an LOAD_FILE_TASK with 2nd file to load');const expectedFileLoadingProgress6 = {diff --git a/tsconfig.json b/tsconfig.jsonindex 30383711a..d5e34034f 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -25,7 +25,9 @@},"paths": {"actions": "src/actions",- "actions/*": "src/actions/*"+ "actions/*": "src/actions/*",+ "reducers": "src/reducers",+ "reducers/*": "src/reducers/*"},"include": ["src/actions",