Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

Add `visState.loaders` to let app inject a list of loaders.gl loaders. (#1192)

Ib Green8270f35317ff
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.js
index c26394325..b23b5f97d 100644
--- a/examples/demo-app/src/reducers/index.js
+++ b/examples/demo-app/src/reducers/index.js
@@ -89,6 +89,10 @@ const demoReducer = combineReducers({
exportMapboxAccessToken: AUTH_TOKENS.EXPORT_MAPBOX_TOKEN
}
}
+ },
+ visState: {
+ loaders: [], // Add additional loaders.gl loaders here
+ loadOptions: {} // Add additional loaders.gl loader options here
}
}),
app: appReducer
diff --git a/package.json b/package.json
index 2b39af748..391eea3a1 100644
--- a/package.json
+++ b/package.json
@@ -87,6 +87,7 @@
"@loaders.gl/csv": "^2.2.5",
"@loaders.gl/gltf": "^2.2.5",
"@loaders.gl/json": "^2.2.5",
+ "@loaders.gl/loader-utils": "^2.2.5",
"@loaders.gl/polyfills": "^2.2.5",
"@luma.gl/constants": "^8.2.0",
"@luma.gl/core": "^8.2.0",
diff --git a/src/actions/vis-state-actions.d.ts b/src/actions/vis-state-actions.d.ts
index 2db3777a4..0431ff048 100644
--- a/src/actions/vis-state-actions.d.ts
+++ b/src/actions/vis-state-actions.d.ts
@@ -26,6 +26,9 @@ import {FileCacheItem} from '../processors/file-handler';
import {Layer, LayerConfig, LayerVisConfig} from 'layers';
import {Feature, InteractionConfig} from 'reducers/vis-state-updaters';
import {ValueOf, Merge} from '../reducers/types';
+// TODO - import LoaderObject type from @loaders.gl/core when supported
+// TODO - import LoadOptions type from @loaders.gl/core when supported
+import {LoaderObject} from '@loaders.gl/loader-utils';
export type LayerConfigChangeUpdaterAction = {
oldLayer: Layer;
@@ -199,7 +202,7 @@ export function copyTableColumn(
column: string
): Merge<CopyTableColumnUpdaterAction, {type: ActionTypes.COPY_TABLE_COLUMN}>;
-export type AddDaataToMapOUpdaterptions = {
+export type AddDataToMapUpdaterOptions = {
centrMap?: boolean;
readOnly?: boolean;
keepExistingConfig?: boolean;
diff --git a/src/actions/vis-state-actions.js b/src/actions/vis-state-actions.js
index 003a43a2d..79ab2496c 100644
--- a/src/actions/vis-state-actions.js
+++ b/src/actions/vis-state-actions.js
@@ -582,10 +582,11 @@ export function setMapInfo(info) {
* @returns action
* @public
*/
-export function loadFiles(files) {
+export function loadFiles(files, onFinish) {
return {
type: ActionTypes.LOAD_FILES,
- files
+ files,
+ onFinish
};
}
diff --git a/src/components/modal-container.d.ts b/src/components/modal-container.d.ts
index ae51a5525..88e300e98 100644
--- a/src/components/modal-container.d.ts
+++ b/src/components/modal-container.d.ts
@@ -3,13 +3,13 @@ import * as VisStateActions from '../actions/vis-state-actions';
import * as UIStateActions from '../actions/ui-state-actions';
import * as MapStyleActions from '../actions/map-style-actions';
import * as ProviderActions from '../actions/provider-actions';
-import {OnErrorCallBack, OnSuccessCallBack} from '../actions/provider-actions';
+import {OnErrorCallBack, OnSuccessCallBack} from 'actions/provider-actions';
-import {MapState} from '../reducers/map-state-updaters';
-import {MapStyle} from '../reducers/map-style-updaters';
-import {UiState} from '../reducers/ui-state-updaters';
-import {VisState} from '../reducers/vis-state-updaters';
-import {ProviderState} from '../reducers/provider-state-updaters';
+import {MapState} from 'reducers/map-state-updaters';
+import {MapStyle} from 'reducers/map-style-updaters';
+import {UiState} from 'reducers/ui-state-updaters';
+import {VisState} from 'reducers/vis-state-updaters';
+import {ProviderState} from 'reducers/provider-state-updaters';
import {OnSuccessCallBack} from 'actions';
export type ModalContainerProps = {
@@ -32,7 +32,7 @@ export type ModalContainerProps = {
onLoadCloudMapSuccess: OnSuccessCallBack,
onLoadCloudMapError: OnErrorCallBack,
onExportToCloudSuccess:OnSuccessCallBack,
- onExportToCloudError: OnErrorCallBack,
+ onExportToCloudError: OnErrorCallBack
};
export default function ModalContainerFactory(): React.Component<ModelContainerProps>;
diff --git a/src/components/modal-container.js b/src/components/modal-container.js
index 981e2f686..3588605fd 100644
--- a/src/components/modal-container.js
+++ b/src/components/modal-container.js
@@ -173,8 +173,8 @@ export default function ModalContainerFactory(
this._closeModal();
};
- _onFileUpload = blob => {
- this.props.visStateActions.loadFiles(blob);
+ _onFileUpload = fileList => {
+ this.props.visStateActions.loadFiles(fileList);
};
_onExportImage = () => {
diff --git a/src/processors/file-handler.d.ts b/src/processors/file-handler.d.ts
index f0aa8f572..a95e99c55 100644
--- a/src/processors/file-handler.d.ts
+++ b/src/processors/file-handler.d.ts
@@ -31,6 +31,8 @@ export type FileCacheItem = {
export function readFileInBatches(payload: {
file: File;
fileCache: FileCacheItem[];
+ loaders: LoaderObject;
+ loadOptions: object;
}): AsyncGenerator;
export function processFileData(payload: {
diff --git a/src/processors/file-handler.js b/src/processors/file-handler.js
index a6221ba33..c851e0efe 100644
--- a/src/processors/file-handler.js
+++ b/src/processors/file-handler.js
@@ -132,17 +132,16 @@ export async function* readBatch(asyncIterator, fileName) {
}
}
-export async function readFileInBatches({file, fileCache = []}) {
- const batchIterator = await parseInBatches(
- file,
- [JSONLoader, CSVLoader],
- {
- csv: CSV_LOADER_OPTIONS,
- json: JSON_LOADER_OPTIONS,
- metadata: true
- },
- file.name
- );
+export async function readFileInBatches({file, fileCache = [], loaders = [], loadOptions = {}}) {
+ loaders = [JSONLoader, CSVLoader, ...loaders];
+ loadOptions = {
+ csv: CSV_LOADER_OPTIONS,
+ json: JSON_LOADER_OPTIONS,
+ metadata: true,
+ ...loadOptions
+ };
+
+ const batchIterator = await parseInBatches(file, loaders, loadOptions, file.name);
const progressIterator = makeProgressIterator(batchIterator, {size: file.size});
return readBatch(progressIterator, file.name);
diff --git a/src/reducers/vis-state-updaters.d.ts b/src/reducers/vis-state-updaters.d.ts
index 301cc5aa0..0b443450e 100644
--- a/src/reducers/vis-state-updaters.d.ts
+++ b/src/reducers/vis-state-updaters.d.ts
@@ -3,6 +3,7 @@ import {AddDataToMapOptions, ReceiveMapConfigPayload} from '../actions/actions';
import {ParsedConfig} from '../schemas';
import * as VisStateActions from 'actions/vis-state-actions';
import ActionTypes from 'constants/action-types';
+import {LoaderObject} from '@loaders.gl/loader-utils';
export type HistogramBin = {
x0: number | undefined;
@@ -302,6 +303,8 @@ export type VisState = {
splitMapsToBeMerged?: SplitMap[];
fileLoading: FileLoading | false;
fileLoadingProgress: FileLoadingProgress;
+ loaders?: LoaderObject[];
+ loadOptions?: object;
initialState?: Partial<VisState>;
};
diff --git a/src/reducers/vis-state-updaters.js b/src/reducers/vis-state-updaters.js
index afec88f90..f80409614 100644
--- a/src/reducers/vis-state-updaters.js
+++ b/src/reducers/vis-state-updaters.js
@@ -1312,11 +1312,15 @@ export function loadNextFileUpdater(state) {
progress: {percent: 0, message: 'loading...'}
});
- return withTask(stateWithProgress, makeLoadFileTask(file, nextState.fileLoading.fileCache));
+ const {loaders, loadOptions} = state;
+ return withTask(
+ stateWithProgress,
+ makeLoadFileTask(file, nextState.fileLoading.fileCache, loaders, loadOptions)
+ );
}
-export function makeLoadFileTask(file, fileCache) {
- return LOAD_FILE_TASK({file, fileCache}).bimap(
+export function makeLoadFileTask(file, fileCache, loaders = [], loadOptions = {}) {
+ return LOAD_FILE_TASK({file, fileCache, loaders, loadOptions}).bimap(
// prettier ignore
// success
gen =>
diff --git a/src/tasks/tasks.js b/src/tasks/tasks.js
index 219648c6b..e3277c386 100644
--- a/src/tasks/tasks.js
+++ b/src/tasks/tasks.js
@@ -23,7 +23,8 @@ import {json as requestJson} from 'd3-request';
import {readFileInBatches, processFileData} from '../processors/file-handler';
export const LOAD_FILE_TASK = Task.fromPromise(
- ({file, fileCache}) => readFileInBatches({file, fileCache}),
+ ({file, fileCache, loaders, loadOptions}) =>
+ readFileInBatches({file, fileCache, loaders, loadOptions}),
'LOAD_FILE_TASK'
);
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index 3e1cc5bd6..c12c79a5d 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -4206,17 +4206,19 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.equal(more.length, 0, 'should ceate 1 task');
- const exptectedTask1 = {
+ const expectedTask1 = {
type: 'LOAD_FILE_TASK',
- payload: {file: {type: 'text/csv', name: 'test-file.csv'}, fileCache: []}
+ payload: {
+ file: {type: 'text/csv', name: 'test-file.csv'},
+ fileCache: [],
+ loaders: [],
+ loadOptions: {}
+ }
};
- t.equal(task1.type, exptectedTask1.type, 'should create LOAD_FILE_TASK task');
- t.deepEqual(
- task1.payload,
- exptectedTask1.payload,
- 'should create LOAD_FILE_TASK correct payload'
- );
+ t.comment(JSON.stringify(task1));
+ t.equal(task1.type, expectedTask1.type, 'should create LOAD_FILE_TASK task');
+ t.deepEqual(task1.payload, expectedTask1.payload, 'should create LOAD_FILE_TASK correct payload');
const expectedFileLoading = {
fileCache: [],
@@ -4283,7 +4285,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.equal(task3Err.type, 'LOAD_FILE_TASK', 'should return an LOAD_FILE_TASK');
t.deepEqual(
task3Err.payload,
- {file: {type: 'text/csv', name: 'test-file-2.csv'}, fileCache: []},
+ {
+ file: {type: 'text/csv', name: 'test-file-2.csv'},
+ fileCache: [],
+ loaders: [],
+ loadOptions: {}
+ },
'should return an LOAD_FILE_TASK with 2nd file to load'
);
const expectedErrFileLoadingProgress = {
@@ -4399,7 +4406,12 @@ test('#visStateReducer -> LOAD_FILES', async t => {
t.equal(task6.type, 'LOAD_FILE_TASK', 'should return an LOAD_FILE_TASK');
t.deepEqual(
task6.payload,
- {file: {type: 'text/csv', name: 'test-file-2.csv'}, fileCache: fileProcessResult},
+ {
+ file: {type: 'text/csv', name: 'test-file-2.csv'},
+ fileCache: fileProcessResult,
+ loaders: [],
+ loadOptions: {}
+ },
'should return an LOAD_FILE_TASK with 2nd file to load'
);
const expectedFileLoadingProgress6 = {
diff --git a/tsconfig.json b/tsconfig.json
index 30383711a..d5e34034f 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -25,7 +25,9 @@
},
"paths": {
"actions": "src/actions",
- "actions/*": "src/actions/*"
+ "actions/*": "src/actions/*",
+ "reducers": "src/reducers",
+ "reducers/*": "src/reducers/*"
},
"include": [
"src/actions",