Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[fix] fix for potential freeze during add data pipeline (#3015)
Igor Dykhtae271c8f8cbf6
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex 42d70e506..c6a9f141e 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -10,8 +10,6 @@ import {GEOARROW_EXTENSIONS,GEOARROW_METADATA_KEY} from '@kepler.gl/constants';-import {KeplerTable} from '@kepler.gl/table';-import {Field} from '@kepler.gl/types';import {arrowSchemaToFields,isArrowData,@@ -20,6 +18,8 @@ import {isRowObject,processArrowBatches} from '@kepler.gl/processors';+import {KeplerTable} from '@kepler.gl/table';+import {Field} from '@kepler.gl/types';import {getDuckDB} from '../init';import {@@ -301,6 +301,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {};static getInputDataValidator = function () {+ // In DuckDB mode data is validated later during import.return d => d;};}diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 742568ebc..616356aab 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -22,6 +22,7 @@ import {} from '@kepler.gl/tasks';// Actionsimport {+ addNotification,ActionTypes,CreateNewDatasetSuccessPayload,MapStateActions,@@ -56,6 +57,7 @@ import {arrayInsert,computeSplitMapLayers,adjustValueToFilterDomain,+ errorNotification,featureToFilterValue,filterDatasetCPU,generatePolygonFilter,@@ -2348,15 +2350,37 @@ export const updateVisDataUpdater = (const datasets = toArray(action.datasets);- const allCreateDatasetsTasks = datasets.map(- ({info = {}, ...rest}) => createNewDataEntry({info, ...rest}, state.datasets) || {}- );- // call all Tasks- const datasetTasks = Task.allSettled(allCreateDatasetsTasks).map(results =>- createNewDatasetSuccess({results, addToMapOptions: options})- );+ const createDatasetTasks: Task[] = [];+ const notificationTasks: Task[] = [];++ datasets.forEach(({info = {}, ...rest}, datasetIndex) => {+ const task = createNewDataEntry({info, ...rest}, state.datasets);+ if (task) {+ createDatasetTasks.push(task);+ } else {+ notificationTasks.push(+ ACTION_TASK().map(() =>+ addNotification(+ errorNotification({+ message: `Failed to create a new dataset due to data verification errors`,+ id: `dataset-failed-${datasetIndex}`+ })+ )+ )+ );+ }+ });- return withTask(updatedState, datasetTasks);+ const datasetsAllSettledTask = createDatasetTasks.length+ ? Task.allSettled(createDatasetTasks).map(results =>+ createNewDatasetSuccess({results, addToMapOptions: options})+ )+ : null;++ return withTask(updatedState, [+ ...(datasetsAllSettledTask ? [datasetsAllSettledTask] : []),+ ...notificationTasks+ ]);};export const createNewDatasetSuccessUpdater = (diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.tsindex 47ba97356..759566211 100644--- a/src/table/src/dataset-utils.ts+++ b/src/table/src/dataset-utils.ts@@ -56,7 +56,7 @@ export function getNewDatasetColor(datasets: Datasets): RGBColor {export function createNewDataEntry({info, data, ...opts}: ProtoDataset,datasets: Datasets = {}-): Datasets {+): Datasets | null {const TableClass = getApplicationConfig().table ?? KeplerTable;let dataValidator = validateInputData;if (typeof TableClass.getInputDataValidator === 'function') {@@ -65,7 +65,7 @@ export function createNewDataEntry(const validatedData = dataValidator(data);if (!validatedData) {- return {};+ return null;}// check if dataset already exists, and update it when loading data by batches incrementally