Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] fix for potential freeze during add data pipeline (#3015)

Igor Dykhtae271c8f8cbf6
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index 42d70e506..c6a9f141e 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -10,8 +10,6 @@ import {
GEOARROW_EXTENSIONS,
GEOARROW_METADATA_KEY
} from '@kepler.gl/constants';
-import {KeplerTable} from '@kepler.gl/table';
-import {Field} from '@kepler.gl/types';
import {
arrowSchemaToFields,
isArrowData,
@@ -20,6 +18,8 @@ import {
isRowObject,
processArrowBatches
} from '@kepler.gl/processors';
+import {KeplerTable} from '@kepler.gl/table';
+import {Field} from '@kepler.gl/types';
import {getDuckDB} from '../init';
import {
@@ -301,6 +301,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {
};
static getInputDataValidator = function () {
+ // In DuckDB mode data is validated later during import.
return d => d;
};
}
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 742568ebc..616356aab 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -22,6 +22,7 @@ import {
} from '@kepler.gl/tasks';
// Actions
import {
+ addNotification,
ActionTypes,
CreateNewDatasetSuccessPayload,
MapStateActions,
@@ -56,6 +57,7 @@ import {
arrayInsert,
computeSplitMapLayers,
adjustValueToFilterDomain,
+ errorNotification,
featureToFilterValue,
filterDatasetCPU,
generatePolygonFilter,
@@ -2348,15 +2350,37 @@ export const updateVisDataUpdater = (
const datasets = toArray(action.datasets);
- const allCreateDatasetsTasks = datasets.map(
- ({info = {}, ...rest}) => createNewDataEntry({info, ...rest}, state.datasets) || {}
- );
- // call all Tasks
- const datasetTasks = Task.allSettled(allCreateDatasetsTasks).map(results =>
- createNewDatasetSuccess({results, addToMapOptions: options})
- );
+ const createDatasetTasks: Task[] = [];
+ const notificationTasks: Task[] = [];
+
+ datasets.forEach(({info = {}, ...rest}, datasetIndex) => {
+ const task = createNewDataEntry({info, ...rest}, state.datasets);
+ if (task) {
+ createDatasetTasks.push(task);
+ } else {
+ notificationTasks.push(
+ ACTION_TASK().map(() =>
+ addNotification(
+ errorNotification({
+ message: `Failed to create a new dataset due to data verification errors`,
+ id: `dataset-failed-${datasetIndex}`
+ })
+ )
+ )
+ );
+ }
+ });
- return withTask(updatedState, datasetTasks);
+ const datasetsAllSettledTask = createDatasetTasks.length
+ ? Task.allSettled(createDatasetTasks).map(results =>
+ createNewDatasetSuccess({results, addToMapOptions: options})
+ )
+ : null;
+
+ return withTask(updatedState, [
+ ...(datasetsAllSettledTask ? [datasetsAllSettledTask] : []),
+ ...notificationTasks
+ ]);
};
export const createNewDatasetSuccessUpdater = (
diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.ts
index 47ba97356..759566211 100644
--- a/src/table/src/dataset-utils.ts
+++ b/src/table/src/dataset-utils.ts
@@ -56,7 +56,7 @@ export function getNewDatasetColor(datasets: Datasets): RGBColor {
export function createNewDataEntry(
{info, data, ...opts}: ProtoDataset,
datasets: Datasets = {}
-): Datasets {
+): Datasets | null {
const TableClass = getApplicationConfig().table ?? KeplerTable;
let dataValidator = validateInputData;
if (typeof TableClass.getInputDataValidator === 'function') {
@@ -65,7 +65,7 @@ export function createNewDataEntry(
const validatedData = dataValidator(data);
if (!validatedData) {
- return {};
+ return null;
}
// check if dataset already exists, and update it when loading data by batches incrementally