Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] arrow tables - save timestamps as iso date string (#2953)

Igor Dykhtae95c4e5a47ff
diff --git a/src/schemas/package.json b/src/schemas/package.json
index 222bbed9e..99ce1863d 100644
--- a/src/schemas/package.json
+++ b/src/schemas/package.json
@@ -40,6 +40,7 @@
"@types/keymirror": "^0.1.1",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.pick": "^4.4.6",
+ "apache-arrow": ">=15.0.0",
"global": "^4.3.0",
"keymirror": "^0.1.1",
"lodash.clonedeep": "^4.0.1",
diff --git a/src/schemas/src/dataset-schema.ts b/src/schemas/src/dataset-schema.ts
index d77d1d644..d273fcde7 100644
--- a/src/schemas/src/dataset-schema.ts
+++ b/src/schemas/src/dataset-schema.ts
@@ -3,6 +3,7 @@
import pick from 'lodash.pick';
import {console as globalConsole} from 'global/window';
+import {Type as ArrowTypes} from 'apache-arrow';
import {DATASET_FORMATS} from '@kepler.gl/constants';
import {ProtoDataset, RGBColor, JsonObject} from '@kepler.gl/types';
@@ -10,6 +11,7 @@ import {KeplerTable} from '@kepler.gl/table';
import {VERSIONS} from './versions';
import Schema from './schema';
import {getFieldsFromData, getSampleForTypeAnalyze} from '@kepler.gl/common-utils';
+import {ArrowDataContainer, DataContainerInterface} from '@kepler.gl/utils';
export type SavedField = {
name: string;
@@ -100,6 +102,37 @@ export const propertiesV1 = {
disableDataOperation: null
};
+/**
+ * TODO Consider moving this cast to ArrowDataContainer?
+ * Prepare a data container for export as part of json / html.
+ * 1) Arrow tables can store Timestamps as BigInts, so convert numbers to ISOStrings compatible with Kepler.gl's TIMESTAMP.
+ * @param dataContainer A data container to flatten.
+ * @returns Row based data.
+ */
+const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] => {
+ const allData = dataContainer.flattenData();
+
+ if (dataContainer instanceof ArrowDataContainer) {
+ const numColumns = dataContainer.numColumns();
+
+ for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+ const column = dataContainer.getColumn(columnIndex);
+ const typeId = column.type?.typeId;
+ if (
+ typeId === ArrowTypes.Timestamp ||
+ typeId === ArrowTypes.Date ||
+ typeId === ArrowTypes.Time
+ ) {
+ allData.forEach(row => {
+ row[columnIndex] = new Date(row[columnIndex]).toISOString();
+ });
+ }
+ }
+ }
+
+ return allData;
+};
+
export class DatasetSchema extends Schema {
key = 'dataset';
@@ -107,7 +140,7 @@ export class DatasetSchema extends Schema {
const datasetFlattened = dataset.dataContainer
? {
...dataset,
- allData: dataset.dataContainer.flattenData(),
+ allData: getAllDataForSaving(dataset.dataContainer),
// we use flattenData to save arrow tables,
// but once flattened it's not an arrow file anymore.
metadata: {
diff --git a/yarn.lock b/yarn.lock
index 0c6e73199..170e70c70 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3605,6 +3605,7 @@ __metadata:
"@types/keymirror": "npm:^0.1.1"
"@types/lodash.clonedeep": "npm:^4.5.7"
"@types/lodash.pick": "npm:^4.4.6"
+ apache-arrow: "npm:>=15.0.0"
global: "npm:^4.3.0"
keymirror: "npm:^0.1.1"
lodash.clonedeep: "npm:^4.0.1"