Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[fix] arrow tables - save timestamps as iso date string (#2953)
Igor Dykhtae95c4e5a47ff
diff --git a/src/schemas/package.json b/src/schemas/package.jsonindex 222bbed9e..99ce1863d 100644--- a/src/schemas/package.json+++ b/src/schemas/package.json@@ -40,6 +40,7 @@"@types/keymirror": "^0.1.1","@types/lodash.clonedeep": "^4.5.7","@types/lodash.pick": "^4.4.6",+ "apache-arrow": ">=15.0.0","global": "^4.3.0","keymirror": "^0.1.1","lodash.clonedeep": "^4.0.1",diff --git a/src/schemas/src/dataset-schema.ts b/src/schemas/src/dataset-schema.tsindex d77d1d644..d273fcde7 100644--- a/src/schemas/src/dataset-schema.ts+++ b/src/schemas/src/dataset-schema.ts@@ -3,6 +3,7 @@import pick from 'lodash.pick';import {console as globalConsole} from 'global/window';+import {Type as ArrowTypes} from 'apache-arrow';import {DATASET_FORMATS} from '@kepler.gl/constants';import {ProtoDataset, RGBColor, JsonObject} from '@kepler.gl/types';@@ -10,6 +11,7 @@ import {KeplerTable} from '@kepler.gl/table';import {VERSIONS} from './versions';import Schema from './schema';import {getFieldsFromData, getSampleForTypeAnalyze} from '@kepler.gl/common-utils';+import {ArrowDataContainer, DataContainerInterface} from '@kepler.gl/utils';export type SavedField = {name: string;@@ -100,6 +102,37 @@ export const propertiesV1 = {disableDataOperation: null};+/**+ * TODO Consider moving this cast to ArrowDataContainer?+ * Prepare a data container for export as part of json / html.+ * 1) Arrow tables can store Timestamps as BigInts, so convert numbers to ISOStrings compatible with Kepler.gl's TIMESTAMP.+ * @param dataContainer A data container to flatten.+ * @returns Row based data.+ */+const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] => {+ const allData = dataContainer.flattenData();++ if (dataContainer instanceof ArrowDataContainer) {+ const numColumns = dataContainer.numColumns();++ for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {+ const column = dataContainer.getColumn(columnIndex);+ const typeId = column.type?.typeId;+ if (+ typeId === ArrowTypes.Timestamp ||+ typeId === ArrowTypes.Date ||+ typeId === ArrowTypes.Time+ ) {+ allData.forEach(row => {+ row[columnIndex] = new Date(row[columnIndex]).toISOString();+ });+ }+ }+ }++ return allData;+};+export class DatasetSchema extends Schema {key = 'dataset';@@ -107,7 +140,7 @@ export class DatasetSchema extends Schema {const datasetFlattened = dataset.dataContainer? {...dataset,- allData: dataset.dataContainer.flattenData(),+ allData: getAllDataForSaving(dataset.dataContainer),// we use flattenData to save arrow tables,// but once flattened it's not an arrow file anymore.metadata: {diff --git a/yarn.lock b/yarn.lockindex 0c6e73199..170e70c70 100644--- a/yarn.lock+++ b/yarn.lock@@ -3605,6 +3605,7 @@ __metadata:"@types/keymirror": "npm:^0.1.1""@types/lodash.clonedeep": "npm:^4.5.7""@types/lodash.pick": "npm:^4.4.6"+ apache-arrow: "npm:>=15.0.0"global: "npm:^4.3.0"keymirror: "npm:^0.1.1"lodash.clonedeep: "npm:^4.0.1"