Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] export geoarrow to CSV as geojson (#2988)

Igor Dykhta1216d235beb5
diff --git a/src/reducers/src/export-utils.ts b/src/reducers/src/export-utils.ts
index fc76b95af..f8a79eed3 100644
--- a/src/reducers/src/export-utils.ts
+++ b/src/reducers/src/export-utils.ts
@@ -69,7 +69,7 @@ export function formatCsv(data: DataContainerInterface, fields: Field[]): string
// parse geojson object as string
for (const row of data.rows(true)) {
- formattedData.push(row.map((d, i) => parseFieldValue(d, fields[i].type)));
+ formattedData.push(row.map((d, i) => parseFieldValue(d, fields[i].type, fields[i])));
}
return csvFormatRows(formattedData);
diff --git a/src/utils/package.json b/src/utils/package.json
index c398a5a72..ca7fce754 100644
--- a/src/utils/package.json
+++ b/src/utils/package.json
@@ -35,6 +35,7 @@
"@kepler.gl/common-utils": "3.1.0",
"@kepler.gl/constants": "3.1.0",
"@kepler.gl/types": "3.1.0",
+ "@loaders.gl/arrow": "^4.3.2",
"@luma.gl/constants": "^8.5.20",
"@luma.gl/core": "^8.5.20",
"@mapbox/geo-viewport": "^0.4.1",
diff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.ts
index d7919e49a..00d6edce8 100644
--- a/src/utils/src/data-utils.ts
+++ b/src/utils/src/data-utils.ts
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
+import * as arrow from 'apache-arrow';
import assert from 'assert';
import {format as d3Format} from 'd3-format';
import moment from 'moment-timezone';
+import {parseGeometryFromArrow} from '@loaders.gl/arrow';
+
import {
ALL_FIELD_TYPES,
TOOLTIP_FORMATS,
@@ -18,9 +21,9 @@ import {Field, Millisecond} from '@kepler.gl/types';
import {snapToMarks} from './plot';
import {isPlainObject} from './utils';
-export type FieldFormatter = (value: any) => string;
+export type FieldFormatter = (value: any, field?: Field) => string;
-// We need threat latitude differently otherwise marcator project view throws
+// We need threat latitude differently otherwise mercator project view throws
// a projection matrix error
// Uncaught Error: Pixel project matrix not invertible
// at WebMercatorViewport16.Viewport6 (viewport.js:81:13)
@@ -312,11 +315,21 @@ export const FIELD_DISPLAY_FORMAT: {
: Array.isArray(d)
? `[${String(d)}]`
: '',
- [ALL_FIELD_TYPES.geoarrow]: d => {
- if (d instanceof Uint8Array) {
- return uint8ArrayToHex(d);
+ [ALL_FIELD_TYPES.geoarrow]: (data, field) => {
+ if (data instanceof arrow.Vector) {
+ try {
+ const encoding = field?.metadata?.get('ARROW:extension:name');
+ if (encoding) {
+ const geometry = parseGeometryFromArrow(data, encoding);
+ return JSON.stringify(geometry);
+ }
+ } catch (error) {
+ // ignore for now
+ }
+ } else if (data instanceof Uint8Array) {
+ return uint8ArrayToHex(data);
}
- return d;
+ return data;
},
[ALL_FIELD_TYPES.object]: (value: any) => {
try {
@@ -325,14 +338,14 @@ export const FIELD_DISPLAY_FORMAT: {
return String(value);
}
},
- [ALL_FIELD_TYPES.array]: JSON.stringify,
+ [ALL_FIELD_TYPES.array]: d => JSON.stringify(d),
[ALL_FIELD_TYPES.h3]: defaultFormatter
};
/**
* Parse field value and type and return a string representation
*/
-export const parseFieldValue = (value: any, type: string): string => {
+export const parseFieldValue = (value: any, type: string, field?: Field): string => {
if (!notNullorUndefined(value)) {
return '';
}
@@ -342,7 +355,7 @@ export const parseFieldValue = (value: any, type: string): string => {
if (typeof value === 'bigint') {
return value.toString();
}
- return FIELD_DISPLAY_FORMAT[type] ? FIELD_DISPLAY_FORMAT[type](value) : String(value);
+ return FIELD_DISPLAY_FORMAT[type] ? FIELD_DISPLAY_FORMAT[type](value, field) : String(value);
};
/**
diff --git a/yarn.lock b/yarn.lock
index 63ad79190..2f34fd079 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3633,6 +3633,7 @@ __metadata:
"@kepler.gl/common-utils": "npm:3.1.0"
"@kepler.gl/constants": "npm:3.1.0"
"@kepler.gl/types": "npm:3.1.0"
+ "@loaders.gl/arrow": "npm:^4.3.2"
"@luma.gl/constants": "npm:^8.5.20"
"@luma.gl/core": "npm:^8.5.20"
"@mapbox/geo-viewport": "npm:^0.4.1"