Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[feat] add support for object and array field type (#2120)

Igor Dykhta3d5db39e88f7
diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.ts
index fa22b7359..4e497b2bd 100644
--- a/src/constants/src/default-settings.ts
+++ b/src/constants/src/default-settings.ts
@@ -333,7 +333,9 @@ export const ALL_FIELD_TYPES = keyMirror({
real: null,
string: null,
timestamp: null,
- point: null
+ point: null,
+ array: null,
+ object: null
});
// Data Table
@@ -386,14 +388,16 @@ export const TABLE_OPTION_LIST = [
{value: TABLE_OPTION.COPY, display: 'Copy Column', icon: 'Clipboard'}
];
-const ORANGE = '248, 194, 28';
-const PINK = '231, 189, 194';
+const YELLOW = '248, 194, 28';
+const PINK = '242, 152, 163';
const PURPLE = '160, 106, 206';
const BLUE = '140, 210, 205';
const BLUE2 = '106, 160, 206';
const BLUE3 = '0, 172, 237';
const GREEN = '106, 160, 56';
+const GREEN2 = '74, 165, 150';
const RED = '237, 88, 106';
+const ORANGE = '231, 110, 58';
export const FIELD_TYPE_DISPLAY = {
[ALL_FIELD_TYPES.boolean]: {
@@ -410,11 +414,11 @@ export const FIELD_TYPE_DISPLAY = {
},
[ALL_FIELD_TYPES.integer]: {
label: 'int',
- color: ORANGE
+ color: YELLOW
},
[ALL_FIELD_TYPES.real]: {
label: 'float',
- color: ORANGE
+ color: YELLOW
},
[ALL_FIELD_TYPES.string]: {
label: 'string',
@@ -428,6 +432,14 @@ export const FIELD_TYPE_DISPLAY = {
[ALL_FIELD_TYPES.point]: {
label: 'point',
color: BLUE3
+ },
+ [ALL_FIELD_TYPES.array]: {
+ label: 'array',
+ color: ORANGE
+ },
+ [ALL_FIELD_TYPES.object]: {
+ label: 'object',
+ color: GREEN2
}
};
@@ -545,7 +557,7 @@ export const DEFAULT_AGGREGATION = {
* Define what type of scale operation is allowed on each type of fields
*/
export const FIELD_OPTS = {
- string: {
+ [ALL_FIELD_TYPES.string]: {
type: 'categorical',
scale: {
...ordinalFieldScaleFunctions,
@@ -556,7 +568,7 @@ export const FIELD_OPTS = {
tooltip: []
}
},
- real: {
+ [ALL_FIELD_TYPES.real]: {
type: 'numerical',
scale: {
...linearFieldScaleFunctions,
@@ -571,7 +583,7 @@ export const FIELD_OPTS = {
]
}
},
- timestamp: {
+ [ALL_FIELD_TYPES.timestamp]: {
type: 'time',
scale: {
...linearFieldScaleFunctions,
@@ -586,7 +598,7 @@ export const FIELD_OPTS = {
]
}
},
- integer: {
+ [ALL_FIELD_TYPES.integer]: {
type: 'numerical',
scale: {
...linearFieldScaleFunctions,
@@ -601,7 +613,7 @@ export const FIELD_OPTS = {
]
}
},
- boolean: {
+ [ALL_FIELD_TYPES.boolean]: {
type: 'boolean',
scale: {
...ordinalFieldScaleFunctions,
@@ -612,7 +624,7 @@ export const FIELD_OPTS = {
tooltip: [TOOLTIP_FORMAT_TYPES.NONE, TOOLTIP_FORMAT_TYPES.BOOLEAN]
}
},
- date: {
+ [ALL_FIELD_TYPES.date]: {
type: 'time',
scale: {
...ordinalFieldScaleFunctions,
@@ -623,7 +635,7 @@ export const FIELD_OPTS = {
tooltip: [TOOLTIP_FORMAT_TYPES.NONE, TOOLTIP_FORMAT_TYPES.DATE]
}
},
- geojson: {
+ [ALL_FIELD_TYPES.geojson]: {
type: 'geometry',
scale: {
...notSupportedScaleOpts,
@@ -633,6 +645,28 @@ export const FIELD_OPTS = {
legend: d => '...',
tooltip: []
}
+ },
+ [ALL_FIELD_TYPES.object]: {
+ type: 'categorical',
+ scale: {
+ ...ordinalFieldScaleFunctions,
+ ...ordinalFieldAggrScaleFunctions
+ },
+ format: {
+ legend: d => '...',
+ tooltip: []
+ }
+ },
+ [ALL_FIELD_TYPES.array]: {
+ type: 'categorical',
+ scale: {
+ ...ordinalFieldScaleFunctions,
+ ...ordinalFieldAggrScaleFunctions
+ },
+ format: {
+ legend: d => '...',
+ tooltip: []
+ }
}
};
diff --git a/src/layers/package.json b/src/layers/package.json
index 396cc2771..55d5272e0 100644
--- a/src/layers/package.json
+++ b/src/layers/package.json
@@ -66,7 +66,7 @@
"reselect": "^4.0.0",
"s2-geometry": "^1.2.10",
"styled-components": "^4.1.3",
- "type-analyzer": "0.3.0",
+ "type-analyzer": "0.4.0",
"viewport-mercator-project": "^6.0.0",
"wellknown": "^0.5.0"
},
diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts
index 2794f069d..334883412 100644
--- a/src/processors/src/data-processor.ts
+++ b/src/processors/src/data-processor.ts
@@ -39,6 +39,14 @@ import {Feature} from '@nebula.gl/edit-modes';
// matches empty string
export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
+function tryParseJsonString(str) {
+ try {
+ return JSON.parse(str);
+ } catch (e) {
+ return null;
+ }
+}
+
export const PARSE_FIELD_VALUE_FROM_STRING = {
[ALL_FIELD_TYPES.boolean]: {
valid: (d: unknown): boolean => typeof d === 'boolean',
@@ -60,6 +68,15 @@ export const PARSE_FIELD_VALUE_FROM_STRING = {
valid: (d: unknown): boolean => parseFloat(d) === d,
// Note this will result in NaN for some string
parse: parseFloat
+ },
+ [ALL_FIELD_TYPES.object]: {
+ valid: isPlainObject,
+ parse: tryParseJsonString
+ },
+
+ [ALL_FIELD_TYPES.array]: {
+ valid: Array.isArray,
+ parse: tryParseJsonString
}
};
diff --git a/src/utils/package.json b/src/utils/package.json
index f778c85a2..b6df69ce0 100644
--- a/src/utils/package.json
+++ b/src/utils/package.json
@@ -59,8 +59,8 @@
"moment-timezone": "^0.5.35",
"react": "^16.8.4",
"resize-observer-polyfill": "^1.5.1",
- "type-analyzer": "0.3.0"
-},
+ "type-analyzer": "0.4.0"
+ },
"nyc": {
"sourceMap": false,
"instrument": false
diff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.ts
index ac1a9a601..1b766b87b 100644
--- a/src/utils/src/data-utils.ts
+++ b/src/utils/src/data-utils.ts
@@ -274,7 +274,9 @@ export const FIELD_DISPLAY_FORMAT: {
? JSON.stringify(d)
: Array.isArray(d)
? `[${String(d)}]`
- : ''
+ : '',
+ [ALL_FIELD_TYPES.object]: JSON.stringify,
+ [ALL_FIELD_TYPES.array]: JSON.stringify
};
/**
diff --git a/src/utils/src/dataset-utils.ts b/src/utils/src/dataset-utils.ts
index 9a9f9e571..304f1241e 100644
--- a/src/utils/src/dataset-utils.ts
+++ b/src/utils/src/dataset-utils.ts
@@ -524,10 +524,11 @@ export function analyzerTypeToFieldType(aType: string): string {
case GEOMETRY:
case GEOMETRY_FROM_STRING:
case PAIR_GEOMETRY_FROM_STRING:
+ return ALL_FIELD_TYPES.geojson;
case ARRAY:
+ return ALL_FIELD_TYPES.array;
case OBJECT:
- // TODO: create a new data type for objects and arrays
- return ALL_FIELD_TYPES.geojson;
+ return ALL_FIELD_TYPES.object;
case NUMBER:
case STRING:
case ZIPCODE:
diff --git a/test/fixtures/geojson-style.js b/test/fixtures/geojson-style.js
index f49a34679..96e295923 100644
--- a/test/fixtures/geojson-style.js
+++ b/test/fixtures/geojson-style.js
@@ -122,7 +122,7 @@ export const processedFields = [
displayName: 'fillColor',
format: '',
fieldIdx: 1,
- type: 'geojson',
+ type: 'array',
analyzerType: 'ARRAY',
valueAccessor: values => values[1]
},
diff --git a/test/fixtures/geojson.js b/test/fixtures/geojson.js
index 55797adb9..360630db8 100644
--- a/test/fixtures/geojson.js
+++ b/test/fixtures/geojson.js
@@ -267,7 +267,7 @@ export const fields = [
valueAccessor: values => values[5]
},
{
- type: 'geojson',
+ type: 'object',
name: 'OBJ',
id: 'OBJ',
displayName: 'OBJ',
@@ -561,7 +561,7 @@ export const geoStyleFields = [
displayName: 'fillColor',
format: '',
fieldIdx: 1,
- type: 'geojson',
+ type: 'array',
analyzerType: 'ARRAY',
valueAccessor: values => values[1]
},
@@ -571,7 +571,7 @@ export const geoStyleFields = [
displayName: 'lineColor',
format: '',
fieldIdx: 2,
- type: 'geojson',
+ type: 'array',
analyzerType: 'ARRAY',
valueAccessor: values => values[2]
},
diff --git a/test/fixtures/test-csv-object.js b/test/fixtures/test-csv-object.js
new file mode 100644
index 000000000..104954138
--- /dev/null
+++ b/test/fixtures/test-csv-object.js
@@ -0,0 +1,86 @@
+// Copyright (c) 2023 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+export const rawData = `tooltip,num_array,str_array,bool_array
+"{""b"": 1}","[1,2]","[""a"", ""b"", ""c""]","[true]"
+"{""c"": 1}","[2,3]","[""d"", ""e""]","[false, true]"
+"{""a"": 2}","[3,4]","[""f"", ""g""]","[true]"
+"{""a"": 3}","[4,5,6]","[]","[true]"
+"{""a"": 4}","[5,6]",,"[true]"
+"{""d"": 1}","[6,7]","[""h""]","[false]"
+"{""e"": 1}","[7,8]","[""i"", ""j""]","[true]"
+"{""f"": 1}","[8,9]","[""k""]","[true]"`;
+
+export const csvObjectDataId = 'test-csv-object';
+
+export const objCsvFields = [
+ {
+ type: 'object',
+ name: 'tooltip',
+ id: 'tooltip',
+ displayName: 'tooltip',
+ format: '',
+ fieldIdx: 0,
+ analyzerType: 'OBJECT',
+ valueAccessor: values => values[0]
+ },
+ {
+ type: 'array',
+ name: 'num_array',
+ id: 'num_array',
+ displayName: 'num_array',
+ format: '',
+ fieldIdx: 1,
+ analyzerType: 'ARRAY',
+ valueAccessor: values => values[1]
+ },
+ {
+ type: 'array',
+ name: 'str_array',
+ id: 'str_array',
+ displayName: 'str_array',
+ format: '',
+ fieldIdx: 2,
+ analyzerType: 'ARRAY',
+ valueAccessor: values => values[2]
+ },
+ {
+ type: 'array',
+ name: 'bool_array',
+ id: 'bool_array',
+ displayName: 'bool_array',
+ format: '',
+ fieldIdx: 3,
+ analyzerType: 'ARRAY',
+ valueAccessor: values => values[3]
+ }
+];
+
+export const objCsvRows = [
+ [{b: 1}, [1, 2], ['a', 'b', 'c'], [true]],
+ [{c: 1}, [2, 3], ['d', 'e'], [false, true]],
+ [{a: 2}, [3, 4], ['f', 'g'], [true]],
+ [{a: 3}, [4, 5, 6], [], [true]],
+ [{a: 4}, [5, 6], null, [true]],
+ [{d: 1}, [6, 7], ['h'], [false]],
+ [{e: 1}, [7, 8], ['i', 'j'], [true]],
+ [{f: 1}, [8, 9], ['k'], [true]]
+];
+export default rawData;
diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.js
index 5251ec226..0c843910a 100644
--- a/test/node/utils/data-processor-test.js
+++ b/test/node/utils/data-processor-test.js
@@ -41,7 +41,7 @@ import {
fields as geojsonFields,
rows as geojsonRows
} from 'test/fixtures/geojson';
-
+import testCsvObjectData, {objCsvFields, objCsvRows} from 'test/fixtures/test-csv-object';
import {
parseCsvRowsByFieldType,
processCsvData,
@@ -140,7 +140,7 @@ test('Processor -> processCsvData', t => {
t.end();
});
-test('Processor -> processCsvData: duplicated field name', t => {
+test('Processor -> processCsvData -> duplicated field name', t => {
const testData1 = `column1,column1,column1,column2\na,b,c,d\nc,d,e,f`;
// load sample dataset csv as text
@@ -216,7 +216,7 @@ test('Processor -> processCsvData -> with nulls', t => {
t.end();
});
-test('Processor -> processCsv.wkt', t => {
+test('Processor -> processCsvData -> wkt', t => {
const {fields, rows} = processCsvData(wktCsv);
cmpFields(t, fields, wktCsvFields, 'should find geometry fields as type:geojson');
@@ -229,6 +229,17 @@ test('Processor -> processCsv.wkt', t => {
t.end();
});
+test('Processor -> processCsvData -> w/ array and object', t => {
+ const {fields, rows} = processCsvData(testCsvObjectData);
+ cmpFields(t, fields, objCsvFields, 'should find csv object fields as type:object');
+
+ t.equal(rows.length, objCsvRows.length, 'should have same row length');
+ rows.forEach((r, i) => {
+ t.deepEqual(r, objCsvRows[i], 'should format correct csv object rows');
+ });
+ t.end();
+});
+
test('Processor => processGeojson', t => {
const {fields, rows} = processGeojson(cloneDeep(geojsonData));
diff --git a/yarn.lock b/yarn.lock
index 4b066e553..08f1e6abd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -15856,10 +15856,10 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0:
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=
- version "0.3.0"
- resolved "https://registry.yarnpkg.com/type-analyzer/-/type-analyzer-0.3.0.tgz#b62a226deb4e9ca46b8cc86e14a93344154377d0"
- integrity sha512-C2A/rZcoP03dAmv1/hCo2AjD7q2BHfZEp62Fn52BXgHoAEDVTobRbtR6UcEOhNy7g9/sh17MHq3ThxAp3578zw==
+ version "0.4.0"
+ resolved "https://registry.yarnpkg.com/type-analyzer/-/type-analyzer-0.4.0.tgz#56621f23b832468d358080466a2d41e142c8078a"
+ integrity sha512-vLj0uKCCZYqtkFoQwf7mIvRQopDc2R4mdADLuX1uhtOm5MjtksHMiatI1TW+n0r7vDZLAMgW5VKGKlVTwbcAhg==
type-check@~0.3.2:
version "0.3.2"