Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[feat] add support for object and array field type (#2120)
Igor Dykhta3d5db39e88f7
diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.tsindex fa22b7359..4e497b2bd 100644--- a/src/constants/src/default-settings.ts+++ b/src/constants/src/default-settings.ts@@ -333,7 +333,9 @@ export const ALL_FIELD_TYPES = keyMirror({real: null,string: null,timestamp: null,- point: null+ point: null,+ array: null,+ object: null});// Data Table@@ -386,14 +388,16 @@ export const TABLE_OPTION_LIST = [{value: TABLE_OPTION.COPY, display: 'Copy Column', icon: 'Clipboard'}];-const ORANGE = '248, 194, 28';-const PINK = '231, 189, 194';+const YELLOW = '248, 194, 28';+const PINK = '242, 152, 163';const PURPLE = '160, 106, 206';const BLUE = '140, 210, 205';const BLUE2 = '106, 160, 206';const BLUE3 = '0, 172, 237';const GREEN = '106, 160, 56';+const GREEN2 = '74, 165, 150';const RED = '237, 88, 106';+const ORANGE = '231, 110, 58';export const FIELD_TYPE_DISPLAY = {[ALL_FIELD_TYPES.boolean]: {@@ -410,11 +414,11 @@ export const FIELD_TYPE_DISPLAY = {},[ALL_FIELD_TYPES.integer]: {label: 'int',- color: ORANGE+ color: YELLOW},[ALL_FIELD_TYPES.real]: {label: 'float',- color: ORANGE+ color: YELLOW},[ALL_FIELD_TYPES.string]: {label: 'string',@@ -428,6 +432,14 @@ export const FIELD_TYPE_DISPLAY = {[ALL_FIELD_TYPES.point]: {label: 'point',color: BLUE3+ },+ [ALL_FIELD_TYPES.array]: {+ label: 'array',+ color: ORANGE+ },+ [ALL_FIELD_TYPES.object]: {+ label: 'object',+ color: GREEN2}};@@ -545,7 +557,7 @@ export const DEFAULT_AGGREGATION = {* Define what type of scale operation is allowed on each type of fields*/export const FIELD_OPTS = {- string: {+ [ALL_FIELD_TYPES.string]: {type: 'categorical',scale: {...ordinalFieldScaleFunctions,@@ -556,7 +568,7 @@ export const FIELD_OPTS = {tooltip: []}},- real: {+ [ALL_FIELD_TYPES.real]: {type: 'numerical',scale: {...linearFieldScaleFunctions,@@ -571,7 +583,7 @@ export const FIELD_OPTS = {]}},- timestamp: {+ [ALL_FIELD_TYPES.timestamp]: {type: 'time',scale: {...linearFieldScaleFunctions,@@ -586,7 +598,7 @@ export const FIELD_OPTS = {]}},- integer: {+ [ALL_FIELD_TYPES.integer]: {type: 'numerical',scale: {...linearFieldScaleFunctions,@@ -601,7 +613,7 @@ export const FIELD_OPTS = {]}},- boolean: {+ [ALL_FIELD_TYPES.boolean]: {type: 'boolean',scale: {...ordinalFieldScaleFunctions,@@ -612,7 +624,7 @@ export const FIELD_OPTS = {tooltip: [TOOLTIP_FORMAT_TYPES.NONE, TOOLTIP_FORMAT_TYPES.BOOLEAN]}},- date: {+ [ALL_FIELD_TYPES.date]: {type: 'time',scale: {...ordinalFieldScaleFunctions,@@ -623,7 +635,7 @@ export const FIELD_OPTS = {tooltip: [TOOLTIP_FORMAT_TYPES.NONE, TOOLTIP_FORMAT_TYPES.DATE]}},- geojson: {+ [ALL_FIELD_TYPES.geojson]: {type: 'geometry',scale: {...notSupportedScaleOpts,@@ -633,6 +645,28 @@ export const FIELD_OPTS = {legend: d => '...',tooltip: []}+ },+ [ALL_FIELD_TYPES.object]: {+ type: 'categorical',+ scale: {+ ...ordinalFieldScaleFunctions,+ ...ordinalFieldAggrScaleFunctions+ },+ format: {+ legend: d => '...',+ tooltip: []+ }+ },+ [ALL_FIELD_TYPES.array]: {+ type: 'categorical',+ scale: {+ ...ordinalFieldScaleFunctions,+ ...ordinalFieldAggrScaleFunctions+ },+ format: {+ legend: d => '...',+ tooltip: []+ }}};diff --git a/src/layers/package.json b/src/layers/package.jsonindex 396cc2771..55d5272e0 100644--- a/src/layers/package.json+++ b/src/layers/package.json@@ -66,7 +66,7 @@"reselect": "^4.0.0","s2-geometry": "^1.2.10","styled-components": "^4.1.3",- "type-analyzer": "0.3.0",+ "type-analyzer": "0.4.0","viewport-mercator-project": "^6.0.0","wellknown": "^0.5.0"},diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.tsindex 2794f069d..334883412 100644--- a/src/processors/src/data-processor.ts+++ b/src/processors/src/data-processor.ts@@ -39,6 +39,14 @@ import {Feature} from '@nebula.gl/edit-modes';// matches empty stringexport const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;+function tryParseJsonString(str) {+ try {+ return JSON.parse(str);+ } catch (e) {+ return null;+ }+}+export const PARSE_FIELD_VALUE_FROM_STRING = {[ALL_FIELD_TYPES.boolean]: {valid: (d: unknown): boolean => typeof d === 'boolean',@@ -60,6 +68,15 @@ export const PARSE_FIELD_VALUE_FROM_STRING = {valid: (d: unknown): boolean => parseFloat(d) === d,// Note this will result in NaN for some stringparse: parseFloat+ },+ [ALL_FIELD_TYPES.object]: {+ valid: isPlainObject,+ parse: tryParseJsonString+ },++ [ALL_FIELD_TYPES.array]: {+ valid: Array.isArray,+ parse: tryParseJsonString}};diff --git a/src/utils/package.json b/src/utils/package.jsonindex f778c85a2..b6df69ce0 100644--- a/src/utils/package.json+++ b/src/utils/package.json@@ -59,8 +59,8 @@"moment-timezone": "^0.5.35","react": "^16.8.4","resize-observer-polyfill": "^1.5.1",- "type-analyzer": "0.3.0"-},+ "type-analyzer": "0.4.0"+ },"nyc": {"sourceMap": false,"instrument": falsediff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.tsindex ac1a9a601..1b766b87b 100644--- a/src/utils/src/data-utils.ts+++ b/src/utils/src/data-utils.ts@@ -274,7 +274,9 @@ export const FIELD_DISPLAY_FORMAT: {? JSON.stringify(d): Array.isArray(d)? `[${String(d)}]`- : ''+ : '',+ [ALL_FIELD_TYPES.object]: JSON.stringify,+ [ALL_FIELD_TYPES.array]: JSON.stringify};/**diff --git a/src/utils/src/dataset-utils.ts b/src/utils/src/dataset-utils.tsindex 9a9f9e571..304f1241e 100644--- a/src/utils/src/dataset-utils.ts+++ b/src/utils/src/dataset-utils.ts@@ -524,10 +524,11 @@ export function analyzerTypeToFieldType(aType: string): string {case GEOMETRY:case GEOMETRY_FROM_STRING:case PAIR_GEOMETRY_FROM_STRING:+ return ALL_FIELD_TYPES.geojson;case ARRAY:+ return ALL_FIELD_TYPES.array;case OBJECT:- // TODO: create a new data type for objects and arrays- return ALL_FIELD_TYPES.geojson;+ return ALL_FIELD_TYPES.object;case NUMBER:case STRING:case ZIPCODE:diff --git a/test/fixtures/geojson-style.js b/test/fixtures/geojson-style.jsindex f49a34679..96e295923 100644--- a/test/fixtures/geojson-style.js+++ b/test/fixtures/geojson-style.js@@ -122,7 +122,7 @@ export const processedFields = [displayName: 'fillColor',format: '',fieldIdx: 1,- type: 'geojson',+ type: 'array',analyzerType: 'ARRAY',valueAccessor: values => values[1]},diff --git a/test/fixtures/geojson.js b/test/fixtures/geojson.jsindex 55797adb9..360630db8 100644--- a/test/fixtures/geojson.js+++ b/test/fixtures/geojson.js@@ -267,7 +267,7 @@ export const fields = [valueAccessor: values => values[5]},{- type: 'geojson',+ type: 'object',name: 'OBJ',id: 'OBJ',displayName: 'OBJ',@@ -561,7 +561,7 @@ export const geoStyleFields = [displayName: 'fillColor',format: '',fieldIdx: 1,- type: 'geojson',+ type: 'array',analyzerType: 'ARRAY',valueAccessor: values => values[1]},@@ -571,7 +571,7 @@ export const geoStyleFields = [displayName: 'lineColor',format: '',fieldIdx: 2,- type: 'geojson',+ type: 'array',analyzerType: 'ARRAY',valueAccessor: values => values[2]},diff --git a/test/fixtures/test-csv-object.js b/test/fixtures/test-csv-object.jsnew file mode 100644index 000000000..104954138--- /dev/null+++ b/test/fixtures/test-csv-object.js@@ -0,0 +1,86 @@+// Copyright (c) 2023 Uber Technologies, Inc.+//+// Permission is hereby granted, free of charge, to any person obtaining a copy+// of this software and associated documentation files (the "Software"), to deal+// in the Software without restriction, including without limitation the rights+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+// copies of the Software, and to permit persons to whom the Software is+// furnished to do so, subject to the following conditions:+//+// The above copyright notice and this permission notice shall be included in+// all copies or substantial portions of the Software.+//+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+// THE SOFTWARE.++export const rawData = `tooltip,num_array,str_array,bool_array+"{""b"": 1}","[1,2]","[""a"", ""b"", ""c""]","[true]"+"{""c"": 1}","[2,3]","[""d"", ""e""]","[false, true]"+"{""a"": 2}","[3,4]","[""f"", ""g""]","[true]"+"{""a"": 3}","[4,5,6]","[]","[true]"+"{""a"": 4}","[5,6]",,"[true]"+"{""d"": 1}","[6,7]","[""h""]","[false]"+"{""e"": 1}","[7,8]","[""i"", ""j""]","[true]"+"{""f"": 1}","[8,9]","[""k""]","[true]"`;++export const csvObjectDataId = 'test-csv-object';++export const objCsvFields = [+ {+ type: 'object',+ name: 'tooltip',+ id: 'tooltip',+ displayName: 'tooltip',+ format: '',+ fieldIdx: 0,+ analyzerType: 'OBJECT',+ valueAccessor: values => values[0]+ },+ {+ type: 'array',+ name: 'num_array',+ id: 'num_array',+ displayName: 'num_array',+ format: '',+ fieldIdx: 1,+ analyzerType: 'ARRAY',+ valueAccessor: values => values[1]+ },+ {+ type: 'array',+ name: 'str_array',+ id: 'str_array',+ displayName: 'str_array',+ format: '',+ fieldIdx: 2,+ analyzerType: 'ARRAY',+ valueAccessor: values => values[2]+ },+ {+ type: 'array',+ name: 'bool_array',+ id: 'bool_array',+ displayName: 'bool_array',+ format: '',+ fieldIdx: 3,+ analyzerType: 'ARRAY',+ valueAccessor: values => values[3]+ }+];++export const objCsvRows = [+ [{b: 1}, [1, 2], ['a', 'b', 'c'], [true]],+ [{c: 1}, [2, 3], ['d', 'e'], [false, true]],+ [{a: 2}, [3, 4], ['f', 'g'], [true]],+ [{a: 3}, [4, 5, 6], [], [true]],+ [{a: 4}, [5, 6], null, [true]],+ [{d: 1}, [6, 7], ['h'], [false]],+ [{e: 1}, [7, 8], ['i', 'j'], [true]],+ [{f: 1}, [8, 9], ['k'], [true]]+];+export default rawData;diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.jsindex 5251ec226..0c843910a 100644--- a/test/node/utils/data-processor-test.js+++ b/test/node/utils/data-processor-test.js@@ -41,7 +41,7 @@ import {fields as geojsonFields,rows as geojsonRows} from 'test/fixtures/geojson';-+import testCsvObjectData, {objCsvFields, objCsvRows} from 'test/fixtures/test-csv-object';import {parseCsvRowsByFieldType,processCsvData,@@ -140,7 +140,7 @@ test('Processor -> processCsvData', t => {t.end();});-test('Processor -> processCsvData: duplicated field name', t => {+test('Processor -> processCsvData -> duplicated field name', t => {const testData1 = `column1,column1,column1,column2\na,b,c,d\nc,d,e,f`;// load sample dataset csv as text@@ -216,7 +216,7 @@ test('Processor -> processCsvData -> with nulls', t => {t.end();});-test('Processor -> processCsv.wkt', t => {+test('Processor -> processCsvData -> wkt', t => {const {fields, rows} = processCsvData(wktCsv);cmpFields(t, fields, wktCsvFields, 'should find geometry fields as type:geojson');@@ -229,6 +229,17 @@ test('Processor -> processCsv.wkt', t => {t.end();});+test('Processor -> processCsvData -> w/ array and object', t => {+ const {fields, rows} = processCsvData(testCsvObjectData);+ cmpFields(t, fields, objCsvFields, 'should find csv object fields as type:object');++ t.equal(rows.length, objCsvRows.length, 'should have same row length');+ rows.forEach((r, i) => {+ t.deepEqual(r, objCsvRows[i], 'should format correct csv object rows');+ });+ t.end();+});+test('Processor => processGeojson', t => {const {fields, rows} = processGeojson(cloneDeep(geojsonData));diff --git a/yarn.lock b/yarn.lockindex 4b066e553..08f1e6abd 100644--- a/yarn.lock+++ b/yarn.lock@@ -15856,10 +15856,10 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0:resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=- version "0.3.0"- resolved "https://registry.yarnpkg.com/type-analyzer/-/type-analyzer-0.3.0.tgz#b62a226deb4e9ca46b8cc86e14a93344154377d0"- integrity sha512-C2A/rZcoP03dAmv1/hCo2AjD7q2BHfZEp62Fn52BXgHoAEDVTobRbtR6UcEOhNy7g9/sh17MHq3ThxAp3578zw==+ version "0.4.0"+ resolved "https://registry.yarnpkg.com/type-analyzer/-/type-analyzer-0.4.0.tgz#56621f23b832468d358080466a2d41e142c8078a"+ integrity sha512-vLj0uKCCZYqtkFoQwf7mIvRQopDc2R4mdADLuX1uhtOm5MjtksHMiatI1TW+n0r7vDZLAMgW5VKGKlVTwbcAhg==type-check@~0.3.2:version "0.3.2"