Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Enhancement] Choose the default field to be integer if no reals are present (#1409)
Shan Heb97b58a93147
diff --git a/src/utils/dataset-utils.js b/src/utils/dataset-utils.jsindex 9fddc7c37..9ab110081 100644--- a/src/utils/dataset-utils.js+++ b/src/utils/dataset-utils.js@@ -91,29 +91,165 @@ export function createNewDataEntry({info, data, metadata}, datasets = {}) {};}+/**+ * Field name prefixes and suffixes which should not be considered+ * as metrics. Fields will still be included if a 'metric word'+ * is found on the field name, however.+ */+const EXCLUDED_DEFAULT_FIELDS = [+ // Serial numbers and identification numbers+ '_id',+ 'id',+ 'index',+ 'uuid',+ 'guid',+ 'uid',+ 'gid',+ 'serial',+ // Geographic IDs are unlikely to be interesting to color+ 'zip',+ 'code',+ 'post',+ 'region',+ 'fips',+ 'cbgs',+ 'h3',+ 's2',+ // Geographic coords (but not z/elevation/altitude+ // since that might be a metric)+ 'lat',+ 'lon',+ 'lng',+ 'latitude',+ 'longitude',+ '_x',+ '_y'+];++/**+ * Prefixes and suffixes that indicate a field is a metric.+ *+ * Note that these are in order of preference, first being+ * most preferred.+ */+const METRIC_DEFAULT_FIELDS = [+ 'metric',+ 'value',+ 'sum',+ 'count',+ 'unique',+ 'mean',+ 'mode',+ 'median',+ 'max',+ 'min',+ 'deviation',+ 'variance',+ 'p99',+ 'p95',+ 'p75',+ 'p50',+ 'p25',+ 'p05',+ // Abbreviations are less preferred+ 'cnt',+ 'val'+];+/*** Choose a field to use as the default color field of a layer.*- * Right now this implements a very simple heuristic looking- * for a real-type field that is not lat/lon.+ * The heuristic is:+ *+ * First, exclude fields that are on the exclusion list and don't+ * have names that suggest they contain metrics. Also exclude+ * field names that are blank.+ *+ * Next, look for a field that is of real type and contains one+ * of the preferred names (in order of the preferred names).+ *+ * Next, look for a field that is of integer type and contains+ * one of the preferred names (in order of the preferred names).+ *+ * Next, look for the first field that is of real type (in order+ * of field index).+ *+ * Next, look for the first field that is of integer type (in+ * order of field index).*- * In the future we could consider other things:- * Consider integer fields- * look for highest dynamic range (using a sample of the data)- * Look for particular names to select ("value", "color", etc)- * Look for particular names to avoid ("" - the Pandas index column)+ * It's possible no field will be chosen (i.e. because all fields+ * are strings.)** @param dataset*/export function findDefaultColorField({fields, fieldPairs = []}) {- const defaultField = fields.find(- f =>- f.type === ALL_FIELD_TYPES.real &&+ const fieldsWithoutExcluded = fields.filter(field => {+ if (field.type !== ALL_FIELD_TYPES.real && field.type !== ALL_FIELD_TYPES.integer) {+ // Only select numeric fields.+ return false;+ }+ if (+ fieldPairs.find(+ pair => pair.pair.lat.value === field.name || pair.pair.lng.value === field.name+ )+ ) {// Do not permit lat, lon fields- !fieldPairs.find(pair => pair.pair.lat.value === f.name || pair.pair.lng.value === f.name)- );- if (!defaultField) {- return null;+ return false;+ }++ const normalizedFieldName = field.name.toLowerCase();+ if (normalizedFieldName === '') {+ // Special case excluded name when the name is blank.+ return false;+ }+ const hasExcluded = EXCLUDED_DEFAULT_FIELDS.find(+ f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)+ );+ const hasInclusion = METRIC_DEFAULT_FIELDS.find(+ f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)+ );+ return !hasExcluded || hasInclusion;+ });++ const sortedFields = fieldsWithoutExcluded.sort((left, right) => {+ const normalizedLeft = left.name.toLowerCase();+ const normalizedRight = right.name.toLowerCase();+ const leftHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(+ f => normalizedLeft.startsWith(f) || normalizedLeft.endsWith(f)+ );+ const rightHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(+ f => normalizedRight.startsWith(f) || normalizedRight.endsWith(f)+ );+ if (leftHasInclusion !== rightHasInclusion) {+ if (leftHasInclusion === -1) {+ // Elements that do not have the inclusion list should go after those that do.+ return 1;+ } else if (rightHasInclusion === -1) {+ // Elements that do have the inclusion list should go before those that don't.+ return -1;+ }+ // Compare based on order in the inclusion list+ return leftHasInclusion - rightHasInclusion;+ }++ // Compare based on type+ if (left.type !== right.type) {+ if (left.type === ALL_FIELD_TYPES.real) {+ return -1;+ }+ // left is an integer and right is not+ // and reals come before integers+ return 1;+ }++ // Finally, order based on the order in the datasets columns+ return left.index - right.index;+ });++ if (sortedFields.length) {+ // There was a best match+ return sortedFields[0];}- return defaultField;+ // No matches+ return null;}diff --git a/test/node/utils/dataset-utils-test.js b/test/node/utils/dataset-utils-test.jsindex 6c33d6e1c..fa6625695 100644--- a/test/node/utils/dataset-utils-test.js+++ b/test/node/utils/dataset-utils-test.js@@ -25,27 +25,57 @@ import {processCsvData} from 'processors/data-processor';import csvData from 'test/fixtures/test-layer-data';-test('datasetUtils.findDefaultColorField', t => {- const dataset = createNewDataEntry({- info: {id: 'taro'},- data: processCsvData(csvData)- }).taro;-- const defaultField = findDefaultColorField(dataset);- // Unfortunately lat_1 is not detected as part of a field pair :(- t.equals(defaultField.name, 'lat_1', 'default field name is OK');+const DEFAULT_FIELD_TEST_CASES = [+ {+ name: 'excluded lat',+ csv: csvData,+ expected: 'trip_distance'+ },+ {+ name: 'empty',+ csv: 'a\na',+ expected: null+ },+ {+ name: 'integer only',+ csv: 'a,b\na,0\na,1',+ expected: 'b'+ },+ {+ name: 'integer and real',+ csv: 'a,b,c\na,0,0.5\na,1,0.5',+ expected: 'c'+ },+ {+ name: 'excluded real',+ csv: 'zipcode,b,c\n0.5,0,0.5\n0.5,1,0.5',+ expected: 'c'+ },+ {+ name: 'included real',+ csv: 'zipcode mean,b,c\n0.5,0,0.5\n0.5,1,0.5',+ expected: 'zipcode mean'+ },+ {+ name: 'included real, with inclusion ordering',+ csv: 'zipcode mean,a metric,b,c\n0.5,0.1,0,0.5\n0.5,0.1,1,0.5',+ expected: 'a metric'+ }+];- t.end();-});--test('datasetUtils.findDefaultColorField empty', t => {- const dataset = createNewDataEntry({- info: {id: 'taro'},- data: processCsvData('a\na')- }).taro;-- const defaultField = findDefaultColorField(dataset);- t.notOk(defaultField, 'default field is null');+test('datasetUtils.findDefaultColorField', t => {+ for (const tc of DEFAULT_FIELD_TEST_CASES) {+ const dataset = createNewDataEntry({+ info: {id: 'taro'},+ data: processCsvData(tc.csv)+ }).taro;+ const defaultField = findDefaultColorField(dataset);+ if (!tc.expected) {+ t.notOk(defaultField, `${tc.name}: default field is null`);+ } else {+ t.equals(defaultField.name, tc.expected, `${tc.name}: default field name is OK`);+ }+ }t.end();});