Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(type-analyzer): Scientific-notation columns now load as real instead of text (#3809)

Igor Dykhtaa79923dcc09d
diff --git a/src/common-utils/src/data-type.ts b/src/common-utils/src/data-type.ts
index a12b5d8f2..ecdf7a58c 100644
--- a/src/common-utils/src/data-type.ts
+++ b/src/common-utils/src/data-type.ts
@@ -11,6 +11,33 @@ import {h3IsValid} from './h3-utils';
const H3_ANALYZER_TYPE = 'H3';
+// a plain decimal number, optionally in scientific notation: 12, -0.5, .5, 1.5e+10, 2E-4
+const DECIMAL_NUMBER_REGEX = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
+const INTEGER_REGEX = /^[+-]?\d+$/;
+
+/**
+ * type-analyzer reports NUMBER for values that pass `!isNaN` but match neither its INT nor
+ * its FLOAT regex, which in practice is mostly scientific notation such as `1.5e+10`. Such a
+ * value is safe to read as a real number unless it is a non decimal literal (`0x1A`, also hex
+ * hashes and addresses) or an integer beyond the safe range (64-bit ids), both of which
+ * parseFloat would misread or round.
+ */
+function isRealNumberSample(value: unknown): boolean {
+ if (!notNullorUndefined(value) || value === '') {
+ return true;
+ }
+ if (typeof value === 'number') {
+ return Number.isFinite(value);
+ }
+ if (typeof value !== 'string' || !DECIMAL_NUMBER_REGEX.test(value)) {
+ return false;
+ }
+ const asNumber = Number(value);
+ return (
+ Number.isFinite(asNumber) && (!INTEGER_REGEX.test(value) || Number.isSafeInteger(asNumber))
+ );
+}
+
/**
* type-analyzer uses `isNaN(value)`, which throws on BigInt
* ("Cannot convert a BigInt value to a number"). Arrow/Parquet Int64
@@ -293,6 +320,14 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[]
type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type;
}
+ // read a column of numbers in scientific notation as real numbers instead of strings
+ if (
+ type === AnalyzerDATA_TYPES.NUMBER &&
+ analyzerData.every(row => isRealNumberSample(row[field]))
+ ) {
+ type = AnalyzerDATA_TYPES.FLOAT;
+ }
+
return {
name,
id: name,
diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.js
index 443bdd02f..8d5637f5b 100644
--- a/test/node/utils/data-processor-test.js
+++ b/test/node/utils/data-processor-test.js
@@ -160,6 +160,30 @@ test('Processor -> getFieldsFromData BigInt columns', t => {
t.end();
});
+test('Processor -> getFieldsFromData scientific notation', t => {
+ const data = [
+ {sci: '2.3e+10', small: '1.5E-05', id: '12345678901234567890', hex: '0x1A'},
+ {sci: '1.1e+09', small: null, id: '12345678901234567891', hex: '0xFF'},
+ {sci: '4E+07', small: '3e2', id: '12345678901234567892', hex: '0x10'}
+ ];
+ const fields = getFieldsFromData(data, ['sci', 'small', 'id', 'hex']);
+
+ t.equal(fields[0].type, ALL_FIELD_TYPES.real, 'scientific notation should be real');
+ t.equal(
+ fields[0].analyzerType,
+ DATA_TYPES.FLOAT,
+ 'scientific notation should be analyzed as float'
+ );
+ t.equal(fields[1].type, ALL_FIELD_TYPES.real, 'scientific notation with nulls should be real');
+ t.equal(
+ fields[2].type,
+ ALL_FIELD_TYPES.string,
+ 'integers beyond the safe range should stay string to keep every digit'
+ );
+ t.equal(fields[3].type, ALL_FIELD_TYPES.string, 'hex literals should stay string');
+ t.end();
+});
+
test('Processor -> processCsvData', t => {
t.throws(() => processCsvData(''), 'should throw if csv is empty');
@@ -369,6 +393,20 @@ test('Processor -> processCsvData -> semicolon-separated with numeric data', t =
t.end();
});
+test('Processor -> processCsvData -> scientific notation', t => {
+ const data = 'lat,lng,value\n37.1,-122.1,2.3e+10\n37.2,-122.2,1.1E+09\n37.3,-122.3,4E+07';
+ const {fields, rows} = processCsvData(data);
+
+ t.equal(fields[2].type, ALL_FIELD_TYPES.real, 'should type scientific notation as real');
+ t.deepEqual(
+ rows.map(r => r[2]),
+ [2.3e10, 1.1e9, 4e7],
+ 'should parse scientific notation as numbers'
+ );
+
+ t.end();
+});
+
test('Processor -> processCsvData -> pipe-separated with special characters in values', t => {
const data =
'id|description|url\n1|"hello, world"|http://example.com\n2|"foo; bar"|http://test.org';