Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(type-analyzer): Scientific-notation columns now load as real instead of text (#3809)
Igor Dykhtaa79923dcc09d
diff --git a/src/common-utils/src/data-type.ts b/src/common-utils/src/data-type.tsindex a12b5d8f2..ecdf7a58c 100644--- a/src/common-utils/src/data-type.ts+++ b/src/common-utils/src/data-type.ts@@ -11,6 +11,33 @@ import {h3IsValid} from './h3-utils';const H3_ANALYZER_TYPE = 'H3';+// a plain decimal number, optionally in scientific notation: 12, -0.5, .5, 1.5e+10, 2E-4+const DECIMAL_NUMBER_REGEX = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;+const INTEGER_REGEX = /^[+-]?\d+$/;++/**+ * type-analyzer reports NUMBER for values that pass `!isNaN` but match neither its INT nor+ * its FLOAT regex, which in practice is mostly scientific notation such as `1.5e+10`. Such a+ * value is safe to read as a real number unless it is a non decimal literal (`0x1A`, also hex+ * hashes and addresses) or an integer beyond the safe range (64-bit ids), both of which+ * parseFloat would misread or round.+ */+function isRealNumberSample(value: unknown): boolean {+ if (!notNullorUndefined(value) || value === '') {+ return true;+ }+ if (typeof value === 'number') {+ return Number.isFinite(value);+ }+ if (typeof value !== 'string' || !DECIMAL_NUMBER_REGEX.test(value)) {+ return false;+ }+ const asNumber = Number(value);+ return (+ Number.isFinite(asNumber) && (!INTEGER_REGEX.test(value) || Number.isSafeInteger(asNumber))+ );+}+/*** type-analyzer uses `isNaN(value)`, which throws on BigInt* ("Cannot convert a BigInt value to a number"). Arrow/Parquet Int64@@ -293,6 +320,14 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[]type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type;}+ // read a column of numbers in scientific notation as real numbers instead of strings+ if (+ type === AnalyzerDATA_TYPES.NUMBER &&+ analyzerData.every(row => isRealNumberSample(row[field]))+ ) {+ type = AnalyzerDATA_TYPES.FLOAT;+ }+return {name,id: name,diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.jsindex 443bdd02f..8d5637f5b 100644--- a/test/node/utils/data-processor-test.js+++ b/test/node/utils/data-processor-test.js@@ -160,6 +160,30 @@ test('Processor -> getFieldsFromData BigInt columns', t => {t.end();});+test('Processor -> getFieldsFromData scientific notation', t => {+ const data = [+ {sci: '2.3e+10', small: '1.5E-05', id: '12345678901234567890', hex: '0x1A'},+ {sci: '1.1e+09', small: null, id: '12345678901234567891', hex: '0xFF'},+ {sci: '4E+07', small: '3e2', id: '12345678901234567892', hex: '0x10'}+ ];+ const fields = getFieldsFromData(data, ['sci', 'small', 'id', 'hex']);++ t.equal(fields[0].type, ALL_FIELD_TYPES.real, 'scientific notation should be real');+ t.equal(+ fields[0].analyzerType,+ DATA_TYPES.FLOAT,+ 'scientific notation should be analyzed as float'+ );+ t.equal(fields[1].type, ALL_FIELD_TYPES.real, 'scientific notation with nulls should be real');+ t.equal(+ fields[2].type,+ ALL_FIELD_TYPES.string,+ 'integers beyond the safe range should stay string to keep every digit'+ );+ t.equal(fields[3].type, ALL_FIELD_TYPES.string, 'hex literals should stay string');+ t.end();+});+test('Processor -> processCsvData', t => {t.throws(() => processCsvData(''), 'should throw if csv is empty');@@ -369,6 +393,20 @@ test('Processor -> processCsvData -> semicolon-separated with numeric data', t =t.end();});+test('Processor -> processCsvData -> scientific notation', t => {+ const data = 'lat,lng,value\n37.1,-122.1,2.3e+10\n37.2,-122.2,1.1E+09\n37.3,-122.3,4E+07';+ const {fields, rows} = processCsvData(data);++ t.equal(fields[2].type, ALL_FIELD_TYPES.real, 'should type scientific notation as real');+ t.deepEqual(+ rows.map(r => r[2]),+ [2.3e10, 1.1e9, 4e7],+ 'should parse scientific notation as numbers'+ );++ t.end();+});+test('Processor -> processCsvData -> pipe-separated with special characters in values', t => {const data ='id|description|url\n1|"hello, world"|http://example.com\n2|"foo; bar"|http://test.org';