Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(processors): write valid shapefile DBF numbers (#3848)
Igor Dykhta682a37219c32
diff --git a/src/processors/src/export/shapefile-writer.ts b/src/processors/src/export/shapefile-writer.tsindex 4f36b17d5..468882140 100644--- a/src/processors/src/export/shapefile-writer.ts+++ b/src/processors/src/export/shapefile-writer.ts@@ -267,6 +267,10 @@ interface DbfField {}const TEXT_ENCODER = new TextEncoder();+// This writer emits a dBASE III table (version 0x03), whose numeric fields+// support at most 19 total bytes and 15 digits after the decimal point.+const DBF_NUMERIC_MAX_WIDTH = 19;+const DBF_NUMERIC_MAX_DECIMALS = 15;/** Encode `text` to UTF-8, truncated to at most `maxBytes` without splitting a* multi-byte code point (so non-ASCII attribute values stay valid). */@@ -303,12 +307,40 @@ function dbfFieldName(key: string, taken: Set<string>): string {throw new Error(`Cannot generate a unique DBF field name for "${key}" (>999 collisions).`);}-function decimalPlaces(value: number): number {- if (Number.isInteger(value)) return 0;+/** DBF numeric fields only allow digits, a sign, and a decimal point. */+function decimalText(value: number): string {const text = String(value);+ const match = /^(-?)(\d+)(?:\.(\d+))?e([+-]?\d+)$/i.exec(text);+ if (!match) return text;++ const [, sign, integer, fraction = '', rawExponent] = match;+ const digits = `${integer}${fraction}`;+ const decimalIndex = integer.length + Number(rawExponent);+ if (decimalIndex <= 0) {+ return `${sign}0.${'0'.repeat(-decimalIndex)}${digits}`;+ }+ if (decimalIndex >= digits.length) {+ return `${sign}${digits}${'0'.repeat(decimalIndex - digits.length)}`;+ }+ return `${sign}${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;+}++function decimalPlaces(text: string): number {const dot = text.indexOf('.');if (dot < 0) return 0;- return Math.min(15, text.length - dot - 1);+ return text.length - dot - 1;+}++function formatDbfNumber(value: number, decimals: number): string {+ // Number#toFixed provides the rounding policy and fixed notation below 1e21.+ // Larger values cannot fit a dBASE III numeric field even at zero decimals,+ // but expanding them here lets field planning report their actual width.+ const text =+ Math.abs(value) < 1e21+ ? value.toFixed(decimals)+ : `${decimalText(value)}${decimals > 0 ? `.${'0'.repeat(decimals)}` : ''}`;+ // Do not spend a sign byte on a value that rounded to zero.+ return /^-0(?:\.0+)?$/.test(text) ? text.slice(1) : text;}function planField(key: string, name: string, values: unknown[]): DbfField {@@ -335,19 +367,35 @@ function planField(key: string, name: string, values: unknown[]): DbfField {}if (sawNumber && !sawNonNumber) {- let decimals = 0;+ let requestedDecimals = 0;for (const value of values) {if (typeof value === 'number' && Number.isFinite(value)) {- decimals = Math.max(decimals, decimalPlaces(value));+ requestedDecimals = Math.max(requestedDecimals, decimalPlaces(decimalText(value)));}}- let length = 1;- for (const value of values) {- if (typeof value !== 'number' || !Number.isFinite(value)) continue;- const text = decimals > 0 ? value.toFixed(decimals) : String(value);- length = Math.max(length, text.length);++ // Use the highest common precision that fits every rounded value. Measuring+ // the formatted values on each pass accounts for signs and rounding carry.+ for (+ let decimals = Math.min(requestedDecimals, DBF_NUMERIC_MAX_DECIMALS);+ decimals >= 0;+ decimals -= 1+ ) {+ let length = 1;+ for (const value of values) {+ if (typeof value !== 'number' || !Number.isFinite(value)) continue;+ length = Math.max(length, formatDbfNumber(value, decimals).length);+ }+ if (length <= DBF_NUMERIC_MAX_WIDTH) {+ return {key, name, type: 'N', length, decimals};+ }+ if (decimals === 0) {+ throw new Error(+ `DBF numeric field "${key}" requires ${length} bytes at zero decimal places; ` ++ `dBASE III numeric fields support at most ${DBF_NUMERIC_MAX_WIDTH}.`+ );+ }}- return {key, name, type: 'N', length: Math.min(length, 33), decimals};}// Fall back to text for everything else (strings, mixed types, objects).@@ -371,8 +419,10 @@ function encodeDbfValue(value: unknown, field: DbfField): Uint8Array {if (field.type === 'N') {if (typeof value !== 'number' || !Number.isFinite(value)) return cell;- let text = field.decimals > 0 ? value.toFixed(field.decimals) : String(value);- if (text.length > field.length) text = text.slice(0, field.length);+ const text = formatDbfNumber(value, field.decimals);+ if (text.length > field.length) {+ throw new Error(`DBF numeric value for "${field.key}" exceeds its planned field width.`);+ }// Numbers are right-justified in DBF.const bytes = TEXT_ENCODER.encode(text);cell.set(bytes, field.length - bytes.length);diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.tsindex db80a4f2a..dd690caae 100644--- a/src/processors/src/export/vector-export.spec.ts+++ b/src/processors/src/export/vector-export.spec.ts@@ -3,12 +3,15 @@import {convertWKBToGeometry} from '@loaders.gl/gis';import {convertArrowToSchema} from '@loaders.gl/schema-utils';+import {DBFLoader} from '@loaders.gl/shapefile';import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';import {Field} from '@kepler.gl/types';import {createDataContainer} from '@kepler.gl/utils';import {tableFromIPC} from 'apache-arrow';+import type {FeatureCollection} from 'geojson';import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';+import {writeShapefile} from './shapefile-writer';import {loadSqlJs} from './sql-js';const fields = [@@ -212,6 +215,153 @@ describe('dataset export', () => {).rejects.toThrow('Shapefile export requires all geometries to have the same shape family.');});+ test('rounds noisy and tiny DBF numbers at the maximum supported precision', () => {+ const values = [0.1 + 0.2, 1e-7, -1e-7, 1e-16, -1e-16, null];+ const collection: FeatureCollection = {+ type: 'FeatureCollection',+ features: values.map((value, index) => ({+ type: 'Feature',+ geometry: {type: 'Point', coordinates: [index, index]},+ properties: {value}+ }))+ };++ const {dbf} = writeShapefile(collection);+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);+ const headerLength = view.getUint16(8, true);+ const recordLength = view.getUint16(10, true);+ const fieldLength = dbf[32 + 16];+ const decimalCount = dbf[32 + 17];+ const decoder = new TextDecoder();+ const cells = values.map((_, index) => {+ const start = headerLength + index * recordLength + 1;+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();+ });++ expect({fieldLength, decimalCount}).toEqual({fieldLength: 18, decimalCount: 15});+ expect(cells).toEqual([+ '0.300000000000000',+ '0.000000100000000',+ '-0.000000100000000',+ '0.000000000000000',+ '0.000000000000000',+ ''+ ]);+ expect(cells.filter(Boolean).every(cell => /^-?\d+(?:\.\d+)?$/.test(cell))).toBe(true);++ const parsed = DBFLoader.parseSync?.(dbf.buffer as ArrayBuffer, {+ dbf: {shape: 'rows'}+ }) as Array<{value: number | null}>;+ expect(parsed.map(row => row.value)).toEqual([0.3, 1e-7, -1e-7, 0, 0, null]);+ });++ test('uses the highest common precision that fits mixed magnitudes after rounding', () => {+ const values = [-1234567890123456, 999.9999, -12.3456, null];+ const collection: FeatureCollection = {+ type: 'FeatureCollection',+ features: values.map((value, index) => ({+ type: 'Feature',+ geometry: {type: 'Point', coordinates: [index, index]},+ properties: {value}+ }))+ };++ const {dbf} = writeShapefile(collection);+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);+ const headerLength = view.getUint16(8, true);+ const recordLength = view.getUint16(10, true);+ const fieldLength = dbf[32 + 16];+ const decimalCount = dbf[32 + 17];+ const decoder = new TextDecoder();+ const cells = values.map((_, index) => {+ const start = headerLength + index * recordLength + 1;+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();+ });++ expect({fieldLength, decimalCount}).toEqual({fieldLength: 19, decimalCount: 1});+ expect(cells).toEqual(['-1234567890123456.0', '1000.0', '-12.3', '']);++ const parsed = DBFLoader.parseSync?.(dbf.buffer as ArrayBuffer, {+ dbf: {shape: 'rows'}+ }) as Array<{value: number | null}>;+ expect(parsed.map(row => row.value)).toEqual([-1234567890123456, 1000, -12.3, null]);+ });++ test('keeps large positive and negative integers in numeric fields', () => {+ const values = [1e18, -1e17, null];+ const collection: FeatureCollection = {+ type: 'FeatureCollection',+ features: values.map((value, index) => ({+ type: 'Feature',+ geometry: {type: 'Point', coordinates: [index, index]},+ properties: {value}+ }))+ };++ const {dbf} = writeShapefile(collection);+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);+ const headerLength = view.getUint16(8, true);+ const recordLength = view.getUint16(10, true);+ const fieldLength = dbf[32 + 16];+ const decimalCount = dbf[32 + 17];+ const decoder = new TextDecoder();+ const cells = values.map((_, index) => {+ const start = headerLength + index * recordLength + 1;+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();+ });++ expect({fieldLength, decimalCount}).toEqual({fieldLength: 19, decimalCount: 0});+ expect(cells).toEqual(['1000000000000000000', '-100000000000000000', '']);+ });++ test.each([+ [1e19, 20],+ [-1e18, 20],+ [1e21, 22]+ ])('rejects the DBF number %s when its rounded integer needs %i bytes', (value, width) => {+ const collection: FeatureCollection = {+ type: 'FeatureCollection',+ features: [+ {+ type: 'Feature',+ geometry: {type: 'Point', coordinates: [0, 0]},+ properties: {value}+ }+ ]+ };++ expect(() => writeShapefile(collection)).toThrow(+ `DBF numeric field "value" requires ${width} bytes at zero decimal places; ` ++ 'dBASE III numeric fields support at most 19.'+ );+ });++ test('keeps mixed finite and non-finite values in a DBF text field', () => {+ const values = [1, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, null];+ const collection: FeatureCollection = {+ type: 'FeatureCollection',+ features: values.map((value, index) => ({+ type: 'Feature',+ geometry: {type: 'Point', coordinates: [index, index]},+ properties: {value}+ }))+ };++ const {dbf} = writeShapefile(collection);+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);+ const headerLength = view.getUint16(8, true);+ const recordLength = view.getUint16(10, true);+ const fieldLength = dbf[32 + 16];+ const decoder = new TextDecoder();+ const cells = values.map((_, index) => {+ const start = headerLength + index * recordLength + 1;+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();+ });++ expect(String.fromCharCode(dbf[32 + 11])).toBe('C');+ expect(cells).toEqual(['1', 'NaN', 'Infinity', '-Infinity', '']);+ });+test('writes geoparquet string columns the importer can read', async () => {const geoparquet = await encodeDatasetExport({data: container(),