Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(processors): write valid shapefile DBF numbers (#3848)

Igor Dykhta682a37219c32
diff --git a/src/processors/src/export/shapefile-writer.ts b/src/processors/src/export/shapefile-writer.ts
index 4f36b17d5..468882140 100644
--- a/src/processors/src/export/shapefile-writer.ts
+++ b/src/processors/src/export/shapefile-writer.ts
@@ -267,6 +267,10 @@ interface DbfField {
}
const TEXT_ENCODER = new TextEncoder();
+// This writer emits a dBASE III table (version 0x03), whose numeric fields
+// support at most 19 total bytes and 15 digits after the decimal point.
+const DBF_NUMERIC_MAX_WIDTH = 19;
+const DBF_NUMERIC_MAX_DECIMALS = 15;
/** Encode `text` to UTF-8, truncated to at most `maxBytes` without splitting a
* multi-byte code point (so non-ASCII attribute values stay valid). */
@@ -303,12 +307,40 @@ function dbfFieldName(key: string, taken: Set<string>): string {
throw new Error(`Cannot generate a unique DBF field name for "${key}" (>999 collisions).`);
}
-function decimalPlaces(value: number): number {
- if (Number.isInteger(value)) return 0;
+/** DBF numeric fields only allow digits, a sign, and a decimal point. */
+function decimalText(value: number): string {
const text = String(value);
+ const match = /^(-?)(\d+)(?:\.(\d+))?e([+-]?\d+)$/i.exec(text);
+ if (!match) return text;
+
+ const [, sign, integer, fraction = '', rawExponent] = match;
+ const digits = `${integer}${fraction}`;
+ const decimalIndex = integer.length + Number(rawExponent);
+ if (decimalIndex <= 0) {
+ return `${sign}0.${'0'.repeat(-decimalIndex)}${digits}`;
+ }
+ if (decimalIndex >= digits.length) {
+ return `${sign}${digits}${'0'.repeat(decimalIndex - digits.length)}`;
+ }
+ return `${sign}${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
+}
+
+function decimalPlaces(text: string): number {
const dot = text.indexOf('.');
if (dot < 0) return 0;
- return Math.min(15, text.length - dot - 1);
+ return text.length - dot - 1;
+}
+
+function formatDbfNumber(value: number, decimals: number): string {
+ // Number#toFixed provides the rounding policy and fixed notation below 1e21.
+ // Larger values cannot fit a dBASE III numeric field even at zero decimals,
+ // but expanding them here lets field planning report their actual width.
+ const text =
+ Math.abs(value) < 1e21
+ ? value.toFixed(decimals)
+ : `${decimalText(value)}${decimals > 0 ? `.${'0'.repeat(decimals)}` : ''}`;
+ // Do not spend a sign byte on a value that rounded to zero.
+ return /^-0(?:\.0+)?$/.test(text) ? text.slice(1) : text;
}
function planField(key: string, name: string, values: unknown[]): DbfField {
@@ -335,19 +367,35 @@ function planField(key: string, name: string, values: unknown[]): DbfField {
}
if (sawNumber && !sawNonNumber) {
- let decimals = 0;
+ let requestedDecimals = 0;
for (const value of values) {
if (typeof value === 'number' && Number.isFinite(value)) {
- decimals = Math.max(decimals, decimalPlaces(value));
+ requestedDecimals = Math.max(requestedDecimals, decimalPlaces(decimalText(value)));
}
}
- let length = 1;
- for (const value of values) {
- if (typeof value !== 'number' || !Number.isFinite(value)) continue;
- const text = decimals > 0 ? value.toFixed(decimals) : String(value);
- length = Math.max(length, text.length);
+
+ // Use the highest common precision that fits every rounded value. Measuring
+ // the formatted values on each pass accounts for signs and rounding carry.
+ for (
+ let decimals = Math.min(requestedDecimals, DBF_NUMERIC_MAX_DECIMALS);
+ decimals >= 0;
+ decimals -= 1
+ ) {
+ let length = 1;
+ for (const value of values) {
+ if (typeof value !== 'number' || !Number.isFinite(value)) continue;
+ length = Math.max(length, formatDbfNumber(value, decimals).length);
+ }
+ if (length <= DBF_NUMERIC_MAX_WIDTH) {
+ return {key, name, type: 'N', length, decimals};
+ }
+ if (decimals === 0) {
+ throw new Error(
+ `DBF numeric field "${key}" requires ${length} bytes at zero decimal places; ` +
+ `dBASE III numeric fields support at most ${DBF_NUMERIC_MAX_WIDTH}.`
+ );
+ }
}
- return {key, name, type: 'N', length: Math.min(length, 33), decimals};
}
// Fall back to text for everything else (strings, mixed types, objects).
@@ -371,8 +419,10 @@ function encodeDbfValue(value: unknown, field: DbfField): Uint8Array {
if (field.type === 'N') {
if (typeof value !== 'number' || !Number.isFinite(value)) return cell;
- let text = field.decimals > 0 ? value.toFixed(field.decimals) : String(value);
- if (text.length > field.length) text = text.slice(0, field.length);
+ const text = formatDbfNumber(value, field.decimals);
+ if (text.length > field.length) {
+ throw new Error(`DBF numeric value for "${field.key}" exceeds its planned field width.`);
+ }
// Numbers are right-justified in DBF.
const bytes = TEXT_ENCODER.encode(text);
cell.set(bytes, field.length - bytes.length);
diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.ts
index db80a4f2a..dd690caae 100644
--- a/src/processors/src/export/vector-export.spec.ts
+++ b/src/processors/src/export/vector-export.spec.ts
@@ -3,12 +3,15 @@
import {convertWKBToGeometry} from '@loaders.gl/gis';
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+import {DBFLoader} from '@loaders.gl/shapefile';
import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';
import {Field} from '@kepler.gl/types';
import {createDataContainer} from '@kepler.gl/utils';
import {tableFromIPC} from 'apache-arrow';
+import type {FeatureCollection} from 'geojson';
import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';
+import {writeShapefile} from './shapefile-writer';
import {loadSqlJs} from './sql-js';
const fields = [
@@ -212,6 +215,153 @@ describe('dataset export', () => {
).rejects.toThrow('Shapefile export requires all geometries to have the same shape family.');
});
+ test('rounds noisy and tiny DBF numbers at the maximum supported precision', () => {
+ const values = [0.1 + 0.2, 1e-7, -1e-7, 1e-16, -1e-16, null];
+ const collection: FeatureCollection = {
+ type: 'FeatureCollection',
+ features: values.map((value, index) => ({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [index, index]},
+ properties: {value}
+ }))
+ };
+
+ const {dbf} = writeShapefile(collection);
+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);
+ const headerLength = view.getUint16(8, true);
+ const recordLength = view.getUint16(10, true);
+ const fieldLength = dbf[32 + 16];
+ const decimalCount = dbf[32 + 17];
+ const decoder = new TextDecoder();
+ const cells = values.map((_, index) => {
+ const start = headerLength + index * recordLength + 1;
+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();
+ });
+
+ expect({fieldLength, decimalCount}).toEqual({fieldLength: 18, decimalCount: 15});
+ expect(cells).toEqual([
+ '0.300000000000000',
+ '0.000000100000000',
+ '-0.000000100000000',
+ '0.000000000000000',
+ '0.000000000000000',
+ ''
+ ]);
+ expect(cells.filter(Boolean).every(cell => /^-?\d+(?:\.\d+)?$/.test(cell))).toBe(true);
+
+ const parsed = DBFLoader.parseSync?.(dbf.buffer as ArrayBuffer, {
+ dbf: {shape: 'rows'}
+ }) as Array<{value: number | null}>;
+ expect(parsed.map(row => row.value)).toEqual([0.3, 1e-7, -1e-7, 0, 0, null]);
+ });
+
+ test('uses the highest common precision that fits mixed magnitudes after rounding', () => {
+ const values = [-1234567890123456, 999.9999, -12.3456, null];
+ const collection: FeatureCollection = {
+ type: 'FeatureCollection',
+ features: values.map((value, index) => ({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [index, index]},
+ properties: {value}
+ }))
+ };
+
+ const {dbf} = writeShapefile(collection);
+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);
+ const headerLength = view.getUint16(8, true);
+ const recordLength = view.getUint16(10, true);
+ const fieldLength = dbf[32 + 16];
+ const decimalCount = dbf[32 + 17];
+ const decoder = new TextDecoder();
+ const cells = values.map((_, index) => {
+ const start = headerLength + index * recordLength + 1;
+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();
+ });
+
+ expect({fieldLength, decimalCount}).toEqual({fieldLength: 19, decimalCount: 1});
+ expect(cells).toEqual(['-1234567890123456.0', '1000.0', '-12.3', '']);
+
+ const parsed = DBFLoader.parseSync?.(dbf.buffer as ArrayBuffer, {
+ dbf: {shape: 'rows'}
+ }) as Array<{value: number | null}>;
+ expect(parsed.map(row => row.value)).toEqual([-1234567890123456, 1000, -12.3, null]);
+ });
+
+ test('keeps large positive and negative integers in numeric fields', () => {
+ const values = [1e18, -1e17, null];
+ const collection: FeatureCollection = {
+ type: 'FeatureCollection',
+ features: values.map((value, index) => ({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [index, index]},
+ properties: {value}
+ }))
+ };
+
+ const {dbf} = writeShapefile(collection);
+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);
+ const headerLength = view.getUint16(8, true);
+ const recordLength = view.getUint16(10, true);
+ const fieldLength = dbf[32 + 16];
+ const decimalCount = dbf[32 + 17];
+ const decoder = new TextDecoder();
+ const cells = values.map((_, index) => {
+ const start = headerLength + index * recordLength + 1;
+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();
+ });
+
+ expect({fieldLength, decimalCount}).toEqual({fieldLength: 19, decimalCount: 0});
+ expect(cells).toEqual(['1000000000000000000', '-100000000000000000', '']);
+ });
+
+ test.each([
+ [1e19, 20],
+ [-1e18, 20],
+ [1e21, 22]
+ ])('rejects the DBF number %s when its rounded integer needs %i bytes', (value, width) => {
+ const collection: FeatureCollection = {
+ type: 'FeatureCollection',
+ features: [
+ {
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [0, 0]},
+ properties: {value}
+ }
+ ]
+ };
+
+ expect(() => writeShapefile(collection)).toThrow(
+ `DBF numeric field "value" requires ${width} bytes at zero decimal places; ` +
+ 'dBASE III numeric fields support at most 19.'
+ );
+ });
+
+ test('keeps mixed finite and non-finite values in a DBF text field', () => {
+ const values = [1, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, null];
+ const collection: FeatureCollection = {
+ type: 'FeatureCollection',
+ features: values.map((value, index) => ({
+ type: 'Feature',
+ geometry: {type: 'Point', coordinates: [index, index]},
+ properties: {value}
+ }))
+ };
+
+ const {dbf} = writeShapefile(collection);
+ const view = new DataView(dbf.buffer, dbf.byteOffset, dbf.byteLength);
+ const headerLength = view.getUint16(8, true);
+ const recordLength = view.getUint16(10, true);
+ const fieldLength = dbf[32 + 16];
+ const decoder = new TextDecoder();
+ const cells = values.map((_, index) => {
+ const start = headerLength + index * recordLength + 1;
+ return decoder.decode(dbf.subarray(start, start + fieldLength)).trim();
+ });
+
+ expect(String.fromCharCode(dbf[32 + 11])).toBe('C');
+ expect(cells).toEqual(['1', 'NaN', 'Infinity', '-Infinity', '']);
+ });
+
test('writes geoparquet string columns the importer can read', async () => {
const geoparquet = await encodeDatasetExport({
data: container(),