Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(duckdb): infer missing row data column types (#3822)

Igor Dykhta647f2217d261
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index c2f06e386..b2086bbb9 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -4,6 +4,7 @@
import * as arrow from 'apache-arrow';
import {
+ ALL_FIELD_TYPES,
DatasetType,
DATASET_FORMATS,
GEOARROW_EXTENSIONS,
@@ -25,10 +26,12 @@ import {
DatabaseConnection,
compactArrowTable,
isArrowTable,
- isArrowVector
+ isArrowVector,
+ timeToUnixMilli
} from '@kepler.gl/utils';
import {
+ fieldTypeToColumnType,
processCsvRowObject,
processGeojson,
processKeplerglJSONforDuckDb,
@@ -108,23 +111,53 @@ export class KeplerGlDuckDbTable extends KeplerTable {
try {
const {fields, rows} = data;
+ const columnTypes = fields.map(field => {
+ const explicitDuckDBColumnType = (field as typeof field & {duckDBColumnType?: string})
+ .duckDBColumnType;
+ const duckDBColumnType = explicitDuckDBColumnType ?? fieldTypeToColumnType(field.type);
+
+ if (!duckDBColumnType) {
+ throw new Error(
+ `Unsupported field type "${field.type}" for DuckDB column "${field.name}"`
+ );
+ }
+
+ return {
+ duckDBColumnType,
+ normalizeTimestamp: explicitDuckDBColumnType === undefined
+ };
+ });
+
// DATASET_FORMATS.keplergl loads data as any[][] instead of [{}]
// TODO potential memory usage explosion: original data, new object and then DuckDb table
- const rowsForDb = Array.isArray(rows[0])
- ? rows.map(row => {
- const newRow = {};
- row.forEach((value, index) => {
- newRow[fields[index].name] = value;
- });
- return newRow;
- })
- : rows;
+ const rowsForDb = rows.map(row => {
+ const isArrayRow = Array.isArray(row);
+ const newRow: Record<string, unknown> = isArrayRow ? {} : {...row};
+
+ fields.forEach((field, index) => {
+ const value = isArrayRow ? row[index] : row[field.name];
+ const timestamp =
+ columnTypes[index].normalizeTimestamp &&
+ field.type === ALL_FIELD_TYPES.timestamp &&
+ value !== null &&
+ value !== undefined
+ ? timeToUnixMilli(value as string | number | Date, field.format || '')
+ : null;
+
+ newRow[field.name] =
+ timestamp !== null && Number.isFinite(timestamp)
+ ? new Date(timestamp).toISOString()
+ : value;
+ });
+
+ return newRow;
+ });
await db.registerFileText(this.id, JSON.stringify(rowsForDb));
const columns = fields.reduce((acc, field, index) => {
- // @ts-expect-error TODO extend fields to contain duckDBColumnType
- return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${field.duckDBColumnType}'`;
+ const {duckDBColumnType} = columnTypes[index];
+ return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${duckDBColumnType}'`;
}, '');
const createTableSql = `
diff --git a/test/node/utils/duckdb-table-test.js b/test/node/utils/duckdb-table-test.js
new file mode 100644
index 000000000..75ffe0d90
--- /dev/null
+++ b/test/node/utils/duckdb-table-test.js
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import test from 'tape';
+
+import {KeplerGlDuckDbTable} from '../../../src/duckdb/src/table/duckdb-table';
+
+const importRowData = (data, {db, c}) =>
+ KeplerGlDuckDbTable.prototype.importRowData.call(
+ {id: 'sample-data', label: 'Sample Data'},
+ {data, db, c}
+ );
+
+test('KeplerGlDuckDbTable -> importRowData maps ordinary Kepler field types', async t => {
+ let registeredFile;
+ let query;
+ const db = {
+ registerFileText: async (name, contents) => {
+ registeredFile = {name, contents};
+ }
+ };
+ const c = {
+ query: async sql => {
+ query = sql;
+ }
+ };
+ const data = {
+ fields: [
+ {name: 'id', type: 'integer'},
+ {name: 'label', type: 'string'},
+ {name: 'pickup_time', type: 'timestamp', format: 'YYYY-M-D H:m:s'},
+ {name: 'epoch_seconds', type: 'timestamp', format: 'x', duckDBColumnType: 'INTEGER'},
+ {name: 'metadata', type: 'string', duckDBColumnType: 'JSON'}
+ ],
+ rows: [[1, 'one', '2015-01-15 19:05:40 +00:00', 1477647118, {source: 'sample'}]]
+ };
+
+ await importRowData(data, {db, c});
+
+ t.deepEqual(
+ registeredFile,
+ {
+ name: 'sample-data',
+ contents: JSON.stringify([
+ {
+ id: 1,
+ label: 'one',
+ pickup_time: '2015-01-15T19:05:40.000Z',
+ epoch_seconds: 1477647118,
+ metadata: {source: 'sample'}
+ }
+ ])
+ },
+ 'registers row arrays as objects with normalized timestamps'
+ );
+ t.ok(query.includes("'id': 'INTEGER'"), 'maps integer fields to DuckDB INTEGER');
+ t.ok(query.includes("'label': 'VARCHAR'"), 'maps string fields to DuckDB VARCHAR');
+ t.ok(query.includes("'pickup_time': 'TIMESTAMP'"), 'maps timestamp fields to TIMESTAMP');
+ t.ok(query.includes("'epoch_seconds': 'INTEGER'"), 'preserves a numeric timestamp column type');
+ t.ok(query.includes("'metadata': 'JSON'"), 'preserves an explicit DuckDB column type');
+ t.notOk(query.includes('undefined'), 'does not interpolate an undefined column type');
+ t.end();
+});
+
+test('KeplerGlDuckDbTable -> importRowData rejects unsupported field types', async t => {
+ let queried = false;
+ const db = {registerFileText: async () => {}};
+ const c = {
+ query: async () => {
+ queried = true;
+ }
+ };
+
+ try {
+ await importRowData(
+ {
+ fields: [{name: 'value', type: 'unsupported'}],
+ rows: [{value: 'one'}]
+ },
+ {db, c}
+ );
+ t.fail('should reject an unsupported field type');
+ } catch (error) {
+ t.equal(
+ error.message,
+ 'Unsupported field type "unsupported" for DuckDB column "value"',
+ 'reports the unsupported field and column'
+ );
+ }
+
+ t.notOk(queried, 'does not run invalid SQL');
+ t.end();
+});
diff --git a/test/node/utils/index.js b/test/node/utils/index.js
index 1643774b1..3c5119958 100644
--- a/test/node/utils/index.js
+++ b/test/node/utils/index.js
@@ -35,6 +35,7 @@ import './dom-to-image';
import './wms-utils-test';
import './effect-utils-test';
import './chart-aggregation-test';
+import './duckdb-table-test';
import './duckdb-utils-test';
import './annotation-utils-test';
import './slider-scale-utils-test';