Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(duckdb): infer missing row data column types (#3822)
Igor Dykhta647f2217d261
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex c2f06e386..b2086bbb9 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -4,6 +4,7 @@import * as arrow from 'apache-arrow';import {+ ALL_FIELD_TYPES,DatasetType,DATASET_FORMATS,GEOARROW_EXTENSIONS,@@ -25,10 +26,12 @@ import {DatabaseConnection,compactArrowTable,isArrowTable,- isArrowVector+ isArrowVector,+ timeToUnixMilli} from '@kepler.gl/utils';import {+ fieldTypeToColumnType,processCsvRowObject,processGeojson,processKeplerglJSONforDuckDb,@@ -108,23 +111,53 @@ export class KeplerGlDuckDbTable extends KeplerTable {try {const {fields, rows} = data;+ const columnTypes = fields.map(field => {+ const explicitDuckDBColumnType = (field as typeof field & {duckDBColumnType?: string})+ .duckDBColumnType;+ const duckDBColumnType = explicitDuckDBColumnType ?? fieldTypeToColumnType(field.type);++ if (!duckDBColumnType) {+ throw new Error(+ `Unsupported field type "${field.type}" for DuckDB column "${field.name}"`+ );+ }++ return {+ duckDBColumnType,+ normalizeTimestamp: explicitDuckDBColumnType === undefined+ };+ });+// DATASET_FORMATS.keplergl loads data as any[][] instead of [{}]// TODO potential memory usage explosion: original data, new object and then DuckDb table- const rowsForDb = Array.isArray(rows[0])- ? rows.map(row => {- const newRow = {};- row.forEach((value, index) => {- newRow[fields[index].name] = value;- });- return newRow;- })- : rows;+ const rowsForDb = rows.map(row => {+ const isArrayRow = Array.isArray(row);+ const newRow: Record<string, unknown> = isArrayRow ? {} : {...row};++ fields.forEach((field, index) => {+ const value = isArrayRow ? row[index] : row[field.name];+ const timestamp =+ columnTypes[index].normalizeTimestamp &&+ field.type === ALL_FIELD_TYPES.timestamp &&+ value !== null &&+ value !== undefined+ ? timeToUnixMilli(value as string | number | Date, field.format || '')+ : null;++ newRow[field.name] =+ timestamp !== null && Number.isFinite(timestamp)+ ? new Date(timestamp).toISOString()+ : value;+ });++ return newRow;+ });await db.registerFileText(this.id, JSON.stringify(rowsForDb));const columns = fields.reduce((acc, field, index) => {- // @ts-expect-error TODO extend fields to contain duckDBColumnType- return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${field.duckDBColumnType}'`;+ const {duckDBColumnType} = columnTypes[index];+ return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${duckDBColumnType}'`;}, '');const createTableSql = `diff --git a/test/node/utils/duckdb-table-test.js b/test/node/utils/duckdb-table-test.jsnew file mode 100644index 000000000..75ffe0d90--- /dev/null+++ b/test/node/utils/duckdb-table-test.js@@ -0,0 +1,93 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import test from 'tape';++import {KeplerGlDuckDbTable} from '../../../src/duckdb/src/table/duckdb-table';++const importRowData = (data, {db, c}) =>+ KeplerGlDuckDbTable.prototype.importRowData.call(+ {id: 'sample-data', label: 'Sample Data'},+ {data, db, c}+ );++test('KeplerGlDuckDbTable -> importRowData maps ordinary Kepler field types', async t => {+ let registeredFile;+ let query;+ const db = {+ registerFileText: async (name, contents) => {+ registeredFile = {name, contents};+ }+ };+ const c = {+ query: async sql => {+ query = sql;+ }+ };+ const data = {+ fields: [+ {name: 'id', type: 'integer'},+ {name: 'label', type: 'string'},+ {name: 'pickup_time', type: 'timestamp', format: 'YYYY-M-D H:m:s'},+ {name: 'epoch_seconds', type: 'timestamp', format: 'x', duckDBColumnType: 'INTEGER'},+ {name: 'metadata', type: 'string', duckDBColumnType: 'JSON'}+ ],+ rows: [[1, 'one', '2015-01-15 19:05:40 +00:00', 1477647118, {source: 'sample'}]]+ };++ await importRowData(data, {db, c});++ t.deepEqual(+ registeredFile,+ {+ name: 'sample-data',+ contents: JSON.stringify([+ {+ id: 1,+ label: 'one',+ pickup_time: '2015-01-15T19:05:40.000Z',+ epoch_seconds: 1477647118,+ metadata: {source: 'sample'}+ }+ ])+ },+ 'registers row arrays as objects with normalized timestamps'+ );+ t.ok(query.includes("'id': 'INTEGER'"), 'maps integer fields to DuckDB INTEGER');+ t.ok(query.includes("'label': 'VARCHAR'"), 'maps string fields to DuckDB VARCHAR');+ t.ok(query.includes("'pickup_time': 'TIMESTAMP'"), 'maps timestamp fields to TIMESTAMP');+ t.ok(query.includes("'epoch_seconds': 'INTEGER'"), 'preserves a numeric timestamp column type');+ t.ok(query.includes("'metadata': 'JSON'"), 'preserves an explicit DuckDB column type');+ t.notOk(query.includes('undefined'), 'does not interpolate an undefined column type');+ t.end();+});++test('KeplerGlDuckDbTable -> importRowData rejects unsupported field types', async t => {+ let queried = false;+ const db = {registerFileText: async () => {}};+ const c = {+ query: async () => {+ queried = true;+ }+ };++ try {+ await importRowData(+ {+ fields: [{name: 'value', type: 'unsupported'}],+ rows: [{value: 'one'}]+ },+ {db, c}+ );+ t.fail('should reject an unsupported field type');+ } catch (error) {+ t.equal(+ error.message,+ 'Unsupported field type "unsupported" for DuckDB column "value"',+ 'reports the unsupported field and column'+ );+ }++ t.notOk(queried, 'does not run invalid SQL');+ t.end();+});diff --git a/test/node/utils/index.js b/test/node/utils/index.jsindex 1643774b1..3c5119958 100644--- a/test/node/utils/index.js+++ b/test/node/utils/index.js@@ -35,6 +35,7 @@ import './dom-to-image';import './wms-utils-test';import './effect-utils-test';import './chart-aggregation-test';+import './duckdb-table-test';import './duckdb-utils-test';import './annotation-utils-test';import './slider-scale-utils-test';