Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[fix] DuckDB - cast BigInts to Double by default (#3120)
Igor Dykhtaf57173d35004
diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsxindex d568dc3ef..d91ee9954 100644--- a/src/duckdb/src/components/sql-panel.tsx+++ b/src/duckdb/src/components/sql-panel.tsx@@ -19,10 +19,9 @@ import {SchemaPanel, SchemaSuggestion} from './schema-panel';import {PreviewDataPanel, QueryResult} from './preview-data-panel';import {- constructST_asWKBQuery,+ castDuckDBTypesForKepler,getDuckDBColumnTypes,getDuckDBColumnTypesMap,- getGeometryColumns,setGeoArrowWKBExtension,splitSqlStatements,checkIsSelectQuery,@@ -209,10 +208,9 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {// 2) query duckdb types and detect candidate columns for ST_asWKB transform.const duckDbColumns = await getDuckDBColumnTypes(connection, tempTableName);tableDuckDBTypes = getDuckDBColumnTypesMap(duckDbColumns);- const columnsToConvertToWKB = getGeometryColumns(duckDbColumns);- // 3) query GEOMETRY columns as WKB.- const adjustedQuery = constructST_asWKBQuery(tempTableName, columnsToConvertToWKB);+ // 3) cast GEOMETRY columns to WKB and BigInt to double.+ const adjustedQuery = castDuckDBTypesForKepler(tempTableName, duckDbColumns);arrowResult = await connection.query(adjustedQuery);// 4) set geoarrow extension for the arrow table as DuckDB doesn't support the geoarrow extension.diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.tsindex 22c762e83..386677f2e 100644--- a/src/duckdb/src/table/duckdb-table-utils.ts+++ b/src/duckdb/src/table/duckdb-table-utils.ts@@ -62,28 +62,29 @@ export function getDuckDBColumnTypesMap(columns: DuckDBColumnDesc[]) {/*** Constructs an SQL query to select all columns from a given table,- * converting specified columns to Well-Known Binary (WKB) format using ST_AsWKB.+ * converting specified columns to Well-Known Binary (WKB) format using ST_AsWKB,+ * and casting BIGINT columns to DOUBLE if specified.* @param tableName The name of the table from which to select data.- * @param columnsToConvertToWKB An array of column names that should be converted to WKB format.+ * @param columns An array of column descriptors, each with a type and name.+ * @param options Optional parameters to control the conversion behavior.* @returns The constructed SQL query.*/-export function constructST_asWKBQuery(tableName: string, columnsToConvertToWKB: string[]): string {- const exclude =- columnsToConvertToWKB.length > 0 ? `EXCLUDE ${columnsToConvertToWKB.join(', ')}` : '';- const asWKB =- columnsToConvertToWKB.length > 0- ? `, ${columnsToConvertToWKB.map(column => `ST_AsWKB(${column}) as ${column}`).join(', ')}`- : '';- return `SELECT * ${exclude} ${asWKB} FROM '${tableName}'`;-}+export function castDuckDBTypesForKepler(+ tableName: string,+ columns: DuckDBColumnDesc[],+ options = {geometryToWKB: true, bigIntToDouble: true}+): string {+ const modifiedColumns = columns.map(column => {+ const {name, type} = column;+ if (type === 'GEOMETRY' && options.geometryToWKB) {+ return `ST_AsWKB(${name}) as ${name}`;+ } else if (type === 'BIGINT' && options.bigIntToDouble) {+ return `CAST(${name} AS DOUBLE) as ${name}`;+ }+ return name;+ });-/**- * Finds the names of columns that have a GEOMETRY type.- * @param columns An array of column descriptors from a DuckDB table.- * @returns An array of column names that are of type GEOMETRY.- */-export function getGeometryColumns(columns: DuckDBColumnDesc[]): string[] {- return columns.filter(column => column.type === 'GEOMETRY').map(column => column.name);+ return `SELECT ${modifiedColumns.join(', ')} FROM '${tableName}'`;}/**diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex aa24bf3ec..9e112d0d6 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -38,11 +38,10 @@ import {} from './duckdb-table-utils';import {- constructST_asWKBQuery,+ castDuckDBTypesForKepler,dropTableIfExists,getDuckDBColumnTypes,getDuckDBColumnTypesMap,- getGeometryColumns,removeUnsupportedExtensions,restoreArrowTable,restoreUnsupportedExtensions@@ -240,8 +239,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {const duckDbColumns = await getDuckDBColumnTypes(c, tableName);const tableDuckDBTypes = getDuckDBColumnTypesMap(duckDbColumns);- const columnsToConvertToWKB = getGeometryColumns(duckDbColumns);- const adjustedQuery = constructST_asWKBQuery(tableName, columnsToConvertToWKB);+ const adjustedQuery = castDuckDBTypesForKepler(tableName, duckDbColumns);const arrowResult = await c.query(adjustedQuery);// TODO if format is an arrow table then just use the original one, instead of the new table from the query?