Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] DuckDB - cast BigInts to Double by default (#3120)

Igor Dykhtaf57173d35004
diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsx
index d568dc3ef..d91ee9954 100644
--- a/src/duckdb/src/components/sql-panel.tsx
+++ b/src/duckdb/src/components/sql-panel.tsx
@@ -19,10 +19,9 @@ import {SchemaPanel, SchemaSuggestion} from './schema-panel';
import {PreviewDataPanel, QueryResult} from './preview-data-panel';
import {
- constructST_asWKBQuery,
+ castDuckDBTypesForKepler,
getDuckDBColumnTypes,
getDuckDBColumnTypesMap,
- getGeometryColumns,
setGeoArrowWKBExtension,
splitSqlStatements,
checkIsSelectQuery,
@@ -209,10 +208,9 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
// 2) query duckdb types and detect candidate columns for ST_asWKB transform.
const duckDbColumns = await getDuckDBColumnTypes(connection, tempTableName);
tableDuckDBTypes = getDuckDBColumnTypesMap(duckDbColumns);
- const columnsToConvertToWKB = getGeometryColumns(duckDbColumns);
- // 3) query GEOMETRY columns as WKB.
- const adjustedQuery = constructST_asWKBQuery(tempTableName, columnsToConvertToWKB);
+ // 3) cast GEOMETRY columns to WKB and BigInt to double.
+ const adjustedQuery = castDuckDBTypesForKepler(tempTableName, duckDbColumns);
arrowResult = await connection.query(adjustedQuery);
// 4) set geoarrow extension for the arrow table as DuckDB doesn't support the geoarrow extension.
diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.ts
index 22c762e83..386677f2e 100644
--- a/src/duckdb/src/table/duckdb-table-utils.ts
+++ b/src/duckdb/src/table/duckdb-table-utils.ts
@@ -62,28 +62,29 @@ export function getDuckDBColumnTypesMap(columns: DuckDBColumnDesc[]) {
/**
* Constructs an SQL query to select all columns from a given table,
- * converting specified columns to Well-Known Binary (WKB) format using ST_AsWKB.
+ * converting specified columns to Well-Known Binary (WKB) format using ST_AsWKB,
+ * and casting BIGINT columns to DOUBLE if specified.
* @param tableName The name of the table from which to select data.
- * @param columnsToConvertToWKB An array of column names that should be converted to WKB format.
+ * @param columns An array of column descriptors, each with a type and name.
+ * @param options Optional parameters to control the conversion behavior.
* @returns The constructed SQL query.
*/
-export function constructST_asWKBQuery(tableName: string, columnsToConvertToWKB: string[]): string {
- const exclude =
- columnsToConvertToWKB.length > 0 ? `EXCLUDE ${columnsToConvertToWKB.join(', ')}` : '';
- const asWKB =
- columnsToConvertToWKB.length > 0
- ? `, ${columnsToConvertToWKB.map(column => `ST_AsWKB(${column}) as ${column}`).join(', ')}`
- : '';
- return `SELECT * ${exclude} ${asWKB} FROM '${tableName}'`;
-}
+export function castDuckDBTypesForKepler(
+ tableName: string,
+ columns: DuckDBColumnDesc[],
+ options = {geometryToWKB: true, bigIntToDouble: true}
+): string {
+ const modifiedColumns = columns.map(column => {
+ const {name, type} = column;
+ if (type === 'GEOMETRY' && options.geometryToWKB) {
+ return `ST_AsWKB(${name}) as ${name}`;
+ } else if (type === 'BIGINT' && options.bigIntToDouble) {
+ return `CAST(${name} AS DOUBLE) as ${name}`;
+ }
+ return name;
+ });
-/**
- * Finds the names of columns that have a GEOMETRY type.
- * @param columns An array of column descriptors from a DuckDB table.
- * @returns An array of column names that are of type GEOMETRY.
- */
-export function getGeometryColumns(columns: DuckDBColumnDesc[]): string[] {
- return columns.filter(column => column.type === 'GEOMETRY').map(column => column.name);
+ return `SELECT ${modifiedColumns.join(', ')} FROM '${tableName}'`;
}
/**
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index aa24bf3ec..9e112d0d6 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -38,11 +38,10 @@ import {
} from './duckdb-table-utils';
import {
- constructST_asWKBQuery,
+ castDuckDBTypesForKepler,
dropTableIfExists,
getDuckDBColumnTypes,
getDuckDBColumnTypesMap,
- getGeometryColumns,
removeUnsupportedExtensions,
restoreArrowTable,
restoreUnsupportedExtensions
@@ -240,8 +239,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {
const duckDbColumns = await getDuckDBColumnTypes(c, tableName);
const tableDuckDBTypes = getDuckDBColumnTypesMap(duckDbColumns);
- const columnsToConvertToWKB = getGeometryColumns(duckDbColumns);
- const adjustedQuery = constructST_asWKBQuery(tableName, columnsToConvertToWKB);
+ const adjustedQuery = castDuckDBTypesForKepler(tableName, duckDbColumns);
const arrowResult = await c.query(adjustedQuery);
// TODO if format is an arrow table then just use the original one, instead of the new table from the query?