Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[feat] export duckdb column logic (#3170)

Igor Dykhtaf2001983af5e
diff --git a/src/duckdb/src/index.ts b/src/duckdb/src/index.ts
index 4ba6300fc..b716e1a16 100644
--- a/src/duckdb/src/index.ts
+++ b/src/duckdb/src/index.ts
@@ -7,3 +7,8 @@ import {keplerGlDuckDBPlugin} from './plugin';
export {restoreGeoarrowMetadata, KeplerGlDuckDbTable} from './table/duckdb-table';
export * from './table/duckdb-table-utils';
export default keplerGlDuckDBPlugin;
+export {
+ consolidateFieldTypes,
+ extendFieldsWithDuckDBColumnType,
+ fieldTypeToColumnType
+} from './processors/data-processor';
diff --git a/src/duckdb/src/processors/data-processor.ts b/src/duckdb/src/processors/data-processor.ts
index d245bcb34..1197f5ccb 100644
--- a/src/duckdb/src/processors/data-processor.ts
+++ b/src/duckdb/src/processors/data-processor.ts
@@ -171,7 +171,7 @@ function cleanUpFalsyCsvValue(rows: RowData): void {
}
// align type analyzer types with DuckDB csv auto detected column types
-function consolidateFieldTypes(fields, schema) {
+export function consolidateFieldTypes(fields, schema) {
return fields.map(field => {
const columnName = field.name;
@@ -255,3 +255,27 @@ export function processGeojson(rawData: unknown): ProcessorResult {
// fields: []
};
}
+
+/**
+ * A utility function to extend existing fields with native DuckDB column type.
+ * @param fields Array of fields to extend.
+ * @param typeOverrides An optional mapping of DuckDB column types to override.
+ * @returns An array of fields with the DuckDB column type extended.
+ */
+export function extendFieldsWithDuckDBColumnType(
+ fields: ProtoDatasetField[],
+ typeOverrides: Record<string, string>
+): Promise<ProtoDatasetField[]> {
+ const schema = fields.reduce((acc, field) => {
+ acc[field.name] = fieldTypeToColumnType(field.type);
+ return acc;
+ }, {});
+
+ const updatedFields = consolidateFieldTypes(fields, schema);
+ updatedFields.forEach(field => {
+ if (typeOverrides[field.duckDBColumnType]) {
+ field.duckDBColumnType = typeOverrides[field.duckDBColumnType];
+ }
+ });
+ return updatedFields;
+}