Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[chore] pass duckDB adapter via application config (#3023)
Igor Dykhta224975a25fff
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.jsindex 282ab87a2..27c5054d6 100644--- a/examples/demo-app/src/reducers/index.js+++ b/examples/demo-app/src/reducers/index.js@@ -13,6 +13,9 @@ import KeplerGlSchema from '@kepler.gl/schemas';import {KeplerTable} from '@kepler.gl/table';import {getApplicationConfig} from '@kepler.gl/utils';+// import {getApplicationConfig, initApplicationConfig} from '@kepler.gl/utils';+// import keplerGlDuckdbPlugin, {KeplerGlDuckDbTable, DuckDBWasmAdapter} from '@kepler.gl/duckdb';+import {INIT,LOAD_MAP_SAMPLE_FILE,@@ -26,6 +29,26 @@ import {import {CLOUD_PROVIDERS_CONFIGURATION} from '../constants/default-settings';import {generateHashId} from '../utils/strings';+// initialize kepler demo-app with DuckDB plugin+/*+initApplicationConfig({+ // Custom UI for DuckDB+ plugins: [keplerGlDuckdbPlugin],+ // async data ingestion to DuckDb+ table: KeplerGlDuckDbTable,+ // setup database for DuckDB plugin+ database: new DuckDBWasmAdapter({+ config: {+ query: {+ castBigIntToDouble: true+ }+ }+ }),+ // progressive loading is sync, doesn't wait properly for a dataset to be created in DuckDB+ useArrowProgressiveLoading: false+});+*/+const {DEFAULT_MAP_CONTROLS} = uiStateUpdaters;// INITIAL_APP_STATEdiff --git a/src/duckdb/src/adapters/duckdb-wasm-adapter.ts b/src/duckdb/src/adapters/duckdb-wasm-adapter.tsnew file mode 100644index 000000000..e383d1713--- /dev/null+++ b/src/duckdb/src/adapters/duckdb-wasm-adapter.ts@@ -0,0 +1,125 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import * as arrow from 'apache-arrow';+import * as duckdb from '@duckdb/duckdb-wasm';+import {AsyncDuckDB, DuckDBConfig, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';++import {DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';++import {logElapsedTime} from '../utils/perf';++class Connection implements DatabaseConnection {+ private connection: AsyncDuckDBConnection;++ constructor(connection: AsyncDuckDBConnection) {+ this.connection = connection;+ }++ async query(statement: string): Promise<arrow.Table> {+ return this.connection.query(statement);+ }++ async insertArrowTable(arrowTable: arrow.Table, params: {name: string}): Promise<void> {+ await this.connection.insertArrowTable(arrowTable, params);+ }++ async close() {+ await this.connection.close();+ }+}++type DuckDBWasmAdapterProps =+ | {+ debug?: boolean;+ config?: DuckDBConfig;+ }+ | Promise<AsyncDuckDB>;++export class DuckDBWasmAdapter implements DatabaseAdapter {+ private duckDB: Promise<AsyncDuckDB>;++ constructor(options: DuckDBWasmAdapterProps) {+ // pass existing AsyncDuckDB object created elsewhere+ if (options instanceof Promise || options instanceof AsyncDuckDB) {+ this.duckDB = options as any;+ return;+ }++ // or create a new AsyncDuckDB object+ const {debug = false, config} = options || {};+ this.duckDB = initializeDuckDbWasm(config, debug);+ }++ async connect() {+ const db = await this.duckDB;+ const c = await db.connect();+ return new Connection(c);+ }++ async registerFileText(filename: string, content) {+ const db = await this.duckDB;+ await db.registerFileText(filename, content);+ }++ async registerFileHandle(+ name: string,+ handle: any,+ protocol: duckdb.DuckDBDataProtocol,+ directIO: boolean+ ): Promise<void> {+ const db = await this.duckDB;+ await db.registerFileHandle(name, handle, protocol, directIO);+ }+}++/**+ * Initialize DuckDB with a browser-specific Wasm bundle.+ */+const initializeDuckDbWasm = async (+ config?: DuckDBConfig,+ debug?: boolean+): Promise<AsyncDuckDB> => {+ const start = performance.now();++ // Select a bundle based on browser checks+ const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();+ const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);+ if (!bundle.mainWorker) {+ throw new Error('Failed to initialize DuckDB');+ }++ const worker_url = URL.createObjectURL(+ new Blob([`importScripts("${bundle.mainWorker}");`], {+ type: 'text/javascript'+ })+ );++ // Instantiate the async version of DuckDB-wasm+ const worker = new Worker(worker_url);+ const logger = debug ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();+ const db = new AsyncDuckDB(logger, worker);+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);+ URL.revokeObjectURL(worker_url);++ if (config) {+ if (config.path) {+ const res = await fetch(config.path);+ const buffer = await res.arrayBuffer();+ const fileNameMatch = config.path.match(/[^/]*$/);+ if (fileNameMatch) {+ config.path = fileNameMatch[0];+ }+ await db.registerFileBuffer(config.path, new Uint8Array(buffer));+ }+ await db.open(config);+ }++ if (debug) {+ logElapsedTime('DuckDB initialized', start);+ if (config) {+ console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);+ }+ }+ return db;+};diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsxindex 439cdb18d..473faffe3 100644--- a/src/duckdb/src/components/schema-panel.tsx+++ b/src/duckdb/src/components/schema-panel.tsx@@ -4,14 +4,13 @@import React, {useCallback, useEffect, useState} from 'react';import {useSelector} from 'react-redux';import styled from 'styled-components';-import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';import {LoadingSpinner, Icons} from '@kepler.gl/components';import {arrowSchemaToFields} from '@kepler.gl/processors';import {VisState} from '@kepler.gl/schemas';+import {getApplicationConfig, DatabaseConnection} from '@kepler.gl/utils';import {Tree, DatasetNode, ColumnNode, TreeNodeData} from './tree';-import {getDuckDB} from '../init';import {getDuckDBColumnTypes, getDuckDBColumnTypesMap} from '../table/duckdb-table-utils';// TODO note that demo state is available in demo-app, but not when add modules to dependencies in a custom map@@ -40,7 +39,7 @@ const StyledLoadingSpinnerWrapper = styled.div`height: 100%;`;-async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {+async function getColumnSchema(connection: DatabaseConnection, tableName: string) {const columnResult = await connection.query(`Select * from '${tableName}' LIMIT 1;`);const columnDescribe = await getDuckDBColumnTypes(connection, tableName);@@ -130,7 +129,11 @@ export const SchemaPanel = ({const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);const getTableSchema = useCallback(async () => {- const db = await getDuckDB();+ const db = getApplicationConfig().database;+ if (!db) {+ console.error('The database is not configured properly.');+ return;+ }const c = await db.connect();const tableResult = await c.query('SHOW TABLES;');diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsxindex baed42cab..d568dc3ef 100644--- a/src/duckdb/src/components/sql-panel.tsx+++ b/src/duckdb/src/components/sql-panel.tsx@@ -12,12 +12,12 @@ import {generateHashId} from '@kepler.gl/common-utils';import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';import {arrowSchemaToFields} from '@kepler.gl/processors';import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';-import {isAppleDevice} from '@kepler.gl/utils';+import {isAppleDevice, getApplicationConfig} from '@kepler.gl/utils';import MonacoEditor from './monaco-editor';import {SchemaPanel, SchemaSuggestion} from './schema-panel';import {PreviewDataPanel, QueryResult} from './preview-data-panel';-import {getDuckDB} from '../init';+import {constructST_asWKBQuery,getDuckDBColumnTypes,@@ -184,7 +184,11 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {return;}- const db = await getDuckDB();+ const db = getApplicationConfig().database;+ if (!db) {+ setError(new Error('The database is not configured properly.'));+ return;+ }const connection = await db.connect();// TODO find a cheap way to get DuckDb types with a single query to a remote resource - temp table? cte?diff --git a/src/duckdb/src/index.ts b/src/duckdb/src/index.tsindex d151344fb..1520ff5ea 100644--- a/src/duckdb/src/index.ts+++ b/src/duckdb/src/index.ts@@ -1,7 +1,8 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project+export {DuckDBWasmAdapter} from './adapters/duckdb-wasm-adapter';+export {SqlPanel} from './components';import {keplerGlDuckDBPlugin} from './plugin';export {KeplerGlDuckDbTable} from './table/duckdb-table';-export {SqlPanel} from './components';export default keplerGlDuckDBPlugin;diff --git a/src/duckdb/src/init.ts b/src/duckdb/src/init.tsdeleted file mode 100644index d173c9576..000000000--- a/src/duckdb/src/init.ts+++ /dev/null@@ -1,92 +0,0 @@-// SPDX-License-Identifier: MIT-// Copyright contributors to the kepler.gl project--// [email protected]:holdenmatt/duckdb-wasm-kit.git-import * as duckdb from '@duckdb/duckdb-wasm';-import {AsyncDuckDB, DuckDBConfig} from '@duckdb/duckdb-wasm';-import {logElapsedTime} from './utils/perf';--export let DEBUG: boolean | undefined;--let DuckDB: Promise<AsyncDuckDB> | undefined;--/**- * Initialize DuckDB, ensuring we only initialize it once.- *- * @param debug If true, log DuckDB logs and elapsed times to the console.- * @param config An optional DuckDBConfig object.- */-export async function initializeDuckDb(options?: {- debug?: boolean;- config?: DuckDBConfig;-}): Promise<AsyncDuckDB> {- const {debug = false, config} = options || {};- DEBUG = debug;-- if (DuckDB === undefined) {- DuckDB = _initializeDuckDb(config);- }- return DuckDB;-}--/**- * Initialize DuckDB with a browser-specific Wasm bundle.- */-const _initializeDuckDb = async (config?: DuckDBConfig): Promise<AsyncDuckDB> => {- const start = performance.now();-- // Select a bundle based on browser checks- const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();- const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);- if (!bundle.mainWorker) {- throw new Error('Failed to initialize DuckDB');- }-- const worker_url = URL.createObjectURL(- new Blob([`importScripts("${bundle.mainWorker}");`], {- type: 'text/javascript'- })- );-- // Instantiate the async version of DuckDB-wasm- const worker = new Worker(worker_url);- const logger = DEBUG ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();- const db = new AsyncDuckDB(logger, worker);- await db.instantiate(bundle.mainModule, bundle.pthreadWorker);- URL.revokeObjectURL(worker_url);-- if (config) {- if (config.path) {- const res = await fetch(config.path);- const buffer = await res.arrayBuffer();- const fileNameMatch = config.path.match(/[^/]*$/);- if (fileNameMatch) {- config.path = fileNameMatch[0];- }- await db.registerFileBuffer(config.path, new Uint8Array(buffer));- }- await db.open(config);- }-- if (DEBUG) {- logElapsedTime('DuckDB initialized', start);- if (config) {- console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);- }- }- return db;-};--/**- * Get the instance of DuckDB, initializing it if needed.- *- * Typically `useDuckDB` is used in React components instead, but this- * method provides access outside of React contexts.- */-export const getDuckDB = async (): Promise<AsyncDuckDB> => {- if (DuckDB) {- return DuckDB;- } else {- return await initializeDuckDb();- }-};diff --git a/src/duckdb/src/plugin.ts b/src/duckdb/src/plugin.tsindex fe9f0d07b..7377e9d20 100644--- a/src/duckdb/src/plugin.ts+++ b/src/duckdb/src/plugin.ts@@ -1,18 +1,9 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import {initializeDuckDb} from './init';-export const keplerGlDuckDBPlugin = {name: 'duckdb',async init() {- await initializeDuckDb({- config: {- query: {- castBigIntToDouble: true- }- }- });console.log('kepler.gl DuckDB Plugin initialized');}};diff --git a/src/duckdb/src/processors/data-processor.ts b/src/duckdb/src/processors/data-processor.tsindex 3bceb78a8..d245bcb34 100644--- a/src/duckdb/src/processors/data-processor.ts+++ b/src/duckdb/src/processors/data-processor.ts@@ -13,8 +13,7 @@ import {import {ProtoDatasetField} from '@kepler.gl/types';import {ALL_FIELD_TYPES, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';import {processKeplerglJSON} from '@kepler.gl/processors';--import {getDuckDB} from '../init';+import {getApplicationConfig} from '@kepler.gl/utils';export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;@@ -211,7 +210,11 @@ async function sniffCsvSchema(sample: RowData) {headerRow);- const db = await getDuckDB();+ const db = getApplicationConfig().database;+ if (!db) {+ console.error('The database is not configured properly.');+ return;+ }const c = await db.connect();const fileName = generateHashId();diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.tsindex 635ced37e..1c8274f00 100644--- a/src/duckdb/src/table/duckdb-table-utils.ts+++ b/src/duckdb/src/table/duckdb-table-utils.ts@@ -11,12 +11,11 @@import * as arrow from 'apache-arrow';import {DataType} from 'apache-arrow/type';-import {AsyncDuckDBConnection, DuckDBDataProtocol} from '@duckdb/duckdb-wasm';+import {DuckDBDataProtocol} from '@duckdb/duckdb-wasm';import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';import {ProtoDatasetField} from '@kepler.gl/types';--import {getDuckDB} from '../init';+import {DatabaseConnection, getApplicationConfig} from '@kepler.gl/utils';export const SUPPORTED_DUCKDB_DROP_EXTENSIONS = ['arrow', 'csv', 'geojson', 'json', 'parquet'];@@ -29,7 +28,7 @@ export type DuckDBColumnDesc = {name: string; type: string};* @returns An array of column names and DuckDB types.*/export async function getDuckDBColumnTypes(- connection: AsyncDuckDBConnection,+ connection: DatabaseConnection,tableName: string): Promise<DuckDBColumnDesc[]> {const resDescribe = await connection.query(`DESCRIBE "${tableName}";`);@@ -256,7 +255,7 @@ export function isGeoArrowMultiPolygon(type: DataType) {* @returns Resolves to `true` if the query is a SELECT statement, otherwise `false`.*/export async function checkIsSelectQuery(- connection: AsyncDuckDBConnection,+ connection: DatabaseConnection,query: string): Promise<boolean> {try {@@ -400,7 +399,7 @@ export function removeSQLComments(sql: string): string {* @returns A promise that resolves when the operation is complete.* @throws Logs an error if the table drop operation fails.*/-export const dropTableIfExists = async (connection: AsyncDuckDBConnection, tableName: string) => {+export const dropTableIfExists = async (connection: DatabaseConnection, tableName: string) => {try {await connection.query(`DROP TABLE IF EXISTS "${tableName}";`);} catch (error) {@@ -421,7 +420,10 @@ export async function tableFromFile(file: File | null): Promise<null | Error> {return new Error("File Drag & Drop: File extension isn't supported");}- const db = await getDuckDB();+ const db = await getApplicationConfig().database;+ if (!db) {+ return new Error('The database is not configured properly.');+ }const c = await db.connect();let error: Error | null = null;diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex c6a9f141e..7959e87a9 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -2,7 +2,6 @@// Copyright contributors to the kepler.gl projectimport * as arrow from 'apache-arrow';-import {AsyncDuckDB, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';import {DatasetType,@@ -20,8 +19,8 @@ import {} from '@kepler.gl/processors';import {KeplerTable} from '@kepler.gl/table';import {Field} from '@kepler.gl/types';+import {getApplicationConfig, DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';-import {getDuckDB} from '../init';import {processCsvRowObject,processGeojson,@@ -75,8 +74,8 @@ const SUGGESTED_GEOM_COLUMNS = {type ImportDataToDuckProps = {data: ProcessorResult & {arrowSchema: arrow.Schema};- db: AsyncDuckDB;- c: AsyncDuckDBConnection;+ db: DatabaseAdapter;+ c: DatabaseConnection;};type ImportDataToDuckResult = {@@ -196,7 +195,11 @@ export class KeplerGlDuckDbTable extends KeplerTable {* @returns {Promise<{fields: Field[], cols: any[]}>}*/protected async createTableAndGetArrow(data): Promise<{fields: any[]; cols: arrow.Vector[]}> {- const db = await getDuckDB();+ const db = await getApplicationConfig().database;+ if (!db) {+ console.error('The database is not configured properly.');+ return {fields: [], cols: []};+ }const c = await db.connect();const tableName = this.label;diff --git a/src/utils/src/application-config-types.ts b/src/utils/src/application-config-types.tsnew file mode 100644index 000000000..722a4415b--- /dev/null+++ b/src/utils/src/application-config-types.ts@@ -0,0 +1,16 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import * as arrow from 'apache-arrow';++export interface DatabaseConnection {+ query(statement: string): Promise<arrow.Table>;+ insertArrowTable(arrowTable: arrow.Table, {name}: {name: string}): Promise<void>;+ close();+}++export interface DatabaseAdapter {+ connect(): Promise<DatabaseConnection>;+ registerFileText(name: string, text: string): Promise<void>;+ registerFileHandle(name: string, handle: any, protocol: number, directIO: boolean): Promise<void>;+}diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.tsindex 195a6841d..40144ef70 100644--- a/src/utils/src/application-config.ts+++ b/src/utils/src/application-config.ts@@ -5,6 +5,8 @@ import {MapLib, MapRef} from 'react-map-gl';import type {BaseMapLibraryType} from '@kepler.gl/constants';+import type {DatabaseAdapter} from './application-config-types';+export type MapLibInstance = MapLib<any>;export type GetMapRef = ReturnType<MapRef['getMap']>;@@ -34,6 +36,7 @@ export type KeplerApplicationConfig = {// KeplerTable alternative// TODO improve typing by exporting KeplerTable interface to @kepler.gl/typestable?: any;+ database?: DatabaseAdapter | null;// Disable progressive loading for arrow filesuseArrowProgressiveLoading?: boolean;@@ -75,6 +78,7 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {// The default table class is KeplerTable.// TODO include KeplerTable here when the circular dependency with @kepler.gl/table and @kepler.gl/utils are resolved.table: null,+ database: null,useArrowProgressiveLoading: true,showReleaseBanner: true,diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex 4d92b3aa0..e33e65be3 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -180,6 +180,7 @@ export type {MapLibInstance,GetMapRef} from './application-config';+export type {DatabaseAdapter, DatabaseConnection} from './application-config-types';// Browser utilsexport {isAppleDevice} from './browser-utils';