Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[chore] pass duckDB adapter via application config (#3023)

Igor Dykhta224975a25fff
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.js
index 282ab87a2..27c5054d6 100644
--- a/examples/demo-app/src/reducers/index.js
+++ b/examples/demo-app/src/reducers/index.js
@@ -13,6 +13,9 @@ import KeplerGlSchema from '@kepler.gl/schemas';
import {KeplerTable} from '@kepler.gl/table';
import {getApplicationConfig} from '@kepler.gl/utils';
+// import {getApplicationConfig, initApplicationConfig} from '@kepler.gl/utils';
+// import keplerGlDuckdbPlugin, {KeplerGlDuckDbTable, DuckDBWasmAdapter} from '@kepler.gl/duckdb';
+
import {
INIT,
LOAD_MAP_SAMPLE_FILE,
@@ -26,6 +29,26 @@ import {
import {CLOUD_PROVIDERS_CONFIGURATION} from '../constants/default-settings';
import {generateHashId} from '../utils/strings';
+// initialize kepler demo-app with DuckDB plugin
+/*
+initApplicationConfig({
+ // Custom UI for DuckDB
+ plugins: [keplerGlDuckdbPlugin],
+ // async data ingestion to DuckDb
+ table: KeplerGlDuckDbTable,
+ // setup database for DuckDB plugin
+ database: new DuckDBWasmAdapter({
+ config: {
+ query: {
+ castBigIntToDouble: true
+ }
+ }
+ }),
+ // progressive loading is sync, doesn't wait properly for a dataset to be created in DuckDB
+ useArrowProgressiveLoading: false
+});
+*/
+
const {DEFAULT_MAP_CONTROLS} = uiStateUpdaters;
// INITIAL_APP_STATE
diff --git a/src/duckdb/src/adapters/duckdb-wasm-adapter.ts b/src/duckdb/src/adapters/duckdb-wasm-adapter.ts
new file mode 100644
index 000000000..e383d1713
--- /dev/null
+++ b/src/duckdb/src/adapters/duckdb-wasm-adapter.ts
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import * as arrow from 'apache-arrow';
+import * as duckdb from '@duckdb/duckdb-wasm';
+import {AsyncDuckDB, DuckDBConfig, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
+
+import {DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';
+
+import {logElapsedTime} from '../utils/perf';
+
+class Connection implements DatabaseConnection {
+ private connection: AsyncDuckDBConnection;
+
+ constructor(connection: AsyncDuckDBConnection) {
+ this.connection = connection;
+ }
+
+ async query(statement: string): Promise<arrow.Table> {
+ return this.connection.query(statement);
+ }
+
+ async insertArrowTable(arrowTable: arrow.Table, params: {name: string}): Promise<void> {
+ await this.connection.insertArrowTable(arrowTable, params);
+ }
+
+ async close() {
+ await this.connection.close();
+ }
+}
+
+type DuckDBWasmAdapterProps =
+ | {
+ debug?: boolean;
+ config?: DuckDBConfig;
+ }
+ | Promise<AsyncDuckDB>;
+
+export class DuckDBWasmAdapter implements DatabaseAdapter {
+ private duckDB: Promise<AsyncDuckDB>;
+
+ constructor(options: DuckDBWasmAdapterProps) {
+ // pass existing AsyncDuckDB object created elsewhere
+ if (options instanceof Promise || options instanceof AsyncDuckDB) {
+ this.duckDB = options as any;
+ return;
+ }
+
+ // or create a new AsyncDuckDB object
+ const {debug = false, config} = options || {};
+ this.duckDB = initializeDuckDbWasm(config, debug);
+ }
+
+ async connect() {
+ const db = await this.duckDB;
+ const c = await db.connect();
+ return new Connection(c);
+ }
+
+ async registerFileText(filename: string, content) {
+ const db = await this.duckDB;
+ await db.registerFileText(filename, content);
+ }
+
+ async registerFileHandle(
+ name: string,
+ handle: any,
+ protocol: duckdb.DuckDBDataProtocol,
+ directIO: boolean
+ ): Promise<void> {
+ const db = await this.duckDB;
+ await db.registerFileHandle(name, handle, protocol, directIO);
+ }
+}
+
+/**
+ * Initialize DuckDB with a browser-specific Wasm bundle.
+ */
+const initializeDuckDbWasm = async (
+ config?: DuckDBConfig,
+ debug?: boolean
+): Promise<AsyncDuckDB> => {
+ const start = performance.now();
+
+ // Select a bundle based on browser checks
+ const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+ const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+ if (!bundle.mainWorker) {
+ throw new Error('Failed to initialize DuckDB');
+ }
+
+ const worker_url = URL.createObjectURL(
+ new Blob([`importScripts("${bundle.mainWorker}");`], {
+ type: 'text/javascript'
+ })
+ );
+
+ // Instantiate the async version of DuckDB-wasm
+ const worker = new Worker(worker_url);
+ const logger = debug ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();
+ const db = new AsyncDuckDB(logger, worker);
+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+ URL.revokeObjectURL(worker_url);
+
+ if (config) {
+ if (config.path) {
+ const res = await fetch(config.path);
+ const buffer = await res.arrayBuffer();
+ const fileNameMatch = config.path.match(/[^/]*$/);
+ if (fileNameMatch) {
+ config.path = fileNameMatch[0];
+ }
+ await db.registerFileBuffer(config.path, new Uint8Array(buffer));
+ }
+ await db.open(config);
+ }
+
+ if (debug) {
+ logElapsedTime('DuckDB initialized', start);
+ if (config) {
+ console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);
+ }
+ }
+ return db;
+};
diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsx
index 439cdb18d..473faffe3 100644
--- a/src/duckdb/src/components/schema-panel.tsx
+++ b/src/duckdb/src/components/schema-panel.tsx
@@ -4,14 +4,13 @@
import React, {useCallback, useEffect, useState} from 'react';
import {useSelector} from 'react-redux';
import styled from 'styled-components';
-import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
import {LoadingSpinner, Icons} from '@kepler.gl/components';
import {arrowSchemaToFields} from '@kepler.gl/processors';
import {VisState} from '@kepler.gl/schemas';
+import {getApplicationConfig, DatabaseConnection} from '@kepler.gl/utils';
import {Tree, DatasetNode, ColumnNode, TreeNodeData} from './tree';
-import {getDuckDB} from '../init';
import {getDuckDBColumnTypes, getDuckDBColumnTypesMap} from '../table/duckdb-table-utils';
// TODO note that demo state is available in demo-app, but not when add modules to dependencies in a custom map
@@ -40,7 +39,7 @@ const StyledLoadingSpinnerWrapper = styled.div`
height: 100%;
`;
-async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {
+async function getColumnSchema(connection: DatabaseConnection, tableName: string) {
const columnResult = await connection.query(`Select * from '${tableName}' LIMIT 1;`);
const columnDescribe = await getDuckDBColumnTypes(connection, tableName);
@@ -130,7 +129,11 @@ export const SchemaPanel = ({
const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);
const getTableSchema = useCallback(async () => {
- const db = await getDuckDB();
+ const db = getApplicationConfig().database;
+ if (!db) {
+ console.error('The database is not configured properly.');
+ return;
+ }
const c = await db.connect();
const tableResult = await c.query('SHOW TABLES;');
diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsx
index baed42cab..d568dc3ef 100644
--- a/src/duckdb/src/components/sql-panel.tsx
+++ b/src/duckdb/src/components/sql-panel.tsx
@@ -12,12 +12,12 @@ import {generateHashId} from '@kepler.gl/common-utils';
import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
import {arrowSchemaToFields} from '@kepler.gl/processors';
import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';
-import {isAppleDevice} from '@kepler.gl/utils';
+import {isAppleDevice, getApplicationConfig} from '@kepler.gl/utils';
import MonacoEditor from './monaco-editor';
import {SchemaPanel, SchemaSuggestion} from './schema-panel';
import {PreviewDataPanel, QueryResult} from './preview-data-panel';
-import {getDuckDB} from '../init';
+
import {
constructST_asWKBQuery,
getDuckDBColumnTypes,
@@ -184,7 +184,11 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
return;
}
- const db = await getDuckDB();
+ const db = getApplicationConfig().database;
+ if (!db) {
+ setError(new Error('The database is not configured properly.'));
+ return;
+ }
const connection = await db.connect();
// TODO find a cheap way to get DuckDb types with a single query to a remote resource - temp table? cte?
diff --git a/src/duckdb/src/index.ts b/src/duckdb/src/index.ts
index d151344fb..1520ff5ea 100644
--- a/src/duckdb/src/index.ts
+++ b/src/duckdb/src/index.ts
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
+export {DuckDBWasmAdapter} from './adapters/duckdb-wasm-adapter';
+export {SqlPanel} from './components';
import {keplerGlDuckDBPlugin} from './plugin';
export {KeplerGlDuckDbTable} from './table/duckdb-table';
-export {SqlPanel} from './components';
export default keplerGlDuckDBPlugin;
diff --git a/src/duckdb/src/init.ts b/src/duckdb/src/init.ts
deleted file mode 100644
index d173c9576..000000000
--- a/src/duckdb/src/init.ts
+++ /dev/null
@@ -1,92 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright contributors to the kepler.gl project
-
-// [email protected]:holdenmatt/duckdb-wasm-kit.git
-import * as duckdb from '@duckdb/duckdb-wasm';
-import {AsyncDuckDB, DuckDBConfig} from '@duckdb/duckdb-wasm';
-import {logElapsedTime} from './utils/perf';
-
-export let DEBUG: boolean | undefined;
-
-let DuckDB: Promise<AsyncDuckDB> | undefined;
-
-/**
- * Initialize DuckDB, ensuring we only initialize it once.
- *
- * @param debug If true, log DuckDB logs and elapsed times to the console.
- * @param config An optional DuckDBConfig object.
- */
-export async function initializeDuckDb(options?: {
- debug?: boolean;
- config?: DuckDBConfig;
-}): Promise<AsyncDuckDB> {
- const {debug = false, config} = options || {};
- DEBUG = debug;
-
- if (DuckDB === undefined) {
- DuckDB = _initializeDuckDb(config);
- }
- return DuckDB;
-}
-
-/**
- * Initialize DuckDB with a browser-specific Wasm bundle.
- */
-const _initializeDuckDb = async (config?: DuckDBConfig): Promise<AsyncDuckDB> => {
- const start = performance.now();
-
- // Select a bundle based on browser checks
- const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
- const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
- if (!bundle.mainWorker) {
- throw new Error('Failed to initialize DuckDB');
- }
-
- const worker_url = URL.createObjectURL(
- new Blob([`importScripts("${bundle.mainWorker}");`], {
- type: 'text/javascript'
- })
- );
-
- // Instantiate the async version of DuckDB-wasm
- const worker = new Worker(worker_url);
- const logger = DEBUG ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();
- const db = new AsyncDuckDB(logger, worker);
- await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
- URL.revokeObjectURL(worker_url);
-
- if (config) {
- if (config.path) {
- const res = await fetch(config.path);
- const buffer = await res.arrayBuffer();
- const fileNameMatch = config.path.match(/[^/]*$/);
- if (fileNameMatch) {
- config.path = fileNameMatch[0];
- }
- await db.registerFileBuffer(config.path, new Uint8Array(buffer));
- }
- await db.open(config);
- }
-
- if (DEBUG) {
- logElapsedTime('DuckDB initialized', start);
- if (config) {
- console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);
- }
- }
- return db;
-};
-
-/**
- * Get the instance of DuckDB, initializing it if needed.
- *
- * Typically `useDuckDB` is used in React components instead, but this
- * method provides access outside of React contexts.
- */
-export const getDuckDB = async (): Promise<AsyncDuckDB> => {
- if (DuckDB) {
- return DuckDB;
- } else {
- return await initializeDuckDb();
- }
-};
diff --git a/src/duckdb/src/plugin.ts b/src/duckdb/src/plugin.ts
index fe9f0d07b..7377e9d20 100644
--- a/src/duckdb/src/plugin.ts
+++ b/src/duckdb/src/plugin.ts
@@ -1,18 +1,9 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import {initializeDuckDb} from './init';
-
export const keplerGlDuckDBPlugin = {
name: 'duckdb',
async init() {
- await initializeDuckDb({
- config: {
- query: {
- castBigIntToDouble: true
- }
- }
- });
console.log('kepler.gl DuckDB Plugin initialized');
}
};
diff --git a/src/duckdb/src/processors/data-processor.ts b/src/duckdb/src/processors/data-processor.ts
index 3bceb78a8..d245bcb34 100644
--- a/src/duckdb/src/processors/data-processor.ts
+++ b/src/duckdb/src/processors/data-processor.ts
@@ -13,8 +13,7 @@ import {
import {ProtoDatasetField} from '@kepler.gl/types';
import {ALL_FIELD_TYPES, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';
import {processKeplerglJSON} from '@kepler.gl/processors';
-
-import {getDuckDB} from '../init';
+import {getApplicationConfig} from '@kepler.gl/utils';
export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
@@ -211,7 +210,11 @@ async function sniffCsvSchema(sample: RowData) {
headerRow
);
- const db = await getDuckDB();
+ const db = getApplicationConfig().database;
+ if (!db) {
+ console.error('The database is not configured properly.');
+ return;
+ }
const c = await db.connect();
const fileName = generateHashId();
diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.ts
index 635ced37e..1c8274f00 100644
--- a/src/duckdb/src/table/duckdb-table-utils.ts
+++ b/src/duckdb/src/table/duckdb-table-utils.ts
@@ -11,12 +11,11 @@
import * as arrow from 'apache-arrow';
import {DataType} from 'apache-arrow/type';
-import {AsyncDuckDBConnection, DuckDBDataProtocol} from '@duckdb/duckdb-wasm';
+import {DuckDBDataProtocol} from '@duckdb/duckdb-wasm';
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
import {ProtoDatasetField} from '@kepler.gl/types';
-
-import {getDuckDB} from '../init';
+import {DatabaseConnection, getApplicationConfig} from '@kepler.gl/utils';
export const SUPPORTED_DUCKDB_DROP_EXTENSIONS = ['arrow', 'csv', 'geojson', 'json', 'parquet'];
@@ -29,7 +28,7 @@ export type DuckDBColumnDesc = {name: string; type: string};
* @returns An array of column names and DuckDB types.
*/
export async function getDuckDBColumnTypes(
- connection: AsyncDuckDBConnection,
+ connection: DatabaseConnection,
tableName: string
): Promise<DuckDBColumnDesc[]> {
const resDescribe = await connection.query(`DESCRIBE "${tableName}";`);
@@ -256,7 +255,7 @@ export function isGeoArrowMultiPolygon(type: DataType) {
* @returns Resolves to `true` if the query is a SELECT statement, otherwise `false`.
*/
export async function checkIsSelectQuery(
- connection: AsyncDuckDBConnection,
+ connection: DatabaseConnection,
query: string
): Promise<boolean> {
try {
@@ -400,7 +399,7 @@ export function removeSQLComments(sql: string): string {
* @returns A promise that resolves when the operation is complete.
* @throws Logs an error if the table drop operation fails.
*/
-export const dropTableIfExists = async (connection: AsyncDuckDBConnection, tableName: string) => {
+export const dropTableIfExists = async (connection: DatabaseConnection, tableName: string) => {
try {
await connection.query(`DROP TABLE IF EXISTS "${tableName}";`);
} catch (error) {
@@ -421,7 +420,10 @@ export async function tableFromFile(file: File | null): Promise<null | Error> {
return new Error("File Drag & Drop: File extension isn't supported");
}
- const db = await getDuckDB();
+ const db = await getApplicationConfig().database;
+ if (!db) {
+ return new Error('The database is not configured properly.');
+ }
const c = await db.connect();
let error: Error | null = null;
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index c6a9f141e..7959e87a9 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -2,7 +2,6 @@
// Copyright contributors to the kepler.gl project
import * as arrow from 'apache-arrow';
-import {AsyncDuckDB, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
import {
DatasetType,
@@ -20,8 +19,8 @@ import {
} from '@kepler.gl/processors';
import {KeplerTable} from '@kepler.gl/table';
import {Field} from '@kepler.gl/types';
+import {getApplicationConfig, DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';
-import {getDuckDB} from '../init';
import {
processCsvRowObject,
processGeojson,
@@ -75,8 +74,8 @@ const SUGGESTED_GEOM_COLUMNS = {
type ImportDataToDuckProps = {
data: ProcessorResult & {arrowSchema: arrow.Schema};
- db: AsyncDuckDB;
- c: AsyncDuckDBConnection;
+ db: DatabaseAdapter;
+ c: DatabaseConnection;
};
type ImportDataToDuckResult = {
@@ -196,7 +195,11 @@ export class KeplerGlDuckDbTable extends KeplerTable {
* @returns {Promise<{fields: Field[], cols: any[]}>}
*/
protected async createTableAndGetArrow(data): Promise<{fields: any[]; cols: arrow.Vector[]}> {
- const db = await getDuckDB();
+ const db = await getApplicationConfig().database;
+ if (!db) {
+ console.error('The database is not configured properly.');
+ return {fields: [], cols: []};
+ }
const c = await db.connect();
const tableName = this.label;
diff --git a/src/utils/src/application-config-types.ts b/src/utils/src/application-config-types.ts
new file mode 100644
index 000000000..722a4415b
--- /dev/null
+++ b/src/utils/src/application-config-types.ts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import * as arrow from 'apache-arrow';
+
+export interface DatabaseConnection {
+ query(statement: string): Promise<arrow.Table>;
+ insertArrowTable(arrowTable: arrow.Table, {name}: {name: string}): Promise<void>;
+ close();
+}
+
+export interface DatabaseAdapter {
+ connect(): Promise<DatabaseConnection>;
+ registerFileText(name: string, text: string): Promise<void>;
+ registerFileHandle(name: string, handle: any, protocol: number, directIO: boolean): Promise<void>;
+}
diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts
index 195a6841d..40144ef70 100644
--- a/src/utils/src/application-config.ts
+++ b/src/utils/src/application-config.ts
@@ -5,6 +5,8 @@ import {MapLib, MapRef} from 'react-map-gl';
import type {BaseMapLibraryType} from '@kepler.gl/constants';
+import type {DatabaseAdapter} from './application-config-types';
+
export type MapLibInstance = MapLib<any>;
export type GetMapRef = ReturnType<MapRef['getMap']>;
@@ -34,6 +36,7 @@ export type KeplerApplicationConfig = {
// KeplerTable alternative
// TODO improve typing by exporting KeplerTable interface to @kepler.gl/types
table?: any;
+ database?: DatabaseAdapter | null;
// Disable progressive loading for arrow files
useArrowProgressiveLoading?: boolean;
@@ -75,6 +78,7 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {
// The default table class is KeplerTable.
// TODO include KeplerTable here when the circular dependency with @kepler.gl/table and @kepler.gl/utils are resolved.
table: null,
+ database: null,
useArrowProgressiveLoading: true,
showReleaseBanner: true,
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index 4d92b3aa0..e33e65be3 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -180,6 +180,7 @@ export type {
MapLibInstance,
GetMapRef
} from './application-config';
+export type {DatabaseAdapter, DatabaseConnection} from './application-config-types';
// Browser utils
export {isAppleDevice} from './browser-utils';