Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[feat] duckdb module updates (#2927)

Igor Dykhtace23c7668cfe
diff --git a/.eslintrc.js b/.eslintrc.js
index cf3e6758e..04b42c4bb 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -38,7 +38,8 @@ module.exports = {
'consistent-return': 0,
'comma-dangle': 1,
'enzyme-deprecation/no-shallow': 2,
- 'enzyme-deprecation/no-mount': 2
+ 'enzyme-deprecation/no-mount': 2,
+ 'no-constant-condition': ['error', {checkLoops: false}]
},
overrides: [
{
diff --git a/src/duckdb/package.json b/src/duckdb/package.json
index 4f5519bf5..cfc776aea 100644
--- a/src/duckdb/package.json
+++ b/src/duckdb/package.json
@@ -2,7 +2,7 @@
"name": "@kepler.gl/duckdb",
"author": "Shan He <[email protected]>",
"version": "3.1.0-alpha.5",
- "description": "kepler.gl duckDB plugin",
+ "description": "DuckDB plugin for Kepler.gl",
"license": "MIT",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -25,8 +25,17 @@
"umd"
],
"dependencies": {
+ "@duckdb/duckdb-wasm": "^1.28.0",
+ "@kepler.gl/common-utils": "3.1.0-alpha.5",
"@kepler.gl/constants": "3.1.0-alpha.5",
- "@kepler.gl/types": "3.1.0-alpha.5"
+ "@kepler.gl/processors": "3.1.0-alpha.5",
+ "@kepler.gl/table": "3.1.0-alpha.5",
+ "@kepler.gl/types": "3.1.0-alpha.5",
+ "@monaco-editor/react": "^4.6.0",
+ "@radix-ui/react-collapsible": "^1.1.0",
+ "apache-arrow": ">=15.0.0",
+ "monaco-editor": "^0.52.0",
+ "react-resizable-panels": "^2.1.7"
},
"nyc": {
"sourceMap": false,
diff --git a/src/duckdb/src/components/index.tsx b/src/duckdb/src/components/index.tsx
new file mode 100644
index 000000000..4d289b8de
--- /dev/null
+++ b/src/duckdb/src/components/index.tsx
@@ -0,0 +1 @@
+export * from './sql-panel';
diff --git a/src/duckdb/src/components/monaco-editor.tsx b/src/duckdb/src/components/monaco-editor.tsx
new file mode 100644
index 000000000..08dd6f6ec
--- /dev/null
+++ b/src/duckdb/src/components/monaco-editor.tsx
@@ -0,0 +1,189 @@
+import React, {useCallback, useMemo, useRef} from 'react';
+import Editor, {OnChange, OnMount} from '@monaco-editor/react';
+import * as monaco from 'monaco-editor';
+// import {tableSchema as DEFAULT_SCHEMA} from './table-schema';
+import uniq from 'lodash.uniq';
+import uniqBy from 'lodash.uniqby';
+
+const MONACO_OPTIONS: monaco.editor.IStandaloneEditorConstructionOptions = {
+ minimap: {enabled: false},
+ language: 'sql',
+ contextmenu: false,
+ renderLineHighlight: 'none',
+ scrollBeyondLastLine: false,
+ scrollbar: {alwaysConsumeMouseWheel: false},
+ overviewRulerLanes: 0,
+ automaticLayout: true,
+ acceptSuggestionOnEnter: 'on',
+ quickSuggestionsDelay: 400,
+ matchOnWordStartOnly: false,
+ tabCompletion: 'off',
+ lineNumbers: 'off'
+};
+
+function parseSqlAndFindTableNameAndAliases(sql: string) {
+ const regex = /\b(?:FROM|JOIN)\s+([^\s.]+(?:\.[^\s.]+)?)\s*(?:AS)?\s*([^\s,]+)?/gi;
+ const tables: {table_name: string; alias: string}[] = [];
+
+ while (true) {
+ const match = regex.exec(sql);
+ if (!match) {
+ break;
+ }
+ const table_name = match[1];
+ if (!/\(/.test(table_name)) {
+ // exclude function calls
+ let alias = match[2] as string | null;
+ if (alias && /on|where|inner|left|right|join/.test(alias)) {
+ alias = null;
+ }
+ tables.push({
+ table_name,
+ alias: alias || table_name
+ });
+ }
+ }
+
+ return tables;
+}
+
+interface MonacoEditorProps {
+ code: string;
+ isReadOnly?: boolean;
+ onChange: OnChange;
+ onRunQuery: () => void;
+ tableSchema?: {table_name: string; column_name: string}[];
+}
+
+const MonacoEditor: React.FC<MonacoEditorProps> = ({
+ onRunQuery,
+ onChange,
+ code,
+ tableSchema,
+ isReadOnly
+}) => {
+ // private editor?: monaco.editor.IStandaloneCodeEditor;
+ const schemaTableNames = useMemo(
+ () => (tableSchema ? uniq(tableSchema.map(d => d.table_name)) : []),
+ [tableSchema]
+ );
+ const schemaTableNamesSet = useMemo(() => new Set(schemaTableNames), [schemaTableNames]);
+ const handleRunQueryRef = useRef(onRunQuery);
+ handleRunQueryRef.current = onRunQuery;
+
+ const handleEditorDidMount: OnMount = useCallback(
+ editor => {
+ // this.editor = editor;
+ editor.focus();
+
+ editor.addAction({
+ id: 'run-query',
+ label: 'Run Query',
+ keybindings: [
+ monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter,
+ monaco.KeyMod.Shift | monaco.KeyCode.Enter
+ ],
+ contextMenuGroupId: 'custom',
+ contextMenuOrder: 0,
+ run: () => handleRunQueryRef.current()
+ });
+
+ monaco.languages.registerCompletionItemProvider('*', {
+ provideCompletionItems: (model, position, context, cancelationToken) => {
+ const suggestions: monaco.languages.CompletionItem[] = [
+ {
+ label: 'myCustomSnippet',
+ kind: monaco.languages.CompletionItemKind.Snippet,
+ insertText: 'This is a piece of custom code',
+ insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
+ documentation: 'This is a piece of custom code'
+ // TODO: range is missing
+ } as monaco.languages.CompletionItem
+ ];
+
+ const fullQueryText = model.getValue();
+
+ const tableNamesAndAliases = new Map<string, string>(
+ parseSqlAndFindTableNameAndAliases(fullQueryText).map(({table_name, alias}) => [
+ alias,
+ table_name
+ ])
+ );
+
+ const thisLine = model.getValueInRange({
+ startLineNumber: position.lineNumber,
+ startColumn: 1,
+ endLineNumber: position.lineNumber,
+ endColumn: position.column
+ });
+ const thisToken = thisLine.trim().split(' ').slice(-1)?.[0] || '';
+
+ const lastTokenBeforeSpace = /\s?(\w+)\s+\w+$/.exec(thisLine.trim())?.[1];
+ const lastTokenBeforeDot = /(\w+)\.\w*$/.exec(thisToken)?.[1];
+
+ // console.log(tableNamesAndAliases, thisToken, lastTokenBeforeSpace, lastTokenBeforeDot);
+
+ if (lastTokenBeforeSpace && /from|join|update|into/.test(lastTokenBeforeSpace)) {
+ suggestions.push(
+ ...schemaTableNames.map(
+ table_name =>
+ ({
+ label: table_name,
+ kind: monaco.languages.CompletionItemKind.Field,
+ insertText: table_name
+ // TODO: range is missing
+ } as monaco.languages.CompletionItem)
+ )
+ );
+ }
+
+ if (lastTokenBeforeDot) {
+ let table_name = null as string | null;
+ if (schemaTableNamesSet.has(lastTokenBeforeDot)) {
+ table_name = lastTokenBeforeDot;
+ } else if (tableNamesAndAliases.get(lastTokenBeforeDot)) {
+ table_name = tableNamesAndAliases.get(lastTokenBeforeDot) as string;
+ }
+ if (table_name && tableSchema) {
+ suggestions.push(
+ ...tableSchema
+ .filter(d => d.table_name === table_name)
+ .map(
+ ({table_name, column_name}) =>
+ ({
+ label: column_name,
+ kind: monaco.languages.CompletionItemKind.Field,
+ insertText: column_name
+ // TODO: range is missing
+ } as monaco.languages.CompletionItem)
+ )
+ );
+ }
+ }
+
+ return {
+ suggestions: uniqBy(suggestions, s => s.insertText)
+ };
+ }
+ });
+ },
+ [tableSchema]
+ );
+
+ return (
+ <Editor
+ height="100%"
+ theme="vs-dark"
+ defaultLanguage="sql"
+ defaultValue={code}
+ onChange={onChange}
+ onMount={handleEditorDidMount}
+ options={{
+ ...MONACO_OPTIONS,
+ readOnly: isReadOnly
+ }}
+ />
+ );
+};
+
+export default MonacoEditor;
diff --git a/src/duckdb/src/components/preview-data-panel.tsx b/src/duckdb/src/components/preview-data-panel.tsx
new file mode 100644
index 000000000..d152840f8
--- /dev/null
+++ b/src/duckdb/src/components/preview-data-panel.tsx
@@ -0,0 +1,158 @@
+// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.
+
+import React, {useCallback, useMemo, useState, CSSProperties} from 'react';
+
+import {DataTable, renderedSize} from '@kepler.gl/components';
+import {parseFieldValue, createDataContainer} from '@kepler.gl/utils';
+import {arrowSchemaToFields} from '@kepler.gl/processors';
+import {DataForm} from '@kepler.gl/utils';
+import {withTheme} from 'styled-components';
+
+type BaseComponentProps = {
+ className?: string;
+ style?: CSSProperties;
+};
+
+const DEFAULT_ROWS_TO_CALCULATE_PREVIEW = 100;
+// min Cellsize should take into account option button and field token
+const minCellSize = 80;
+// option buttons and field token
+const optionButtonWidth = 20;
+const pinButton = 20;
+const cellPadding = 20;
+
+export type ColMeta = {
+ [key: string]: {
+ colIdx: number;
+ name: string;
+ displayName: string;
+ type: string;
+ };
+};
+
+export type DataTableStyle = {
+ minCellSize?: number;
+ cellPadding?: number;
+ fontSize?: number;
+ font?: string;
+ optionsButton?: number;
+};
+
+export type PreviewDataPanelProps = BaseComponentProps & {
+ result: any;
+ rowsToCalculatePreview?: number;
+ theme?: any;
+ setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;
+ defaultPinnedColumns?: string[];
+ dataTableStyle: DataTableStyle;
+ onAddResultToMap: (result: any) => void;
+};
+
+const PreviewDataPanelWOTheme: React.FC<PreviewDataPanelProps> = ({
+ result,
+ rowsToCalculatePreview = DEFAULT_ROWS_TO_CALCULATE_PREVIEW,
+ setColumnDisplayFormat,
+ defaultPinnedColumns = [],
+ theme
+}) => {
+ const [pinnedColumns, setPinnedColumns] = useState<string[]>(defaultPinnedColumns);
+ const fields = useMemo(() => arrowSchemaToFields(result.schema), [result.schema]);
+ const dataContainer = useMemo(() => {
+ // const fields = arrowSchemaToFields(result.schema);
+
+ const cols = [...Array(result.numCols).keys()].map(i => result.getChildAt(i));
+
+ const dataContainer = createDataContainer(cols, {
+ fields,
+ inputDataFormat: DataForm.COLS_ARRAY
+ });
+ return dataContainer;
+ }, [result, fields]);
+
+ const columns = useMemo(() => fields.map(f => f.name), [fields]);
+ const colMeta = useMemo(
+ () =>
+ fields.reduce(
+ (acc, {name, displayName, type, displayFormat}, colIdx) => ({
+ ...acc,
+ [name]: {
+ // because '' || 'aaa' = 'aaa'
+ name: displayName !== undefined ? displayName : name,
+ displayName,
+ displayFormat,
+ type,
+ colIdx
+ }
+ }),
+ {}
+ ),
+ [fields]
+ );
+ const copyTableColumn = useCallback(
+ column => {
+ const {colIdx, type} = colMeta[column];
+ const text = dataContainer
+ .mapIndex(row => parseFieldValue(dataContainer.valueAt(row.index, colIdx), type))
+ .join('\n');
+ navigator?.clipboard.writeText(text);
+ },
+ [colMeta, dataContainer]
+ );
+ const pinTableColumn = useCallback(
+ column =>
+ pinnedColumns.includes(column)
+ ? setPinnedColumns(pinnedColumns.filter(c => c !== column))
+ : setPinnedColumns([...pinnedColumns, column]),
+ [pinnedColumns]
+ );
+
+ // TODO Potentially costly operation for non row based data containers. Revisit sorting below.
+ const dataTableStyle = useMemo(
+ () => ({
+ minCellSize,
+ cellPadding,
+ optionsButton:
+ theme.fieldTokenWidth + theme.fieldTokenRightMargin + optionButtonWidth + pinButton,
+ fontSize: theme.cellFontSize,
+ font: theme.fontFamily
+ }),
+ [theme]
+ );
+ const cellSizeCache = useMemo(() => {
+ return columns.reduce((acc, column) => {
+ const {colIdx, displayName, type} = colMeta[column];
+ return {
+ ...acc,
+ [column]: renderedSize({
+ text: {
+ dataContainer,
+ column: displayName
+ },
+ colIdx,
+ type,
+ numRowsToCalculate: rowsToCalculatePreview,
+ ...dataTableStyle
+ })
+ };
+ }, {});
+ }, [columns, colMeta, dataContainer, rowsToCalculatePreview, dataTableStyle]);
+
+ return (
+ <DataTable
+ colMeta={colMeta}
+ columns={columns}
+ cellSizeCache={cellSizeCache}
+ dataContainer={dataContainer}
+ pinnedColumns={pinnedColumns}
+ // sortColumn={sortColumnConfig}
+ copyTableColumn={copyTableColumn}
+ pinTableColumn={pinTableColumn}
+ // sortTableColumn={setTableSortColumn}
+ setColumnDisplayFormat={setColumnDisplayFormat ?? (() => null)}
+ />
+ );
+};
+
+export const PreviewDataPanel = withTheme(
+ PreviewDataPanelWOTheme
+) as React.FC<PreviewDataPanelProps>;
diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsx
new file mode 100644
index 000000000..93b5195ec
--- /dev/null
+++ b/src/duckdb/src/components/schema-panel.tsx
@@ -0,0 +1,107 @@
+import React, {useCallback, useEffect, useState} from 'react';
+import {useSelector} from 'react-redux';
+import styled from 'styled-components';
+import {arrowDataTypeToFieldType} from '@kepler.gl/utils';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {VisState} from '@kepler.gl/schemas';
+import {Tree, DatasetNode, ColumnNode, TreeNodeData} from './tree';
+import {getDuckDB} from '../init';
+
+// TODO note that demo state is available in demo-app, but not when add modules to dependencies in a custom map
+type State = {
+ demo?: {
+ keplerGl: {
+ map: {
+ visState: VisState;
+ };
+ };
+ };
+};
+
+const StyledSchemaPanel = styled.div`
+ color: ${props => props.theme.textColor};
+ font-size: 12px;
+ padding: 12px;
+ font-family: ${props => props.theme.fontFamily};
+`;
+
+async function getColumnSchema(c, name) {
+ const columnResult = await c.query(`Select * from '${name}' LIMIT 1;`);
+
+ return {
+ key: name,
+ object: {
+ type: 'dataset',
+ tableName: name
+ },
+ children: columnResult.schema.fields.map(field => {
+ const isGeoArrowColumn = field.metadata.get('ARROW:extension:name')?.startsWith('geoarrow');
+ return {
+ key: field.name,
+ object: {
+ type: 'column',
+ name: field.name,
+ arrowType: field.type,
+ fieldType: isGeoArrowColumn
+ ? ALL_FIELD_TYPES.geoarrow
+ : arrowDataTypeToFieldType(field.type)
+ }
+ };
+ })
+ };
+}
+
+function getSchemaSuggestion(result) {
+ return result.reduce((accu, data) => {
+ const columns = data.children.map(child => ({
+ column_name: child.key,
+ table_name: data.key
+ }));
+ return accu.concat(columns);
+ }, []);
+}
+export const SchemaPanel = ({setTableSchema}) => {
+ const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);
+ const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);
+ useEffect(() => {
+ getTableSchema();
+ }, [datasets]);
+ const getTableSchema = useCallback(async () => {
+ const db = await getDuckDB();
+ const c = await db.connect();
+
+ const tableResult = await c.query('SHOW TABLES;');
+
+ const tableNames = tableResult.getChildAt(0)?.toJSON();
+
+ const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));
+ const tableSchema = getSchemaSuggestion(result);
+
+ setColumnSchemas(result);
+ setTableSchema(tableSchema);
+ c.close();
+ }, [setColumnSchemas, setTableSchema]);
+
+ return (
+ <StyledSchemaPanel>
+ {columnSchemas?.length ? (
+ columnSchemas.map(data => (
+ <Tree
+ key={data.key}
+ treeData={data}
+ renderNode={node => {
+ if (node.object.type === 'dataset') {
+ return <DatasetNode node={node} />;
+ } else if (node.object.type === 'column') {
+ return <ColumnNode node={node} />;
+ }
+ return null;
+ }}
+ />
+ ))
+ ) : (
+ <div>No tables found</div>
+ )}
+ </StyledSchemaPanel>
+ );
+};
diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsx
new file mode 100644
index 000000000..586c2f171
--- /dev/null
+++ b/src/duckdb/src/components/sql-panel.tsx
@@ -0,0 +1,264 @@
+import * as arrow from 'apache-arrow';
+import React, {useCallback, useState, useEffect} from 'react';
+import {useDispatch} from 'react-redux';
+import styled from 'styled-components';
+import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';
+import MonacoEditor from './monaco-editor';
+import {SchemaPanel} from './schema-panel';
+import {PreviewDataPanel} from './preview-data-panel';
+import {getDuckDB} from '../init';
+import {Button, IconButton, Tooltip} from '@kepler.gl/components';
+import {generateHashId} from '@kepler.gl/common-utils';
+import {addDataToMap} from '@kepler.gl/actions';
+import {Icons, LoadingSpinner} from '@kepler.gl/components';
+import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';
+import {isAppleDevice} from '@kepler.gl/utils';
+import {arrowSchemaToFields} from '@kepler.gl/processors';
+
+const StyledSqlPanel = styled.div`
+ display: flex;
+ height: 100%;
+ background-color: ${sidePanelBg};
+ border-left: 1px solid #333;
+`;
+
+const StyledSqlActions = styled.div`
+ flex: 0 0 50px;
+ display: flex;
+ padding-top: 12px;
+ align-items: center;
+ flex-direction: column;
+ justify-content: flex-start;
+ border-bottom: 1px solid #333;
+`;
+
+const StyledSqlEditor = styled.div`
+ display: flex;
+ flex-direction: row;
+ flex: 1;
+ min-width: 0; // Prevents flex child from overflowing
+ height: 100%;
+`;
+
+const StyledEditorPanel = styled.div`
+ background-color: ${sidePanelBg};
+ display: flex;
+ height: 100%;
+ flex-direction: row;
+ gap: 5px;
+ flex-grow: 1;
+`;
+
+const StyledDataPanel = styled.div`
+ display: flex;
+ flex-direction: column;
+ height: 100%;
+ background: white;
+`;
+
+const StyledEditor = styled.div<{isRunning: boolean}>`
+ display: flex;
+ padding: 12px 12px 12px 0;
+ background-color: rgb(30, 30, 30);
+ opacity: ${props => (props.isRunning ? 0.5 : 1)};
+ width: 100%;
+ height: 100%;
+ min-height: 0; // Prevents flex child from overflowing
+`;
+
+const StyledResultActions = styled.div`
+ background-color: white;
+ padding: 2px 12px;
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ font-size: 12px;
+`;
+
+const StyledResizeHandle = styled(PanelResizeHandle)`
+ background-color: ${panelBorderColor};
+ width: 4px;
+ cursor: col-resize;
+
+ &:hover {
+ background-color: #555;
+ }
+`;
+
+const StyledVerticalResizeHandle = styled(PanelResizeHandle)`
+ background-color: ${panelBorderColor};
+ height: 4px;
+ cursor: row-resize;
+
+ &:hover {
+ background-color: #555;
+ }
+`;
+
+const StyledLoadingContainer = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ height: 100%;
+`;
+
+const StyledErrorContainer = styled.pre`
+ padding: 12px;
+ color: red;
+ font-size: 12px;
+ background-color: ${sidePanelBg};
+ height: 100%;
+ overflow: auto;
+`;
+
+type SqlPanelProps = {
+ initialSql?: string;
+};
+
+export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
+ const [sql, setSql] = useState(() => {
+ const params = new URLSearchParams(window.location.search);
+ return params.get('sql') || initialSql;
+ });
+ const [result, setResult] = useState<null | arrow.Table>(null);
+ const [error, setError] = useState<Error | null>(null);
+ const [counter, setCounter] = useState(0);
+ const [tableSchema, setTableSchema] = useState([]);
+ const [isRunning, setIsRunning] = useState(false);
+ const [isMac] = useState(() => isAppleDevice());
+ const dispatch = useDispatch();
+
+ useEffect(() => {
+ const currentUrl = new URL(window.location.href);
+ if (sql) {
+ currentUrl.searchParams.set('sql', sql);
+ } else {
+ currentUrl.searchParams.delete('sql');
+ }
+ window.history.replaceState({}, '', currentUrl.toString());
+ }, [sql]);
+
+ const runQuery = useCallback(async () => {
+ setIsRunning(true);
+ try {
+ if (!sql?.trim()) {
+ setError(new Error('Query is empty'));
+ return;
+ }
+
+ const db = await getDuckDB();
+ const connection = await db.connect();
+
+ const arrowResult = await connection.query(sql);
+ setResult(arrowResult);
+ setError(null);
+
+ connection.close();
+ } catch (e) {
+ setError(e as Error);
+ } finally {
+ setIsRunning(false);
+ }
+ }, [sql]);
+
+ const onChange = useCallback(
+ value => {
+ setSql(value);
+ },
+ [setSql]
+ );
+
+ const onAddResultToMap = useCallback(() => {
+ if (!result) return;
+
+ const keplerFields = arrowSchemaToFields(result.schema);
+
+ const datasetToAdd = {
+ data: {
+ fields: keplerFields,
+ // TODO type AddDataToMapPayload -> rows -> + arrow.Table
+ rows: result as any
+ },
+ info: {
+ id: generateHashId(),
+ label: `Query Result${counter > 0 ? ` (${counter})` : ''}`,
+ format: 'arrow'
+ }
+ };
+
+ dispatch(addDataToMap({datasets: [datasetToAdd]}));
+ setCounter(counter + 1);
+ }, [result, counter, dispatch]);
+
+ return (
+ <StyledSqlPanel>
+ <PanelGroup direction="horizontal">
+ <Panel defaultSize={20} minSize={15}>
+ <SchemaPanel setTableSchema={setTableSchema} />
+ </Panel>
+
+ <StyledResizeHandle />
+
+ <Panel minSize={40}>
+ <StyledSqlEditor>
+ <PanelGroup direction="vertical">
+ <Panel defaultSize={30} className="editor-panel">
+ <StyledEditorPanel>
+ <StyledSqlActions>
+ <IconButton
+ data-tip
+ data-for="run-query-tooltip"
+ onClick={runQuery}
+ disabled={isRunning || !sql}
+ >
+ <Icons.Play height="18px" />
+ <Tooltip id="run-query-tooltip" place="top" effect="solid">
+ Run Query ({isMac ? '⌘Enter' : 'Ctrl+Enter'} or Shift+Enter)
+ </Tooltip>
+ </IconButton>
+ </StyledSqlActions>
+ <StyledEditor isRunning={isRunning}>
+ <MonacoEditor
+ isReadOnly={isRunning}
+ onRunQuery={runQuery}
+ onChange={onChange}
+ tableSchema={tableSchema}
+ code={sql}
+ />
+ </StyledEditor>
+ </StyledEditorPanel>
+ </Panel>
+
+ <StyledVerticalResizeHandle />
+ <Panel>
+ {isRunning ? (
+ <StyledLoadingContainer>
+ <LoadingSpinner />
+ </StyledLoadingContainer>
+ ) : error ? (
+ <StyledErrorContainer>
+ <div>{error.message}</div>
+ </StyledErrorContainer>
+ ) : result ? (
+ <StyledDataPanel>
+ <StyledResultActions>
+ <Button secondary onClick={onAddResultToMap}>
+ Add to Map
+ </Button>
+ <div>{result.numRows} rows</div>
+ </StyledResultActions>
+ <PreviewDataPanel
+ result={result}
+ dataTableStyle={{}}
+ onAddResultToMap={onAddResultToMap}
+ />
+ </StyledDataPanel>
+ ) : null}
+ </Panel>
+ </PanelGroup>
+ </StyledSqlEditor>
+ </Panel>
+ </PanelGroup>
+ </StyledSqlPanel>
+ );
+};
diff --git a/src/duckdb/src/components/tree.tsx b/src/duckdb/src/components/tree.tsx
new file mode 100644
index 000000000..11267da43
--- /dev/null
+++ b/src/duckdb/src/components/tree.tsx
@@ -0,0 +1,106 @@
+// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.
+
+import {Collapsible, CollapsibleTrigger, CollapsibleContent} from '@radix-ui/react-collapsible';
+import styled from 'styled-components';
+import React, {useEffect, useState} from 'react';
+import classNames from 'classnames';
+
+import {Icons, FieldToken} from '@kepler.gl/components';
+
+export type TreeNodeData<T> = {
+ key: string;
+ object: T;
+ children?: TreeNodeData<T>[];
+ isOpen?: boolean;
+};
+
+export type TreeProps<T> = {
+ className?: string;
+ treeData: TreeNodeData<T>;
+ renderNode: TreeNodeProps<T>['renderNode'];
+};
+
+// TODO: consider using react-vtree/react-window for virtualization
+
+/**
+ * Component that renders a generic tree.
+ * @param treeData - The tree data.
+ * @param renderNode - A function that renders a tree node.
+ */
+export function Tree<T>(props: TreeProps<T>): React.ReactElement {
+ const {treeData, renderNode} = props;
+ return (
+ <div className="flex flex-col">
+ <TreeNode<T> treeData={treeData} renderNode={renderNode} />
+ </div>
+ );
+}
+
+export type TreeNodeProps<T> = {
+ treeData: TreeNodeData<T>;
+ renderNode: (node: TreeNodeData<T>, isOpen: boolean) => JSX.Element | null;
+};
+
+const StyledCollapsibleTriggerContent = styled.div`
+ display: flex;
+ width: 100%;
+ cursor: pointer;
+ align-items: center;
+`;
+/**
+ * Component that renders a tree node.
+ */
+function TreeNode<T>(props: TreeNodeProps<T>): JSX.Element | null {
+ const {treeData, renderNode} = props;
+ const {children} = treeData;
+ const [isOpen, setIsOpen] = useState(Boolean(treeData.isOpen));
+ useEffect(() => {
+ setIsOpen(Boolean(treeData.isOpen));
+ }, [treeData.isOpen]);
+ if (!children) {
+ return renderNode(treeData, isOpen);
+ }
+ return (
+ <Collapsible open={isOpen} onOpenChange={setIsOpen}>
+ <CollapsibleTrigger className="w-full" asChild>
+ <StyledCollapsibleTriggerContent>
+ <Icons.ArrowRight
+ className={classNames('flex-shrink-0 text-gray-500', {'rotate-90 transform': isOpen})}
+ height="18px"
+ />
+ {renderNode(treeData, isOpen)}
+ </StyledCollapsibleTriggerContent>
+ </CollapsibleTrigger>
+ <CollapsibleContent className="pl-4">
+ {isOpen
+ ? children.map(child => (
+ <TreeNode<T> key={child.key} treeData={child} renderNode={renderNode} />
+ ))
+ : null}
+ </CollapsibleContent>
+ </Collapsible>
+ );
+}
+
+export const DatasetNode = ({node}) => {
+ return <div>{node.object.tableName}</div>;
+};
+
+const StyledColumnNode = styled.div`
+ display: flex;
+ align-items: center;
+ padding: 4px 0;
+ margin-left: 16px;
+
+ .column-name {
+ margin-left: 8px;
+ }
+`;
+export const ColumnNode = ({node}) => {
+ return (
+ <StyledColumnNode>
+ <FieldToken type={node.object.fieldType} />
+ <div className="column-name">{node.object.name}</div>
+ </StyledColumnNode>
+ );
+};
diff --git a/src/duckdb/src/index.ts b/src/duckdb/src/index.ts
index 762aa28fe..59735e2b1 100644
--- a/src/duckdb/src/index.ts
+++ b/src/duckdb/src/index.ts
@@ -1,6 +1,4 @@
-// SPDX-License-Identifier: MIT
-// Copyright contributors to the kepler.gl project
-
-const DUCK = 0;
-
-export default DUCK;
+import {keplerGlDuckDBPlugin} from './plugin';
+export {KeplerGlDuckDbTable} from './table/duckdb-table';
+export {SqlPanel} from './components';
+export default keplerGlDuckDBPlugin;
diff --git a/src/duckdb/src/init.ts b/src/duckdb/src/init.ts
new file mode 100644
index 000000000..bf83fe33b
--- /dev/null
+++ b/src/duckdb/src/init.ts
@@ -0,0 +1,86 @@
+// [email protected]:holdenmatt/duckdb-wasm-kit.git
+import * as duckdb from '@duckdb/duckdb-wasm';
+import {AsyncDuckDB, DuckDBConfig} from '@duckdb/duckdb-wasm';
+import {logElapsedTime} from './utils/perf';
+
+export let DEBUG: boolean | undefined;
+
+let DuckDB: Promise<AsyncDuckDB> | undefined;
+
+/**
+ * Initialize DuckDB, ensuring we only initialize it once.
+ *
+ * @param debug If true, log DuckDB logs and elapsed times to the console.
+ * @param config An optional DuckDBConfig object.
+ */
+export async function initializeDuckDb(options?: {
+ debug?: boolean;
+ config?: DuckDBConfig;
+}): Promise<AsyncDuckDB> {
+ const {debug = false, config} = options || {};
+ DEBUG = debug;
+
+ if (DuckDB === undefined) {
+ DuckDB = _initializeDuckDb(config);
+ }
+ return DuckDB;
+}
+
+/**
+ * Initialize DuckDB with a browser-specific Wasm bundle.
+ */
+const _initializeDuckDb = async (config?: DuckDBConfig): Promise<AsyncDuckDB> => {
+ const start = performance.now();
+
+ // Select a bundle based on browser checks
+ const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+ const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+
+ const worker_url = URL.createObjectURL(
+ new Blob([`importScripts("${bundle.mainWorker!}");`], {
+ type: 'text/javascript'
+ })
+ );
+
+ // Instantiate the async version of DuckDB-wasm
+ const worker = new Worker(worker_url);
+ const logger = DEBUG ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();
+ const db = new AsyncDuckDB(logger, worker);
+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+ URL.revokeObjectURL(worker_url);
+
+ if (config) {
+ if (config.path) {
+ const res = await fetch(config.path);
+ const buffer = await res.arrayBuffer();
+ const fileNameMatch = config.path.match(/[^/]*$/);
+ if (fileNameMatch) {
+ config.path = fileNameMatch[0];
+ }
+ await db.registerFileBuffer(config.path, new Uint8Array(buffer));
+ }
+ await db.open(config);
+ }
+
+ if (DEBUG) {
+ logElapsedTime('DuckDB initialized', start);
+ if (config) {
+ console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);
+ }
+ }
+ return db;
+};
+
+/**
+ * Get the instance of DuckDB, initializing it if needed.
+ *
+ * Typically `useDuckDB` is used in React components instead, but this
+ * method provides access outside of React contexts.
+ */
+export const getDuckDB = async (): Promise<AsyncDuckDB> => {
+ if (DuckDB) {
+ return DuckDB;
+ } else {
+ return await initializeDuckDb();
+ }
+};
diff --git a/src/duckdb/src/plugin.ts b/src/duckdb/src/plugin.ts
new file mode 100644
index 000000000..f0b139933
--- /dev/null
+++ b/src/duckdb/src/plugin.ts
@@ -0,0 +1,8 @@
+import {initializeDuckDb} from './init';
+
+export const keplerGlDuckDBPlugin = {
+ async init() {
+ await initializeDuckDb();
+ console.log('kepler.gl DuckDB Plugin initialized');
+ }
+};
diff --git a/src/duckdb/src/processors/data-processor.ts b/src/duckdb/src/processors/data-processor.ts
new file mode 100644
index 000000000..9e6de3fb3
--- /dev/null
+++ b/src/duckdb/src/processors/data-processor.ts
@@ -0,0 +1,252 @@
+import * as arrow from 'apache-arrow';
+import normalize from '@mapbox/geojson-normalize';
+
+import {
+ getSampleForTypeAnalyze,
+ getFieldsFromData,
+ generateHashId,
+ notNullorUndefined
+} from '@kepler.gl/common-utils';
+import {ProtoDatasetField} from '@kepler.gl/types';
+import {ALL_FIELD_TYPES, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';
+import {processKeplerglJSON} from '@kepler.gl/processors';
+
+import {getDuckDB} from '../init';
+
+export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
+
+type RowsAsArray = any[][];
+type RowsAsObject = Record<string, unknown>[];
+type RowData = RowsAsArray | RowsAsObject;
+
+export type ProcessorResult = {
+ cols?: arrow.Vector[];
+ rows: RowData;
+ fields: ProtoDatasetField[];
+};
+
+enum COLUMN_TYPES {
+ GEOMETRY = 'GEOMETRY',
+ BOOLEAN = 'BOOLEAN',
+ INTEGER = 'INTEGER',
+ FLOAT = 'FLOAT',
+ TIME = 'TIME',
+ DATE = 'DATE',
+ TIMESTAMP = 'TIMESTAMP',
+ VARCHAR = 'VARCHAR'
+}
+// @ts-expect-error https://duckdb.org/docs/data/csv/auto_detection.html#type-detection
+const COLUMN_TYPES_PRIORITIES = [
+ 'GEOMETRY',
+ 'BOOLEAN',
+ 'INTEGER',
+ 'FLOAT',
+ 'TIME',
+ 'DATE',
+ 'TIMESTAMP',
+ 'VARCHAR'
+];
+
+export function columnTypeToFieldType(columnType: string): string | null {
+ switch (columnType) {
+ case COLUMN_TYPES.BOOLEAN:
+ return ALL_FIELD_TYPES.boolean;
+ case COLUMN_TYPES.DATE:
+ return ALL_FIELD_TYPES.date;
+ case COLUMN_TYPES.INTEGER:
+ return ALL_FIELD_TYPES.integer;
+ case COLUMN_TYPES.FLOAT:
+ return ALL_FIELD_TYPES.real;
+ case COLUMN_TYPES.VARCHAR:
+ return ALL_FIELD_TYPES.string;
+ case COLUMN_TYPES.TIMESTAMP:
+ return ALL_FIELD_TYPES.timestamp;
+ case COLUMN_TYPES.GEOMETRY:
+ return ALL_FIELD_TYPES.geojson;
+ default:
+ // is there any other type we didn't cover?
+ console.warn(`Unsupported DuckDB Column type: ${columnType}`);
+ return null;
+ }
+}
+
+export function fieldTypeToColumnType(fieldType: string): string | null {
+ switch (fieldType) {
+ case ALL_FIELD_TYPES.boolean:
+ return COLUMN_TYPES.BOOLEAN;
+ case ALL_FIELD_TYPES.date:
+ return COLUMN_TYPES.DATE;
+ case ALL_FIELD_TYPES.integer:
+ return COLUMN_TYPES.INTEGER;
+ case ALL_FIELD_TYPES.real:
+ return COLUMN_TYPES.FLOAT;
+ case ALL_FIELD_TYPES.string:
+ return COLUMN_TYPES.VARCHAR;
+ case ALL_FIELD_TYPES.timestamp:
+ return COLUMN_TYPES.TIMESTAMP;
+ case ALL_FIELD_TYPES.geojson:
+ return COLUMN_TYPES.GEOMETRY;
+ default:
+ // is there any other type we didn't cover?
+ console.warn(`Unsupported Field type for DuckDb: ${fieldType}`);
+ return null;
+ }
+}
+
+/*
+ * Process uploaded exported Keplergl json map.
+ * We need to populate field with proper DuckDb compatible type.
+ */
+// TODO: merge with logic from processCsvRowObject. Different formats: [[]] vs [{}]
+export async function processKeplerglJSONforDuckDb(
+ keplerJson
+): Promise<ReturnType<typeof processKeplerglJSON>> {
+ const res = processKeplerglJSON(keplerJson);
+
+ const datasets = res?.datasets || [];
+ for (const dataset of datasets) {
+ const rowsAll = dataset.data.rows;
+ const fieldsAll = dataset.data.fields;
+
+ const header = fieldsAll.map(f => f.name);
+ const sample = getSampleForTypeAnalyze({fields: header, rows: rowsAll});
+
+ const schema =
+ sample.length > 0
+ ? await sniffCsvSchema(sample)
+ : fieldsAll.reduce((acc, field) => {
+ acc[field.name] = fieldTypeToColumnType(field.type);
+ return acc;
+ }, {});
+ const fieldsUpd = consolidateFieldTypes(dataset.data.fields, schema);
+
+ dataset.data.fields = fieldsAll.map((f, i) => {
+ return {...fieldsUpd[i], ...f};
+ });
+ }
+
+ return res;
+}
+
+/*
+ * Process uploaded csv returned by loaders.gl as row object and string value
+ */
+export async function processCsvRowObject(
+ rawData: Record<string, string | null>[]
+): Promise<ProcessorResult> {
+ if (!Array.isArray(rawData) || !rawData.length) {
+ return {
+ fields: [],
+ rows: []
+ };
+ }
+
+ const header = Object.keys(rawData[0]); // [lat, lng, value]
+
+ // row object can still contain values like `Null` or `N/A`
+ cleanUpFalsyCsvValue(rawData);
+ const sample = getSampleForTypeAnalyze({fields: header, rows: rawData});
+ // add sample data to duckdb and get schema
+ const schema = await sniffCsvSchema(sample);
+
+ const fieldsWAnalyzerType = getFieldsFromData(sample, header);
+
+ const fields = consolidateFieldTypes(fieldsWAnalyzerType, schema);
+
+ return {rows: rawData, fields};
+}
+
+function cleanUpFalsyCsvValue(rows: RowData): void {
+ const re = new RegExp(CSV_NULLS, 'g');
+ for (let i = 0; i < rows.length; i++) {
+ Object.keys(rows[i]).forEach(key => {
+ // here we parse empty data as null
+ if (typeof rows[i][key] === 'string' && (rows[i][key] as string).match(re)) {
+ rows[i][key] = null;
+ }
+ });
+ }
+}
+
+// align type analyzer types with DuckDB csv auto detected column types
+function consolidateFieldTypes(fields, schema) {
+ return fields.map(field => {
+ const columnName = field.name;
+
+ const detectedColumnType = schema[columnName];
+
+ // TODO columnTypeToFieldType tranforms detected timestamps to string,
+ // completely breaking time filter logic.
+ // const fieldType = columnTypeToFieldType(detectedColumnType) || field.type;
+
+ return {
+ ...field,
+ // type: fieldType,
+ duckDBColumnType: detectedColumnType
+ };
+ });
+}
+
+function toCSVRow(row: unknown[]): string {
+ return `${row
+ .map(r => {
+ const rToStr = notNullorUndefined(r) ? String(r).replace(/"/g, '\\"') : '';
+ return rToStr.includes(',') ? `"${rToStr}"` : rToStr;
+ })
+ .join(',')}\n`;
+}
+
+// Use DucckDB to detect csv column schema
+async function sniffCsvSchema(sample: RowData) {
+ if (!Array.isArray(sample) || sample.length < 0) {
+ return;
+ }
+ const headerRow = toCSVRow(Object.keys(sample[0]));
+
+ const csvContent = (sample as (any[] | Record<string, unknown>)[]).reduce(
+ (accu: string, row) => `${accu}${toCSVRow(Object.values(row))}`,
+ headerRow
+ );
+
+ const db = await getDuckDB();
+ const c = await db.connect();
+
+ const fileName = generateHashId();
+ await db.registerFileText(`${fileName}-${generateHashId}.csv`, csvContent);
+
+ // https://duckdb.org/docs/data/csv/auto_detection.html
+ const result = await c.query(`
+ FROM sniff_csv('${fileName}-${generateHashId}.csv',
+ sample_size = 500,
+ auto_type_candidates = ['FLOAT', 'INTEGER', 'TIMESTAMP', 'DATE', 'TIME', 'VARCHAR', 'BOOLEAN']);
+ `);
+
+ const schemaResult = result.toArray().map(row => row.toJSON());
+
+ const columns = schemaResult[0].Columns;
+ const schema = {};
+ for (let i = 0; i < columns.length; i++) {
+ schema[columns.get(i).name] = columns.get(i).type;
+ }
+
+ return schema;
+}
+
+export function processGeojson(rawData: unknown): ProcessorResult {
+ const normalizedGeojson = normalize(rawData);
+
+ if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {
+ const error = new Error(
+ `Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](${GUIDES_FILE_FORMAT_DOC})`
+ );
+ throw error;
+ // fail to normalize geojson
+ }
+ // console.log(normalizedGeojson);
+
+ return {
+ rows: normalizedGeojson,
+ // TODO get fields to preserve field names?
+ fields: []
+ };
+}
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
new file mode 100644
index 000000000..1b604b2f1
--- /dev/null
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -0,0 +1,258 @@
+import * as arrow from 'apache-arrow';
+import {AsyncDuckDB, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
+
+import {DatasetType, DATASET_FORMATS} from '@kepler.gl/constants';
+import {KeplerTable} from '@kepler.gl/table';
+import {Field, ProtoDatasetField} from '@kepler.gl/types';
+import {
+ arrowSchemaToFields,
+ isArrowData,
+ isGeoJson,
+ isKeplerGlMap,
+ isRowObject,
+ processArrowBatches
+} from '@kepler.gl/processors';
+
+import {getDuckDB} from '../init';
+import {
+ processCsvRowObject,
+ processGeojson,
+ processKeplerglJSONforDuckDb,
+ ProcessorResult
+} from '../processors/data-processor';
+
+// TODO format geojson, all binary geogson columns should be tranformed to wkb
+const GEOMETRY_COLUMN = 'geom';
+
+type ImportDataToDuckProps = {
+ data: ProcessorResult;
+ db: AsyncDuckDB;
+ c: AsyncDuckDBConnection;
+};
+
+/**
+ * Creates an arrow table from arrow vectors and fields.
+ * @param columns An array of arrow vectors.
+ * @param fields An array of fields per arrow vector.
+ * @returns An arrow table.
+ */
+// TODO: consider using original drag&dropped arrow table, as it could contain extra metadata, not passed to the fields.
+const restoreArrowTable = (columns: arrow.Vector[], fields: ProtoDatasetField[]) => {
+ const creaOpts = {};
+ fields.map((field, index) => {
+ creaOpts[field.name] = columns[index];
+ });
+ return new arrow.Table(creaOpts);
+};
+
+const dropIfTableExists = async (c: AsyncDuckDBConnection, tableName: string) => {
+ try {
+ await c.query(`DROP TABLE IF EXISTS "${tableName}";`);
+ } catch (error) {
+ console.error('Dropping table failed', tableName, error);
+ }
+};
+
+export class KeplerGlDuckDbTable extends KeplerTable {
+ constructor(props) {
+ super(props);
+ }
+
+ async importRowData({data, db, c}: ImportDataToDuckProps): Promise<void> {
+ try {
+ const {fields, rows} = data;
+
+ // DATASET_FORMATS.keplergl loads data as any[][] instead of [{}]
+ // TODO potential memory usage explosion: original data, new object and then DuckDb table
+ const rowsForDb = Array.isArray(rows[0])
+ ? rows.map(row => {
+ const newRow = {};
+ row.forEach((value, index) => {
+ newRow[fields[index].name] = value;
+ });
+ return newRow;
+ })
+ : rows;
+
+ await db.registerFileText(this.id, JSON.stringify(rowsForDb));
+
+ const columns = fields.reduce((acc, field, index) => {
+ // @ts-expect-error TODO extend fields to contain duckDBColumnType
+ return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${field.duckDBColumnType}'`;
+ }, '');
+
+ const createTableSql = `
+ CREATE TABLE '${this.label}' AS
+ SELECT *
+ FROM read_json('${this.id}',
+ columns = {${columns}});
+ `;
+ await c.query(createTableSql);
+ } catch (error) {
+ console.log('importRowData', error);
+ }
+ }
+
+ async importGeoJsonData({data, db, c}: ImportDataToDuckProps): Promise<void> {
+ try {
+ const {rows} = data;
+ await db.registerFileText(this.id, JSON.stringify(rows));
+
+ const createTableSql = `
+ install spatial;
+ load spatial;
+ CREATE TABLE '${this.label}' AS
+ SELECT *
+ FROM ST_READ('${this.id}');
+ `;
+ await c.query(createTableSql);
+ } catch (error) {
+ console.error('importGeoJsonData', error);
+ }
+ }
+
+ async importArrowData({data, db, c}: ImportDataToDuckProps): Promise<void> {
+ try {
+ // 1) data.rows contains an arrow table created by Add to Map data from DuckDb query.
+ // 2) arrow table is in cols & fields when a file is dragged & dropped into Add Data To Map dialog.
+ const arrowTable =
+ data.rows instanceof arrow.Table
+ ? data.rows
+ : restoreArrowTable(data.cols || [], data.fields);
+
+ const setupSql = `
+ install spatial;
+ load spatial;
+ `;
+ await c.query(setupSql);
+ await c.insertArrowTable(arrowTable, {name: this.label});
+ } catch (error) {
+ // Known issues:
+ // 1) Arrow Type with extension name: geoarrow.point and format: +w:2 is not currently supported in DuckDB.
+ console.error('importArrowData', error);
+ }
+ }
+
+ /**
+ * Creates a table from data, returns an arrow table with the data
+ * @param data
+ * @returns {Promise<{fields: Field[], cols: any[]}>}
+ */
+ protected async createTableAndGetArrow(data): Promise<{fields: any[]; cols: arrow.Vector[]}> {
+ const db = await getDuckDB();
+ const c = await db.connect();
+
+ const tableName = this.label;
+ await dropIfTableExists(c, tableName);
+
+ let format = this.metadata.format;
+ if (!format) {
+ // format is missing when we load Kepler.gl examples
+ if (Array.isArray(data.rows?.[0]) || typeof data.rows?.[0] === 'object') {
+ format = DATASET_FORMATS.row;
+ } else if (data.rows?.type === 'FeatureCollection') {
+ format = DATASET_FORMATS.geojson;
+ } else if (data.cols?.[0] instanceof arrow.Vector) {
+ format = DATASET_FORMATS.arrow;
+ }
+ }
+
+ if (format === DATASET_FORMATS.row) {
+ await this.importRowData({data, db, c});
+ } else if (format === DATASET_FORMATS.geojson) {
+ await this.importGeoJsonData({data, db, c});
+ } else if (format === DATASET_FORMATS.arrow) {
+ await this.importArrowData({data, db, c});
+ } else {
+ console.error('Unrecognized format', format);
+ }
+
+ let fields: Field[] = [];
+ let cols: arrow.Vector[] = [];
+
+ try {
+ // We need to transform the binary geometry column to proper wkb.
+ // First check whether such a column exists, then query with ST_AsWKB.
+ const dummyResult = await c.query(`
+ SELECT * FROM '${tableName}' LIMIT 0;
+ `);
+ const hasBinaryGeoField = Boolean(
+ dummyResult.schema.fields.find(
+ f => f.name === GEOMETRY_COLUMN && arrow.DataType.isBinary(f.type)
+ )
+ );
+
+ // TODO don't do this for all columns named 'geom', check whether it's a wkb and not something else!
+ // query and transform geometry column to wkb
+ const arrowResult = await c.query(
+ `SELECT * ${
+ hasBinaryGeoField
+ ? `EXCLUDE ${GEOMETRY_COLUMN}, ST_AsWKB(${GEOMETRY_COLUMN}) as ${GEOMETRY_COLUMN}`
+ : ''
+ } FROM '${tableName}';`
+ );
+ const geoField = arrowResult.schema.fields.find(
+ f => f.name === GEOMETRY_COLUMN && arrow.DataType.isBinary(f.type)
+ );
+ geoField?.metadata.set('ARROW:extension:name', 'geoarrow.wkb');
+
+ fields = data.fields ?? arrowSchemaToFields(arrowResult.schema);
+ cols = [...Array(arrowResult.numCols).keys()]
+ .map(i => arrowResult.getChildAt(i))
+ .filter(col => col) as arrow.Vector[];
+ } catch (error) {
+ // Known issues:
+ // 1) ST_AsWKB binder error, no type match
+ console.error('DuckDb table: binary geometry column to wkb', error);
+ }
+
+ await c.close();
+
+ return {fields, cols};
+ }
+
+ async importData({data}: {data: ProcessorResult}): Promise<void> {
+ // VectorTile is a special case, ignore for now
+ if (this.type === DatasetType.VECTOR_TILE) {
+ super.importData({data: {...data, rows: []}});
+ return;
+ }
+
+ const {fields, cols} = await this.createTableAndGetArrow(data);
+
+ super.importData({data: {fields, cols, rows: []}});
+ }
+
+ async update(data) {
+ // VectorTile is a special case, ignore for now
+ if (this.type === DatasetType.VECTOR_TILE) {
+ super.importData({data});
+ return this;
+ }
+
+ const {cols} = await this.createTableAndGetArrow(data);
+
+ return super.update({cols, rows: [], fields: []});
+ }
+
+ static getFileProcessor = function (data: any, inputFormat?: string) {
+ let processor;
+ let format;
+ if (inputFormat === DATASET_FORMATS.arrow || isArrowData(data)) {
+ format = DATASET_FORMATS.arrow;
+ // TODO injest arrow files from disk without loaders
+ processor = processArrowBatches;
+ } else if (inputFormat === DATASET_FORMATS.keplergl || isKeplerGlMap(data)) {
+ format = DATASET_FORMATS.keplergl;
+ processor = processKeplerglJSONforDuckDb;
+ } else if (inputFormat === DATASET_FORMATS.row || isRowObject(data)) {
+ // csv file goes here
+ format = DATASET_FORMATS.row;
+ processor = processCsvRowObject; // directly import json object into duckdb-wasm
+ } else if (inputFormat === DATASET_FORMATS.geojson || isGeoJson(data)) {
+ format = DATASET_FORMATS.geojson;
+ processor = processGeojson;
+ }
+ return {processor, format};
+ };
+}
diff --git a/src/duckdb/src/utils/perf.ts b/src/duckdb/src/utils/perf.ts
new file mode 100644
index 000000000..c6e807029
--- /dev/null
+++ b/src/duckdb/src/utils/perf.ts
@@ -0,0 +1,44 @@
+// [email protected]:holdenmatt/duckdb-wasm-kit.git
+const FgRed = '\x1b[31m';
+const ResetColors = '\x1b[0m';
+
+/**
+ * Format a time interval between start/end timestamps.
+ *
+ * If no end time is given, use the current time.
+ */
+const formatElapsedTime = (
+ label: string,
+ start: number,
+ end: number | undefined = undefined
+): string => {
+ const endTime = end ?? performance.now();
+ const elapsed = endTime - start;
+
+ let timeString: string;
+ switch (true) {
+ case elapsed >= 1000:
+ timeString = `${(elapsed / 1000).toFixed(1)}s`;
+ break;
+ case elapsed >= 1:
+ timeString = `${elapsed.toFixed(0)}ms`;
+ break;
+ default:
+ timeString = `${elapsed.toFixed(3)}ms`;
+ break;
+ }
+
+ const message = `${FgRed}[${timeString}] ${ResetColors}${label}`;
+ return message;
+};
+
+/**
+ * Print the elapsed time to console.debug.
+ */
+export const logElapsedTime = (
+ label: string,
+ start: number,
+ end: number | undefined = undefined
+) => {
+ console.debug(formatElapsedTime(label, start, end));
+};
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index ab51dfe4d..77b9aa06c 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -151,7 +151,9 @@ class KeplerTable<F extends Field = Field> {
disableDataOperation?: boolean;
// table-injected metadata
- metadata: object;
+ metadata: Record<string, any>;
+
+ getFileProcessor?: (data: any, inputFormat?: string) => {data: any; format: string};
constructor({
info,
@@ -202,7 +204,7 @@ class KeplerTable<F extends Field = Field> {
this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);
}
- importData({data}: {data: ProtoDataset['data']}) {
+ async importData({data}: {data: ProtoDataset['data']}) {
const dataContainerData = data.cols ? data.cols : data.rows;
const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;
@@ -236,7 +238,7 @@ class KeplerTable<F extends Field = Field> {
* update table with new data
* @param data - new data e.g. the arrow data with new batches loaded
*/
- update(data: ProtoDataset['data']) {
+ async update(data: ProtoDataset['data']) {
const dataContainerData = data.cols ? data.cols : data.rows;
this.dataContainer.update?.(dataContainerData);
this.allIndexes = this.dataContainer.getPlainIndex();
diff --git a/yarn.lock b/yarn.lock
index 719be5255..df2151de3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2177,6 +2177,15 @@ __metadata:
languageName: node
linkType: hard
+"@duckdb/duckdb-wasm@npm:^1.28.0":
+ version: 1.29.0
+ resolution: "@duckdb/duckdb-wasm@npm:1.29.0"
+ dependencies:
+ apache-arrow: "npm:^17.0.0"
+ checksum: 10c0/caa550b09433adb709f997241ef00fd7ee93dc4cb4a03838d9d978536fdc640c80ef0f2c294248c688e4a38231be42287e4aba655e13d94bcd9aff4767a404a0
+ languageName: node
+ linkType: hard
+
"@emotion/is-prop-valid@npm:1.2.1":
version: 1.2.1
resolution: "@emotion/is-prop-valid@npm:1.2.1"
@@ -3396,8 +3405,17 @@ __metadata:
version: 0.0.0-use.local
resolution: "@kepler.gl/duckdb@workspace:src/duckdb"
dependencies:
+ "@duckdb/duckdb-wasm": "npm:^1.28.0"
+ "@kepler.gl/common-utils": "npm:3.1.0-alpha.5"
"@kepler.gl/constants": "npm:3.1.0-alpha.5"
+ "@kepler.gl/processors": "npm:3.1.0-alpha.5"
+ "@kepler.gl/table": "npm:3.1.0-alpha.5"
"@kepler.gl/types": "npm:3.1.0-alpha.5"
+ "@monaco-editor/react": "npm:^4.6.0"
+ "@radix-ui/react-collapsible": "npm:^1.1.0"
+ apache-arrow: "npm:>=15.0.0"
+ monaco-editor: "npm:^0.52.0"
+ react-resizable-panels: "npm:^2.1.7"
languageName: unknown
linkType: soft
@@ -5044,6 +5062,30 @@ __metadata:
languageName: node
linkType: hard
+"@monaco-editor/loader@npm:^1.4.0":
+ version: 1.4.0
+ resolution: "@monaco-editor/loader@npm:1.4.0"
+ dependencies:
+ state-local: "npm:^1.0.6"
+ peerDependencies:
+ monaco-editor: ">= 0.21.0 < 1"
+ checksum: 10c0/68938350adf2f42363a801d87f5d00c87d397d4cba7041141af10a9216bd35c85209b4723a26d56cb32e68eef61471deda2a450f8892891118fbdce7fa1d987d
+ languageName: node
+ linkType: hard
+
+"@monaco-editor/react@npm:^4.6.0":
+ version: 4.6.0
+ resolution: "@monaco-editor/react@npm:4.6.0"
+ dependencies:
+ "@monaco-editor/loader": "npm:^1.4.0"
+ peerDependencies:
+ monaco-editor: ">= 0.25.0 < 1"
+ react: ^16.8.0 || ^17.0.0 || ^18.0.0
+ react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0
+ checksum: 10c0/231e9a9b66a530db326f6732de0ebffcce6b79dcfaf4948923d78b9a3d5e2a04b7a06e1f85bbbca45a5ae15c107a124e4c5c46cabadc20a498fb5f2d05f7f379
+ languageName: node
+ linkType: hard
+
"@nebula.gl/edit-modes@npm:1.0.2-alpha.1":
version: 1.0.2-alpha.1
resolution: "@nebula.gl/edit-modes@npm:1.0.2-alpha.1"
@@ -6665,6 +6707,175 @@ __metadata:
languageName: node
linkType: hard
+"@radix-ui/primitive@npm:1.1.1":
+ version: 1.1.1
+ resolution: "@radix-ui/primitive@npm:1.1.1"
+ checksum: 10c0/6457bd8d1aa4ecb948e5d2a2484fc570698b2ab472db6d915a8f1eec04823f80423efa60b5ba840f0693bec2ca380333cc5f3b52586b40f407d9f572f9261f8d
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-collapsible@npm:^1.1.0":
+ version: 1.1.2
+ resolution: "@radix-ui/react-collapsible@npm:1.1.2"
+ dependencies:
+ "@radix-ui/primitive": "npm:1.1.1"
+ "@radix-ui/react-compose-refs": "npm:1.1.1"
+ "@radix-ui/react-context": "npm:1.1.1"
+ "@radix-ui/react-id": "npm:1.1.0"
+ "@radix-ui/react-presence": "npm:1.1.2"
+ "@radix-ui/react-primitive": "npm:2.0.1"
+ "@radix-ui/react-use-controllable-state": "npm:1.1.0"
+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ "@types/react-dom": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ "@types/react-dom":
+ optional: true
+ checksum: 10c0/8a725539c0c259ea53a0e35d4ddd3acca42cab5113fd537758450ad1e76f0b757423f18aca29364f963bef4f0624d57feb32bf9d12a3ea6b2c084b523ba65205
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-compose-refs@npm:1.1.1":
+ version: 1.1.1
+ resolution: "@radix-ui/react-compose-refs@npm:1.1.1"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/3e84580024e66e3cc5b9ae79355e787815c1d2a3c7d46e7f47900a29c33751ca24cf4ac8903314957ab1f7788aebe1687e2258641c188cf94653f7ddf8f70627
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-context@npm:1.1.1":
+ version: 1.1.1
+ resolution: "@radix-ui/react-context@npm:1.1.1"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/fc4ace9d79d7954c715ade765e06c95d7e1b12a63a536bcbe842fb904f03f88fc5bd6e38d44bd23243d37a270b4c44380fedddaeeae2d274f0b898a20665aba2
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-id@npm:1.1.0":
+ version: 1.1.0
+ resolution: "@radix-ui/react-id@npm:1.1.0"
+ dependencies:
+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/acf13e29e51ee96336837fc0cfecc306328b20b0e0070f6f0f7aa7a621ded4a1ee5537cfad58456f64bae76caa7f8769231e88dc7dc106197347ee433c275a79
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-presence@npm:1.1.2":
+ version: 1.1.2
+ resolution: "@radix-ui/react-presence@npm:1.1.2"
+ dependencies:
+ "@radix-ui/react-compose-refs": "npm:1.1.1"
+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ "@types/react-dom": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ "@types/react-dom":
+ optional: true
+ checksum: 10c0/0c6fa281368636308044df3be4c1f02733094b5e35ba04f26e610dd1c4315a245ffc758e0e176c444742a7a46f4328af1a9d8181e860175ec39338d06525a78d
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-primitive@npm:2.0.1":
+ version: 2.0.1
+ resolution: "@radix-ui/react-primitive@npm:2.0.1"
+ dependencies:
+ "@radix-ui/react-slot": "npm:1.1.1"
+ peerDependencies:
+ "@types/react": "*"
+ "@types/react-dom": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ "@types/react-dom":
+ optional: true
+ checksum: 10c0/6a562bec14f8e9fbfe0012d6c2932b0e54518fed898fa0622300c463611e77a4ca28a969f0cd484efd6570c01c5665dd6151f736262317d01715bc4da1a7dea6
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-slot@npm:1.1.1":
+ version: 1.1.1
+ resolution: "@radix-ui/react-slot@npm:1.1.1"
+ dependencies:
+ "@radix-ui/react-compose-refs": "npm:1.1.1"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/f3cc71c16529c67a8407a89e0ac13a868cafa0cd05ca185b464db609aa5996a3f00588695518e420bd47ffdb4cc2f76c14cc12ea5a38fc2ca3578a30d2ca58b9
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-use-callback-ref@npm:1.1.0":
+ version: 1.1.0
+ resolution: "@radix-ui/react-use-callback-ref@npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/e954863f3baa151faf89ac052a5468b42650efca924417470efd1bd254b411a94c69c30de2fdbb90187b38cb984795978e12e30423dc41e4309d93d53b66d819
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-use-controllable-state@npm:1.1.0":
+ version: 1.1.0
+ resolution: "@radix-ui/react-use-controllable-state@npm:1.1.0"
+ dependencies:
+ "@radix-ui/react-use-callback-ref": "npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/2af883b5b25822ac226e60a6bfde647c0123a76345052a90219026059b3f7225844b2c13a9a16fba859c1cda5fb3d057f2a04503f71780e607516492db4eb3a1
+ languageName: node
+ linkType: hard
+
+"@radix-ui/react-use-layout-effect@npm:1.1.0":
+ version: 1.1.0
+ resolution: "@radix-ui/react-use-layout-effect@npm:1.1.0"
+ peerDependencies:
+ "@types/react": "*"
+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+ peerDependenciesMeta:
+ "@types/react":
+ optional: true
+ checksum: 10c0/9bf87ece1845c038ed95863cfccf9d75f557c2400d606343bab0ab3192b9806b9840e6aa0a0333fdf3e83cf9982632852192f3e68d7d8367bc8c788dfdf8e62b
+ languageName: node
+ linkType: hard
+
"@react-aria/breadcrumbs@npm:3.5.13":
version: 3.5.13
resolution: "@react-aria/breadcrumbs@npm:3.5.13"
@@ -10906,7 +11117,7 @@ __metadata:
languageName: node
linkType: hard
-"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0":
+"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0, apache-arrow@npm:^17.0.0":
version: 17.0.0
resolution: "apache-arrow@npm:17.0.0"
dependencies:
@@ -22345,6 +22556,13 @@ __metadata:
languageName: node
linkType: hard
+"monaco-editor@npm:^0.52.0":
+ version: 0.52.2
+ resolution: "monaco-editor@npm:0.52.2"
+ checksum: 10c0/5a92da64f1e2ab375c0ce99364137f794d057c97bed10ecc65a08d6e6846804b8ecbd377eacf01e498f7dfbe1b21e8be64f728256681448f0484df90e767b435
+ languageName: node
+ linkType: hard
+
"moo@npm:^0.5.0":
version: 0.5.2
resolution: "moo@npm:0.5.2"
@@ -25409,6 +25627,16 @@ __metadata:
languageName: node
linkType: hard
+"react-resizable-panels@npm:^2.1.7":
+ version: 2.1.7
+ resolution: "react-resizable-panels@npm:2.1.7"
+ peerDependencies:
+ react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc
+ react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc
+ checksum: 10c0/644a57960507b809f571bf8c95f07a04e3aeb9b20405a1d9d8cb6deb72d2462192e6e5e9751237347e7818179a619a6f65de4b355e276334d10b8cfe737ca4e1
+ languageName: node
+ linkType: hard
+
"react-shallow-renderer@npm:^16.15.0":
version: 16.15.0
resolution: "react-shallow-renderer@npm:16.15.0"
@@ -27556,6 +27784,13 @@ __metadata:
languageName: node
linkType: hard
+"state-local@npm:^1.0.6":
+ version: 1.0.7
+ resolution: "state-local@npm:1.0.7"
+ checksum: 10c0/8dc7daeac71844452fafb514a6d6b6f40d7e2b33df398309ea1c7b3948d6110c57f112b7196500a10c54fdde40291488c52c875575670fb5c819602deca48bd9
+ languageName: node
+ linkType: hard
+
"state-toggle@npm:^1.0.0":
version: 1.0.3
resolution: "state-toggle@npm:1.0.3"