Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[feat] duckdb module updates (#2927)
Igor Dykhtace23c7668cfe
diff --git a/.eslintrc.js b/.eslintrc.jsindex cf3e6758e..04b42c4bb 100644--- a/.eslintrc.js+++ b/.eslintrc.js@@ -38,7 +38,8 @@ module.exports = {'consistent-return': 0,'comma-dangle': 1,'enzyme-deprecation/no-shallow': 2,- 'enzyme-deprecation/no-mount': 2+ 'enzyme-deprecation/no-mount': 2,+ 'no-constant-condition': ['error', {checkLoops: false}]},overrides: [{diff --git a/src/duckdb/package.json b/src/duckdb/package.jsonindex 4f5519bf5..cfc776aea 100644--- a/src/duckdb/package.json+++ b/src/duckdb/package.json@@ -2,7 +2,7 @@"name": "@kepler.gl/duckdb","author": "Shan He <[email protected]>","version": "3.1.0-alpha.5",- "description": "kepler.gl duckDB plugin",+ "description": "DuckDB plugin for Kepler.gl","license": "MIT","main": "dist/index.js","types": "dist/index.d.ts",@@ -25,8 +25,17 @@"umd"],"dependencies": {+ "@duckdb/duckdb-wasm": "^1.28.0",+ "@kepler.gl/common-utils": "3.1.0-alpha.5","@kepler.gl/constants": "3.1.0-alpha.5",- "@kepler.gl/types": "3.1.0-alpha.5"+ "@kepler.gl/processors": "3.1.0-alpha.5",+ "@kepler.gl/table": "3.1.0-alpha.5",+ "@kepler.gl/types": "3.1.0-alpha.5",+ "@monaco-editor/react": "^4.6.0",+ "@radix-ui/react-collapsible": "^1.1.0",+ "apache-arrow": ">=15.0.0",+ "monaco-editor": "^0.52.0",+ "react-resizable-panels": "^2.1.7"},"nyc": {"sourceMap": false,diff --git a/src/duckdb/src/components/index.tsx b/src/duckdb/src/components/index.tsxnew file mode 100644index 000000000..4d289b8de--- /dev/null+++ b/src/duckdb/src/components/index.tsx@@ -0,0 +1 @@+export * from './sql-panel';diff --git a/src/duckdb/src/components/monaco-editor.tsx b/src/duckdb/src/components/monaco-editor.tsxnew file mode 100644index 000000000..08dd6f6ec--- /dev/null+++ b/src/duckdb/src/components/monaco-editor.tsx@@ -0,0 +1,189 @@+import React, {useCallback, useMemo, useRef} from 'react';+import Editor, {OnChange, OnMount} from '@monaco-editor/react';+import * as monaco from 'monaco-editor';+// import {tableSchema as DEFAULT_SCHEMA} from './table-schema';+import uniq from 'lodash.uniq';+import uniqBy from 'lodash.uniqby';++const MONACO_OPTIONS: monaco.editor.IStandaloneEditorConstructionOptions = {+ minimap: {enabled: false},+ language: 'sql',+ contextmenu: false,+ renderLineHighlight: 'none',+ scrollBeyondLastLine: false,+ scrollbar: {alwaysConsumeMouseWheel: false},+ overviewRulerLanes: 0,+ automaticLayout: true,+ acceptSuggestionOnEnter: 'on',+ quickSuggestionsDelay: 400,+ matchOnWordStartOnly: false,+ tabCompletion: 'off',+ lineNumbers: 'off'+};++function parseSqlAndFindTableNameAndAliases(sql: string) {+ const regex = /\b(?:FROM|JOIN)\s+([^\s.]+(?:\.[^\s.]+)?)\s*(?:AS)?\s*([^\s,]+)?/gi;+ const tables: {table_name: string; alias: string}[] = [];++ while (true) {+ const match = regex.exec(sql);+ if (!match) {+ break;+ }+ const table_name = match[1];+ if (!/\(/.test(table_name)) {+ // exclude function calls+ let alias = match[2] as string | null;+ if (alias && /on|where|inner|left|right|join/.test(alias)) {+ alias = null;+ }+ tables.push({+ table_name,+ alias: alias || table_name+ });+ }+ }++ return tables;+}++interface MonacoEditorProps {+ code: string;+ isReadOnly?: boolean;+ onChange: OnChange;+ onRunQuery: () => void;+ tableSchema?: {table_name: string; column_name: string}[];+}++const MonacoEditor: React.FC<MonacoEditorProps> = ({+ onRunQuery,+ onChange,+ code,+ tableSchema,+ isReadOnly+}) => {+ // private editor?: monaco.editor.IStandaloneCodeEditor;+ const schemaTableNames = useMemo(+ () => (tableSchema ? uniq(tableSchema.map(d => d.table_name)) : []),+ [tableSchema]+ );+ const schemaTableNamesSet = useMemo(() => new Set(schemaTableNames), [schemaTableNames]);+ const handleRunQueryRef = useRef(onRunQuery);+ handleRunQueryRef.current = onRunQuery;++ const handleEditorDidMount: OnMount = useCallback(+ editor => {+ // this.editor = editor;+ editor.focus();++ editor.addAction({+ id: 'run-query',+ label: 'Run Query',+ keybindings: [+ monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter,+ monaco.KeyMod.Shift | monaco.KeyCode.Enter+ ],+ contextMenuGroupId: 'custom',+ contextMenuOrder: 0,+ run: () => handleRunQueryRef.current()+ });++ monaco.languages.registerCompletionItemProvider('*', {+ provideCompletionItems: (model, position, context, cancelationToken) => {+ const suggestions: monaco.languages.CompletionItem[] = [+ {+ label: 'myCustomSnippet',+ kind: monaco.languages.CompletionItemKind.Snippet,+ insertText: 'This is a piece of custom code',+ insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,+ documentation: 'This is a piece of custom code'+ // TODO: range is missing+ } as monaco.languages.CompletionItem+ ];++ const fullQueryText = model.getValue();++ const tableNamesAndAliases = new Map<string, string>(+ parseSqlAndFindTableNameAndAliases(fullQueryText).map(({table_name, alias}) => [+ alias,+ table_name+ ])+ );++ const thisLine = model.getValueInRange({+ startLineNumber: position.lineNumber,+ startColumn: 1,+ endLineNumber: position.lineNumber,+ endColumn: position.column+ });+ const thisToken = thisLine.trim().split(' ').slice(-1)?.[0] || '';++ const lastTokenBeforeSpace = /\s?(\w+)\s+\w+$/.exec(thisLine.trim())?.[1];+ const lastTokenBeforeDot = /(\w+)\.\w*$/.exec(thisToken)?.[1];++ // console.log(tableNamesAndAliases, thisToken, lastTokenBeforeSpace, lastTokenBeforeDot);++ if (lastTokenBeforeSpace && /from|join|update|into/.test(lastTokenBeforeSpace)) {+ suggestions.push(+ ...schemaTableNames.map(+ table_name =>+ ({+ label: table_name,+ kind: monaco.languages.CompletionItemKind.Field,+ insertText: table_name+ // TODO: range is missing+ } as monaco.languages.CompletionItem)+ )+ );+ }++ if (lastTokenBeforeDot) {+ let table_name = null as string | null;+ if (schemaTableNamesSet.has(lastTokenBeforeDot)) {+ table_name = lastTokenBeforeDot;+ } else if (tableNamesAndAliases.get(lastTokenBeforeDot)) {+ table_name = tableNamesAndAliases.get(lastTokenBeforeDot) as string;+ }+ if (table_name && tableSchema) {+ suggestions.push(+ ...tableSchema+ .filter(d => d.table_name === table_name)+ .map(+ ({table_name, column_name}) =>+ ({+ label: column_name,+ kind: monaco.languages.CompletionItemKind.Field,+ insertText: column_name+ // TODO: range is missing+ } as monaco.languages.CompletionItem)+ )+ );+ }+ }++ return {+ suggestions: uniqBy(suggestions, s => s.insertText)+ };+ }+ });+ },+ [tableSchema]+ );++ return (+ <Editor+ height="100%"+ theme="vs-dark"+ defaultLanguage="sql"+ defaultValue={code}+ onChange={onChange}+ onMount={handleEditorDidMount}+ options={{+ ...MONACO_OPTIONS,+ readOnly: isReadOnly+ }}+ />+ );+};++export default MonacoEditor;diff --git a/src/duckdb/src/components/preview-data-panel.tsx b/src/duckdb/src/components/preview-data-panel.tsxnew file mode 100644index 000000000..d152840f8--- /dev/null+++ b/src/duckdb/src/components/preview-data-panel.tsx@@ -0,0 +1,158 @@+// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.++import React, {useCallback, useMemo, useState, CSSProperties} from 'react';++import {DataTable, renderedSize} from '@kepler.gl/components';+import {parseFieldValue, createDataContainer} from '@kepler.gl/utils';+import {arrowSchemaToFields} from '@kepler.gl/processors';+import {DataForm} from '@kepler.gl/utils';+import {withTheme} from 'styled-components';++type BaseComponentProps = {+ className?: string;+ style?: CSSProperties;+};++const DEFAULT_ROWS_TO_CALCULATE_PREVIEW = 100;+// min Cellsize should take into account option button and field token+const minCellSize = 80;+// option buttons and field token+const optionButtonWidth = 20;+const pinButton = 20;+const cellPadding = 20;++export type ColMeta = {+ [key: string]: {+ colIdx: number;+ name: string;+ displayName: string;+ type: string;+ };+};++export type DataTableStyle = {+ minCellSize?: number;+ cellPadding?: number;+ fontSize?: number;+ font?: string;+ optionsButton?: number;+};++export type PreviewDataPanelProps = BaseComponentProps & {+ result: any;+ rowsToCalculatePreview?: number;+ theme?: any;+ setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;+ defaultPinnedColumns?: string[];+ dataTableStyle: DataTableStyle;+ onAddResultToMap: (result: any) => void;+};++const PreviewDataPanelWOTheme: React.FC<PreviewDataPanelProps> = ({+ result,+ rowsToCalculatePreview = DEFAULT_ROWS_TO_CALCULATE_PREVIEW,+ setColumnDisplayFormat,+ defaultPinnedColumns = [],+ theme+}) => {+ const [pinnedColumns, setPinnedColumns] = useState<string[]>(defaultPinnedColumns);+ const fields = useMemo(() => arrowSchemaToFields(result.schema), [result.schema]);+ const dataContainer = useMemo(() => {+ // const fields = arrowSchemaToFields(result.schema);++ const cols = [...Array(result.numCols).keys()].map(i => result.getChildAt(i));++ const dataContainer = createDataContainer(cols, {+ fields,+ inputDataFormat: DataForm.COLS_ARRAY+ });+ return dataContainer;+ }, [result, fields]);++ const columns = useMemo(() => fields.map(f => f.name), [fields]);+ const colMeta = useMemo(+ () =>+ fields.reduce(+ (acc, {name, displayName, type, displayFormat}, colIdx) => ({+ ...acc,+ [name]: {+ // because '' || 'aaa' = 'aaa'+ name: displayName !== undefined ? displayName : name,+ displayName,+ displayFormat,+ type,+ colIdx+ }+ }),+ {}+ ),+ [fields]+ );+ const copyTableColumn = useCallback(+ column => {+ const {colIdx, type} = colMeta[column];+ const text = dataContainer+ .mapIndex(row => parseFieldValue(dataContainer.valueAt(row.index, colIdx), type))+ .join('\n');+ navigator?.clipboard.writeText(text);+ },+ [colMeta, dataContainer]+ );+ const pinTableColumn = useCallback(+ column =>+ pinnedColumns.includes(column)+ ? setPinnedColumns(pinnedColumns.filter(c => c !== column))+ : setPinnedColumns([...pinnedColumns, column]),+ [pinnedColumns]+ );++ // TODO Potentially costly operation for non row based data containers. Revisit sorting below.+ const dataTableStyle = useMemo(+ () => ({+ minCellSize,+ cellPadding,+ optionsButton:+ theme.fieldTokenWidth + theme.fieldTokenRightMargin + optionButtonWidth + pinButton,+ fontSize: theme.cellFontSize,+ font: theme.fontFamily+ }),+ [theme]+ );+ const cellSizeCache = useMemo(() => {+ return columns.reduce((acc, column) => {+ const {colIdx, displayName, type} = colMeta[column];+ return {+ ...acc,+ [column]: renderedSize({+ text: {+ dataContainer,+ column: displayName+ },+ colIdx,+ type,+ numRowsToCalculate: rowsToCalculatePreview,+ ...dataTableStyle+ })+ };+ }, {});+ }, [columns, colMeta, dataContainer, rowsToCalculatePreview, dataTableStyle]);++ return (+ <DataTable+ colMeta={colMeta}+ columns={columns}+ cellSizeCache={cellSizeCache}+ dataContainer={dataContainer}+ pinnedColumns={pinnedColumns}+ // sortColumn={sortColumnConfig}+ copyTableColumn={copyTableColumn}+ pinTableColumn={pinTableColumn}+ // sortTableColumn={setTableSortColumn}+ setColumnDisplayFormat={setColumnDisplayFormat ?? (() => null)}+ />+ );+};++export const PreviewDataPanel = withTheme(+ PreviewDataPanelWOTheme+) as React.FC<PreviewDataPanelProps>;diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsxnew file mode 100644index 000000000..93b5195ec--- /dev/null+++ b/src/duckdb/src/components/schema-panel.tsx@@ -0,0 +1,107 @@+import React, {useCallback, useEffect, useState} from 'react';+import {useSelector} from 'react-redux';+import styled from 'styled-components';+import {arrowDataTypeToFieldType} from '@kepler.gl/utils';+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {VisState} from '@kepler.gl/schemas';+import {Tree, DatasetNode, ColumnNode, TreeNodeData} from './tree';+import {getDuckDB} from '../init';++// TODO note that demo state is available in demo-app, but not when add modules to dependencies in a custom map+type State = {+ demo?: {+ keplerGl: {+ map: {+ visState: VisState;+ };+ };+ };+};++const StyledSchemaPanel = styled.div`+ color: ${props => props.theme.textColor};+ font-size: 12px;+ padding: 12px;+ font-family: ${props => props.theme.fontFamily};+`;++async function getColumnSchema(c, name) {+ const columnResult = await c.query(`Select * from '${name}' LIMIT 1;`);++ return {+ key: name,+ object: {+ type: 'dataset',+ tableName: name+ },+ children: columnResult.schema.fields.map(field => {+ const isGeoArrowColumn = field.metadata.get('ARROW:extension:name')?.startsWith('geoarrow');+ return {+ key: field.name,+ object: {+ type: 'column',+ name: field.name,+ arrowType: field.type,+ fieldType: isGeoArrowColumn+ ? ALL_FIELD_TYPES.geoarrow+ : arrowDataTypeToFieldType(field.type)+ }+ };+ })+ };+}++function getSchemaSuggestion(result) {+ return result.reduce((accu, data) => {+ const columns = data.children.map(child => ({+ column_name: child.key,+ table_name: data.key+ }));+ return accu.concat(columns);+ }, []);+}+export const SchemaPanel = ({setTableSchema}) => {+ const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);+ const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);+ useEffect(() => {+ getTableSchema();+ }, [datasets]);+ const getTableSchema = useCallback(async () => {+ const db = await getDuckDB();+ const c = await db.connect();++ const tableResult = await c.query('SHOW TABLES;');++ const tableNames = tableResult.getChildAt(0)?.toJSON();++ const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));+ const tableSchema = getSchemaSuggestion(result);++ setColumnSchemas(result);+ setTableSchema(tableSchema);+ c.close();+ }, [setColumnSchemas, setTableSchema]);++ return (+ <StyledSchemaPanel>+ {columnSchemas?.length ? (+ columnSchemas.map(data => (+ <Tree+ key={data.key}+ treeData={data}+ renderNode={node => {+ if (node.object.type === 'dataset') {+ return <DatasetNode node={node} />;+ } else if (node.object.type === 'column') {+ return <ColumnNode node={node} />;+ }+ return null;+ }}+ />+ ))+ ) : (+ <div>No tables found</div>+ )}+ </StyledSchemaPanel>+ );+};diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsxnew file mode 100644index 000000000..586c2f171--- /dev/null+++ b/src/duckdb/src/components/sql-panel.tsx@@ -0,0 +1,264 @@+import * as arrow from 'apache-arrow';+import React, {useCallback, useState, useEffect} from 'react';+import {useDispatch} from 'react-redux';+import styled from 'styled-components';+import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';+import MonacoEditor from './monaco-editor';+import {SchemaPanel} from './schema-panel';+import {PreviewDataPanel} from './preview-data-panel';+import {getDuckDB} from '../init';+import {Button, IconButton, Tooltip} from '@kepler.gl/components';+import {generateHashId} from '@kepler.gl/common-utils';+import {addDataToMap} from '@kepler.gl/actions';+import {Icons, LoadingSpinner} from '@kepler.gl/components';+import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';+import {isAppleDevice} from '@kepler.gl/utils';+import {arrowSchemaToFields} from '@kepler.gl/processors';++const StyledSqlPanel = styled.div`+ display: flex;+ height: 100%;+ background-color: ${sidePanelBg};+ border-left: 1px solid #333;+`;++const StyledSqlActions = styled.div`+ flex: 0 0 50px;+ display: flex;+ padding-top: 12px;+ align-items: center;+ flex-direction: column;+ justify-content: flex-start;+ border-bottom: 1px solid #333;+`;++const StyledSqlEditor = styled.div`+ display: flex;+ flex-direction: row;+ flex: 1;+ min-width: 0; // Prevents flex child from overflowing+ height: 100%;+`;++const StyledEditorPanel = styled.div`+ background-color: ${sidePanelBg};+ display: flex;+ height: 100%;+ flex-direction: row;+ gap: 5px;+ flex-grow: 1;+`;++const StyledDataPanel = styled.div`+ display: flex;+ flex-direction: column;+ height: 100%;+ background: white;+`;++const StyledEditor = styled.div<{isRunning: boolean}>`+ display: flex;+ padding: 12px 12px 12px 0;+ background-color: rgb(30, 30, 30);+ opacity: ${props => (props.isRunning ? 0.5 : 1)};+ width: 100%;+ height: 100%;+ min-height: 0; // Prevents flex child from overflowing+`;++const StyledResultActions = styled.div`+ background-color: white;+ padding: 2px 12px;+ display: flex;+ justify-content: space-between;+ align-items: center;+ font-size: 12px;+`;++const StyledResizeHandle = styled(PanelResizeHandle)`+ background-color: ${panelBorderColor};+ width: 4px;+ cursor: col-resize;++ &:hover {+ background-color: #555;+ }+`;++const StyledVerticalResizeHandle = styled(PanelResizeHandle)`+ background-color: ${panelBorderColor};+ height: 4px;+ cursor: row-resize;++ &:hover {+ background-color: #555;+ }+`;++const StyledLoadingContainer = styled.div`+ display: flex;+ justify-content: center;+ align-items: center;+ height: 100%;+`;++const StyledErrorContainer = styled.pre`+ padding: 12px;+ color: red;+ font-size: 12px;+ background-color: ${sidePanelBg};+ height: 100%;+ overflow: auto;+`;++type SqlPanelProps = {+ initialSql?: string;+};++export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {+ const [sql, setSql] = useState(() => {+ const params = new URLSearchParams(window.location.search);+ return params.get('sql') || initialSql;+ });+ const [result, setResult] = useState<null | arrow.Table>(null);+ const [error, setError] = useState<Error | null>(null);+ const [counter, setCounter] = useState(0);+ const [tableSchema, setTableSchema] = useState([]);+ const [isRunning, setIsRunning] = useState(false);+ const [isMac] = useState(() => isAppleDevice());+ const dispatch = useDispatch();++ useEffect(() => {+ const currentUrl = new URL(window.location.href);+ if (sql) {+ currentUrl.searchParams.set('sql', sql);+ } else {+ currentUrl.searchParams.delete('sql');+ }+ window.history.replaceState({}, '', currentUrl.toString());+ }, [sql]);++ const runQuery = useCallback(async () => {+ setIsRunning(true);+ try {+ if (!sql?.trim()) {+ setError(new Error('Query is empty'));+ return;+ }++ const db = await getDuckDB();+ const connection = await db.connect();++ const arrowResult = await connection.query(sql);+ setResult(arrowResult);+ setError(null);++ connection.close();+ } catch (e) {+ setError(e as Error);+ } finally {+ setIsRunning(false);+ }+ }, [sql]);++ const onChange = useCallback(+ value => {+ setSql(value);+ },+ [setSql]+ );++ const onAddResultToMap = useCallback(() => {+ if (!result) return;++ const keplerFields = arrowSchemaToFields(result.schema);++ const datasetToAdd = {+ data: {+ fields: keplerFields,+ // TODO type AddDataToMapPayload -> rows -> + arrow.Table+ rows: result as any+ },+ info: {+ id: generateHashId(),+ label: `Query Result${counter > 0 ? ` (${counter})` : ''}`,+ format: 'arrow'+ }+ };++ dispatch(addDataToMap({datasets: [datasetToAdd]}));+ setCounter(counter + 1);+ }, [result, counter, dispatch]);++ return (+ <StyledSqlPanel>+ <PanelGroup direction="horizontal">+ <Panel defaultSize={20} minSize={15}>+ <SchemaPanel setTableSchema={setTableSchema} />+ </Panel>++ <StyledResizeHandle />++ <Panel minSize={40}>+ <StyledSqlEditor>+ <PanelGroup direction="vertical">+ <Panel defaultSize={30} className="editor-panel">+ <StyledEditorPanel>+ <StyledSqlActions>+ <IconButton+ data-tip+ data-for="run-query-tooltip"+ onClick={runQuery}+ disabled={isRunning || !sql}+ >+ <Icons.Play height="18px" />+ <Tooltip id="run-query-tooltip" place="top" effect="solid">+ Run Query ({isMac ? '⌘Enter' : 'Ctrl+Enter'} or Shift+Enter)+ </Tooltip>+ </IconButton>+ </StyledSqlActions>+ <StyledEditor isRunning={isRunning}>+ <MonacoEditor+ isReadOnly={isRunning}+ onRunQuery={runQuery}+ onChange={onChange}+ tableSchema={tableSchema}+ code={sql}+ />+ </StyledEditor>+ </StyledEditorPanel>+ </Panel>++ <StyledVerticalResizeHandle />+ <Panel>+ {isRunning ? (+ <StyledLoadingContainer>+ <LoadingSpinner />+ </StyledLoadingContainer>+ ) : error ? (+ <StyledErrorContainer>+ <div>{error.message}</div>+ </StyledErrorContainer>+ ) : result ? (+ <StyledDataPanel>+ <StyledResultActions>+ <Button secondary onClick={onAddResultToMap}>+ Add to Map+ </Button>+ <div>{result.numRows} rows</div>+ </StyledResultActions>+ <PreviewDataPanel+ result={result}+ dataTableStyle={{}}+ onAddResultToMap={onAddResultToMap}+ />+ </StyledDataPanel>+ ) : null}+ </Panel>+ </PanelGroup>+ </StyledSqlEditor>+ </Panel>+ </PanelGroup>+ </StyledSqlPanel>+ );+};diff --git a/src/duckdb/src/components/tree.tsx b/src/duckdb/src/components/tree.tsxnew file mode 100644index 000000000..11267da43--- /dev/null+++ b/src/duckdb/src/components/tree.tsx@@ -0,0 +1,106 @@+// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.++import {Collapsible, CollapsibleTrigger, CollapsibleContent} from '@radix-ui/react-collapsible';+import styled from 'styled-components';+import React, {useEffect, useState} from 'react';+import classNames from 'classnames';++import {Icons, FieldToken} from '@kepler.gl/components';++export type TreeNodeData<T> = {+ key: string;+ object: T;+ children?: TreeNodeData<T>[];+ isOpen?: boolean;+};++export type TreeProps<T> = {+ className?: string;+ treeData: TreeNodeData<T>;+ renderNode: TreeNodeProps<T>['renderNode'];+};++// TODO: consider using react-vtree/react-window for virtualization++/**+ * Component that renders a generic tree.+ * @param treeData - The tree data.+ * @param renderNode - A function that renders a tree node.+ */+export function Tree<T>(props: TreeProps<T>): React.ReactElement {+ const {treeData, renderNode} = props;+ return (+ <div className="flex flex-col">+ <TreeNode<T> treeData={treeData} renderNode={renderNode} />+ </div>+ );+}++export type TreeNodeProps<T> = {+ treeData: TreeNodeData<T>;+ renderNode: (node: TreeNodeData<T>, isOpen: boolean) => JSX.Element | null;+};++const StyledCollapsibleTriggerContent = styled.div`+ display: flex;+ width: 100%;+ cursor: pointer;+ align-items: center;+`;+/**+ * Component that renders a tree node.+ */+function TreeNode<T>(props: TreeNodeProps<T>): JSX.Element | null {+ const {treeData, renderNode} = props;+ const {children} = treeData;+ const [isOpen, setIsOpen] = useState(Boolean(treeData.isOpen));+ useEffect(() => {+ setIsOpen(Boolean(treeData.isOpen));+ }, [treeData.isOpen]);+ if (!children) {+ return renderNode(treeData, isOpen);+ }+ return (+ <Collapsible open={isOpen} onOpenChange={setIsOpen}>+ <CollapsibleTrigger className="w-full" asChild>+ <StyledCollapsibleTriggerContent>+ <Icons.ArrowRight+ className={classNames('flex-shrink-0 text-gray-500', {'rotate-90 transform': isOpen})}+ height="18px"+ />+ {renderNode(treeData, isOpen)}+ </StyledCollapsibleTriggerContent>+ </CollapsibleTrigger>+ <CollapsibleContent className="pl-4">+ {isOpen+ ? children.map(child => (+ <TreeNode<T> key={child.key} treeData={child} renderNode={renderNode} />+ ))+ : null}+ </CollapsibleContent>+ </Collapsible>+ );+}++export const DatasetNode = ({node}) => {+ return <div>{node.object.tableName}</div>;+};++const StyledColumnNode = styled.div`+ display: flex;+ align-items: center;+ padding: 4px 0;+ margin-left: 16px;++ .column-name {+ margin-left: 8px;+ }+`;+export const ColumnNode = ({node}) => {+ return (+ <StyledColumnNode>+ <FieldToken type={node.object.fieldType} />+ <div className="column-name">{node.object.name}</div>+ </StyledColumnNode>+ );+};diff --git a/src/duckdb/src/index.ts b/src/duckdb/src/index.tsindex 762aa28fe..59735e2b1 100644--- a/src/duckdb/src/index.ts+++ b/src/duckdb/src/index.ts@@ -1,6 +1,4 @@-// SPDX-License-Identifier: MIT-// Copyright contributors to the kepler.gl project--const DUCK = 0;--export default DUCK;+import {keplerGlDuckDBPlugin} from './plugin';+export {KeplerGlDuckDbTable} from './table/duckdb-table';+export {SqlPanel} from './components';+export default keplerGlDuckDBPlugin;diff --git a/src/duckdb/src/init.ts b/src/duckdb/src/init.tsnew file mode 100644index 000000000..bf83fe33b--- /dev/null+++ b/src/duckdb/src/init.ts@@ -0,0 +1,86 @@+// [email protected]:holdenmatt/duckdb-wasm-kit.git+import * as duckdb from '@duckdb/duckdb-wasm';+import {AsyncDuckDB, DuckDBConfig} from '@duckdb/duckdb-wasm';+import {logElapsedTime} from './utils/perf';++export let DEBUG: boolean | undefined;++let DuckDB: Promise<AsyncDuckDB> | undefined;++/**+ * Initialize DuckDB, ensuring we only initialize it once.+ *+ * @param debug If true, log DuckDB logs and elapsed times to the console.+ * @param config An optional DuckDBConfig object.+ */+export async function initializeDuckDb(options?: {+ debug?: boolean;+ config?: DuckDBConfig;+}): Promise<AsyncDuckDB> {+ const {debug = false, config} = options || {};+ DEBUG = debug;++ if (DuckDB === undefined) {+ DuckDB = _initializeDuckDb(config);+ }+ return DuckDB;+}++/**+ * Initialize DuckDB with a browser-specific Wasm bundle.+ */+const _initializeDuckDb = async (config?: DuckDBConfig): Promise<AsyncDuckDB> => {+ const start = performance.now();++ // Select a bundle based on browser checks+ const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();+ const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);++ const worker_url = URL.createObjectURL(+ new Blob([`importScripts("${bundle.mainWorker!}");`], {+ type: 'text/javascript'+ })+ );++ // Instantiate the async version of DuckDB-wasm+ const worker = new Worker(worker_url);+ const logger = DEBUG ? new duckdb.ConsoleLogger() : new duckdb.VoidLogger();+ const db = new AsyncDuckDB(logger, worker);+ await db.instantiate(bundle.mainModule, bundle.pthreadWorker);+ URL.revokeObjectURL(worker_url);++ if (config) {+ if (config.path) {+ const res = await fetch(config.path);+ const buffer = await res.arrayBuffer();+ const fileNameMatch = config.path.match(/[^/]*$/);+ if (fileNameMatch) {+ config.path = fileNameMatch[0];+ }+ await db.registerFileBuffer(config.path, new Uint8Array(buffer));+ }+ await db.open(config);+ }++ if (DEBUG) {+ logElapsedTime('DuckDB initialized', start);+ if (config) {+ console.debug(`DuckDbConfig: ${JSON.stringify(config, null, 2)}`);+ }+ }+ return db;+};++/**+ * Get the instance of DuckDB, initializing it if needed.+ *+ * Typically `useDuckDB` is used in React components instead, but this+ * method provides access outside of React contexts.+ */+export const getDuckDB = async (): Promise<AsyncDuckDB> => {+ if (DuckDB) {+ return DuckDB;+ } else {+ return await initializeDuckDb();+ }+};diff --git a/src/duckdb/src/plugin.ts b/src/duckdb/src/plugin.tsnew file mode 100644index 000000000..f0b139933--- /dev/null+++ b/src/duckdb/src/plugin.ts@@ -0,0 +1,8 @@+import {initializeDuckDb} from './init';++export const keplerGlDuckDBPlugin = {+ async init() {+ await initializeDuckDb();+ console.log('kepler.gl DuckDB Plugin initialized');+ }+};diff --git a/src/duckdb/src/processors/data-processor.ts b/src/duckdb/src/processors/data-processor.tsnew file mode 100644index 000000000..9e6de3fb3--- /dev/null+++ b/src/duckdb/src/processors/data-processor.ts@@ -0,0 +1,252 @@+import * as arrow from 'apache-arrow';+import normalize from '@mapbox/geojson-normalize';++import {+ getSampleForTypeAnalyze,+ getFieldsFromData,+ generateHashId,+ notNullorUndefined+} from '@kepler.gl/common-utils';+import {ProtoDatasetField} from '@kepler.gl/types';+import {ALL_FIELD_TYPES, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';+import {processKeplerglJSON} from '@kepler.gl/processors';++import {getDuckDB} from '../init';++export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;++type RowsAsArray = any[][];+type RowsAsObject = Record<string, unknown>[];+type RowData = RowsAsArray | RowsAsObject;++export type ProcessorResult = {+ cols?: arrow.Vector[];+ rows: RowData;+ fields: ProtoDatasetField[];+};++enum COLUMN_TYPES {+ GEOMETRY = 'GEOMETRY',+ BOOLEAN = 'BOOLEAN',+ INTEGER = 'INTEGER',+ FLOAT = 'FLOAT',+ TIME = 'TIME',+ DATE = 'DATE',+ TIMESTAMP = 'TIMESTAMP',+ VARCHAR = 'VARCHAR'+}+// @ts-expect-error https://duckdb.org/docs/data/csv/auto_detection.html#type-detection+const COLUMN_TYPES_PRIORITIES = [+ 'GEOMETRY',+ 'BOOLEAN',+ 'INTEGER',+ 'FLOAT',+ 'TIME',+ 'DATE',+ 'TIMESTAMP',+ 'VARCHAR'+];++export function columnTypeToFieldType(columnType: string): string | null {+ switch (columnType) {+ case COLUMN_TYPES.BOOLEAN:+ return ALL_FIELD_TYPES.boolean;+ case COLUMN_TYPES.DATE:+ return ALL_FIELD_TYPES.date;+ case COLUMN_TYPES.INTEGER:+ return ALL_FIELD_TYPES.integer;+ case COLUMN_TYPES.FLOAT:+ return ALL_FIELD_TYPES.real;+ case COLUMN_TYPES.VARCHAR:+ return ALL_FIELD_TYPES.string;+ case COLUMN_TYPES.TIMESTAMP:+ return ALL_FIELD_TYPES.timestamp;+ case COLUMN_TYPES.GEOMETRY:+ return ALL_FIELD_TYPES.geojson;+ default:+ // is there any other type we didn't cover?+ console.warn(`Unsupported DuckDB Column type: ${columnType}`);+ return null;+ }+}++export function fieldTypeToColumnType(fieldType: string): string | null {+ switch (fieldType) {+ case ALL_FIELD_TYPES.boolean:+ return COLUMN_TYPES.BOOLEAN;+ case ALL_FIELD_TYPES.date:+ return COLUMN_TYPES.DATE;+ case ALL_FIELD_TYPES.integer:+ return COLUMN_TYPES.INTEGER;+ case ALL_FIELD_TYPES.real:+ return COLUMN_TYPES.FLOAT;+ case ALL_FIELD_TYPES.string:+ return COLUMN_TYPES.VARCHAR;+ case ALL_FIELD_TYPES.timestamp:+ return COLUMN_TYPES.TIMESTAMP;+ case ALL_FIELD_TYPES.geojson:+ return COLUMN_TYPES.GEOMETRY;+ default:+ // is there any other type we didn't cover?+ console.warn(`Unsupported Field type for DuckDb: ${fieldType}`);+ return null;+ }+}++/*+ * Process uploaded exported Keplergl json map.+ * We need to populate field with proper DuckDb compatible type.+ */+// TODO: merge with logic from processCsvRowObject. Different formats: [[]] vs [{}]+export async function processKeplerglJSONforDuckDb(+ keplerJson+): Promise<ReturnType<typeof processKeplerglJSON>> {+ const res = processKeplerglJSON(keplerJson);++ const datasets = res?.datasets || [];+ for (const dataset of datasets) {+ const rowsAll = dataset.data.rows;+ const fieldsAll = dataset.data.fields;++ const header = fieldsAll.map(f => f.name);+ const sample = getSampleForTypeAnalyze({fields: header, rows: rowsAll});++ const schema =+ sample.length > 0+ ? await sniffCsvSchema(sample)+ : fieldsAll.reduce((acc, field) => {+ acc[field.name] = fieldTypeToColumnType(field.type);+ return acc;+ }, {});+ const fieldsUpd = consolidateFieldTypes(dataset.data.fields, schema);++ dataset.data.fields = fieldsAll.map((f, i) => {+ return {...fieldsUpd[i], ...f};+ });+ }++ return res;+}++/*+ * Process uploaded csv returned by loaders.gl as row object and string value+ */+export async function processCsvRowObject(+ rawData: Record<string, string | null>[]+): Promise<ProcessorResult> {+ if (!Array.isArray(rawData) || !rawData.length) {+ return {+ fields: [],+ rows: []+ };+ }++ const header = Object.keys(rawData[0]); // [lat, lng, value]++ // row object can still contain values like `Null` or `N/A`+ cleanUpFalsyCsvValue(rawData);+ const sample = getSampleForTypeAnalyze({fields: header, rows: rawData});+ // add sample data to duckdb and get schema+ const schema = await sniffCsvSchema(sample);++ const fieldsWAnalyzerType = getFieldsFromData(sample, header);++ const fields = consolidateFieldTypes(fieldsWAnalyzerType, schema);++ return {rows: rawData, fields};+}++function cleanUpFalsyCsvValue(rows: RowData): void {+ const re = new RegExp(CSV_NULLS, 'g');+ for (let i = 0; i < rows.length; i++) {+ Object.keys(rows[i]).forEach(key => {+ // here we parse empty data as null+ if (typeof rows[i][key] === 'string' && (rows[i][key] as string).match(re)) {+ rows[i][key] = null;+ }+ });+ }+}++// align type analyzer types with DuckDB csv auto detected column types+function consolidateFieldTypes(fields, schema) {+ return fields.map(field => {+ const columnName = field.name;++ const detectedColumnType = schema[columnName];++ // TODO columnTypeToFieldType tranforms detected timestamps to string,+ // completely breaking time filter logic.+ // const fieldType = columnTypeToFieldType(detectedColumnType) || field.type;++ return {+ ...field,+ // type: fieldType,+ duckDBColumnType: detectedColumnType+ };+ });+}++function toCSVRow(row: unknown[]): string {+ return `${row+ .map(r => {+ const rToStr = notNullorUndefined(r) ? String(r).replace(/"/g, '\\"') : '';+ return rToStr.includes(',') ? `"${rToStr}"` : rToStr;+ })+ .join(',')}\n`;+}++// Use DucckDB to detect csv column schema+async function sniffCsvSchema(sample: RowData) {+ if (!Array.isArray(sample) || sample.length < 0) {+ return;+ }+ const headerRow = toCSVRow(Object.keys(sample[0]));++ const csvContent = (sample as (any[] | Record<string, unknown>)[]).reduce(+ (accu: string, row) => `${accu}${toCSVRow(Object.values(row))}`,+ headerRow+ );++ const db = await getDuckDB();+ const c = await db.connect();++ const fileName = generateHashId();+ await db.registerFileText(`${fileName}-${generateHashId}.csv`, csvContent);++ // https://duckdb.org/docs/data/csv/auto_detection.html+ const result = await c.query(`+ FROM sniff_csv('${fileName}-${generateHashId}.csv',+ sample_size = 500,+ auto_type_candidates = ['FLOAT', 'INTEGER', 'TIMESTAMP', 'DATE', 'TIME', 'VARCHAR', 'BOOLEAN']);+ `);++ const schemaResult = result.toArray().map(row => row.toJSON());++ const columns = schemaResult[0].Columns;+ const schema = {};+ for (let i = 0; i < columns.length; i++) {+ schema[columns.get(i).name] = columns.get(i).type;+ }++ return schema;+}++export function processGeojson(rawData: unknown): ProcessorResult {+ const normalizedGeojson = normalize(rawData);++ if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {+ const error = new Error(+ `Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](${GUIDES_FILE_FORMAT_DOC})`+ );+ throw error;+ // fail to normalize geojson+ }+ // console.log(normalizedGeojson);++ return {+ rows: normalizedGeojson,+ // TODO get fields to preserve field names?+ fields: []+ };+}diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsnew file mode 100644index 000000000..1b604b2f1--- /dev/null+++ b/src/duckdb/src/table/duckdb-table.ts@@ -0,0 +1,258 @@+import * as arrow from 'apache-arrow';+import {AsyncDuckDB, AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';++import {DatasetType, DATASET_FORMATS} from '@kepler.gl/constants';+import {KeplerTable} from '@kepler.gl/table';+import {Field, ProtoDatasetField} from '@kepler.gl/types';+import {+ arrowSchemaToFields,+ isArrowData,+ isGeoJson,+ isKeplerGlMap,+ isRowObject,+ processArrowBatches+} from '@kepler.gl/processors';++import {getDuckDB} from '../init';+import {+ processCsvRowObject,+ processGeojson,+ processKeplerglJSONforDuckDb,+ ProcessorResult+} from '../processors/data-processor';++// TODO format geojson, all binary geogson columns should be tranformed to wkb+const GEOMETRY_COLUMN = 'geom';++type ImportDataToDuckProps = {+ data: ProcessorResult;+ db: AsyncDuckDB;+ c: AsyncDuckDBConnection;+};++/**+ * Creates an arrow table from arrow vectors and fields.+ * @param columns An array of arrow vectors.+ * @param fields An array of fields per arrow vector.+ * @returns An arrow table.+ */+// TODO: consider using original drag&dropped arrow table, as it could contain extra metadata, not passed to the fields.+const restoreArrowTable = (columns: arrow.Vector[], fields: ProtoDatasetField[]) => {+ const creaOpts = {};+ fields.map((field, index) => {+ creaOpts[field.name] = columns[index];+ });+ return new arrow.Table(creaOpts);+};++const dropIfTableExists = async (c: AsyncDuckDBConnection, tableName: string) => {+ try {+ await c.query(`DROP TABLE IF EXISTS "${tableName}";`);+ } catch (error) {+ console.error('Dropping table failed', tableName, error);+ }+};++export class KeplerGlDuckDbTable extends KeplerTable {+ constructor(props) {+ super(props);+ }++ async importRowData({data, db, c}: ImportDataToDuckProps): Promise<void> {+ try {+ const {fields, rows} = data;++ // DATASET_FORMATS.keplergl loads data as any[][] instead of [{}]+ // TODO potential memory usage explosion: original data, new object and then DuckDb table+ const rowsForDb = Array.isArray(rows[0])+ ? rows.map(row => {+ const newRow = {};+ row.forEach((value, index) => {+ newRow[fields[index].name] = value;+ });+ return newRow;+ })+ : rows;++ await db.registerFileText(this.id, JSON.stringify(rowsForDb));++ const columns = fields.reduce((acc, field, index) => {+ // @ts-expect-error TODO extend fields to contain duckDBColumnType+ return `${acc}${index > 0 ? ',' : ''} '${field.name}': '${field.duckDBColumnType}'`;+ }, '');++ const createTableSql = `+ CREATE TABLE '${this.label}' AS+ SELECT *+ FROM read_json('${this.id}',+ columns = {${columns}});+ `;+ await c.query(createTableSql);+ } catch (error) {+ console.log('importRowData', error);+ }+ }++ async importGeoJsonData({data, db, c}: ImportDataToDuckProps): Promise<void> {+ try {+ const {rows} = data;+ await db.registerFileText(this.id, JSON.stringify(rows));++ const createTableSql = `+ install spatial;+ load spatial;+ CREATE TABLE '${this.label}' AS+ SELECT *+ FROM ST_READ('${this.id}');+ `;+ await c.query(createTableSql);+ } catch (error) {+ console.error('importGeoJsonData', error);+ }+ }++ async importArrowData({data, db, c}: ImportDataToDuckProps): Promise<void> {+ try {+ // 1) data.rows contains an arrow table created by Add to Map data from DuckDb query.+ // 2) arrow table is in cols & fields when a file is dragged & dropped into Add Data To Map dialog.+ const arrowTable =+ data.rows instanceof arrow.Table+ ? data.rows+ : restoreArrowTable(data.cols || [], data.fields);++ const setupSql = `+ install spatial;+ load spatial;+ `;+ await c.query(setupSql);+ await c.insertArrowTable(arrowTable, {name: this.label});+ } catch (error) {+ // Known issues:+ // 1) Arrow Type with extension name: geoarrow.point and format: +w:2 is not currently supported in DuckDB.+ console.error('importArrowData', error);+ }+ }++ /**+ * Creates a table from data, returns an arrow table with the data+ * @param data+ * @returns {Promise<{fields: Field[], cols: any[]}>}+ */+ protected async createTableAndGetArrow(data): Promise<{fields: any[]; cols: arrow.Vector[]}> {+ const db = await getDuckDB();+ const c = await db.connect();++ const tableName = this.label;+ await dropIfTableExists(c, tableName);++ let format = this.metadata.format;+ if (!format) {+ // format is missing when we load Kepler.gl examples+ if (Array.isArray(data.rows?.[0]) || typeof data.rows?.[0] === 'object') {+ format = DATASET_FORMATS.row;+ } else if (data.rows?.type === 'FeatureCollection') {+ format = DATASET_FORMATS.geojson;+ } else if (data.cols?.[0] instanceof arrow.Vector) {+ format = DATASET_FORMATS.arrow;+ }+ }++ if (format === DATASET_FORMATS.row) {+ await this.importRowData({data, db, c});+ } else if (format === DATASET_FORMATS.geojson) {+ await this.importGeoJsonData({data, db, c});+ } else if (format === DATASET_FORMATS.arrow) {+ await this.importArrowData({data, db, c});+ } else {+ console.error('Unrecognized format', format);+ }++ let fields: Field[] = [];+ let cols: arrow.Vector[] = [];++ try {+ // We need to transform the binary geometry column to proper wkb.+ // First check whether such a column exists, then query with ST_AsWKB.+ const dummyResult = await c.query(`+ SELECT * FROM '${tableName}' LIMIT 0;+ `);+ const hasBinaryGeoField = Boolean(+ dummyResult.schema.fields.find(+ f => f.name === GEOMETRY_COLUMN && arrow.DataType.isBinary(f.type)+ )+ );++ // TODO don't do this for all columns named 'geom', check whether it's a wkb and not something else!+ // query and transform geometry column to wkb+ const arrowResult = await c.query(+ `SELECT * ${+ hasBinaryGeoField+ ? `EXCLUDE ${GEOMETRY_COLUMN}, ST_AsWKB(${GEOMETRY_COLUMN}) as ${GEOMETRY_COLUMN}`+ : ''+ } FROM '${tableName}';`+ );+ const geoField = arrowResult.schema.fields.find(+ f => f.name === GEOMETRY_COLUMN && arrow.DataType.isBinary(f.type)+ );+ geoField?.metadata.set('ARROW:extension:name', 'geoarrow.wkb');++ fields = data.fields ?? arrowSchemaToFields(arrowResult.schema);+ cols = [...Array(arrowResult.numCols).keys()]+ .map(i => arrowResult.getChildAt(i))+ .filter(col => col) as arrow.Vector[];+ } catch (error) {+ // Known issues:+ // 1) ST_AsWKB binder error, no type match+ console.error('DuckDb table: binary geometry column to wkb', error);+ }++ await c.close();++ return {fields, cols};+ }++ async importData({data}: {data: ProcessorResult}): Promise<void> {+ // VectorTile is a special case, ignore for now+ if (this.type === DatasetType.VECTOR_TILE) {+ super.importData({data: {...data, rows: []}});+ return;+ }++ const {fields, cols} = await this.createTableAndGetArrow(data);++ super.importData({data: {fields, cols, rows: []}});+ }++ async update(data) {+ // VectorTile is a special case, ignore for now+ if (this.type === DatasetType.VECTOR_TILE) {+ super.importData({data});+ return this;+ }++ const {cols} = await this.createTableAndGetArrow(data);++ return super.update({cols, rows: [], fields: []});+ }++ static getFileProcessor = function (data: any, inputFormat?: string) {+ let processor;+ let format;+ if (inputFormat === DATASET_FORMATS.arrow || isArrowData(data)) {+ format = DATASET_FORMATS.arrow;+ // TODO injest arrow files from disk without loaders+ processor = processArrowBatches;+ } else if (inputFormat === DATASET_FORMATS.keplergl || isKeplerGlMap(data)) {+ format = DATASET_FORMATS.keplergl;+ processor = processKeplerglJSONforDuckDb;+ } else if (inputFormat === DATASET_FORMATS.row || isRowObject(data)) {+ // csv file goes here+ format = DATASET_FORMATS.row;+ processor = processCsvRowObject; // directly import json object into duckdb-wasm+ } else if (inputFormat === DATASET_FORMATS.geojson || isGeoJson(data)) {+ format = DATASET_FORMATS.geojson;+ processor = processGeojson;+ }+ return {processor, format};+ };+}diff --git a/src/duckdb/src/utils/perf.ts b/src/duckdb/src/utils/perf.tsnew file mode 100644index 000000000..c6e807029--- /dev/null+++ b/src/duckdb/src/utils/perf.ts@@ -0,0 +1,44 @@+// [email protected]:holdenmatt/duckdb-wasm-kit.git+const FgRed = '\x1b[31m';+const ResetColors = '\x1b[0m';++/**+ * Format a time interval between start/end timestamps.+ *+ * If no end time is given, use the current time.+ */+const formatElapsedTime = (+ label: string,+ start: number,+ end: number | undefined = undefined+): string => {+ const endTime = end ?? performance.now();+ const elapsed = endTime - start;++ let timeString: string;+ switch (true) {+ case elapsed >= 1000:+ timeString = `${(elapsed / 1000).toFixed(1)}s`;+ break;+ case elapsed >= 1:+ timeString = `${elapsed.toFixed(0)}ms`;+ break;+ default:+ timeString = `${elapsed.toFixed(3)}ms`;+ break;+ }++ const message = `${FgRed}[${timeString}] ${ResetColors}${label}`;+ return message;+};++/**+ * Print the elapsed time to console.debug.+ */+export const logElapsedTime = (+ label: string,+ start: number,+ end: number | undefined = undefined+) => {+ console.debug(formatElapsedTime(label, start, end));+};diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex ab51dfe4d..77b9aa06c 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -151,7 +151,9 @@ class KeplerTable<F extends Field = Field> {disableDataOperation?: boolean;// table-injected metadata- metadata: object;+ metadata: Record<string, any>;++ getFileProcessor?: (data: any, inputFormat?: string) => {data: any; format: string};constructor({info,@@ -202,7 +204,7 @@ class KeplerTable<F extends Field = Field> {this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);}- importData({data}: {data: ProtoDataset['data']}) {+ async importData({data}: {data: ProtoDataset['data']}) {const dataContainerData = data.cols ? data.cols : data.rows;const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;@@ -236,7 +238,7 @@ class KeplerTable<F extends Field = Field> {* update table with new data* @param data - new data e.g. the arrow data with new batches loaded*/- update(data: ProtoDataset['data']) {+ async update(data: ProtoDataset['data']) {const dataContainerData = data.cols ? data.cols : data.rows;this.dataContainer.update?.(dataContainerData);this.allIndexes = this.dataContainer.getPlainIndex();diff --git a/yarn.lock b/yarn.lockindex 719be5255..df2151de3 100644--- a/yarn.lock+++ b/yarn.lock@@ -2177,6 +2177,15 @@ __metadata:languageName: nodelinkType: hard+"@duckdb/duckdb-wasm@npm:^1.28.0":+ version: 1.29.0+ resolution: "@duckdb/duckdb-wasm@npm:1.29.0"+ dependencies:+ apache-arrow: "npm:^17.0.0"+ checksum: 10c0/caa550b09433adb709f997241ef00fd7ee93dc4cb4a03838d9d978536fdc640c80ef0f2c294248c688e4a38231be42287e4aba655e13d94bcd9aff4767a404a0+ languageName: node+ linkType: hard+"@emotion/is-prop-valid@npm:1.2.1":version: 1.2.1resolution: "@emotion/is-prop-valid@npm:1.2.1"@@ -3396,8 +3405,17 @@ __metadata:version: 0.0.0-use.localresolution: "@kepler.gl/duckdb@workspace:src/duckdb"dependencies:+ "@duckdb/duckdb-wasm": "npm:^1.28.0"+ "@kepler.gl/common-utils": "npm:3.1.0-alpha.5""@kepler.gl/constants": "npm:3.1.0-alpha.5"+ "@kepler.gl/processors": "npm:3.1.0-alpha.5"+ "@kepler.gl/table": "npm:3.1.0-alpha.5""@kepler.gl/types": "npm:3.1.0-alpha.5"+ "@monaco-editor/react": "npm:^4.6.0"+ "@radix-ui/react-collapsible": "npm:^1.1.0"+ apache-arrow: "npm:>=15.0.0"+ monaco-editor: "npm:^0.52.0"+ react-resizable-panels: "npm:^2.1.7"languageName: unknownlinkType: soft@@ -5044,6 +5062,30 @@ __metadata:languageName: nodelinkType: hard+"@monaco-editor/loader@npm:^1.4.0":+ version: 1.4.0+ resolution: "@monaco-editor/loader@npm:1.4.0"+ dependencies:+ state-local: "npm:^1.0.6"+ peerDependencies:+ monaco-editor: ">= 0.21.0 < 1"+ checksum: 10c0/68938350adf2f42363a801d87f5d00c87d397d4cba7041141af10a9216bd35c85209b4723a26d56cb32e68eef61471deda2a450f8892891118fbdce7fa1d987d+ languageName: node+ linkType: hard++"@monaco-editor/react@npm:^4.6.0":+ version: 4.6.0+ resolution: "@monaco-editor/react@npm:4.6.0"+ dependencies:+ "@monaco-editor/loader": "npm:^1.4.0"+ peerDependencies:+ monaco-editor: ">= 0.25.0 < 1"+ react: ^16.8.0 || ^17.0.0 || ^18.0.0+ react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0+ checksum: 10c0/231e9a9b66a530db326f6732de0ebffcce6b79dcfaf4948923d78b9a3d5e2a04b7a06e1f85bbbca45a5ae15c107a124e4c5c46cabadc20a498fb5f2d05f7f379+ languageName: node+ linkType: hard+"@nebula.gl/edit-modes@npm:1.0.2-alpha.1":version: 1.0.2-alpha.1resolution: "@nebula.gl/edit-modes@npm:1.0.2-alpha.1"@@ -6665,6 +6707,175 @@ __metadata:languageName: nodelinkType: hard+"@radix-ui/primitive@npm:1.1.1":+ version: 1.1.1+ resolution: "@radix-ui/primitive@npm:1.1.1"+ checksum: 10c0/6457bd8d1aa4ecb948e5d2a2484fc570698b2ab472db6d915a8f1eec04823f80423efa60b5ba840f0693bec2ca380333cc5f3b52586b40f407d9f572f9261f8d+ languageName: node+ linkType: hard++"@radix-ui/react-collapsible@npm:^1.1.0":+ version: 1.1.2+ resolution: "@radix-ui/react-collapsible@npm:1.1.2"+ dependencies:+ "@radix-ui/primitive": "npm:1.1.1"+ "@radix-ui/react-compose-refs": "npm:1.1.1"+ "@radix-ui/react-context": "npm:1.1.1"+ "@radix-ui/react-id": "npm:1.1.0"+ "@radix-ui/react-presence": "npm:1.1.2"+ "@radix-ui/react-primitive": "npm:2.0.1"+ "@radix-ui/react-use-controllable-state": "npm:1.1.0"+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ "@types/react-dom": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ "@types/react-dom":+ optional: true+ checksum: 10c0/8a725539c0c259ea53a0e35d4ddd3acca42cab5113fd537758450ad1e76f0b757423f18aca29364f963bef4f0624d57feb32bf9d12a3ea6b2c084b523ba65205+ languageName: node+ linkType: hard++"@radix-ui/react-compose-refs@npm:1.1.1":+ version: 1.1.1+ resolution: "@radix-ui/react-compose-refs@npm:1.1.1"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/3e84580024e66e3cc5b9ae79355e787815c1d2a3c7d46e7f47900a29c33751ca24cf4ac8903314957ab1f7788aebe1687e2258641c188cf94653f7ddf8f70627+ languageName: node+ linkType: hard++"@radix-ui/react-context@npm:1.1.1":+ version: 1.1.1+ resolution: "@radix-ui/react-context@npm:1.1.1"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/fc4ace9d79d7954c715ade765e06c95d7e1b12a63a536bcbe842fb904f03f88fc5bd6e38d44bd23243d37a270b4c44380fedddaeeae2d274f0b898a20665aba2+ languageName: node+ linkType: hard++"@radix-ui/react-id@npm:1.1.0":+ version: 1.1.0+ resolution: "@radix-ui/react-id@npm:1.1.0"+ dependencies:+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/acf13e29e51ee96336837fc0cfecc306328b20b0e0070f6f0f7aa7a621ded4a1ee5537cfad58456f64bae76caa7f8769231e88dc7dc106197347ee433c275a79+ languageName: node+ linkType: hard++"@radix-ui/react-presence@npm:1.1.2":+ version: 1.1.2+ resolution: "@radix-ui/react-presence@npm:1.1.2"+ dependencies:+ "@radix-ui/react-compose-refs": "npm:1.1.1"+ "@radix-ui/react-use-layout-effect": "npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ "@types/react-dom": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ "@types/react-dom":+ optional: true+ checksum: 10c0/0c6fa281368636308044df3be4c1f02733094b5e35ba04f26e610dd1c4315a245ffc758e0e176c444742a7a46f4328af1a9d8181e860175ec39338d06525a78d+ languageName: node+ linkType: hard++"@radix-ui/react-primitive@npm:2.0.1":+ version: 2.0.1+ resolution: "@radix-ui/react-primitive@npm:2.0.1"+ dependencies:+ "@radix-ui/react-slot": "npm:1.1.1"+ peerDependencies:+ "@types/react": "*"+ "@types/react-dom": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ "@types/react-dom":+ optional: true+ checksum: 10c0/6a562bec14f8e9fbfe0012d6c2932b0e54518fed898fa0622300c463611e77a4ca28a969f0cd484efd6570c01c5665dd6151f736262317d01715bc4da1a7dea6+ languageName: node+ linkType: hard++"@radix-ui/react-slot@npm:1.1.1":+ version: 1.1.1+ resolution: "@radix-ui/react-slot@npm:1.1.1"+ dependencies:+ "@radix-ui/react-compose-refs": "npm:1.1.1"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/f3cc71c16529c67a8407a89e0ac13a868cafa0cd05ca185b464db609aa5996a3f00588695518e420bd47ffdb4cc2f76c14cc12ea5a38fc2ca3578a30d2ca58b9+ languageName: node+ linkType: hard++"@radix-ui/react-use-callback-ref@npm:1.1.0":+ version: 1.1.0+ resolution: "@radix-ui/react-use-callback-ref@npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/e954863f3baa151faf89ac052a5468b42650efca924417470efd1bd254b411a94c69c30de2fdbb90187b38cb984795978e12e30423dc41e4309d93d53b66d819+ languageName: node+ linkType: hard++"@radix-ui/react-use-controllable-state@npm:1.1.0":+ version: 1.1.0+ resolution: "@radix-ui/react-use-controllable-state@npm:1.1.0"+ dependencies:+ "@radix-ui/react-use-callback-ref": "npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/2af883b5b25822ac226e60a6bfde647c0123a76345052a90219026059b3f7225844b2c13a9a16fba859c1cda5fb3d057f2a04503f71780e607516492db4eb3a1+ languageName: node+ linkType: hard++"@radix-ui/react-use-layout-effect@npm:1.1.0":+ version: 1.1.0+ resolution: "@radix-ui/react-use-layout-effect@npm:1.1.0"+ peerDependencies:+ "@types/react": "*"+ react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc+ peerDependenciesMeta:+ "@types/react":+ optional: true+ checksum: 10c0/9bf87ece1845c038ed95863cfccf9d75f557c2400d606343bab0ab3192b9806b9840e6aa0a0333fdf3e83cf9982632852192f3e68d7d8367bc8c788dfdf8e62b+ languageName: node+ linkType: hard+"@react-aria/breadcrumbs@npm:3.5.13":version: 3.5.13resolution: "@react-aria/breadcrumbs@npm:3.5.13"@@ -10906,7 +11117,7 @@ __metadata:languageName: nodelinkType: hard-"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0":+"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0, apache-arrow@npm:^17.0.0":version: 17.0.0resolution: "apache-arrow@npm:17.0.0"dependencies:@@ -22345,6 +22556,13 @@ __metadata:languageName: nodelinkType: hard+"monaco-editor@npm:^0.52.0":+ version: 0.52.2+ resolution: "monaco-editor@npm:0.52.2"+ checksum: 10c0/5a92da64f1e2ab375c0ce99364137f794d057c97bed10ecc65a08d6e6846804b8ecbd377eacf01e498f7dfbe1b21e8be64f728256681448f0484df90e767b435+ languageName: node+ linkType: hard+"moo@npm:^0.5.0":version: 0.5.2resolution: "moo@npm:0.5.2"@@ -25409,6 +25627,16 @@ __metadata:languageName: nodelinkType: hard+"react-resizable-panels@npm:^2.1.7":+ version: 2.1.7+ resolution: "react-resizable-panels@npm:2.1.7"+ peerDependencies:+ react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc+ react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc+ checksum: 10c0/644a57960507b809f571bf8c95f07a04e3aeb9b20405a1d9d8cb6deb72d2462192e6e5e9751237347e7818179a619a6f65de4b355e276334d10b8cfe737ca4e1+ languageName: node+ linkType: hard+"react-shallow-renderer@npm:^16.15.0":version: 16.15.0resolution: "react-shallow-renderer@npm:16.15.0"@@ -27556,6 +27784,13 @@ __metadata:languageName: nodelinkType: hard+"state-local@npm:^1.0.6":+ version: 1.0.7+ resolution: "state-local@npm:1.0.7"+ checksum: 10c0/8dc7daeac71844452fafb514a6d6b6f40d7e2b33df398309ea1c7b3948d6110c57f112b7196500a10c54fdde40291488c52c875575670fb5c819602deca48bd9+ languageName: node+ linkType: hard+"state-toggle@npm:^1.0.0":version: 1.0.3resolution: "state-toggle@npm:1.0.3"