Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[feat] DuckDb plugin: drag and drop file directly as table (#2952)
Igor Dykhta9762dc3798c8
diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsxindex 06d4808d5..e5bbd47ce 100644--- a/src/duckdb/src/components/schema-panel.tsx+++ b/src/duckdb/src/components/schema-panel.tsx@@ -6,6 +6,7 @@ import {useSelector} from 'react-redux';import styled from 'styled-components';import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';+import {LoadingSpinner, Icons} from '@kepler.gl/components';import {arrowSchemaToFields} from '@kepler.gl/processors';import {VisState} from '@kepler.gl/schemas';@@ -29,6 +30,14 @@ const StyledSchemaPanel = styled.div`font-size: 12px;padding: 12px;font-family: ${props => props.theme.fontFamily};+ height: 100%;+`;++const StyledLoadingSpinnerWrapper = styled.div`+ display: flex;+ justify-content: center;+ align-items: center;+ height: 100%;`;async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {@@ -57,16 +66,61 @@ async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: str};}-function getSchemaSuggestion(result) {+export type SchemaSuggestion = {column_name: string; table_name: string};++function getSchemaSuggestion(result: {key: string; children: {key: string}[]}[]) {return result.reduce((accu, data) => {const columns = data.children.map(child => ({column_name: child.key,table_name: data.key}));return accu.concat(columns);- }, []);+ }, [] as SchemaSuggestion[]);}-export const SchemaPanel = ({setTableSchema}) => {++type SchemaPanelProps = {+ setTableSchema: (tableSchema: SchemaSuggestion[]) => void;+ droppedFile: File | null;+};++const StyledSchemaPanelDropMessage = styled.div`+ display: flex;+ justify-content: center;+ align-items: center;+ height: 100%;+ flex-direction: column;+ text-align: center;++ div {+ margin: 5px;+ }+ .header {+ font-size: 15px;+ }+ .bold {+ font-weight: 700;+ }+`;++const StyledAddIcon = styled(Icons.Add)`+ display: inline;+ margin-top: -3px;+`;++export const SchemaPanelDropMessage = () => {+ return (+ <StyledSchemaPanelDropMessage>+ <div className="header">+ <StyledAddIcon /> Add files to DuckDB+ </div>+ <div className="bold">Supported formats: </div>+ <div>.csv, .json, .geojson, .parquet, .arrow</div>+ <div>Files you add will stay local to your browser.</div>+ </StyledSchemaPanelDropMessage>+ );+};++export const SchemaPanel = ({setTableSchema, droppedFile}: SchemaPanelProps) => {const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);@@ -76,7 +130,7 @@ export const SchemaPanel = ({setTableSchema}) => {const tableResult = await c.query('SHOW TABLES;');- const tableNames = tableResult.getChildAt(0)?.toJSON();+ const tableNames: string[] | undefined = tableResult.getChildAt(0)?.toJSON();const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));const tableSchema = getSchemaSuggestion(result);@@ -88,7 +142,7 @@ export const SchemaPanel = ({setTableSchema}) => {useEffect(() => {getTableSchema();- }, [datasets, getTableSchema]);+ }, [datasets, droppedFile, getTableSchema]);return (<StyledSchemaPanel>@@ -107,8 +161,12 @@ export const SchemaPanel = ({setTableSchema}) => {}}/>))+ ) : droppedFile ? (+ <StyledLoadingSpinnerWrapper>+ <LoadingSpinner />+ </StyledLoadingSpinnerWrapper>) : (- <div>No tables found</div>+ <SchemaPanelDropMessage />)}</StyledSchemaPanel>);diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsxindex 4a2788851..5a0518422 100644--- a/src/duckdb/src/components/sql-panel.tsx+++ b/src/duckdb/src/components/sql-panel.tsx@@ -2,20 +2,20 @@// Copyright contributors to the kepler.gl projectimport * as arrow from 'apache-arrow';-import React, {useCallback, useState, useEffect} from 'react';+import React, {useCallback, useState, useEffect, useRef} from 'react';import {useDispatch} from 'react-redux';import styled from 'styled-components';import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';import {addDataToMap} from '@kepler.gl/actions';import {generateHashId} from '@kepler.gl/common-utils';-import {Button, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';+import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';import {arrowSchemaToFields} from '@kepler.gl/processors';import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';import {isAppleDevice} from '@kepler.gl/utils';import MonacoEditor from './monaco-editor';-import {SchemaPanel} from './schema-panel';+import {SchemaPanel, SchemaSuggestion} from './schema-panel';import {PreviewDataPanel, QueryResult} from './preview-data-panel';import {getDuckDB} from '../init';import {@@ -26,7 +26,9 @@ import {setGeoArrowWKBExtension,splitSqlStatements,checkIsSelectQuery,- removeSQLComments+ removeSQLComments,+ tableFromFile,+ SUPPORTED_DUCKDB_DROP_EXTENSIONS} from '../table/duckdb-table-utils';const StyledSqlPanel = styled.div`@@ -125,6 +127,20 @@ const StyledErrorContainer = styled.pre`overflow: auto;`;+interface StyledDragPanelProps {+ dragOver?: boolean;+}++const StyledFileDropArea = styled(FileDrop)<StyledDragPanelProps>`+ height: 100%;+ border-width: 1px;+ border: 1px ${props => (props.dragOver ? 'solid' : 'dashed')}+ ${props => (props.dragOver ? props.theme.subtextColorLT : 'transparent')};+ .file-drop-target {+ height: 100%;+ }+`;+type SqlPanelProps = {initialSql?: string;};@@ -136,14 +152,18 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {const params = new URLSearchParams(window.location.search);return params.get('sql') || initialSql;});+ const [droppedFile, setDroppedFile] = useState<File | null>(null);+ const [dragState, setDragState] = useState(false);const [result, setResult] = useState<null | QueryResult>(null);const [error, setError] = useState<Error | null>(null);const [counter, setCounter] = useState(0);- const [tableSchema, setTableSchema] = useState([]);+ const [tableSchema, setTableSchema] = useState<SchemaSuggestion[]>([]);const [isRunning, setIsRunning] = useState(false);const [isMac] = useState(() => isAppleDevice());const dispatch = useDispatch();+ const droppedFileAreaRef = useRef(null);+useEffect(() => {const currentUrl = new URL(window.location.href);if (sql) {@@ -244,11 +264,73 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {setCounter(counter + 1);}, [result, counter, dispatch]);+ const isValidFileType = useCallback(filename => {+ const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => filename.endsWith(ext));+ return Boolean(fileExt);+ }, []);++ const createTableFromDroppedFile = useCallback(async (droppedFile: File | null) => {+ if (droppedFile) {+ const error = await tableFromFile(droppedFile);+ if (error) {+ setError(error);+ } else {+ setError(null);+ }+ }++ setDroppedFile(null);+ setDragState(false);+ }, []);++ useEffect(() => {+ createTableFromDroppedFile(droppedFile);+ }, [droppedFile, createTableFromDroppedFile]);++ const handleFileInput = useCallback(+ (fileList: FileList, event: DragEvent) => {+ if (event) {+ event.preventDefault();+ event.stopPropagation();+ }++ const files = [...fileList].filter(Boolean);++ const disableExtensionFilter = false;++ const filesToLoad: File[] = [];+ const errorFiles: string[] = [];+ for (const file of files) {+ if (disableExtensionFilter || isValidFileType(file.name)) {+ filesToLoad.push(file);+ } else {+ errorFiles.push(file.name);+ }+ }++ if (filesToLoad.length > 0) {+ setDroppedFile(filesToLoad[0]);+ } else if (errorFiles.length > 0) {+ setError(new Error(`Unsupported file formats: ${errorFiles.join(', ')}`));+ }+ },+ [isValidFileType]+ );+return (<StyledSqlPanel><PanelGroup direction="horizontal"><Panel defaultSize={20} minSize={15} style={SCHEMA_PANEL_STYLE}>- <SchemaPanel setTableSchema={setTableSchema} />+ <StyledFileDropArea+ dragOver={dragState}+ onDragOver={() => setDragState(true)}+ onDragLeave={() => setDragState(false)}+ frame={droppedFileAreaRef.current || document}+ onDrop={handleFileInput}+ className="file-uploader__file-drop"+ >+ <SchemaPanel setTableSchema={setTableSchema} droppedFile={droppedFile} />+ </StyledFileDropArea></Panel><StyledResizeHandle />@@ -284,7 +366,7 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {</Panel><StyledVerticalResizeHandle />- <Panel>+ <Panel className="preview-panel">{isRunning ? (<StyledLoadingContainer><LoadingSpinner />diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.tsindex 1b9aa44df..5874552b7 100644--- a/src/duckdb/src/table/duckdb-table-utils.ts+++ b/src/duckdb/src/table/duckdb-table-utils.ts@@ -11,11 +11,15 @@import * as arrow from 'apache-arrow';import {DataType} from 'apache-arrow/type';-import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';+import {AsyncDuckDBConnection, DuckDBDataProtocol} from '@duckdb/duckdb-wasm';import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';import {ProtoDatasetField} from '@kepler.gl/types';+import {getDuckDB} from '../init';++export const SUPPORTED_DUCKDB_DROP_EXTENSIONS = ['arrow', 'csv', 'geojson', 'json', 'parquet'];+export type DuckDBColumnDesc = {name: string; type: string};/**@@ -403,3 +407,106 @@ export const dropTableIfExists = async (connection: AsyncDuckDBConnection, tableconsole.error('Dropping table failed', tableName, error);}};++/**+ * Imports a file into DuckDB as a table, supporting multiple formats from SUPPORTED_DUCKDB_DROP_EXTENSIONS.+ * @param file The file to be imported.+ * @returns A promise that resolves when the file has been processed into a DuckDB table.+ */+export async function tableFromFile(file: File | null): Promise<null | Error> {+ if (!file) return new Error('File Drag & Drop: No file');++ const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => file.name.endsWith(ext));+ if (!fileExt) {+ return new Error("File Drag & Drop: File extension isn't supported");+ }++ const db = await getDuckDB();+ const c = await db.connect();++ let error: Error | null = null;++ try {+ const tableName = sanitizeDuckDBTableName(file.name);+ const sourceName = 'temp_file_handle';++ c.query(`install spatial;+ load spatial;`);++ if (fileExt === 'arrow') {+ const arrayBuffer = await file.arrayBuffer();+ const uint8Array = new Uint8Array(arrayBuffer);+ const arrowTable = arrow.tableFromIPC(uint8Array);++ await c.insertArrowTable(arrowTable, {name: tableName});+ } else {+ await db.registerFileHandle(sourceName, file, DuckDBDataProtocol.BROWSER_FILEREADER, true);++ if (fileExt === 'csv') {+ await c.query(`+ CREATE TABLE '${tableName}' AS+ SELECT *+ FROM read_csv('${sourceName}', header = true, auto_detect = true, sample_size = -1);+ `);+ } else if (fileExt === 'json') {+ await c.query(`+ CREATE TABLE '${tableName}' AS+ SELECT *+ FROM read_json_auto('${sourceName}');+ `);+ } else if (fileExt === 'geojson') {+ await c.query(`+ CREATE TABLE '${tableName}' AS+ SELECT *+ FROM ST_READ('${sourceName}', keep_wkb = TRUE);+ `);+ } else if (fileExt === 'parquet') {+ await c.query(`+ CREATE TABLE '${tableName}' AS+ SELECT *+ FROM read_parquet('${sourceName}')+ `);+ }+ }+ } catch (errorData) {+ if (errorData instanceof Error) {+ const message = errorData.message || '';+ // output more readable errors for known issues+ if (message.includes('Arrow Type with extension name: geoarrow')) {+ error = new Error(+ 'The GeoArrow extensions are not implemented in the connected DuckDB version.'+ );+ } else if (message.includes("Geoparquet column 'geometry' does not have geometry types")) {+ error = new Error(+ `Invalid Input Error: Geoparquet column 'geometry' does not have geometry types.+Possible reasons:+ - Old .parquet files that don't match the Parquet format specification.+ - Unsupported compression.`+ );+ }+ }++ if (!error) {+ error = errorData as Error;+ }+ }++ c.close();++ return error;+}++/**+ * Sanitizes a file name to be a valid DuckDB table name.+ * @param fileName The input file name to be sanitized.+ * @returns A valid DuckDB table name.+ */+export function sanitizeDuckDBTableName(fileName: string): string {+ // Replace invalid characters with underscores+ let name = fileName.replace(/[^a-zA-Z0-9_]/g, '_');+ // Ensure it doesn't start with a digit+ if (/^\d/.test(name)) {+ name = 't_' + name;+ }+ return name || 'default_table';+}