Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[feat] DuckDb plugin: drag and drop file directly as table (#2952)

Igor Dykhta9762dc3798c8
diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsx
index 06d4808d5..e5bbd47ce 100644
--- a/src/duckdb/src/components/schema-panel.tsx
+++ b/src/duckdb/src/components/schema-panel.tsx
@@ -6,6 +6,7 @@ import {useSelector} from 'react-redux';
import styled from 'styled-components';
import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
+import {LoadingSpinner, Icons} from '@kepler.gl/components';
import {arrowSchemaToFields} from '@kepler.gl/processors';
import {VisState} from '@kepler.gl/schemas';
@@ -29,6 +30,14 @@ const StyledSchemaPanel = styled.div`
font-size: 12px;
padding: 12px;
font-family: ${props => props.theme.fontFamily};
+ height: 100%;
+`;
+
+const StyledLoadingSpinnerWrapper = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ height: 100%;
`;
async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {
@@ -57,16 +66,61 @@ async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: str
};
}
-function getSchemaSuggestion(result) {
+export type SchemaSuggestion = {column_name: string; table_name: string};
+
+function getSchemaSuggestion(result: {key: string; children: {key: string}[]}[]) {
return result.reduce((accu, data) => {
const columns = data.children.map(child => ({
column_name: child.key,
table_name: data.key
}));
return accu.concat(columns);
- }, []);
+ }, [] as SchemaSuggestion[]);
}
-export const SchemaPanel = ({setTableSchema}) => {
+
+type SchemaPanelProps = {
+ setTableSchema: (tableSchema: SchemaSuggestion[]) => void;
+ droppedFile: File | null;
+};
+
+const StyledSchemaPanelDropMessage = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ height: 100%;
+ flex-direction: column;
+ text-align: center;
+
+ div {
+ margin: 5px;
+ }
+ .header {
+ font-size: 15px;
+ }
+ .bold {
+ font-weight: 700;
+ }
+`;
+
+const StyledAddIcon = styled(Icons.Add)`
+ display: inline;
+ margin-top: -3px;
+`;
+
+export const SchemaPanelDropMessage = () => {
+ return (
+ <StyledSchemaPanelDropMessage>
+ <div className="header">
+ <StyledAddIcon /> Add files to DuckDB
+ </div>
+ <div className="bold">Supported formats: </div>
+ <div>.csv, .json, .geojson, .parquet, .arrow</div>
+ <div>Files you add will stay local to your browser.</div>
+ </StyledSchemaPanelDropMessage>
+ );
+};
+
+export const SchemaPanel = ({setTableSchema, droppedFile}: SchemaPanelProps) => {
const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);
const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);
@@ -76,7 +130,7 @@ export const SchemaPanel = ({setTableSchema}) => {
const tableResult = await c.query('SHOW TABLES;');
- const tableNames = tableResult.getChildAt(0)?.toJSON();
+ const tableNames: string[] | undefined = tableResult.getChildAt(0)?.toJSON();
const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));
const tableSchema = getSchemaSuggestion(result);
@@ -88,7 +142,7 @@ export const SchemaPanel = ({setTableSchema}) => {
useEffect(() => {
getTableSchema();
- }, [datasets, getTableSchema]);
+ }, [datasets, droppedFile, getTableSchema]);
return (
<StyledSchemaPanel>
@@ -107,8 +161,12 @@ export const SchemaPanel = ({setTableSchema}) => {
}}
/>
))
+ ) : droppedFile ? (
+ <StyledLoadingSpinnerWrapper>
+ <LoadingSpinner />
+ </StyledLoadingSpinnerWrapper>
) : (
- <div>No tables found</div>
+ <SchemaPanelDropMessage />
)}
</StyledSchemaPanel>
);
diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsx
index 4a2788851..5a0518422 100644
--- a/src/duckdb/src/components/sql-panel.tsx
+++ b/src/duckdb/src/components/sql-panel.tsx
@@ -2,20 +2,20 @@
// Copyright contributors to the kepler.gl project
import * as arrow from 'apache-arrow';
-import React, {useCallback, useState, useEffect} from 'react';
+import React, {useCallback, useState, useEffect, useRef} from 'react';
import {useDispatch} from 'react-redux';
import styled from 'styled-components';
import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';
import {addDataToMap} from '@kepler.gl/actions';
import {generateHashId} from '@kepler.gl/common-utils';
-import {Button, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
+import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
import {arrowSchemaToFields} from '@kepler.gl/processors';
import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';
import {isAppleDevice} from '@kepler.gl/utils';
import MonacoEditor from './monaco-editor';
-import {SchemaPanel} from './schema-panel';
+import {SchemaPanel, SchemaSuggestion} from './schema-panel';
import {PreviewDataPanel, QueryResult} from './preview-data-panel';
import {getDuckDB} from '../init';
import {
@@ -26,7 +26,9 @@ import {
setGeoArrowWKBExtension,
splitSqlStatements,
checkIsSelectQuery,
- removeSQLComments
+ removeSQLComments,
+ tableFromFile,
+ SUPPORTED_DUCKDB_DROP_EXTENSIONS
} from '../table/duckdb-table-utils';
const StyledSqlPanel = styled.div`
@@ -125,6 +127,20 @@ const StyledErrorContainer = styled.pre`
overflow: auto;
`;
+interface StyledDragPanelProps {
+ dragOver?: boolean;
+}
+
+const StyledFileDropArea = styled(FileDrop)<StyledDragPanelProps>`
+ height: 100%;
+ border-width: 1px;
+ border: 1px ${props => (props.dragOver ? 'solid' : 'dashed')}
+ ${props => (props.dragOver ? props.theme.subtextColorLT : 'transparent')};
+ .file-drop-target {
+ height: 100%;
+ }
+`;
+
type SqlPanelProps = {
initialSql?: string;
};
@@ -136,14 +152,18 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
const params = new URLSearchParams(window.location.search);
return params.get('sql') || initialSql;
});
+ const [droppedFile, setDroppedFile] = useState<File | null>(null);
+ const [dragState, setDragState] = useState(false);
const [result, setResult] = useState<null | QueryResult>(null);
const [error, setError] = useState<Error | null>(null);
const [counter, setCounter] = useState(0);
- const [tableSchema, setTableSchema] = useState([]);
+ const [tableSchema, setTableSchema] = useState<SchemaSuggestion[]>([]);
const [isRunning, setIsRunning] = useState(false);
const [isMac] = useState(() => isAppleDevice());
const dispatch = useDispatch();
+ const droppedFileAreaRef = useRef(null);
+
useEffect(() => {
const currentUrl = new URL(window.location.href);
if (sql) {
@@ -244,11 +264,73 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
setCounter(counter + 1);
}, [result, counter, dispatch]);
+ const isValidFileType = useCallback(filename => {
+ const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => filename.endsWith(ext));
+ return Boolean(fileExt);
+ }, []);
+
+ const createTableFromDroppedFile = useCallback(async (droppedFile: File | null) => {
+ if (droppedFile) {
+ const error = await tableFromFile(droppedFile);
+ if (error) {
+ setError(error);
+ } else {
+ setError(null);
+ }
+ }
+
+ setDroppedFile(null);
+ setDragState(false);
+ }, []);
+
+ useEffect(() => {
+ createTableFromDroppedFile(droppedFile);
+ }, [droppedFile, createTableFromDroppedFile]);
+
+ const handleFileInput = useCallback(
+ (fileList: FileList, event: DragEvent) => {
+ if (event) {
+ event.preventDefault();
+ event.stopPropagation();
+ }
+
+ const files = [...fileList].filter(Boolean);
+
+ const disableExtensionFilter = false;
+
+ const filesToLoad: File[] = [];
+ const errorFiles: string[] = [];
+ for (const file of files) {
+ if (disableExtensionFilter || isValidFileType(file.name)) {
+ filesToLoad.push(file);
+ } else {
+ errorFiles.push(file.name);
+ }
+ }
+
+ if (filesToLoad.length > 0) {
+ setDroppedFile(filesToLoad[0]);
+ } else if (errorFiles.length > 0) {
+ setError(new Error(`Unsupported file formats: ${errorFiles.join(', ')}`));
+ }
+ },
+ [isValidFileType]
+ );
+
return (
<StyledSqlPanel>
<PanelGroup direction="horizontal">
<Panel defaultSize={20} minSize={15} style={SCHEMA_PANEL_STYLE}>
- <SchemaPanel setTableSchema={setTableSchema} />
+ <StyledFileDropArea
+ dragOver={dragState}
+ onDragOver={() => setDragState(true)}
+ onDragLeave={() => setDragState(false)}
+ frame={droppedFileAreaRef.current || document}
+ onDrop={handleFileInput}
+ className="file-uploader__file-drop"
+ >
+ <SchemaPanel setTableSchema={setTableSchema} droppedFile={droppedFile} />
+ </StyledFileDropArea>
</Panel>
<StyledResizeHandle />
@@ -284,7 +366,7 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
</Panel>
<StyledVerticalResizeHandle />
- <Panel>
+ <Panel className="preview-panel">
{isRunning ? (
<StyledLoadingContainer>
<LoadingSpinner />
diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.ts
index 1b9aa44df..5874552b7 100644
--- a/src/duckdb/src/table/duckdb-table-utils.ts
+++ b/src/duckdb/src/table/duckdb-table-utils.ts
@@ -11,11 +11,15 @@
import * as arrow from 'apache-arrow';
import {DataType} from 'apache-arrow/type';
-import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
+import {AsyncDuckDBConnection, DuckDBDataProtocol} from '@duckdb/duckdb-wasm';
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
import {ProtoDatasetField} from '@kepler.gl/types';
+import {getDuckDB} from '../init';
+
+export const SUPPORTED_DUCKDB_DROP_EXTENSIONS = ['arrow', 'csv', 'geojson', 'json', 'parquet'];
+
export type DuckDBColumnDesc = {name: string; type: string};
/**
@@ -403,3 +407,106 @@ export const dropTableIfExists = async (connection: AsyncDuckDBConnection, table
console.error('Dropping table failed', tableName, error);
}
};
+
+/**
+ * Imports a file into DuckDB as a table, supporting multiple formats from SUPPORTED_DUCKDB_DROP_EXTENSIONS.
+ * @param file The file to be imported.
+ * @returns A promise that resolves when the file has been processed into a DuckDB table.
+ */
+export async function tableFromFile(file: File | null): Promise<null | Error> {
+ if (!file) return new Error('File Drag & Drop: No file');
+
+ const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => file.name.endsWith(ext));
+ if (!fileExt) {
+ return new Error("File Drag & Drop: File extension isn't supported");
+ }
+
+ const db = await getDuckDB();
+ const c = await db.connect();
+
+ let error: Error | null = null;
+
+ try {
+ const tableName = sanitizeDuckDBTableName(file.name);
+ const sourceName = 'temp_file_handle';
+
+ c.query(`install spatial;
+ load spatial;`);
+
+ if (fileExt === 'arrow') {
+ const arrayBuffer = await file.arrayBuffer();
+ const uint8Array = new Uint8Array(arrayBuffer);
+ const arrowTable = arrow.tableFromIPC(uint8Array);
+
+ await c.insertArrowTable(arrowTable, {name: tableName});
+ } else {
+ await db.registerFileHandle(sourceName, file, DuckDBDataProtocol.BROWSER_FILEREADER, true);
+
+ if (fileExt === 'csv') {
+ await c.query(`
+ CREATE TABLE '${tableName}' AS
+ SELECT *
+ FROM read_csv('${sourceName}', header = true, auto_detect = true, sample_size = -1);
+ `);
+ } else if (fileExt === 'json') {
+ await c.query(`
+ CREATE TABLE '${tableName}' AS
+ SELECT *
+ FROM read_json_auto('${sourceName}');
+ `);
+ } else if (fileExt === 'geojson') {
+ await c.query(`
+ CREATE TABLE '${tableName}' AS
+ SELECT *
+ FROM ST_READ('${sourceName}', keep_wkb = TRUE);
+ `);
+ } else if (fileExt === 'parquet') {
+ await c.query(`
+ CREATE TABLE '${tableName}' AS
+ SELECT *
+ FROM read_parquet('${sourceName}')
+ `);
+ }
+ }
+ } catch (errorData) {
+ if (errorData instanceof Error) {
+ const message = errorData.message || '';
+ // output more readable errors for known issues
+ if (message.includes('Arrow Type with extension name: geoarrow')) {
+ error = new Error(
+ 'The GeoArrow extensions are not implemented in the connected DuckDB version.'
+ );
+ } else if (message.includes("Geoparquet column 'geometry' does not have geometry types")) {
+ error = new Error(
+ `Invalid Input Error: Geoparquet column 'geometry' does not have geometry types.
+Possible reasons:
+ - Old .parquet files that don't match the Parquet format specification.
+ - Unsupported compression.`
+ );
+ }
+ }
+
+ if (!error) {
+ error = errorData as Error;
+ }
+ }
+
+ c.close();
+
+ return error;
+}
+
+/**
+ * Sanitizes a file name to be a valid DuckDB table name.
+ * @param fileName The input file name to be sanitized.
+ * @returns A valid DuckDB table name.
+ */
+export function sanitizeDuckDBTableName(fileName: string): string {
+ // Replace invalid characters with underscores
+ let name = fileName.replace(/[^a-zA-Z0-9_]/g, '_');
+ // Ensure it doesn't start with a digit
+ if (/^\d/.test(name)) {
+ name = 't_' + name;
+ }
+ return name || 'default_table';
+}