Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

feat(table-import): add DuckDB Parquet imports

Abeautifulsnow02f27dc294c5
diff --git a/agents/drivers/duckdb/src/runtime.rs b/agents/drivers/duckdb/src/runtime.rs
index 9d70df898a..e8dde087f3 100644
--- a/agents/drivers/duckdb/src/runtime.rs
+++ b/agents/drivers/duckdb/src/runtime.rs
@@ -50,13 +50,17 @@ impl DuckDbWorkerSession {
pub fn execute(&mut self, params: DuckDbWorkerExecuteParams) -> Result<db::QueryResult, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
- let result = crate::query::duckdb_execute_for_database(
- &locked,
- &self.attached_names,
- params.database.as_deref(),
- &params.sql,
- params.max_rows,
- )?;
+ let result = if params.preserve_insertion_order {
+ execute_with_preserved_insertion_order(&locked, &self.attached_names, &params)?
+ } else {
+ crate::query::duckdb_execute_for_database(
+ &locked,
+ &self.attached_names,
+ params.database.as_deref(),
+ &params.sql,
+ params.max_rows,
+ )?
+ };
if let Some(name) = crate::sql::attached_name_from_attach_sql(&params.sql) {
if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) {
self.attached_names.push(name);
@@ -191,6 +195,53 @@ impl DuckDbWorkerSession {
}
}
+fn execute_with_preserved_insertion_order(
+ connection: &duckdb::Connection,
+ attached_names: &[String],
+ params: &DuckDbWorkerExecuteParams,
+) -> Result<db::QueryResult, String> {
+ let current: String = connection
+ .query_row("SELECT CAST(current_setting('preserve_insertion_order') AS VARCHAR)", [], |row| row.get(0))
+ .map_err(|error| format!("Failed to read DuckDB preserve_insertion_order setting: {error}"))?;
+ let was_enabled = current
+ .parse::<bool>()
+ .map_err(|error| format!("Invalid DuckDB preserve_insertion_order setting {current:?}: {error}"))?;
+ if was_enabled {
+ return crate::query::duckdb_execute_for_database(
+ connection,
+ attached_names,
+ params.database.as_deref(),
+ &params.sql,
+ params.max_rows,
+ );
+ }
+
+ // LIMIT/OFFSET Parquet pages are separate scans. Temporarily enabling this
+ // setting makes every page use the same source order without changing the
+ // user's connection-level setting after the request completes.
+ connection
+ .execute_batch("SET preserve_insertion_order = true")
+ .map_err(|error| format!("Failed to enable DuckDB insertion-order preservation: {error}"))?;
+ let result = crate::query::duckdb_execute_for_database(
+ connection,
+ attached_names,
+ params.database.as_deref(),
+ &params.sql,
+ params.max_rows,
+ );
+ let restore = connection.execute_batch("SET preserve_insertion_order = false");
+
+ match (result, restore) {
+ (result, Ok(())) => result,
+ (Ok(_), Err(error)) => {
+ Err(format!("DuckDB query succeeded but restoring preserve_insertion_order failed: {error}"))
+ }
+ (Err(query_error), Err(restore_error)) => {
+ Err(format!("{query_error}; restoring preserve_insertion_order also failed: {restore_error}"))
+ }
+ }
+}
+
fn expand_tilde(path: &str) -> String {
if !path.starts_with('~') {
return path.to_string();
@@ -497,13 +548,69 @@ mod tests {
.expect("connect");
let result = session
- .execute(DuckDbWorkerExecuteParams { sql: "SELECT 1 AS value".to_string(), database: None, max_rows: None })
+ .execute(DuckDbWorkerExecuteParams {
+ sql: "SELECT 1 AS value".to_string(),
+ database: None,
+ max_rows: None,
+ preserve_insertion_order: false,
+ })
.expect("execute");
assert_eq!(result.columns, vec!["value"]);
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
}
+ #[test]
+ fn worker_session_scopes_insertion_order_preservation() {
+ let mut session = DuckDbWorkerSession::default();
+ session
+ .connect(DuckDbWorkerConnectParams {
+ path: ":memory:".to_string(),
+ attached_databases: Vec::new(),
+ init_script: Some("SET preserve_insertion_order = false".to_string()),
+ })
+ .expect("connect");
+
+ let disabled = session
+ .execute(DuckDbWorkerExecuteParams {
+ sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
+ database: None,
+ max_rows: None,
+ preserve_insertion_order: false,
+ })
+ .expect("read initial setting");
+ assert_eq!(disabled.rows, vec![vec![serde_json::json!(false)]]);
+
+ let preserved = session
+ .execute(DuckDbWorkerExecuteParams {
+ sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
+ database: None,
+ max_rows: None,
+ preserve_insertion_order: true,
+ })
+ .expect("execute with order preservation");
+ assert_eq!(preserved.rows, vec![vec![serde_json::json!(true)]]);
+
+ session
+ .execute(DuckDbWorkerExecuteParams {
+ sql: "SELECT * FROM missing_table".to_string(),
+ database: None,
+ max_rows: None,
+ preserve_insertion_order: true,
+ })
+ .expect_err("missing table should fail");
+
+ let restored = session
+ .execute(DuckDbWorkerExecuteParams {
+ sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
+ database: None,
+ max_rows: None,
+ preserve_insertion_order: false,
+ })
+ .expect("read restored setting");
+ assert_eq!(restored.rows, vec![vec![serde_json::json!(false)]]);
+ }
+
#[test]
fn worker_session_lists_tables_and_columns() {
let mut session = DuckDbWorkerSession::default();
@@ -522,6 +629,7 @@ mod tests {
.to_string(),
database: None,
max_rows: None,
+ preserve_insertion_order: false,
})
.expect("create table");
@@ -566,6 +674,7 @@ mod tests {
sql: "CREATE VIEW active_orders AS SELECT 1 AS id".to_string(),
database: None,
max_rows: None,
+ preserve_insertion_order: false,
})
.expect("create view");
@@ -713,6 +822,7 @@ mod tests {
sql: format!("ATTACH '{}' AS \"sales db\";", attached_path.to_string_lossy().replace('\'', "''")),
database: None,
max_rows: None,
+ preserve_insertion_order: false,
})
.expect("attach sql");
diff --git a/agents/drivers/duckdb/src/wire.rs b/agents/drivers/duckdb/src/wire.rs
index d276132a43..24e03a29f0 100644
--- a/agents/drivers/duckdb/src/wire.rs
+++ b/agents/drivers/duckdb/src/wire.rs
@@ -271,6 +271,9 @@ pub struct DuckDbWorkerExecuteParams {
pub database: Option<String>,
#[serde(default)]
pub max_rows: Option<usize>,
+ /// Temporarily preserve source row order while executing this query.
+ #[serde(default)]
+ pub preserve_insertion_order: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
diff --git a/agents/drivers/duckdb/tests/duckdb_worker_process.rs b/agents/drivers/duckdb/tests/duckdb_worker_process.rs
index 39d2c026fd..568d852aec 100644
--- a/agents/drivers/duckdb/tests/duckdb_worker_process.rs
+++ b/agents/drivers/duckdb/tests/duckdb_worker_process.rs
@@ -351,6 +351,7 @@ async fn worker_process_exits_when_stdin_closes_during_active_query() {
sql: "SELECT sum(sin(i::DOUBLE) * cos(i::DOUBLE / 3.0)) FROM range(100000000000) AS t(i)".to_string(),
database: None,
max_rows: Some(10),
+ preserve_insertion_order: false,
},
)
.expect("execute request"),
@@ -508,10 +509,14 @@ async fn worker_shutdown_checkpoints_and_removes_wal() {
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(&wal_path);
- let client =
- DuckDbWorkerClient::open_with_executable(executable.clone(), db_path.to_string_lossy().to_string(), Vec::new(), None)
- .await
- .expect("worker process connects");
+ let client = DuckDbWorkerClient::open_with_executable(
+ executable.clone(),
+ db_path.to_string_lossy().to_string(),
+ Vec::new(),
+ None,
+ )
+ .await
+ .expect("worker process connects");
client
.execute(
None,
diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue
index 57c6c9a8b9..2c4c371c8b 100644
--- a/apps/desktop/src/components/import/TableImportDialog.vue
+++ b/apps/desktop/src/components/import/TableImportDialog.vue
@@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
+ { value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" },
];
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
@@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
+const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb");
+const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value));
+const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]);
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
// ordinary INSERT/error behavior instead of approximating an upsert.
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
@@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
if (lower.endsWith(".json")) return "json";
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
if (lower.endsWith(".sql")) return "sql";
+ if (lower.endsWith(".parquet")) return "parquet";
return "csv";
}
@@ -671,6 +676,8 @@ async function loadTargetColumns() {
async function previewSelectedImportFile(fileOrPath: string | File) {
const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath);
return api.previewTableImportFile(input.fileOrPath, {
+ connectionId: props.prefillConnectionId,
+ database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: sourceFormat.value,
parseOptions: parseOptions.value,
@@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
}
function assignSelectedSource(source: string | File) {
+ const detectedFormat = detectFormat(typeof source === "string" ? source : source.name);
+ if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) {
+ selectedSource.value = null;
+ preview.value = null;
+ errorMessage.value = t("tableImport.parquetOnlyDuckdb");
+ return;
+ }
batchTasks.value = [];
selectedSource.value = source;
preview.value = null;
@@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) {
progress.value = null;
errorMessage.value = "";
const name = typeof source === "string" ? source : source.name;
- sourceFormat.value = detectFormat(name);
+ sourceFormat.value = detectedFormat;
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
if (!newTableName.value.trim()) {
newTableName.value = suggestedTableName(name);
@@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) {
const tasks: BatchImportTask[] = [];
const usedNames = new Set<string>();
const formats = sources.map((source) => detectFormat(sourceName(source)));
+ if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) {
+ errorMessage.value = t("tableImport.parquetOnlyDuckdb");
+ loadingPreview.value = false;
+ return;
+ }
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
try {
for (const [index, source] of sources.entries()) {
const format = formats[index]!;
const initialPreview = await api.previewTableImportFile(source, {
+ connectionId: props.prefillConnectionId,
+ database: props.prefillDatabase || "",
sourceFormat: format,
parseOptions: taskParseOptions(format),
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
@@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) {
const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName;
const taskPreview = effectiveSheetName
? await api.previewTableImportFile(input.fileOrPath, {
+ connectionId: props.prefillConnectionId,
+ database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: format,
parseOptions: taskParseOptions(format, effectiveSheetName),
@@ -809,11 +832,12 @@ async function selectFile() {
const selected = await open({
multiple: targetMode.value === "create",
filters: [
- { name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
+ { name: "Data files", extensions: importFileExtensions.value },
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
{ name: "JSON", extensions: ["json"] },
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
{ name: "SQL", extensions: ["sql"] },
+ ...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []),
],
});
if (!selected) return;
@@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() {
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source);
const nextPreview = await api.previewTableImportFile(input.fileOrPath, {
+ connectionId: props.prefillConnectionId,
+ database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: task.format,
parseOptions: taskParseOptions(task.format, task.sheetName),
@@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
- <input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
+ <input ref="fileInput" type="file" :accept="importFileExtensions.map((extension) => `.${extension}`).join(',')" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
<span class="min-w-0 truncate text-sm font-medium">
@@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => {
{{ t("tableImport.selectFile") }}
</Button>
</div>
- <div class="grid grid-cols-6 gap-2">
- <button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
+ <div
+ data-testid="table-import-format-options"
+ class="grid grid-cols-2 gap-2 sm:grid-cols-3"
+ :class="{
+ 'lg:grid-cols-6': availableFormatOptions.length === 6,
+ 'lg:grid-cols-7': availableFormatOptions.length === 7,
+ }"
+ >
+ <button v-for="format in availableFormatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
<div class="mt-1 text-[11px] leading-snug text-muted-foreground">{{ t(format.descriptionKey) }}</div>
@@ -1347,7 +1380,7 @@ watch(rawProgressPercent, (percent) => {
<SelectValue />
</SelectTrigger>
<SelectContent>
- <SelectItem v-for="format in formatOptions" :key="format.value" :value="format.value">
+ <SelectItem v-for="format in availableFormatOptions" :key="format.value" :value="format.value">
{{ t(format.labelKey) }}
</SelectItem>
</SelectContent>
diff --git a/apps/desktop/src/components/import/__tests__/TableImportDialog.batchSelection.spec.ts b/apps/desktop/src/components/import/__tests__/TableImportDialog.batchSelection.spec.ts
index 1ae48f11e9..21809cb90c 100644
--- a/apps/desktop/src/components/import/__tests__/TableImportDialog.batchSelection.spec.ts
+++ b/apps/desktop/src/components/import/__tests__/TableImportDialog.batchSelection.spec.ts
@@ -27,6 +27,7 @@ vi.mock("@/stores/connectionStore", () => ({
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
+ if (id === "duckdb-1") return { id, name: "DuckDB", db_type: "duckdb" };
return undefined;
},
ensureConnected: mocks.ensureConnected,
@@ -198,7 +199,7 @@ async function flushAsyncUpdates() {
}
}
-async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
+async function mountSourceDialog(connectionId = "connection-1") {
const container = document.createElement("div");
document.body.append(container);
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
@@ -206,12 +207,20 @@ async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connec
app.use(i18n);
app.mount(container);
await flushAsyncUpdates();
+}
+
+async function selectFiles(files: File[]) {
const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
Object.defineProperty(input, "files", { configurable: true, value: files });
input.dispatchEvent(new Event("change", { bubbles: true }));
await flushAsyncUpdates();
}
+async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
+ await mountSourceDialog(connectionId);
+ await selectFiles(files);
+}
+
function button(text: string) {
const result = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
expect(result, `button ${text}`).toBeTruthy();
@@ -270,6 +279,26 @@ afterEach(() => {
});
describe("TableImportDialog batch selection", () => {
+ it("uses all desktop grid columns when Parquet is unavailable", async () => {
+ await mountSourceDialog();
+
+ const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
+ expect(formats?.classList.contains("lg:grid-cols-6")).toBe(true);
+ expect(formats?.classList.contains("lg:grid-cols-7")).toBe(false);
+ });
+
+ it("offers Parquet for DuckDB and forwards the connection context to preview", async () => {
+ mocks.previewTableImportFile.mockImplementation(() => Promise.resolve({ ...delimitedPreview(), fileName: "rows.parquet", fileType: "parquet", sourceFingerprint: "rows-parquet" }));
+ await mountSourceDialog("duckdb-1");
+
+ const fileInput = document.body.querySelector<HTMLInputElement>('input[type="file"]');
+ const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
+ expect(fileInput?.accept).toContain(".parquet");
+ expect(formats?.classList.contains("lg:grid-cols-7")).toBe(true);
+ await selectFiles([new File(["parquet"], "rows.parquet")]);
+ expect(mocks.previewTableImportFile).toHaveBeenCalledWith(expect.any(File), expect.objectContaining({ connectionId: "duckdb-1", database: "main", sourceFormat: "parquet" }));
+ });
+
it("lets the import setup dialog be minimized and restored before import starts", async () => {
await mountDialog();
expect(button("Next").disabled).toBe(false);
diff --git a/apps/desktop/src/i18n/locales/az.ts b/apps/desktop/src/i18n/locales/az.ts
index 11dccf8c90..905adf1e20 100644
--- a/apps/desktop/src/i18n/locales/az.ts
+++ b/apps/desktop/src/i18n/locales/az.ts
@@ -6273,6 +6273,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL skriptlərində INSERT əmrləri",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB üçün Parquet faylları",
+ parquetOnlyDuckdb: "Parquet idxalı yalnız DuckDB bağlantıları üçün dəstəklənir",
previewRows: "Sətirlərin önbaxışı",
delimiter: "Ayırıcı",
encoding: "Kodlaşdırma",
diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts
index 72aa9f8819..026ce181f0 100644
--- a/apps/desktop/src/i18n/locales/en.ts
+++ b/apps/desktop/src/i18n/locales/en.ts
@@ -6881,6 +6881,9 @@ export default {
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "INSERT statements in SQL scripts",
+ formatParquet: "Parquet",
+ formatParquetDescription: "Parquet files for DuckDB",
+ parquetOnlyDuckdb: "Parquet import is only supported for DuckDB connections",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Encoding",
diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts
index e921ce64a1..45d0adc76d 100644
--- a/apps/desktop/src/i18n/locales/es.ts
+++ b/apps/desktop/src/i18n/locales/es.ts
@@ -6311,6 +6311,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Sentencias INSERT en scripts SQL",
+ formatParquet: "Parquet",
+ formatParquetDescription: "Archivos Parquet para DuckDB",
+ parquetOnlyDuckdb: "La importación de Parquet solo es compatible con conexiones DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codificación",
diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts
index 71ebf8d46e..066efb71aa 100644
--- a/apps/desktop/src/i18n/locales/it.ts
+++ b/apps/desktop/src/i18n/locales/it.ts
@@ -6233,6 +6233,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Istruzioni INSERT negli script SQL",
+ formatParquet: "Parquet",
+ formatParquetDescription: "File Parquet per DuckDB",
+ parquetOnlyDuckdb: "L'importazione Parquet è supportata solo per le connessioni DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codifica",
diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts
index 2e5762dcf6..f449a8c70f 100644
--- a/apps/desktop/src/i18n/locales/ja.ts
+++ b/apps/desktop/src/i18n/locales/ja.ts
@@ -6252,6 +6252,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL スクリプトの INSERT ステートメント",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB 用の Parquet ファイル",
+ parquetOnlyDuckdb: "Parquet のインポートは DuckDB 接続でのみサポートされています",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "文字コード",
diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts
index 62ae43dadc..b8cc10b0d2 100644
--- a/apps/desktop/src/i18n/locales/ko.ts
+++ b/apps/desktop/src/i18n/locales/ko.ts
@@ -5963,6 +5963,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 스크립트의 INSERT 문",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB용 Parquet 파일",
+ parquetOnlyDuckdb: "Parquet 가져오기는 DuckDB 연결에서만 지원됩니다",
previewRows: "미리보기 행",
delimiter: "구분자",
encoding: "인코딩",
diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts
index d9f7a6a01a..0fa6017f57 100644
--- a/apps/desktop/src/i18n/locales/pt-BR.ts
+++ b/apps/desktop/src/i18n/locales/pt-BR.ts
@@ -6235,6 +6235,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Instruções INSERT em scripts SQL",
+ formatParquet: "Parquet",
+ formatParquetDescription: "Arquivos Parquet para DuckDB",
+ parquetOnlyDuckdb: "A importação de Parquet é compatível apenas com conexões DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codificação",
diff --git a/apps/desktop/src/i18n/locales/ru.ts b/apps/desktop/src/i18n/locales/ru.ts
index 77cff903b3..b107f3af09 100644
--- a/apps/desktop/src/i18n/locales/ru.ts
+++ b/apps/desktop/src/i18n/locales/ru.ts
@@ -7400,6 +7400,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Операторы INSERT в скриптах SQL",
+ formatParquet: "Parquet",
+ formatParquetDescription: "Файлы Parquet для DuckDB",
+ parquetOnlyDuckdb: "Импорт Parquet поддерживается только для подключений DuckDB",
previewRows: "Строк предпросмотра",
delimiter: "Разделитель",
encoding: "Кодировка",
diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts
index 4657dbfcad..99bfb6b789 100644
--- a/apps/desktop/src/i18n/locales/tr.ts
+++ b/apps/desktop/src/i18n/locales/tr.ts
@@ -6155,6 +6155,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL betiklerindeki INSERT ifadeleri",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB için Parquet dosyaları",
+ parquetOnlyDuckdb: "Parquet içe aktarma yalnızca DuckDB bağlantılarında desteklenir",
previewRows: "Satırları önizle",
delimiter: "Ayırıcı",
encoding: "Kodlama",
diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts
index 6a30a373ed..515e968b6f 100644
--- a/apps/desktop/src/i18n/locales/zh-CN.ts
+++ b/apps/desktop/src/i18n/locales/zh-CN.ts
@@ -6848,6 +6848,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS、XLSX、XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 脚本中的 INSERT 语句",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB 的 Parquet 文件",
+ parquetOnlyDuckdb: "Parquet 导入仅支持 DuckDB 连接",
previewRows: "预览行数",
delimiter: "分隔符",
encoding: "文件编码",
diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts
index b5bb357f3b..1714981ada 100644
--- a/apps/desktop/src/i18n/locales/zh-TW.ts
+++ b/apps/desktop/src/i18n/locales/zh-TW.ts
@@ -5542,6 +5542,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS、XLSX、XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 指令檔中的 INSERT 語句",
+ formatParquet: "Parquet",
+ formatParquetDescription: "DuckDB 的 Parquet 檔案",
+ parquetOnlyDuckdb: "Parquet 匯入僅支援 DuckDB 連線",
previewRows: "預覽列數",
delimiter: "分隔符",
encoding: "檔案編碼",
diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts
index 6adfcffc09..ef5a3e9013 100644
--- a/apps/desktop/src/lib/backend/http.ts
+++ b/apps/desktop/src/lib/backend/http.ts
@@ -2888,6 +2888,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
sourceRef: options.sourceRef,
+ connectionId: options.connectionId,
+ database: options.database,
sourceFormat: options.sourceFormat,
parseOptions: options.parseOptions,
previewLimit: options.previewLimit,
@@ -2898,6 +2900,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
}
const formData = new FormData();
formData.append("file", fileOrPath);
+ if (options.connectionId) formData.append("connectionId", options.connectionId);
+ if (options.database != null) formData.append("database", options.database);
if (options.sourceFormat) formData.append("sourceFormat", options.sourceFormat);
if (options.parseOptions) formData.append("parseOptions", JSON.stringify(options.parseOptions));
if (options.previewLimit != null) formData.append("previewLimit", String(options.previewLimit));
diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts
index 0f1f243fab..6184f1fabb 100644
--- a/apps/desktop/src/lib/backend/tauri.ts
+++ b/apps/desktop/src/lib/backend/tauri.ts
@@ -5447,7 +5447,7 @@ export type TableImportMode = "append" | "truncate";
export type TableImportConflictPolicy = "error" | "skip" | "updateExisting";
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
-export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
+export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql" | "parquet";
export type TableImportJsonShape = "auto" | "objects" | "arrays";
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
@@ -5474,6 +5474,8 @@ export interface TableImportParseOptions {
export interface TableImportPreviewRequest {
filePath: string;
+ connectionId?: string | null;
+ database?: string | null;
sourceRef?: string | null;
sourceFormat?: TableImportSourceFormat | null;
parseOptions?: TableImportParseOptions | null;
diff --git a/crates/dbx-core/examples/table_import_bench.rs b/crates/dbx-core/examples/table_import_bench.rs
index 62ec89823d..ef3d33c823 100644
--- a/crates/dbx-core/examples/table_import_bench.rs
+++ b/crates/dbx-core/examples/table_import_bench.rs
@@ -127,6 +127,8 @@ async fn benchmark_file(
let preview_started = Instant::now();
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path_text.to_string(),
+ connection_id: None,
+ database: None,
source_ref: None,
source_format: Some(format),
parse_options: parse_options.clone(),
diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs
index 9fa660d35a..3a6b8c6b79 100644
--- a/crates/dbx-core/src/data/table_import.rs
+++ b/crates/dbx-core/src/data/table_import.rs
@@ -24,6 +24,7 @@ use sqlparser::dialect::{GenericDialect, MsSqlDialect, MySqlDialect, OracleDiale
use sqlparser::parser::Parser;
use crate::connection::{task_client_session_id, AppState, PoolKind};
+use crate::db;
use crate::models::connection::DatabaseType;
use crate::sql::SqlParsingOptions;
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
@@ -135,6 +136,7 @@ pub enum TableImportSourceFormat {
Json,
Excel,
Sql,
+ Parquet,
}
impl TableImportSourceFormat {
@@ -146,6 +148,7 @@ impl TableImportSourceFormat {
TableImportSourceFormat::Json => "json",
TableImportSourceFormat::Excel => "excel",
TableImportSourceFormat::Sql => "sql",
+ TableImportSourceFormat::Parquet => "parquet",
}
}
@@ -237,6 +240,10 @@ impl Default for TableImportParseOptions {
pub struct TableImportPreviewRequest {
pub file_path: String,
#[serde(default)]
+ pub connection_id: Option<String>,
+ #[serde(default)]
+ pub database: Option<String>,
+ #[serde(default)]
pub source_ref: Option<String>,
#[serde(default)]
pub source_format: Option<TableImportSourceFormat>,
@@ -447,6 +454,7 @@ pub enum ImportFileKind {
Json,
Xlsx,
Sql,
+ Parquet,
}
impl ImportFileKind {
@@ -458,6 +466,7 @@ impl ImportFileKind {
ImportFileKind::Json => "json",
ImportFileKind::Xlsx => "xlsx",
ImportFileKind::Sql => "sql",
+ ImportFileKind::Parquet => "parquet",
}
}
}
@@ -476,6 +485,8 @@ pub fn import_file_kind(path: &str) -> Result<ImportFileKind, String> {
Ok(ImportFileKind::Xlsx)
} else if lower.ends_with(".sql") {
Ok(ImportFileKind::Sql)
+ } else if lower.ends_with(".parquet") {
+ Ok(ImportFileKind::Parquet)
} else {
Err("Unsupported import file type".to_string())
}
@@ -489,6 +500,7 @@ pub fn source_format_for_path(path: &str) -> Result<TableImportSourceFormat, Str
ImportFileKind::Json => TableImportSourceFormat::Json,
ImportFileKind::Xlsx => TableImportSourceFormat::Excel,
ImportFileKind::Sql => TableImportSourceFormat::Sql,
+ ImportFileKind::Parquet => TableImportSourceFormat::Parquet,
})
}
@@ -501,6 +513,117 @@ pub fn effective_source_format(
.ok_or_else(|| "Unsupported import file type".to_string())
}
+#[cfg(feature = "duckdb-sidecar")]
+#[derive(Clone)]
+struct DuckDbImportContext {
+ client: Arc<db::duckdb_worker_process::DuckDbWorkerClient>,
+ database: Option<String>,
+ query_timeout: Option<Duration>,
+}
+
+#[cfg(not(feature = "duckdb-sidecar"))]
+#[derive(Clone)]
+struct DuckDbImportContext;
+
+#[cfg(feature = "duckdb-sidecar")]
+impl DuckDbImportContext {
+ async fn execute(&self, sql: String, max_rows: Option<usize>) -> Result<db::QueryResult, String> {
+ self.client.execute(self.database.clone(), sql, max_rows, None, self.query_timeout).await
+ }
+
+ async fn execute_preserving_insertion_order(
+ &self,
+ sql: String,
+ max_rows: Option<usize>,
+ ) -> Result<db::QueryResult, String> {
+ // Each Parquet page is an independent query. Ask the worker to scope
+ // DuckDB's order-preservation setting to that request so OFFSET refers
+ // to the same source order even when the connection configured it off.
+ self.client
+ .execute_preserving_insertion_order(self.database.clone(), sql, max_rows, None, self.query_timeout)
+ .await
+ }
+}
+
+#[cfg(feature = "duckdb-sidecar")]
+async fn duckdb_import_context_for_pool(
+ state: &AppState,
+ pool_key: &str,
+ database: &str,
+) -> Result<DuckDbImportContext, String> {
+ let pool = state.pool_handle(pool_key).await.ok_or_else(|| "Connection pool not found".to_string())?;
+ let client = match pool {
+ PoolKind::DuckDbWorker(client) => client,
+ _ => return Err("Parquet import requires a DuckDB connection".to_string()),
+ };
+ let query_timeout = crate::query::operation_budget_for_pool_key(state, pool_key, None).await.query_timeout;
+ Ok(DuckDbImportContext {
+ client,
+ database: (!database.trim().is_empty()).then_some(database.to_string()),
+ query_timeout,
+ })
+}
+
+async fn duckdb_import_context_for_source(
+ state: &AppState,
+ pool_key: &str,
+ db_type: &DatabaseType,
+ database: &str,
+ source_format: TableImportSourceFormat,
+) -> Result<Option<DuckDbImportContext>, String> {
+ if source_format != TableImportSourceFormat::Parquet {
+ return Ok(None);
+ }
+ if *db_type != DatabaseType::DuckDb {
+ return Err("Parquet import is only supported for DuckDB connections".to_string());
+ }
+ #[cfg(feature = "duckdb-sidecar")]
+ {
+ return duckdb_import_context_for_pool(state, pool_key, database).await.map(Some);
+ }
+ #[cfg(not(feature = "duckdb-sidecar"))]
+ {
+ let _ = (state, pool_key, database);
+ Err("DuckDB worker support is not compiled in this build".to_string())
+ }
+}
+
+fn duckdb_sql_string_literal(value: &str) -> String {
+ format!("'{}'", value.replace('\'', "''"))
+}
+
+fn duckdb_parquet_scan_sql(path: &str, limit: usize, offset: Option<usize>) -> String {
+ let mut sql = format!("SELECT * FROM read_parquet({}) LIMIT {}", duckdb_sql_string_literal(path), limit);
+ if let Some(offset) = offset {
+ sql.push_str(&format!(" OFFSET {offset}"));
+ }
+ sql
+}
+
+fn duckdb_parquet_count_sql(path: &str) -> String {
+ format!("SELECT COUNT(*) AS dbx_total_rows FROM read_parquet({})", duckdb_sql_string_literal(path))
+}
+
+fn duckdb_count_result(result: &db::QueryResult) -> Result<usize, String> {
+ let value = result
+ .rows
+ .first()
+ .and_then(|row| row.first())
+ .ok_or_else(|| "DuckDB Parquet row-count query returned no result".to_string())?;
+ match value {
+ serde_json::Value::Number(number) => number
+ .as_u64()
+ .and_then(|count| usize::try_from(count).ok())
+ .ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
+ serde_json::Value::String(value) => value
+ .parse::<u64>()
+ .ok()
+ .and_then(|count| usize::try_from(count).ok())
+ .ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
+ _ => Err("DuckDB Parquet row-count query returned an invalid value".to_string()),
+ }
+}
+
pub fn normalize_header(value: &str, index: usize) -> String {
let trimmed = value.trim();
if trimmed.is_empty() {
@@ -2172,10 +2295,99 @@ fn stream_json_rows_to_channel(
///
/// 分隔文本与 Excel 仍然先把行解析进内存;`.sql` 脚本与 JSON 改用流式行来源
/// 增量产出,内存不再随文件体积增长,因此二者都没有 100 MB 的体积上限。
+// Keep Parquet decoding in the DuckDB worker so compression and logical/nested type handling
+// stay identical to normal DuckDB queries; the import pipeline only owns mapping and writes.
+#[cfg(feature = "duckdb-sidecar")]
+struct DuckDbParquetRowStream {
+ context: DuckDbImportContext,
+ file_path: String,
+ columns: Vec<String>,
+ total_rows: usize,
+ next_offset: usize,
+ pending: Option<Vec<Vec<serde_json::Value>>>,
+}
+
+#[cfg(feature = "duckdb-sidecar")]
+impl DuckDbParquetRowStream {
+ async fn open(context: DuckDbImportContext, file_path: &str, batch_size: usize) -> Result<Self, String> {
+ let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(file_path), Some(1)).await?)?;
+ if total_rows == 0 {
+ return Err("Parquet file has no rows".to_string());
+ }
+ let first_batch_size = batch_size.max(1);
+ let first = context
+ .execute_preserving_insertion_order(
+ duckdb_parquet_scan_sql(file_path, first_batch_size, None),
+ Some(first_batch_size),
+ )
+ .await?;
+ if total_rows > 0 && first.rows.is_empty() {
+ return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
+ }
+ Ok(Self {
+ context,
+ file_path: file_path.to_string(),
+ columns: first.columns,
+ total_rows,
+ next_offset: first.rows.len(),
+ pending: (!first.rows.is_empty()).then_some(first.rows),
+ })
+ }
+
+ fn columns(&self) -> Vec<String> {
+ self.columns.clone()
+ }
+
+ fn total_rows(&self) -> usize {
+ self.total_rows
+ }
+
+ async fn next_batch(&mut self, max_rows: usize) -> Result<Option<Vec<Vec<serde_json::Value>>>, String> {
+ if let Some(rows) = self.pending.take() {
+ return Ok(Some(rows));
+ }
+ if self.next_offset >= self.total_rows {
+ return Ok(None);
+ }
+ let limit = max_rows.max(1);
+ // The worker protocol exposes bounded query results but no cursor API. LIMIT/OFFSET keeps
+ // each response bounded and lets the existing import writer retain its batch semantics.
+ let result = self
+ .context
+ .execute_preserving_insertion_order(
+ duckdb_parquet_scan_sql(&self.file_path, limit, Some(self.next_offset)),
+ Some(limit),
+ )
+ .await?;
+ if result.rows.is_empty() {
+ return Err(format!(
+ "DuckDB Parquet scan ended before the expected row count (read {}, expected {})",
+ self.next_offset, self.total_rows
+ ));
+ }
+ self.next_offset = self.next_offset.saturating_add(result.rows.len());
+ Ok(Some(result.rows))
+ }
+}
+
enum ImportRowSource {
- Materialized { columns: Vec<String>, rows: std::vec::IntoIter<Vec<serde_json::Value>>, total_rows: usize },
- Sql { stream: Box<SqlImportRowStream>, bytes_read: Arc<AtomicU64>, pending: Option<Vec<Vec<serde_json::Value>>> },
- Json { stream: Box<JsonImportRowStream> },
+ Materialized {
+ columns: Vec<String>,
+ rows: std::vec::IntoIter<Vec<serde_json::Value>>,
+ total_rows: usize,
+ },
+ Sql {
+ stream: Box<SqlImportRowStream>,
+ bytes_read: Arc<AtomicU64>,
+ pending: Option<Vec<Vec<serde_json::Value>>>,
+ },
+ Json {
+ stream: Box<JsonImportRowStream>,
+ },
+ #[cfg(feature = "duckdb-sidecar")]
+ DuckDb {
+ stream: Box<DuckDbParquetRowStream>,
+ },
}
impl ImportRowSource {
@@ -2189,6 +2401,7 @@ impl ImportRowSource {
parse_options: &TableImportParseOptions,
text_source_columns: HashSet<String>,
first_batch_rows: usize,
+ duckdb_context: Option<DuckDbImportContext>,
) -> Result<Self, String> {
if source_format == TableImportSourceFormat::Sql {
let bytes_read = Arc::new(AtomicU64::new(0));
@@ -2200,6 +2413,12 @@ impl ImportRowSource {
let stream = JsonImportRowStream::open(file_path, parse_options, first_batch_rows).await?;
return Ok(Self::Json { stream: Box::new(stream) });
}
+ #[cfg(feature = "duckdb-sidecar")]
+ if source_format == TableImportSourceFormat::Parquet {
+ let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
+ let stream = DuckDbParquetRowStream::open(context, file_path, first_batch_rows).await?;
+ return Ok(Self::DuckDb { stream: Box::new(stream) });
+ }
let parsed = parse_import_file_with_options_and_text_columns(
file_path,
Some(source_format),
@@ -2218,6 +2437,8 @@ impl ImportRowSource {
stream.columns().ok_or_else(|| "No INSERT statements found in SQL file".to_string())
}
Self::Json { stream, .. } => Ok(stream.columns.clone()),
+ #[cfg(feature = "duckdb-sidecar")]
+ Self::DuckDb { stream } => Ok(stream.columns()),
}
}
@@ -2226,6 +2447,8 @@ impl ImportRowSource {
Self::Materialized { total_rows, .. } => *total_rows,
Self::Sql { stream, .. } => stream.total_rows(),
Self::Json { stream, .. } => stream.total_rows,
+ #[cfg(feature = "duckdb-sidecar")]
+ Self::DuckDb { stream } => stream.total_rows(),
}
}
@@ -2243,6 +2466,8 @@ impl ImportRowSource {
Self::Materialized { .. } => total_bytes,
Self::Sql { bytes_read, .. } => bytes_read.load(Ordering::Relaxed).min(total_bytes),
Self::Json { stream, .. } => stream.bytes_read.load(Ordering::Relaxed).min(total_bytes),
+ #[cfg(feature = "duckdb-sidecar")]
+ Self::DuckDb { .. } => total_bytes,
}
}
@@ -2259,6 +2484,8 @@ impl ImportRowSource {
stream.next_batch(max_rows).await
}
Self::Json { stream, .. } => stream.receiver.recv().await.transpose(),
+ #[cfg(feature = "duckdb-sidecar")]
+ Self::DuckDb { stream } => stream.next_batch(max_rows).await,
}
}
}
@@ -4178,7 +4405,11 @@ pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportF
fn ensure_non_streaming_file_size(path: &str, format: TableImportSourceFormat) -> Result<(), String> {
// 分隔文本、SQL 脚本与 JSON 都是流式解析,内存占用不随文件体积增长,没有体积上限。
- if format.is_delimited() || format == TableImportSourceFormat::Sql || format == TableImportSourceFormat::Json {
+ if format.is_delimited()
+ || format == TableImportSourceFormat::Sql
+ || format == TableImportSourceFormat::Json
+ || format == TableImportSourceFormat::Parquet
+ {
return Ok(());
}
let metadata = std::fs::metadata(path).map_err(|e| e.to_string())?;
@@ -4247,6 +4478,7 @@ async fn parse_import_file_with_options_and_text_columns(
.await
.map_err(|e| e.to_string())?
}
+ TableImportSourceFormat::Parquet => Err("Parquet import requires a DuckDB connection".to_string()),
}
}
@@ -4291,6 +4523,52 @@ async fn parse_import_preview_file_with_options(
Ok((parsed, true, sheets))
}
+#[cfg(feature = "duckdb-sidecar")]
+async fn parse_duckdb_parquet_preview(
+ context: &DuckDbImportContext,
+ path: &str,
+ preview_limit: usize,
+) -> Result<ParsedImportFile, String> {
+ let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(path), Some(1)).await?)?;
+ if total_rows == 0 {
+ return Err("Parquet file has no rows".to_string());
+ }
+ let preview_limit = preview_limit.max(1);
+ let result = context
+ .execute_preserving_insertion_order(duckdb_parquet_scan_sql(path, preview_limit, None), Some(preview_limit))
+ .await?;
+ if result.columns.is_empty() {
+ return Err("Parquet file has no columns".to_string());
+ }
+ if total_rows > 0 && result.rows.is_empty() {
+ return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
+ }
+ Ok(ParsedImportFile { columns: result.columns, rows: result.rows, total_rows, effective_encoding: None })
+}
+
+async fn parse_import_preview_file_with_context(
+ path: &str,
+ format: TableImportSourceFormat,
+ options: &TableImportParseOptions,
+ preview_limit: usize,
+ duckdb_context: Option<&DuckDbImportContext>,
+) -> Result<(ParsedImportFile, bool, Vec<String>), String> {
+ if format == TableImportSourceFormat::Parquet {
+ #[cfg(feature = "duckdb-sidecar")]
+ {
+ let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
+ let parsed = parse_duckdb_parquet_preview(context, path, preview_limit).await?;
+ return Ok((parsed, true, Vec::new()));
+ }
+ #[cfg(not(feature = "duckdb-sidecar"))]
+ {
+ let _ = (path, options, preview_limit, duckdb_context);
+ return Err("DuckDB worker support is not compiled in this build".to_string());
+ }
+ }
+ parse_import_preview_file_with_options(path, format, options, preview_limit).await
+}
+
pub async fn parse_import_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
parse_import_file_with_options(path, None, &TableImportParseOptions::default(), preview_limit).await
}
@@ -6164,15 +6442,17 @@ fn validated_prepared_import_source(
})
}
-pub async fn preview_table_import_file_with_request(
+async fn preview_table_import_file_with_context(
request: TableImportPreviewRequest,
+ duckdb_context: Option<&DuckDbImportContext>,
) -> Result<TableImportPreview, String> {
let format = effective_source_format(&request.file_path, request.source_format)?;
- let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_options(
+ let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_context(
&request.file_path,
format,
&request.parse_options,
request.preview_limit.unwrap_or(DEFAULT_PREVIEW_LIMIT),
+ duckdb_context,
)
.await?;
let metadata = tokio::fs::metadata(&request.file_path).await.map_err(|e| e.to_string())?;
@@ -6198,9 +6478,50 @@ pub async fn preview_table_import_file_with_request(
})
}
+pub async fn preview_table_import_file_with_request(
+ request: TableImportPreviewRequest,
+) -> Result<TableImportPreview, String> {
+ preview_table_import_file_with_context(request, None).await
+}
+
+pub async fn preview_table_import_file_with_state(
+ state: &AppState,
+ request: TableImportPreviewRequest,
+) -> Result<TableImportPreview, String> {
+ let format = effective_source_format(&request.file_path, request.source_format)?;
+ if format != TableImportSourceFormat::Parquet {
+ return preview_table_import_file_with_request(request).await;
+ }
+ let connection_id = request
+ .connection_id
+ .as_deref()
+ .filter(|value| !value.trim().is_empty())
+ .ok_or_else(|| "Parquet import preview requires a DuckDB connection".to_string())?
+ .to_string();
+ let db_type = crate::transfer::get_db_type(state, &connection_id).await?;
+ if db_type != DatabaseType::DuckDb {
+ return Err("Parquet import is only supported for DuckDB connections".to_string());
+ }
+ let database = request.database.clone().filter(|value| !value.trim().is_empty());
+ let client_session_id = format!("table-import-preview-{}", uuid::Uuid::new_v4());
+ let pool_key =
+ state.get_or_create_pool_for_session(&connection_id, database.as_deref(), Some(&client_session_id)).await?;
+ let context =
+ duckdb_import_context_for_source(state, &pool_key, &db_type, database.as_deref().unwrap_or_default(), format)
+ .await;
+ let result = match context {
+ Ok(context) => preview_table_import_file_with_context(request, context.as_ref()).await,
+ Err(error) => Err(error),
+ };
+ let _ = state.detach_client_session_pool(&connection_id, database.as_deref(), &client_session_id).await;
+ result
+}
+
pub async fn preview_table_import_file_core(file_path: &str) -> Result<TableImportPreview, String> {
preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: file_path.to_string(),
+ connection_id: None,
+ database: None,
source_ref: None,
source_format: None,
parse_options: TableImportParseOptions::default(),
@@ -6731,6 +7052,11 @@ where
format!("Import source is no longer available: {error}"),
));
}
+ let duckdb_context =
+ match duckdb_import_context_for_source(state, pool_key, db_type, &request.database, source_format).await {
+ Ok(context) => context,
+ Err(error) => return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error)),
+ };
let import_sql_hard_limit = mysql_import_sql_hard_limit(state, pool_key).await;
let prepared_source = validated_prepared_import_source(request, source_format);
let prepared_source_total_exact =
@@ -6831,11 +7157,12 @@ where
{
prepared
} else {
- match parse_import_preview_file_with_options(
+ match parse_import_preview_file_with_context(
&request.file_path,
source_format,
&import_parse_options,
CREATE_TABLE_INFERENCE_ROWS,
+ duckdb_context.as_ref(),
)
.await
{
@@ -7769,6 +8096,7 @@ where
&import_parse_options,
text_source_columns,
effective_batch_size,
+ duckdb_context.clone(),
)
.await
{
@@ -8056,6 +8384,30 @@ mod tests {
.collect()
}
+ #[test]
+ fn parquet_extension_maps_to_the_duckdb_source_format() {
+ assert_eq!(source_format_for_path("sales.PARQUET").unwrap(), TableImportSourceFormat::Parquet);
+ assert_eq!(import_file_kind("sales.parquet").unwrap(), ImportFileKind::Parquet);
+ }
+
+ #[test]
+ fn parquet_scan_sql_escapes_paths_and_adds_offset_after_limit() {
+ let sql = duckdb_parquet_scan_sql(r"C:\data\customer's.parquet", 250, Some(500));
+
+ assert_eq!(sql, "SELECT * FROM read_parquet('C:\\data\\customer''s.parquet') LIMIT 250 OFFSET 500");
+ }
+
+ #[test]
+ fn parquet_source_size_is_not_rejected_by_the_materialized_file_limit() {
+ let path =
+ std::env::temp_dir().join(format!("dbx-table-import-parquet-limit-{}.parquet", uuid::Uuid::new_v4()));
+ let file = File::create(&path).unwrap();
+ file.set_len(MAX_NON_STREAMING_IMPORT_BYTES + 1).unwrap();
+
+ ensure_non_streaming_file_size(&path.to_string_lossy(), TableImportSourceFormat::Parquet).unwrap();
+ let _ = std::fs::remove_file(path);
+ }
+
#[test]
fn table_import_progress_and_summary_report_elapsed_ms() {
let started_at = std::time::Instant::now() - std::time::Duration::from_millis(25);
@@ -9248,6 +9600,8 @@ mod tests {
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path.to_string_lossy().to_string(),
+ connection_id: None,
+ database: None,
source_ref: None,
source_format: Some(TableImportSourceFormat::Csv),
parse_options: TableImportParseOptions::default(),
@@ -12845,6 +13199,8 @@ mod tests {
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
let error = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path.to_string_lossy().to_string(),
+ connection_id: None,
+ database: None,
source_ref: Some("missing".to_string()),
source_format: Some(TableImportSourceFormat::Csv),
parse_options: TableImportParseOptions::default(),
diff --git a/crates/dbx-drivers/src/db/duckdb_worker_process.rs b/crates/dbx-drivers/src/db/duckdb_worker_process.rs
index 4330e56e67..ee63d0b0fe 100644
--- a/crates/dbx-drivers/src/db/duckdb_worker_process.rs
+++ b/crates/dbx-drivers/src/db/duckdb_worker_process.rs
@@ -209,6 +209,35 @@ impl DuckDbWorkerClient {
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
+ ) -> Result<db::QueryResult, DuckDbWorkerError> {
+ self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, false).await
+ }
+
+ /// Executes a query with DuckDB insertion-order preservation enabled for
+ /// the duration of that request. This is used by Parquet pagination:
+ /// independent LIMIT/OFFSET scans must see the same source order even
+ /// when the connection's init script disabled `preserve_insertion_order`.
+ pub async fn execute_preserving_insertion_order(
+ &self,
+ database: Option<String>,
+ sql: String,
+ max_rows: Option<usize>,
+ cancel_token: Option<CancellationToken>,
+ query_timeout: Option<Duration>,
+ ) -> Result<db::QueryResult, String> {
+ self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, true)
+ .await
+ .map_err(|error| error.message)
+ }
+
+ async fn execute_typed_with_options(
+ &self,
+ database: Option<String>,
+ sql: String,
+ max_rows: Option<usize>,
+ cancel_token: Option<CancellationToken>,
+ query_timeout: Option<Duration>,
+ preserve_insertion_order: bool,
) -> Result<db::QueryResult, DuckDbWorkerError> {
let _query_guard = self.inner.query_lock.lock().await;
let client = self.clone();
@@ -228,7 +257,7 @@ impl DuckDbWorkerClient {
match client
.send_request_structured::<db::QueryResult>(
DuckDbWorkerMethod::Execute,
- DuckDbWorkerExecuteParams { sql, database, max_rows },
+ DuckDbWorkerExecuteParams { sql, database, max_rows, preserve_insertion_order },
None,
)
.await
diff --git a/crates/dbx-drivers/src/db/duckdb_worker_protocol.rs b/crates/dbx-drivers/src/db/duckdb_worker_protocol.rs
index 45898bbda3..1acbd640e6 100644
--- a/crates/dbx-drivers/src/db/duckdb_worker_protocol.rs
+++ b/crates/dbx-drivers/src/db/duckdb_worker_protocol.rs
@@ -116,6 +116,13 @@ pub struct DuckDbWorkerExecuteParams {
pub database: Option<String>,
#[serde(default)]
pub max_rows: Option<usize>,
+ /// Temporarily preserve source row order while executing this query.
+ ///
+ /// Parquet imports page through the same file with LIMIT/OFFSET. The
+ /// worker must opt into stable insertion order for every page when a
+ /// connection init script disabled DuckDB's default.
+ #[serde(default)]
+ pub preserve_insertion_order: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
diff --git a/crates/dbx-web/src/routes/table_import.rs b/crates/dbx-web/src/routes/table_import.rs
index 1000193ed4..b17acdcb3f 100644
--- a/crates/dbx-web/src/routes/table_import.rs
+++ b/crates/dbx-web/src/routes/table_import.rs
@@ -88,6 +88,10 @@ pub struct CancelImportRequest {
pub struct PreviewUploadedImportRequest {
pub source_ref: String,
#[serde(default)]
+ pub connection_id: Option<String>,
+ #[serde(default)]
+ pub database: Option<String>,
+ #[serde(default)]
pub source_format: Option<TableImportSourceFormat>,
#[serde(default)]
pub parse_options: TableImportParseOptions,
@@ -110,6 +114,8 @@ pub async fn preview_import(
cleanup_expired_import_uploads(&tmp_dir, Duration::from_secs(24 * 60 * 60));
let mut uploaded_file: Option<(String, PathBuf)> = None;
+ let mut connection_id: Option<String> = None;
+ let mut database: Option<String> = None;
let mut source_format: Option<TableImportSourceFormat> = None;
let mut parse_options = TableImportParseOptions::default();
let mut preview_limit: Option<usize> = None;
@@ -146,6 +152,8 @@ pub async fn preview_import(
}
};
match name.as_str() {
+ "connectionId" => connection_id = Some(value),
+ "database" => database = Some(value),
"sourceFormat" => {
source_format = match serde_json::from_value(serde_json::Value::String(value)) {
Ok(source_format) => Some(source_format),
@@ -174,13 +182,18 @@ pub async fn preview_import(
if let Some((source_ref, file_path)) = uploaded_file {
let file_path_str = file_path.to_string_lossy().to_string();
- let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
- file_path: file_path_str,
- source_ref: Some(source_ref),
- source_format,
- parse_options,
- preview_limit,
- })
+ let preview = table_import::preview_table_import_file_with_state(
+ &state.app,
+ TableImportPreviewRequest {
+ file_path: file_path_str,
+ connection_id,
+ database,
+ source_ref: Some(source_ref),
+ source_format,
+ parse_options,
+ preview_limit,
+ },
+ )
.await;
let preview = match preview {
Ok(preview) => preview,
@@ -207,13 +220,18 @@ pub async fn preview_uploaded_import(
Json(request): Json<PreviewUploadedImportRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let file_path = uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref)?;
- let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
- file_path: file_path.to_string_lossy().to_string(),
- source_ref: Some(request.source_ref),
- source_format: request.source_format,
- parse_options: request.parse_options,
- preview_limit: request.preview_limit,
- })
+ let preview = table_import::preview_table_import_file_with_state(
+ &state.app,
+ TableImportPreviewRequest {
+ file_path: file_path.to_string_lossy().to_string(),
+ connection_id: request.connection_id,
+ database: request.database,
+ source_ref: Some(request.source_ref),
+ source_format: request.source_format,
+ parse_options: request.parse_options,
+ preview_limit: request.preview_limit,
+ },
+ )
.await
.map_err(AppError::from)?;
serde_json::to_value(preview).map(Json).map_err(|error| AppError::from(error.to_string()))
diff --git a/docs/content/docs/table-import.cn.mdx b/docs/content/docs/table-import.cn.mdx
index 0c5bf7e908..42f993aaea 100644
--- a/docs/content/docs/table-import.cn.mdx
+++ b/docs/content/docs/table-import.cn.mdx
@@ -1,6 +1,6 @@
---
title: 表数据导入
-description: 通过向导将 CSV、TSV、文本、JSON、Excel 或 SQL 数据导入现有表,或按文件批量创建新表。
+description: 通过向导将 CSV、TSV、文本、JSON、Excel、SQL 或 DuckDB Parquet 数据导入现有表,或按文件批量创建新表。
---
表数据导入用于把文件数据写入数据库表。DBX 会先解析和预览文件,再让你确认解析选项、目标表、字段映射和执行方式,而不是选中文件后立即写入。
@@ -28,6 +28,7 @@ MongoDB Collection 不走这条关系型导入流程。请在 Collection 文档
| JSON | 支持单个对象、对象数组或数组行;可以指定对象或数组结构 |
| Excel | 支持 `.xlsx`、`.xlsm` 和 `.xls`,可选择工作表 |
| SQL | 包含字面量 `INSERT INTO ... VALUES` 语句的 `.sql` 脚本,每个文件对应一张表 |
+| Parquet | `.parquet` 文件,仅在目标连接为 DuckDB 时提供 |
SQL 导入只接受字面量 `INSERT INTO ... VALUES` 行;表达式、`REPLACE`、`INSERT IGNORE`、`INSERT ... SELECT` 等无法无损改写的内容会被拒绝,以保留数据和语句语义。
@@ -35,6 +36,8 @@ SQL 导入按分块流式读取脚本,不把整个文件读进内存,因此
Desktop 直接读取所选本地文件;Docker/Web 会先把浏览器选择的文件上传到服务器临时目录,预览和导入结束后释放临时源文件。
+Parquet 导入使用所选 DuckDB 连接的原生 Parquet 读取能力,仍然沿用预览、字段映射、新建表、追加和清空后导入流程;其他数据库类型不会提供 Parquet 导入格式。大型 Parquet 文件按有界批次读取,嵌套值仍受目标字段类型和 DuckDB 类型转换规则约束。
+
## 导入向导
<Steps>
diff --git a/docs/content/docs/table-import.mdx b/docs/content/docs/table-import.mdx
index 0e07072f7b..1eaab90b03 100644
--- a/docs/content/docs/table-import.mdx
+++ b/docs/content/docs/table-import.mdx
@@ -1,6 +1,6 @@
---
title: Table Import
-description: Use a guided workflow to import CSV, TSV, text, JSON, Excel, or SQL data into an existing table or create new tables in batches.
+description: Use a guided workflow to import CSV, TSV, text, JSON, Excel, SQL, or DuckDB Parquet data into an existing table or create new tables in batches.
---
Table Import writes file data into database tables. DBX parses and previews the source first, then asks you to review parsing options, the target, column mappings, and execution settings before any rows are written.
@@ -28,6 +28,7 @@ Create-table mode accepts multiple files at once. Excel workbooks are expanded i
| JSON | A single object, object rows, or array rows; the shape can be selected explicitly |
| Excel | `.xlsx`, `.xlsm`, and `.xls`, with worksheet selection |
| SQL | `.sql` scripts containing literal `INSERT INTO ... VALUES` statements for one table |
+| Parquet | `.parquet` files, available when the target connection is DuckDB |
SQL import accepts only literal `INSERT INTO ... VALUES` rows. Expressions, `REPLACE`, `INSERT IGNORE`, `INSERT ... SELECT`, and other constructs that cannot be rewritten losslessly are rejected so data and statement semantics are preserved.
@@ -35,6 +36,8 @@ SQL import streams the script in chunks instead of loading it into memory, so th
Desktop reads selected local files directly. Docker/Web uploads browser-selected files to temporary server storage, then releases the prepared source after preview and import finish.
+Parquet import uses the selected DuckDB connection's native Parquet reader. The same preview, mapping, new-table, append, and truncate workflow applies, while other database types do not offer Parquet as an import format. Large Parquet sources are read in bounded batches; nested values remain subject to the target column type and DuckDB conversion rules.
+
## Import Wizard
<Steps>
diff --git a/src-tauri/src/commands/table_import.rs b/src-tauri/src/commands/table_import.rs
index e21ada8cf9..128a971d5f 100644
--- a/src-tauri/src/commands/table_import.rs
+++ b/src-tauri/src/commands/table_import.rs
@@ -51,8 +51,11 @@ async fn clear_cancelled(import_id: &str) {
}
#[tauri::command]
-pub async fn preview_table_import_file(request: TableImportPreviewRequest) -> Result<TableImportPreview, String> {
- dbx_core::table_import::preview_table_import_file_with_request(request).await
+pub async fn preview_table_import_file(
+ state: State<'_, Arc<AppState>>,
+ request: TableImportPreviewRequest,
+) -> Result<TableImportPreview, String> {
+ dbx_core::table_import::preview_table_import_file_with_state(&state, request).await
}
#[tauri::command]