25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
fix(duckdb): prevent duplicate and missing rows during export
Acd-kk4a5f0e84bfda
diff --git a/agents/drivers/duckdb/Cargo.lock b/agents/drivers/duckdb/Cargo.lockindex 4de6fb5092..5d11625479 100644--- a/agents/drivers/duckdb/Cargo.lock+++ b/agents/drivers/duckdb/Cargo.lock@@ -3262,6 +3262,7 @@ version = "0.30.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"dependencies = [+ "cc","pkg-config","vcpkg",]diff --git a/agents/drivers/duckdb/Cargo.toml b/agents/drivers/duckdb/Cargo.tomlindex 5b1a243ec9..c391971162 100644--- a/agents/drivers/duckdb/Cargo.toml+++ b/agents/drivers/duckdb/Cargo.toml@@ -32,7 +32,7 @@ serde_json = "1"tokio = { version = "1", features = ["full"] }[dev-dependencies]-dbx-core = { path = "../../../crates/dbx-core", default-features = false, features = ["duckdb-sidecar"] }+dbx-core = { path = "../../../crates/dbx-core", default-features = false, features = ["duckdb-sidecar", "sqlite-bundled", "test-support"] }sysinfo = "0.32"tokio-util = "0.7"uuid = { version = "1", features = ["v4"] }diff --git a/agents/drivers/duckdb/tests/duckdb_worker_process.rs b/agents/drivers/duckdb/tests/duckdb_worker_process.rsindex 568d852aec..61eda0b5c3 100644--- a/agents/drivers/duckdb/tests/duckdb_worker_process.rs+++ b/agents/drivers/duckdb/tests/duckdb_worker_process.rs@@ -16,6 +16,21 @@ use tokio_util::sync::CancellationToken;static TEMP_DB_COUNTER: AtomicU64 = AtomicU64::new(0);+#[path = "../../../../crates/dbx-core/tests/support/duckdb_export.rs"]+mod export_regressions;++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn worker_parquet_exports_preserve_rows_and_project_totals() {+ let _guard = duckdb_worker_process_test_guard().await;+ export_regressions::run_parquet_exports(PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver")));+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn worker_export_snapshots_preserve_values_and_cleanup() {+ let _guard = duckdb_worker_process_test_guard().await;+ export_regressions::run_snapshot_cleanup(PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver")));+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn worker_process_reads_view_source() {let _guard = duckdb_worker_process_test_guard().await;diff --git a/crates/dbx-core/src/data/duckdb_export.rs b/crates/dbx-core/src/data/duckdb_export.rsnew file mode 100644index 0000000000..fcf2dc24c7--- /dev/null+++ b/crates/dbx-core/src/data/duckdb_export.rs@@ -0,0 +1,220 @@+use tokio_util::sync::CancellationToken;++use crate::connection::AppState;+use crate::models::connection::DatabaseType;+use crate::query::{execute_sql_statement_with_options, QueryExecutionOptions};+use crate::types::QueryResult;++/// Cleanup only drops our generated temporary table; a bounded wait keeps a+/// broken worker from hanging export completion, but the wait must not be+/// turned into a kill (see `TemporaryDdlPolicy::Cleanup`).+#[cfg(feature = "duckdb-sidecar")]+const SNAPSHOT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);++/// Timeout policy for the snapshot's own temporary-table DDL. These statements+/// must not inherit the interactive query timeout: materialization scans the+/// whole source query, while cleanup must never restart a shared worker.+#[derive(Clone, Copy)]+enum TemporaryDdlPolicy {+ /// Unbounded, like the previous page-by-page export whose individual page+ /// queries stayed inside the interactive timeout. An explicit user cancel+ /// still aborts it through the cancel token.+ Materialize,+ /// Best-effort cleanup with a bounded wait that only interrupts.+ Cleanup,+}++/// A single execution of an export query, stored in DuckDB's temporary catalog.+/// Independent LIMIT/OFFSET queries can overlap when ORDER BY contains ties,+/// including when preserve_insertion_order is enabled.+pub(super) struct DuckDbExportSnapshot {+ table: String,+ columns: Vec<String>,+ connection_id: String,+ database: String,+ schema: Option<String>,+ options: QueryExecutionOptions,+ pool_key: Option<String>,+}++impl DuckDbExportSnapshot {+ pub(super) fn new(+ connection_id: &str,+ database: &str,+ schema: Option<&str>,+ options: QueryExecutionOptions,+ ) -> Self {+ Self {+ table: format!("__dbx_export_{}", uuid::Uuid::new_v4().simple()),+ columns: Vec::new(),+ connection_id: connection_id.to_string(),+ database: database.to_string(),+ schema: schema.map(str::to_string),+ options,+ pool_key: None,+ }+ }++ fn qualified_table(&self) -> String {+ format!("\"temp\".\"main\".\"{}\"", self.table)+ }++ async fn execute(+ &self,+ state: &AppState,+ sql: &str,+ max_rows: usize,+ cancel_token: Option<CancellationToken>,+ ) -> Result<QueryResult, String> {+ let mut options = self.options.clone();+ options.max_rows = Some(max_rows);+ execute_sql_statement_with_options(+ state,+ &self.connection_id,+ &self.database,+ sql,+ self.schema.as_deref(),+ cancel_token,+ options,+ )+ .await+ }++ /// These statements only create/drop our generated temporary table. Keep+ /// them off the user-SQL write guard, after validating the source query,+ /// so read-only connections can still export without modifying source data.+ async fn execute_temporary_ddl(+ &self,+ state: &AppState,+ sql: &str,+ cancel_token: Option<CancellationToken>,+ policy: TemporaryDdlPolicy,+ ) -> Result<(), String> {+ #[cfg(feature = "duckdb-sidecar")]+ {+ let pool_key = self.pool_key.as_deref().ok_or("DuckDB export connection is not initialized")?;+ let Some(crate::connection::PoolKind::DuckDbWorker(client)) = state.pool_handle(pool_key).await else {+ return Err("DuckDB export connection is closed".into());+ };+ match policy {+ // Materialization is a full scan of the source query and can+ // legitimately take longer than the interactive query timeout;+ // inheriting it would SIGKILL the worker and lose the shared+ // `:memory:` session before the export writes a single row.+ TemporaryDdlPolicy::Materialize => {+ client.execute(Some(self.database.clone()), sql.to_string(), Some(1), cancel_token, None).await?;+ }+ // Cleanup runs on a shared connection: a 5s timeout that killed+ // the worker would discard the whole session, so it may only+ // interrupt the DROP and report the failure.+ TemporaryDdlPolicy::Cleanup => {+ client+ .execute_interrupt_only(+ Some(self.database.clone()),+ sql.to_string(),+ Some(1),+ SNAPSHOT_CLEANUP_TIMEOUT,+ )+ .await?;+ }+ }+ Ok(())+ }+ #[cfg(not(feature = "duckdb-sidecar"))]+ {+ let _ = (state, sql, cancel_token, policy);+ Err("DuckDB worker support is not compiled in this build".into())+ }+ }++ pub(super) async fn materialize(+ &mut self,+ state: &AppState,+ sql: &str,+ row_limit: Option<usize>,+ cancel_token: Option<CancellationToken>,+ ) -> Result<(), String> {+ let statements = crate::sql::split_sql_statements_for_database(sql, DatabaseType::DuckDb);+ if statements.len() != 1 {+ return Err("DuckDB export requires a single query".to_string());+ }+ let sql = &statements[0];+ let pool_key = state+ .get_or_create_pool_for_session(+ &self.connection_id,+ Some(&self.database),+ self.options.client_session_id.as_deref(),+ )+ .await?;+ crate::query::check_read_only_for_connection(state, &pool_key, sql).await?;+ self.pool_key = Some(pool_key);+ // DESCRIBE binds without running the query. Internal names preserve+ // duplicate result labels and user columns named rowid in the output.+ let metadata =+ self.execute(state, &format!("DESCRIBE (\n{sql}\n)"), i32::MAX as usize, cancel_token.clone()).await?;+ if metadata.truncated {+ return Err("DuckDB export column metadata was truncated".to_string());+ }+ self.columns = metadata+ .rows+ .iter()+ .map(|row| {+ row.first()+ .and_then(|cell| cell.as_str())+ .map(str::to_string)+ .ok_or_else(|| "DuckDB export returned invalid column metadata".to_string())+ })+ .collect::<Result<_, _>>()?;+ if self.columns.is_empty() {+ return Err("DuckDB export query returned no columns".to_string());+ }+ let aliases = (0..self.columns.len()).map(|i| format!("\"__dbx_column_{i}\"")).collect::<Vec<_>>().join(", ");+ let limit = row_limit.map(|limit| format!(" LIMIT {limit}")).unwrap_or_default();+ // Capture output order before materialization, independently of how+ // DuckDB subsequently stores temporary-table blocks in parallel.+ let create = format!(+ "CREATE TEMP TABLE \"{}\" (\"__dbx_row_number\", {aliases}) AS \+ SELECT row_number() OVER (), * FROM (\n{sql}\n) AS \"__dbx_source\"{limit}",+ self.table+ );+ self.execute_temporary_ddl(state, &create, cancel_token, TemporaryDdlPolicy::Materialize)+ .await+ .map_err(|error| format!("DuckDB export materialization failed: {error}"))?;+ Ok(())+ }++ pub(super) async fn page(+ &self,+ state: &AppState,+ offset: u64,+ limit: usize,+ cancel_token: Option<CancellationToken>,+ ) -> Result<QueryResult, String> {+ let end = offset.saturating_add(limit as u64);+ let projection = self+ .columns+ .iter()+ .enumerate()+ .map(|(i, column)| format!("\"__dbx_column_{i}\" AS \"{}\"", column.replace('"', "\"\"")))+ .collect::<Vec<_>>()+ .join(", ");+ let sql = format!(+ "SELECT {projection} FROM {} WHERE \"__dbx_row_number\" > {offset} \+ AND \"__dbx_row_number\" <= {end} ORDER BY \"__dbx_row_number\"",+ self.qualified_table()+ );+ self.execute(state, &sql, limit, cancel_token).await+ }++ pub(super) async fn close(self, state: &AppState) {+ // Cleanup must not inherit a cancelled execution id. The qualified name+ // can only refer to this export's temporary table.+ if self.pool_key.is_none() {+ return;+ }+ let sql = format!("DROP TABLE IF EXISTS {}", self.qualified_table());+ if let Err(error) = self.execute_temporary_ddl(state, &sql, None, TemporaryDdlPolicy::Cleanup).await {+ log::error!("Failed to remove DuckDB export snapshot table {}: {error}", self.table);+ }+ }+}diff --git a/crates/dbx-core/src/data/mod.rs b/crates/dbx-core/src/data/mod.rsindex 95017e1043..0d9c44281a 100644--- a/crates/dbx-core/src/data/mod.rs+++ b/crates/dbx-core/src/data/mod.rs@@ -5,6 +5,7 @@ pub mod dameng_dump;pub mod data_compare;pub mod database_export;pub mod docs;+mod duckdb_export;pub mod export_runtime;pub mod grid_clipboard_guard;pub mod mongodb_dump;diff --git a/crates/dbx-core/src/data/query_result_export.rs b/crates/dbx-core/src/data/query_result_export.rsindex b3142e0916..eaf332dcc6 100644--- a/crates/dbx-core/src/data/query_result_export.rs+++ b/crates/dbx-core/src/data/query_result_export.rs@@ -1,3 +1,4 @@+use super::duckdb_export::DuckDbExportSnapshot;use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{BufWriter, Seek, Write};@@ -963,7 +964,20 @@ pub async fn export_query_result_core(on_progress: impl Fn(TableExportProgress),) -> Result<(), String> {let mut session_id: Option<String> = None;- let result = export_query_result_core_inner(state, request, cancel_token, &on_progress, &mut session_id).await;+ let mut duckdb_snapshot = None;+ let result = export_query_result_core_inner(+ state,+ request,+ cancel_token,+ &on_progress,+ &mut session_id,+ &mut duckdb_snapshot,+ )+ .await;++ if let Some(snapshot) = duckdb_snapshot {+ snapshot.close(state).await;+ }if let Some(session_id) = session_id {let _ = close_query_session(@@ -991,6 +1005,7 @@ async fn export_query_result_core_inner(cancel_token: Option<CancellationToken>,on_progress: &impl Fn(TableExportProgress),session_id: &mut Option<String>,+ duckdb_snapshot: &mut Option<DuckDbExportSnapshot>,) -> Result<(), String> {let format = request.format.to_lowercase();if format != "csv" && format != "xlsx" && format != "json" && format != "txt" && format != "sql" {@@ -1049,7 +1064,22 @@ async fn export_query_result_core_inner(let mut rows_exported: u64 = 0;let mut offset: usize = 0;let mut wrote_text_header = false;- let mut keyset_plan = build_keyset_plan(state, request).await;+ if request.database_type == DatabaseType::DuckDb && txn_session_id.is_none() {+ let snapshot = duckdb_snapshot.insert(DuckDbExportSnapshot::new(+ &request.connection_id,+ &request.database,+ request.schema.as_deref(),+ QueryExecutionOptions {+ timeout_secs: request.timeout_secs,+ client_session_id: request.client_session_id.clone(),+ execution_id: request.execution_id.clone(),+ catalog: request.catalog.clone(),+ ..Default::default()+ },+ ));+ snapshot.materialize(state, &request.sql, effective_row_limit, cancel_token.clone()).await?;+ }+ let mut keyset_plan = if duckdb_snapshot.is_none() { build_keyset_plan(state, request).await } else { None };// A Kingbase SQL Server compat TOP query that cannot be offset-paginated is// exported with a single execution whose page size is the enforceable row// bound (concrete TOP count and/or the configured export row limit), then it@@ -1058,7 +1088,7 @@ async fn export_query_result_core_inner(if let Some(error) = streaming_pagination_preflight_error(request,page_size,- keyset_plan.is_some(),+ keyset_plan.is_some() || duckdb_snapshot.is_some(),single_execution_bound.is_some(),use_agent_cursor,) {@@ -1094,27 +1124,28 @@ async fn export_query_result_core_inner(remaining.map_or(page_size, |rem| rem.min(page_size)).max(1)};- let (sql_to_execute, plan_limit, use_agent_result_session, single_execution) =- if let Some(plan) = keyset_plan.as_ref() {- (build_keyset_export_sql(plan, request, this_page), this_page, false, false)- } else {- let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {- sql: request.sql.clone(),- query_base_sql: request.query_base_sql.clone(),- database_type: Some(request.database_type),- pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },- use_agent_cursor,- first_page_uses_actual_sql: true,- execution_hint: None,- });- if let Some(error) = plan.pagination_error {- return Err(error);- }- let Some(plan_limit) = plan.page_limit else {- return Err("Failed to build query pagination plan for export".to_string());- };- (plan.sql_to_execute, plan_limit, plan.use_agent_result_session, plan.single_execution)+ let (sql_to_execute, plan_limit, use_agent_result_session, single_execution) = if duckdb_snapshot.is_some() {+ (String::new(), this_page, false, false)+ } else if let Some(plan) = keyset_plan.as_ref() {+ (build_keyset_export_sql(plan, request, this_page), this_page, false, false)+ } else {+ let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {+ sql: request.sql.clone(),+ query_base_sql: request.query_base_sql.clone(),+ database_type: Some(request.database_type),+ pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },+ use_agent_cursor,+ first_page_uses_actual_sql: true,+ execution_hint: None,+ });+ if let Some(error) = plan.pagination_error {+ return Err(error);+ }+ let Some(plan_limit) = plan.page_limit else {+ return Err("Failed to build query pagination plan for export".to_string());};+ (plan.sql_to_execute, plan_limit, plan.use_agent_result_session, plan.single_execution)+ };let options = if use_agent_result_session {QueryExecutionOptions {@@ -1140,7 +1171,9 @@ async fn export_query_result_core_inner(}};- let page_result = if let Some(txn_session_id) = txn_session_id {+ let page_result = if let Some(snapshot) = duckdb_snapshot.as_ref() {+ snapshot.page(state, offset as u64, this_page, cancel_token.clone()).await+ } else if let Some(txn_session_id) = txn_session_id {execute_in_manual_transaction_with_options(state,txn_session_id,diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rsindex 14973f74db..ce4d764536 100644--- a/crates/dbx-core/src/data/table_export.rs+++ b/crates/dbx-core/src/data/table_export.rs@@ -1,3 +1,4 @@+use super::duckdb_export::DuckDbExportSnapshot;use serde::{Deserialize, Serialize};use serde_json::Value;use std::borrow::Cow;@@ -725,8 +726,12 @@ async fn fetch_table_export_batch(cursor_session: &mut Option<TableExportCursorSession>,table_read_attempted: &mut bool,table_read_completed: &mut bool,+ duckdb_snapshot: Option<&DuckDbExportSnapshot>,cancel_token: CancellationToken,) -> Result<QueryResult, String> {+ if let Some(snapshot) = duckdb_snapshot {+ return snapshot.page(state, offset, active_batch_size, Some(cancel_token)).await;+ }if *table_read_completed {return Ok(QueryResult {columns: col_names.to_vec(),@@ -1579,9 +1584,20 @@ pub async fn export_table_data_core(last_rows_exported.store(progress.rows_exported, Ordering::SeqCst);on_progress(progress);};- let result =- export_table_data_core_inner(state, request, &tracked_progress, cancelled.clone(), cancel_token.clone()).await;+ let mut duckdb_snapshot = None;+ let result = export_table_data_core_inner(+ state,+ request,+ &tracked_progress,+ cancelled.clone(),+ cancel_token.clone(),+ &mut duckdb_snapshot,+ )+ .await;cancel_watcher.abort();+ if let Some(snapshot) = duckdb_snapshot {+ snapshot.close(state).await;+ }let client_session_id = table_export_client_session_id(&request.export_id);let _ = state.close_client_session_pool(&request.connection_id, Some(&request.database), &client_session_id).await;match result {@@ -1606,6 +1622,7 @@ async fn export_table_data_core_inner(on_progress: &impl Fn(TableExportProgress),cancelled: Arc<AtomicBool>,cancel_token: CancellationToken,+ duckdb_snapshot: &mut Option<DuckDbExportSnapshot>,) -> Result<(), String> {// 1. Get database typelet db_type = {@@ -1798,6 +1815,24 @@ async fn export_table_data_core_inner(return Ok(());}+ if db_type == DatabaseType::DuckDb {+ let snapshot = duckdb_snapshot.insert(DuckDbExportSnapshot::new(+ &request.connection_id,+ &request.database,+ request.schema.as_deref(),+ QueryExecutionOptions {+ client_session_id: Some(client_session_id),+ // Bounds the small paged reads only: materialization runs as a+ // full source scan and deliberately ignores the interactive+ // query timeout so it cannot SIGKILL the worker mid-export.+ timeout_secs: Some(table_export_query_timeout_secs(state, &pool_key).await),+ ..Default::default()+ },+ ));+ let sql = table_cursor_sql(request, &sql_context, query_col_names.as_ref(), &column_types, &primary_keys);+ snapshot.materialize(state, &sql, row_limit, Some(cancel_token.clone())).await?;+ }+// 8. Create output filelet mut file = if request.format.eq_ignore_ascii_case("sql") {create_table_export_sql_writer(request)?@@ -1860,6 +1895,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;@@ -1960,6 +1996,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;@@ -2074,6 +2111,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;@@ -2170,6 +2208,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;@@ -2255,6 +2294,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;@@ -2340,6 +2380,7 @@ async fn export_table_data_core_inner(&mut cursor_session,&mut table_read_attempted,&mut table_read_completed,+ duckdb_snapshot.as_ref(),cancel_token.clone(),).await?;diff --git a/crates/dbx-core/tests/live_duckdb_export.rs b/crates/dbx-core/tests/live_duckdb_export.rsnew file mode 100644index 0000000000..9598eacc33--- /dev/null+++ b/crates/dbx-core/tests/live_duckdb_export.rs@@ -0,0 +1,16 @@+#![cfg(feature = "duckdb-sidecar")]++#[path = "support/duckdb_export.rs"]+mod support;++#[test]+#[ignore = "requires DBX_DUCKDB_DRIVER_PATH pointing at a DuckDB worker"]+fn duckdb_parquet_exports_are_consistent() {+ support::run_parquet_exports(std::env::var_os("DBX_DUCKDB_DRIVER_PATH").unwrap().into());+}++#[test]+#[ignore = "requires DBX_DUCKDB_DRIVER_PATH pointing at a DuckDB worker"]+fn duckdb_export_snapshots_preserve_values_and_cleanup() {+ support::run_snapshot_cleanup(std::env::var_os("DBX_DUCKDB_DRIVER_PATH").unwrap().into());+}diff --git a/crates/dbx-core/tests/support/duckdb_export.rs b/crates/dbx-core/tests/support/duckdb_export.rsnew file mode 100644index 0000000000..f3869348db--- /dev/null+++ b/crates/dbx-core/tests/support/duckdb_export.rs@@ -0,0 +1,244 @@+use std::collections::{BTreeMap, BTreeSet};+use std::path::{Path, PathBuf};+use std::sync::Arc;++use dbx_core::connection::{AppState, PoolKind};+use dbx_core::db::duckdb_worker_process::DuckDbWorkerClient;+use dbx_core::models::connection::ConnectionConfig;+use dbx_core::query_result_export::{export_query_result_core, QueryResultExportRequest};+use dbx_core::table_export::{export_table_data_core, TableExportRequest};+use dbx_core::table_import::{parse_csv_bytes, parse_xlsx_file};+use serde_json::{json, Value};+use tokio_util::sync::CancellationToken;++fn run_with_stack<F: std::future::Future<Output = ()> + 'static>(test: impl FnOnce() -> F + Send + 'static) {+ std::thread::Builder::new()+ .stack_size(16 * 1024 * 1024)+ .spawn(move || {+ tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(Box::pin(test()));+ })+ .unwrap()+ .join()+ .unwrap();+}++pub fn run_parquet_exports(driver: PathBuf) {+ run_with_stack(move || parquet_exports_are_consistent(driver));+}++pub fn run_snapshot_cleanup(driver: PathBuf) {+ run_with_stack(move || snapshots_preserve_query_values_and_cleanup(driver));+}++struct Fixture {+ state: AppState,+ client: Arc<DuckDbWorkerClient>,+ id: String,+ dir: PathBuf,+}++impl Fixture {+ async fn new(driver: PathBuf) -> Self {+ let id = format!("duckdb-export-{}", uuid::Uuid::new_v4().simple());+ let dir = std::env::temp_dir().join(&id);+ std::fs::create_dir_all(&dir).unwrap();+ let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();+ let state = AppState::new(storage);+ let config: ConnectionConfig = serde_json::from_value(json!({+ "id": id, "name": id, "db_type": "duckdb", "host": ":memory:",+ "port": 0, "username": "", "password": "", "database": "main",+ "query_timeout_secs": 60, "keepalive_interval_secs": 0+ }))+ .unwrap();+ let client = Arc::new(+ DuckDbWorkerClient::open_with_executable(+ driver,+ ":memory:".into(),+ vec![],+ Some("SET threads = 4; SET preserve_insertion_order = false".into()),+ )+ .await+ .unwrap(),+ );+ state.configs.write().await.insert(id.clone(), config.clone());+ state.insert_connection_pool(id.clone(), PoolKind::DuckDbWorker(client.clone()), &config).await.unwrap();+ Self { state, client, id, dir }+ }++ async fn sql(&self, sql: &str) -> dbx_core::types::QueryResult {+ self.client.execute(None, sql.into(), Some(50), None, None).await.unwrap()+ }++ async fn assert_clean(&self) {+ let result = self.sql(+ "SELECT count(*) FROM duckdb_tables() WHERE database_name = 'temp' AND starts_with(table_name, '__dbx_export_')",+ ).await;+ assert_eq!(cell_number(&result.rows[0][0]), 0);+ assert_eq!(cell_number(&self.sql("SELECT * FROM keep_me").await.rows[0][0]), 42);+ }++ fn query_request(&self, sql: &str, path: &Path, format: &str) -> QueryResultExportRequest {+ serde_json::from_value(json!({+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": self.id,+ "database": "main", "schema": "main", "databaseType": "duckdb",+ "sql": sql, "queryBaseSql": sql, "filePath": path,+ "format": format, "pageSize": 512, "includeSqlSheet": false,+ "numericColumnRightAlign": false+ }))+ .unwrap()+ }++ async fn finish(self) {+ self.client.shutdown().await;+ drop(self.state);+ let _ = std::fs::remove_dir_all(self.dir);+ }+}++fn cell_number(value: &Value) -> i64 {+ value.as_i64().unwrap_or_else(|| cell_text(value).parse::<f64>().unwrap().round() as i64)+}++fn cell_text(value: &Value) -> String {+ value.as_str().map(str::to_string).unwrap_or_else(|| value.to_string())+}++fn verify_export(path: &Path, format: &str) {+ let parsed = if format == "csv" {+ parse_csv_bytes(&std::fs::read(path).unwrap(), 20_001).unwrap()+ } else {+ parse_xlsx_file(path.to_str().unwrap(), 20_001).unwrap()+ };+ assert_eq!(parsed.columns, vec!["rowid", "项目名称", "结算账期", "金额"]);+ let mut ids = BTreeSet::new();+ let mut totals = BTreeMap::<String, i64>::new();+ for row in &parsed.rows {+ let id = cell_number(&row[0]);+ assert!(ids.insert(id), "duplicate id {id} in {}", path.display());+ assert_eq!(cell_text(&row[2]), "2026-09");+ let cents = (cell_text(&row[3]).parse::<f64>().unwrap() * 100.0).round() as i64;+ assert_eq!(cents, id + 1, "amount for id {id}");+ *totals.entry(cell_text(&row[1])).or_default() += cents;+ }+ let expected_ids = (0..20_000).filter(|id| id % 5 != 0).collect::<BTreeSet<_>>();+ let mut expected_totals = BTreeMap::<String, i64>::new();+ for id in &expected_ids {+ let project = if *id < 5_000 {+ "Project A"+ } else if *id < 10_000 {+ "Project B"+ } else {+ "Project C"+ };+ *expected_totals.entry(project.into()).or_default() += id + 1;+ }+ assert_eq!(ids, expected_ids, "missing rows in {}", path.display());+ assert_eq!(totals, expected_totals, "project totals in {}", path.display());+}++async fn parquet_exports_are_consistent(driver: PathBuf) {+ let fixture = Fixture::new(driver).await;+ fixture.sql("CREATE TEMP TABLE keep_me AS SELECT 42").await;+ let parquet = fixture.dir.join("settlements.parquet").to_string_lossy().replace('\\', "/");+ fixture.sql(&format!(+ "COPY (SELECT i AS id, CASE WHEN i < 5000 THEN 'Project A' WHEN i < 10000 THEN 'Project B' ELSE 'Project C' END AS project, \+ CASE WHEN i % 5 = 0 THEN '2025-09' ELSE '2026-09' END AS period, \+ ((i + 1) / 100.0)::DECIMAL(18,2) AS amount FROM range(20000) t(i)) \+ TO '{}' (FORMAT PARQUET, ROW_GROUP_SIZE 2048)", parquet.replace('\'', "''")+ )).await;+ fixture.sql(&format!(+ "CREATE VIEW settlements AS SELECT id AS rowid, project AS \"项目名称\", period AS \"结算账期\", amount AS \"金额\" FROM read_parquet('{}')",+ parquet.replace('\'', "''")+ )).await;+ let sql = "SELECT * FROM settlements WHERE \"结算账期\" LIKE '%2026%' ORDER BY \"结算账期\" DESC";+ for preserve in [false, true] {+ fixture.sql(&format!("SET preserve_insertion_order = {preserve}")).await;+ fixture.state.configs.write().await.get_mut(&fixture.id).unwrap().read_only = preserve;+ for format in ["csv", "xlsx"] {+ for table_export in [true, false] {+ let path = fixture.dir.join(format!("{preserve}-{table_export}.{format}"));+ if table_export {+ let request: TableExportRequest = serde_json::from_value(json!({+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": fixture.id,+ "database": "main", "schema": "main", "tableName": "settlements",+ "filePath": path, "format": format, "numericColumnRightAlign": false,+ "columns": ["rowid", "项目名称", "结算账期", "金额"],+ "columnTypes": ["BIGINT", "VARCHAR", "VARCHAR", "DECIMAL(18,2)"],+ "primaryKeys": [], "skipCount": true, "batchSize": 512,+ "whereInput": "\"结算账期\" LIKE '%2026%'", "orderBy": "\"结算账期\" DESC"+ }))+ .unwrap();+ export_table_data_core(&fixture.state, &request, |_| {}).await.unwrap();+ } else {+ let request = fixture.query_request(sql, &path, format);+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();+ }+ verify_export(&path, format);+ fixture.assert_clean().await;+ }+ }+ }+ fixture.finish().await;+}++async fn snapshots_preserve_query_values_and_cleanup(driver: PathBuf) {+ let fixture = Fixture::new(driver).await;+ fixture.sql("CREATE TEMP TABLE keep_me AS SELECT 42").await;+ fixture.sql("CREATE SEQUENCE calls START 1").await;+ let path = fixture.dir.join("query.csv");+ let sql =+ "SELECT nextval('calls') AS id, i AS dup, -i AS dup, i AS rowid FROM range(2050) t(i); -- trailing comment";+ let request = fixture.query_request(sql, &path, "csv");+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();+ let bytes = std::fs::read(&path).unwrap();+ let text = String::from_utf8(bytes.clone()).unwrap();+ assert_eq!(text.trim_start_matches('\u{feff}').lines().next(), Some("\"id\",\"dup\",\"dup\",\"rowid\""));+ let parsed = parse_csv_bytes(&bytes, 2051).unwrap();+ // The importer assigns unique header names; the CSV itself keeps both labels.+ assert_eq!(parsed.columns, vec!["id", "dup", "dup_1", "rowid"]);+ assert_eq!(parsed.rows.len(), 2050);+ for (i, row) in parsed.rows.iter().enumerate() {+ assert_eq!(+ row.iter().map(cell_number).collect::<Vec<_>>(),+ vec![i as i64 + 1, i as i64, -(i as i64), i as i64]+ );+ }+ assert_eq!(cell_number(&fixture.sql("SELECT nextval('calls')").await.rows[0][0]), 2051);+ fixture.assert_clean().await;++ let mut request = fixture.query_request("SELECT * FROM range(2050) t(id) ORDER BY id DESC", &path, "csv");+ request.row_limit = Some(513);+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();+ let parsed = parse_csv_bytes(&std::fs::read(&path).unwrap(), 2051).unwrap();+ assert_eq!(parsed.rows.len(), 513);+ assert_eq!(cell_number(&parsed.rows[0][0]), 2049);+ assert_eq!(cell_number(&parsed.rows[512][0]), 1537);+ fixture.assert_clean().await;++ let request = fixture.query_request("SELECT * FROM range(2050)", &fixture.dir, "xlsx");+ assert!(export_query_result_core(&fixture.state, &request, None, |_| {}).await.is_err());+ fixture.assert_clean().await;++ fixture.sql("CREATE VIEW source AS SELECT * FROM range(2050) t(id)").await;+ let table_request: TableExportRequest = serde_json::from_value(json!({+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": fixture.id,+ "database": "main", "schema": "main", "tableName": "source",+ "filePath": fixture.dir, "format": "xlsx", "numericColumnRightAlign": false,+ "columns": ["id"], "columnTypes": ["BIGINT"], "primaryKeys": [], "skipCount": true+ }))+ .unwrap();+ assert!(export_table_data_core(&fixture.state, &table_request, |_| {}).await.is_err());+ fixture.assert_clean().await;++ let request = fixture.query_request("SELECT * FROM range(2050)", &path, "csv");+ let token = CancellationToken::new();+ export_query_result_core(&fixture.state, &request, Some(token.clone()), |progress| {+ if progress.rows_exported >= 512 {+ token.cancel();+ }+ })+ .await+ .unwrap();+ fixture.assert_clean().await;+ fixture.finish().await;+}diff --git a/crates/dbx-drivers/src/db/duckdb_worker_process.rs b/crates/dbx-drivers/src/db/duckdb_worker_process.rsindex ee63d0b0fe..2665c21dee 100644--- a/crates/dbx-drivers/src/db/duckdb_worker_process.rs+++ b/crates/dbx-drivers/src/db/duckdb_worker_process.rs@@ -33,9 +33,23 @@ const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);/// `connection.rs` so pool close cancellation cannot cut the checkpoint short.const WORKER_SHUTDOWN_EXIT_WAIT: Duration = Duration::from_millis(2500);const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);+/// Upper bound for the worker to acknowledge an interrupt-only cancel. The+/// caller already reports its own timeout error, so waiting longer than this+/// adds latency without improving the outcome.+const WORKER_INTERRUPT_ACK_TIMEOUT: Duration = Duration::from_secs(2);pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;+/// How a request reacts to cancellation or a timeout. Interactive queries+/// restart the worker because a DuckDB connection stuck inside a statement+/// cannot be reused. Statements whose caller shares the session (`:memory:`+/// contents, temp tables, views) must not: they only interrupt.+#[derive(Clone, Copy)]+enum AbortPolicy {+ KillWorker,+ InterruptOnly,+}+struct PendingRequest {generation: u64,sender: oneshot::Sender<DuckDbWorkerResponse>,@@ -210,7 +224,16 @@ impl DuckDbWorkerClient {cancel_token: Option<CancellationToken>,query_timeout: Option<Duration>,) -> Result<db::QueryResult, DuckDbWorkerError> {- self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, false).await+ self.execute_typed_with_options(+ database,+ sql,+ max_rows,+ cancel_token,+ query_timeout,+ false,+ AbortPolicy::KillWorker,+ )+ .await}/// Executes a query with DuckDB insertion-order preservation enabled for@@ -225,7 +248,32 @@ impl DuckDbWorkerClient {cancel_token: Option<CancellationToken>,query_timeout: Option<Duration>,) -> Result<db::QueryResult, String> {- self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, true)+ self.execute_typed_with_options(+ database,+ sql,+ max_rows,+ cancel_token,+ query_timeout,+ true,+ AbortPolicy::KillWorker,+ )+ .await+ .map_err(|error| error.message)+ }++ /// Executes a best-effort statement with a bounded wait that never restarts+ /// the worker. On timeout the running statement is interrupted and a+ /// timeout error is returned; the worker and its session stay alive. Used+ /// where a SIGKILL would discard session state that outlives the statement+ /// (e.g. dropping an export's temporary table on a shared connection).+ pub async fn execute_interrupt_only(+ &self,+ database: Option<String>,+ sql: String,+ max_rows: Option<usize>,+ timeout: Duration,+ ) -> Result<db::QueryResult, String> {+ self.execute_typed_with_options(database, sql, max_rows, None, Some(timeout), false, AbortPolicy::InterruptOnly).await.map_err(|error| error.message)}@@ -238,10 +286,11 @@ impl DuckDbWorkerClient {cancel_token: Option<CancellationToken>,query_timeout: Option<Duration>,preserve_insertion_order: bool,+ abort_policy: AbortPolicy,) -> Result<db::QueryResult, DuckDbWorkerError> {let _query_guard = self.inner.query_lock.lock().await;let client = self.clone();- // Cancellation and timeout restart the worker via cancel_or_kill below. An ordinary+ // Cancellation and timeout apply `abort_policy` below. An ordinary// query error (parser/binder/catalog/runtime) leaves the worker healthy, so we keep it// alive to preserve session-local state (:memory: contents, temp tables, SET, raw ATTACH).// The one exception is a poisoned connection: duckdb-rs can leave the connection unusable@@ -277,33 +326,62 @@ impl DuckDbWorkerClient {(Some(token), Some(duration)) => {tokio::select! {biased;- _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await,+ _ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,result = &mut future => result,- _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await,+ _ = tokio::time::sleep(duration) => {+ self.abort(crate::execution::timeout_error_for(duration), abort_policy).await+ },}}(Some(token), None) => {tokio::select! {biased;- _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await,+ _ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,result = &mut future => result,}}(None, Some(duration)) => {tokio::select! {result = &mut future => result,- _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await,+ _ = tokio::time::sleep(duration) => {+ self.abort(crate::execution::timeout_error_for(duration), abort_policy).await+ },}}(None, None) => future.await,}}- async fn cancel_or_kill(&self, final_error: String) -> Result<db::QueryResult, DuckDbWorkerError> {- let _ = self.cancel().await;+ /// Applies the request's abort policy and reports the cancellation/timeout+ /// as the operation's final error. Interactive queries restart the worker,+ /// because an unresponsive DuckDB session cannot be reused; best-effort+ /// statements only interrupt so a shared session survives.+ async fn abort(&self, final_error: String, policy: AbortPolicy) -> Result<db::QueryResult, DuckDbWorkerError> {+ match policy {+ AbortPolicy::KillWorker => {+ let _ = self.cancel().await;+ }+ AbortPolicy::InterruptOnly => {+ let _ = self.interrupt().await;+ }+ }Err(DuckDbWorkerError::from(final_error))}+ /// Asks the worker to interrupt the statement it is running without+ /// restarting it. Best effort: a worker that cannot acknowledge in time is+ /// left running rather than escalated to SIGKILL.+ async fn interrupt(&self) -> Result<(), String> {+ self.send_request_structured::<serde_json::Value>(+ DuckDbWorkerMethod::Cancel,+ serde_json::json!({}),+ Some(WORKER_INTERRUPT_ACK_TIMEOUT),+ )+ .await+ .map(|_| ())+ .map_err(|error| error.message)+ }+pub async fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {self.metadata_request(DuckDbWorkerMethod::ListDatabases, serde_json::json!({})).await}