Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(duckdb): prevent duplicate and missing rows during export

Acd-kk4a5f0e84bfda
diff --git a/agents/drivers/duckdb/Cargo.lock b/agents/drivers/duckdb/Cargo.lock
index 4de6fb5092..5d11625479 100644
--- a/agents/drivers/duckdb/Cargo.lock
+++ b/agents/drivers/duckdb/Cargo.lock
@@ -3262,6 +3262,7 @@ version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
dependencies = [
+ "cc",
"pkg-config",
"vcpkg",
]
diff --git a/agents/drivers/duckdb/Cargo.toml b/agents/drivers/duckdb/Cargo.toml
index 5b1a243ec9..c391971162 100644
--- a/agents/drivers/duckdb/Cargo.toml
+++ b/agents/drivers/duckdb/Cargo.toml
@@ -32,7 +32,7 @@ serde_json = "1"
tokio = { version = "1", features = ["full"] }
[dev-dependencies]
-dbx-core = { path = "../../../crates/dbx-core", default-features = false, features = ["duckdb-sidecar"] }
+dbx-core = { path = "../../../crates/dbx-core", default-features = false, features = ["duckdb-sidecar", "sqlite-bundled", "test-support"] }
sysinfo = "0.32"
tokio-util = "0.7"
uuid = { version = "1", features = ["v4"] }
diff --git a/agents/drivers/duckdb/tests/duckdb_worker_process.rs b/agents/drivers/duckdb/tests/duckdb_worker_process.rs
index 568d852aec..61eda0b5c3 100644
--- a/agents/drivers/duckdb/tests/duckdb_worker_process.rs
+++ b/agents/drivers/duckdb/tests/duckdb_worker_process.rs
@@ -16,6 +16,21 @@ use tokio_util::sync::CancellationToken;
static TEMP_DB_COUNTER: AtomicU64 = AtomicU64::new(0);
+#[path = "../../../../crates/dbx-core/tests/support/duckdb_export.rs"]
+mod export_regressions;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn worker_parquet_exports_preserve_rows_and_project_totals() {
+ let _guard = duckdb_worker_process_test_guard().await;
+ export_regressions::run_parquet_exports(PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver")));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn worker_export_snapshots_preserve_values_and_cleanup() {
+ let _guard = duckdb_worker_process_test_guard().await;
+ export_regressions::run_snapshot_cleanup(PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver")));
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_reads_view_source() {
let _guard = duckdb_worker_process_test_guard().await;
diff --git a/crates/dbx-core/src/data/duckdb_export.rs b/crates/dbx-core/src/data/duckdb_export.rs
new file mode 100644
index 0000000000..fcf2dc24c7
--- /dev/null
+++ b/crates/dbx-core/src/data/duckdb_export.rs
@@ -0,0 +1,220 @@
+use tokio_util::sync::CancellationToken;
+
+use crate::connection::AppState;
+use crate::models::connection::DatabaseType;
+use crate::query::{execute_sql_statement_with_options, QueryExecutionOptions};
+use crate::types::QueryResult;
+
+/// Cleanup only drops our generated temporary table; a bounded wait keeps a
+/// broken worker from hanging export completion, but the wait must not be
+/// turned into a kill (see `TemporaryDdlPolicy::Cleanup`).
+#[cfg(feature = "duckdb-sidecar")]
+const SNAPSHOT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+
+/// Timeout policy for the snapshot's own temporary-table DDL. These statements
+/// must not inherit the interactive query timeout: materialization scans the
+/// whole source query, while cleanup must never restart a shared worker.
+#[derive(Clone, Copy)]
+enum TemporaryDdlPolicy {
+ /// Unbounded, like the previous page-by-page export whose individual page
+ /// queries stayed inside the interactive timeout. An explicit user cancel
+ /// still aborts it through the cancel token.
+ Materialize,
+ /// Best-effort cleanup with a bounded wait that only interrupts.
+ Cleanup,
+}
+
+/// A single execution of an export query, stored in DuckDB's temporary catalog.
+/// Independent LIMIT/OFFSET queries can overlap when ORDER BY contains ties,
+/// including when preserve_insertion_order is enabled.
+pub(super) struct DuckDbExportSnapshot {
+ table: String,
+ columns: Vec<String>,
+ connection_id: String,
+ database: String,
+ schema: Option<String>,
+ options: QueryExecutionOptions,
+ pool_key: Option<String>,
+}
+
+impl DuckDbExportSnapshot {
+ pub(super) fn new(
+ connection_id: &str,
+ database: &str,
+ schema: Option<&str>,
+ options: QueryExecutionOptions,
+ ) -> Self {
+ Self {
+ table: format!("__dbx_export_{}", uuid::Uuid::new_v4().simple()),
+ columns: Vec::new(),
+ connection_id: connection_id.to_string(),
+ database: database.to_string(),
+ schema: schema.map(str::to_string),
+ options,
+ pool_key: None,
+ }
+ }
+
+ fn qualified_table(&self) -> String {
+ format!("\"temp\".\"main\".\"{}\"", self.table)
+ }
+
+ async fn execute(
+ &self,
+ state: &AppState,
+ sql: &str,
+ max_rows: usize,
+ cancel_token: Option<CancellationToken>,
+ ) -> Result<QueryResult, String> {
+ let mut options = self.options.clone();
+ options.max_rows = Some(max_rows);
+ execute_sql_statement_with_options(
+ state,
+ &self.connection_id,
+ &self.database,
+ sql,
+ self.schema.as_deref(),
+ cancel_token,
+ options,
+ )
+ .await
+ }
+
+ /// These statements only create/drop our generated temporary table. Keep
+ /// them off the user-SQL write guard, after validating the source query,
+ /// so read-only connections can still export without modifying source data.
+ async fn execute_temporary_ddl(
+ &self,
+ state: &AppState,
+ sql: &str,
+ cancel_token: Option<CancellationToken>,
+ policy: TemporaryDdlPolicy,
+ ) -> Result<(), String> {
+ #[cfg(feature = "duckdb-sidecar")]
+ {
+ let pool_key = self.pool_key.as_deref().ok_or("DuckDB export connection is not initialized")?;
+ let Some(crate::connection::PoolKind::DuckDbWorker(client)) = state.pool_handle(pool_key).await else {
+ return Err("DuckDB export connection is closed".into());
+ };
+ match policy {
+ // Materialization is a full scan of the source query and can
+ // legitimately take longer than the interactive query timeout;
+ // inheriting it would SIGKILL the worker and lose the shared
+ // `:memory:` session before the export writes a single row.
+ TemporaryDdlPolicy::Materialize => {
+ client.execute(Some(self.database.clone()), sql.to_string(), Some(1), cancel_token, None).await?;
+ }
+ // Cleanup runs on a shared connection: a 5s timeout that killed
+ // the worker would discard the whole session, so it may only
+ // interrupt the DROP and report the failure.
+ TemporaryDdlPolicy::Cleanup => {
+ client
+ .execute_interrupt_only(
+ Some(self.database.clone()),
+ sql.to_string(),
+ Some(1),
+ SNAPSHOT_CLEANUP_TIMEOUT,
+ )
+ .await?;
+ }
+ }
+ Ok(())
+ }
+ #[cfg(not(feature = "duckdb-sidecar"))]
+ {
+ let _ = (state, sql, cancel_token, policy);
+ Err("DuckDB worker support is not compiled in this build".into())
+ }
+ }
+
+ pub(super) async fn materialize(
+ &mut self,
+ state: &AppState,
+ sql: &str,
+ row_limit: Option<usize>,
+ cancel_token: Option<CancellationToken>,
+ ) -> Result<(), String> {
+ let statements = crate::sql::split_sql_statements_for_database(sql, DatabaseType::DuckDb);
+ if statements.len() != 1 {
+ return Err("DuckDB export requires a single query".to_string());
+ }
+ let sql = &statements[0];
+ let pool_key = state
+ .get_or_create_pool_for_session(
+ &self.connection_id,
+ Some(&self.database),
+ self.options.client_session_id.as_deref(),
+ )
+ .await?;
+ crate::query::check_read_only_for_connection(state, &pool_key, sql).await?;
+ self.pool_key = Some(pool_key);
+ // DESCRIBE binds without running the query. Internal names preserve
+ // duplicate result labels and user columns named rowid in the output.
+ let metadata =
+ self.execute(state, &format!("DESCRIBE (\n{sql}\n)"), i32::MAX as usize, cancel_token.clone()).await?;
+ if metadata.truncated {
+ return Err("DuckDB export column metadata was truncated".to_string());
+ }
+ self.columns = metadata
+ .rows
+ .iter()
+ .map(|row| {
+ row.first()
+ .and_then(|cell| cell.as_str())
+ .map(str::to_string)
+ .ok_or_else(|| "DuckDB export returned invalid column metadata".to_string())
+ })
+ .collect::<Result<_, _>>()?;
+ if self.columns.is_empty() {
+ return Err("DuckDB export query returned no columns".to_string());
+ }
+ let aliases = (0..self.columns.len()).map(|i| format!("\"__dbx_column_{i}\"")).collect::<Vec<_>>().join(", ");
+ let limit = row_limit.map(|limit| format!(" LIMIT {limit}")).unwrap_or_default();
+ // Capture output order before materialization, independently of how
+ // DuckDB subsequently stores temporary-table blocks in parallel.
+ let create = format!(
+ "CREATE TEMP TABLE \"{}\" (\"__dbx_row_number\", {aliases}) AS \
+ SELECT row_number() OVER (), * FROM (\n{sql}\n) AS \"__dbx_source\"{limit}",
+ self.table
+ );
+ self.execute_temporary_ddl(state, &create, cancel_token, TemporaryDdlPolicy::Materialize)
+ .await
+ .map_err(|error| format!("DuckDB export materialization failed: {error}"))?;
+ Ok(())
+ }
+
+ pub(super) async fn page(
+ &self,
+ state: &AppState,
+ offset: u64,
+ limit: usize,
+ cancel_token: Option<CancellationToken>,
+ ) -> Result<QueryResult, String> {
+ let end = offset.saturating_add(limit as u64);
+ let projection = self
+ .columns
+ .iter()
+ .enumerate()
+ .map(|(i, column)| format!("\"__dbx_column_{i}\" AS \"{}\"", column.replace('"', "\"\"")))
+ .collect::<Vec<_>>()
+ .join(", ");
+ let sql = format!(
+ "SELECT {projection} FROM {} WHERE \"__dbx_row_number\" > {offset} \
+ AND \"__dbx_row_number\" <= {end} ORDER BY \"__dbx_row_number\"",
+ self.qualified_table()
+ );
+ self.execute(state, &sql, limit, cancel_token).await
+ }
+
+ pub(super) async fn close(self, state: &AppState) {
+ // Cleanup must not inherit a cancelled execution id. The qualified name
+ // can only refer to this export's temporary table.
+ if self.pool_key.is_none() {
+ return;
+ }
+ let sql = format!("DROP TABLE IF EXISTS {}", self.qualified_table());
+ if let Err(error) = self.execute_temporary_ddl(state, &sql, None, TemporaryDdlPolicy::Cleanup).await {
+ log::error!("Failed to remove DuckDB export snapshot table {}: {error}", self.table);
+ }
+ }
+}
diff --git a/crates/dbx-core/src/data/mod.rs b/crates/dbx-core/src/data/mod.rs
index 95017e1043..0d9c44281a 100644
--- a/crates/dbx-core/src/data/mod.rs
+++ b/crates/dbx-core/src/data/mod.rs
@@ -5,6 +5,7 @@ pub mod dameng_dump;
pub mod data_compare;
pub mod database_export;
pub mod docs;
+mod duckdb_export;
pub mod export_runtime;
pub mod grid_clipboard_guard;
pub mod mongodb_dump;
diff --git a/crates/dbx-core/src/data/query_result_export.rs b/crates/dbx-core/src/data/query_result_export.rs
index b3142e0916..eaf332dcc6 100644
--- a/crates/dbx-core/src/data/query_result_export.rs
+++ b/crates/dbx-core/src/data/query_result_export.rs
@@ -1,3 +1,4 @@
+use super::duckdb_export::DuckDbExportSnapshot;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{BufWriter, Seek, Write};
@@ -963,7 +964,20 @@ pub async fn export_query_result_core(
on_progress: impl Fn(TableExportProgress),
) -> Result<(), String> {
let mut session_id: Option<String> = None;
- let result = export_query_result_core_inner(state, request, cancel_token, &on_progress, &mut session_id).await;
+ let mut duckdb_snapshot = None;
+ let result = export_query_result_core_inner(
+ state,
+ request,
+ cancel_token,
+ &on_progress,
+ &mut session_id,
+ &mut duckdb_snapshot,
+ )
+ .await;
+
+ if let Some(snapshot) = duckdb_snapshot {
+ snapshot.close(state).await;
+ }
if let Some(session_id) = session_id {
let _ = close_query_session(
@@ -991,6 +1005,7 @@ async fn export_query_result_core_inner(
cancel_token: Option<CancellationToken>,
on_progress: &impl Fn(TableExportProgress),
session_id: &mut Option<String>,
+ duckdb_snapshot: &mut Option<DuckDbExportSnapshot>,
) -> Result<(), String> {
let format = request.format.to_lowercase();
if format != "csv" && format != "xlsx" && format != "json" && format != "txt" && format != "sql" {
@@ -1049,7 +1064,22 @@ async fn export_query_result_core_inner(
let mut rows_exported: u64 = 0;
let mut offset: usize = 0;
let mut wrote_text_header = false;
- let mut keyset_plan = build_keyset_plan(state, request).await;
+ if request.database_type == DatabaseType::DuckDb && txn_session_id.is_none() {
+ let snapshot = duckdb_snapshot.insert(DuckDbExportSnapshot::new(
+ &request.connection_id,
+ &request.database,
+ request.schema.as_deref(),
+ QueryExecutionOptions {
+ timeout_secs: request.timeout_secs,
+ client_session_id: request.client_session_id.clone(),
+ execution_id: request.execution_id.clone(),
+ catalog: request.catalog.clone(),
+ ..Default::default()
+ },
+ ));
+ snapshot.materialize(state, &request.sql, effective_row_limit, cancel_token.clone()).await?;
+ }
+ let mut keyset_plan = if duckdb_snapshot.is_none() { build_keyset_plan(state, request).await } else { None };
// A Kingbase SQL Server compat TOP query that cannot be offset-paginated is
// exported with a single execution whose page size is the enforceable row
// bound (concrete TOP count and/or the configured export row limit), then it
@@ -1058,7 +1088,7 @@ async fn export_query_result_core_inner(
if let Some(error) = streaming_pagination_preflight_error(
request,
page_size,
- keyset_plan.is_some(),
+ keyset_plan.is_some() || duckdb_snapshot.is_some(),
single_execution_bound.is_some(),
use_agent_cursor,
) {
@@ -1094,27 +1124,28 @@ async fn export_query_result_core_inner(
remaining.map_or(page_size, |rem| rem.min(page_size)).max(1)
};
- let (sql_to_execute, plan_limit, use_agent_result_session, single_execution) =
- if let Some(plan) = keyset_plan.as_ref() {
- (build_keyset_export_sql(plan, request, this_page), this_page, false, false)
- } else {
- let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
- sql: request.sql.clone(),
- query_base_sql: request.query_base_sql.clone(),
- database_type: Some(request.database_type),
- pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },
- use_agent_cursor,
- first_page_uses_actual_sql: true,
- execution_hint: None,
- });
- if let Some(error) = plan.pagination_error {
- return Err(error);
- }
- let Some(plan_limit) = plan.page_limit else {
- return Err("Failed to build query pagination plan for export".to_string());
- };
- (plan.sql_to_execute, plan_limit, plan.use_agent_result_session, plan.single_execution)
+ let (sql_to_execute, plan_limit, use_agent_result_session, single_execution) = if duckdb_snapshot.is_some() {
+ (String::new(), this_page, false, false)
+ } else if let Some(plan) = keyset_plan.as_ref() {
+ (build_keyset_export_sql(plan, request, this_page), this_page, false, false)
+ } else {
+ let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
+ sql: request.sql.clone(),
+ query_base_sql: request.query_base_sql.clone(),
+ database_type: Some(request.database_type),
+ pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },
+ use_agent_cursor,
+ first_page_uses_actual_sql: true,
+ execution_hint: None,
+ });
+ if let Some(error) = plan.pagination_error {
+ return Err(error);
+ }
+ let Some(plan_limit) = plan.page_limit else {
+ return Err("Failed to build query pagination plan for export".to_string());
};
+ (plan.sql_to_execute, plan_limit, plan.use_agent_result_session, plan.single_execution)
+ };
let options = if use_agent_result_session {
QueryExecutionOptions {
@@ -1140,7 +1171,9 @@ async fn export_query_result_core_inner(
}
};
- let page_result = if let Some(txn_session_id) = txn_session_id {
+ let page_result = if let Some(snapshot) = duckdb_snapshot.as_ref() {
+ snapshot.page(state, offset as u64, this_page, cancel_token.clone()).await
+ } else if let Some(txn_session_id) = txn_session_id {
execute_in_manual_transaction_with_options(
state,
txn_session_id,
diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rs
index 14973f74db..ce4d764536 100644
--- a/crates/dbx-core/src/data/table_export.rs
+++ b/crates/dbx-core/src/data/table_export.rs
@@ -1,3 +1,4 @@
+use super::duckdb_export::DuckDbExportSnapshot;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::borrow::Cow;
@@ -725,8 +726,12 @@ async fn fetch_table_export_batch(
cursor_session: &mut Option<TableExportCursorSession>,
table_read_attempted: &mut bool,
table_read_completed: &mut bool,
+ duckdb_snapshot: Option<&DuckDbExportSnapshot>,
cancel_token: CancellationToken,
) -> Result<QueryResult, String> {
+ if let Some(snapshot) = duckdb_snapshot {
+ return snapshot.page(state, offset, active_batch_size, Some(cancel_token)).await;
+ }
if *table_read_completed {
return Ok(QueryResult {
columns: col_names.to_vec(),
@@ -1579,9 +1584,20 @@ pub async fn export_table_data_core(
last_rows_exported.store(progress.rows_exported, Ordering::SeqCst);
on_progress(progress);
};
- let result =
- export_table_data_core_inner(state, request, &tracked_progress, cancelled.clone(), cancel_token.clone()).await;
+ let mut duckdb_snapshot = None;
+ let result = export_table_data_core_inner(
+ state,
+ request,
+ &tracked_progress,
+ cancelled.clone(),
+ cancel_token.clone(),
+ &mut duckdb_snapshot,
+ )
+ .await;
cancel_watcher.abort();
+ if let Some(snapshot) = duckdb_snapshot {
+ snapshot.close(state).await;
+ }
let client_session_id = table_export_client_session_id(&request.export_id);
let _ = state.close_client_session_pool(&request.connection_id, Some(&request.database), &client_session_id).await;
match result {
@@ -1606,6 +1622,7 @@ async fn export_table_data_core_inner(
on_progress: &impl Fn(TableExportProgress),
cancelled: Arc<AtomicBool>,
cancel_token: CancellationToken,
+ duckdb_snapshot: &mut Option<DuckDbExportSnapshot>,
) -> Result<(), String> {
// 1. Get database type
let db_type = {
@@ -1798,6 +1815,24 @@ async fn export_table_data_core_inner(
return Ok(());
}
+ if db_type == DatabaseType::DuckDb {
+ let snapshot = duckdb_snapshot.insert(DuckDbExportSnapshot::new(
+ &request.connection_id,
+ &request.database,
+ request.schema.as_deref(),
+ QueryExecutionOptions {
+ client_session_id: Some(client_session_id),
+ // Bounds the small paged reads only: materialization runs as a
+ // full source scan and deliberately ignores the interactive
+ // query timeout so it cannot SIGKILL the worker mid-export.
+ timeout_secs: Some(table_export_query_timeout_secs(state, &pool_key).await),
+ ..Default::default()
+ },
+ ));
+ let sql = table_cursor_sql(request, &sql_context, query_col_names.as_ref(), &column_types, &primary_keys);
+ snapshot.materialize(state, &sql, row_limit, Some(cancel_token.clone())).await?;
+ }
+
// 8. Create output file
let mut file = if request.format.eq_ignore_ascii_case("sql") {
create_table_export_sql_writer(request)?
@@ -1860,6 +1895,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
@@ -1960,6 +1996,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
@@ -2074,6 +2111,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
@@ -2170,6 +2208,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
@@ -2255,6 +2294,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
@@ -2340,6 +2380,7 @@ async fn export_table_data_core_inner(
&mut cursor_session,
&mut table_read_attempted,
&mut table_read_completed,
+ duckdb_snapshot.as_ref(),
cancel_token.clone(),
)
.await?;
diff --git a/crates/dbx-core/tests/live_duckdb_export.rs b/crates/dbx-core/tests/live_duckdb_export.rs
new file mode 100644
index 0000000000..9598eacc33
--- /dev/null
+++ b/crates/dbx-core/tests/live_duckdb_export.rs
@@ -0,0 +1,16 @@
+#![cfg(feature = "duckdb-sidecar")]
+
+#[path = "support/duckdb_export.rs"]
+mod support;
+
+#[test]
+#[ignore = "requires DBX_DUCKDB_DRIVER_PATH pointing at a DuckDB worker"]
+fn duckdb_parquet_exports_are_consistent() {
+ support::run_parquet_exports(std::env::var_os("DBX_DUCKDB_DRIVER_PATH").unwrap().into());
+}
+
+#[test]
+#[ignore = "requires DBX_DUCKDB_DRIVER_PATH pointing at a DuckDB worker"]
+fn duckdb_export_snapshots_preserve_values_and_cleanup() {
+ support::run_snapshot_cleanup(std::env::var_os("DBX_DUCKDB_DRIVER_PATH").unwrap().into());
+}
diff --git a/crates/dbx-core/tests/support/duckdb_export.rs b/crates/dbx-core/tests/support/duckdb_export.rs
new file mode 100644
index 0000000000..f3869348db
--- /dev/null
+++ b/crates/dbx-core/tests/support/duckdb_export.rs
@@ -0,0 +1,244 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use dbx_core::connection::{AppState, PoolKind};
+use dbx_core::db::duckdb_worker_process::DuckDbWorkerClient;
+use dbx_core::models::connection::ConnectionConfig;
+use dbx_core::query_result_export::{export_query_result_core, QueryResultExportRequest};
+use dbx_core::table_export::{export_table_data_core, TableExportRequest};
+use dbx_core::table_import::{parse_csv_bytes, parse_xlsx_file};
+use serde_json::{json, Value};
+use tokio_util::sync::CancellationToken;
+
+fn run_with_stack<F: std::future::Future<Output = ()> + 'static>(test: impl FnOnce() -> F + Send + 'static) {
+ std::thread::Builder::new()
+ .stack_size(16 * 1024 * 1024)
+ .spawn(move || {
+ tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(Box::pin(test()));
+ })
+ .unwrap()
+ .join()
+ .unwrap();
+}
+
+pub fn run_parquet_exports(driver: PathBuf) {
+ run_with_stack(move || parquet_exports_are_consistent(driver));
+}
+
+pub fn run_snapshot_cleanup(driver: PathBuf) {
+ run_with_stack(move || snapshots_preserve_query_values_and_cleanup(driver));
+}
+
+struct Fixture {
+ state: AppState,
+ client: Arc<DuckDbWorkerClient>,
+ id: String,
+ dir: PathBuf,
+}
+
+impl Fixture {
+ async fn new(driver: PathBuf) -> Self {
+ let id = format!("duckdb-export-{}", uuid::Uuid::new_v4().simple());
+ let dir = std::env::temp_dir().join(&id);
+ std::fs::create_dir_all(&dir).unwrap();
+ let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let config: ConnectionConfig = serde_json::from_value(json!({
+ "id": id, "name": id, "db_type": "duckdb", "host": ":memory:",
+ "port": 0, "username": "", "password": "", "database": "main",
+ "query_timeout_secs": 60, "keepalive_interval_secs": 0
+ }))
+ .unwrap();
+ let client = Arc::new(
+ DuckDbWorkerClient::open_with_executable(
+ driver,
+ ":memory:".into(),
+ vec![],
+ Some("SET threads = 4; SET preserve_insertion_order = false".into()),
+ )
+ .await
+ .unwrap(),
+ );
+ state.configs.write().await.insert(id.clone(), config.clone());
+ state.insert_connection_pool(id.clone(), PoolKind::DuckDbWorker(client.clone()), &config).await.unwrap();
+ Self { state, client, id, dir }
+ }
+
+ async fn sql(&self, sql: &str) -> dbx_core::types::QueryResult {
+ self.client.execute(None, sql.into(), Some(50), None, None).await.unwrap()
+ }
+
+ async fn assert_clean(&self) {
+ let result = self.sql(
+ "SELECT count(*) FROM duckdb_tables() WHERE database_name = 'temp' AND starts_with(table_name, '__dbx_export_')",
+ ).await;
+ assert_eq!(cell_number(&result.rows[0][0]), 0);
+ assert_eq!(cell_number(&self.sql("SELECT * FROM keep_me").await.rows[0][0]), 42);
+ }
+
+ fn query_request(&self, sql: &str, path: &Path, format: &str) -> QueryResultExportRequest {
+ serde_json::from_value(json!({
+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": self.id,
+ "database": "main", "schema": "main", "databaseType": "duckdb",
+ "sql": sql, "queryBaseSql": sql, "filePath": path,
+ "format": format, "pageSize": 512, "includeSqlSheet": false,
+ "numericColumnRightAlign": false
+ }))
+ .unwrap()
+ }
+
+ async fn finish(self) {
+ self.client.shutdown().await;
+ drop(self.state);
+ let _ = std::fs::remove_dir_all(self.dir);
+ }
+}
+
+fn cell_number(value: &Value) -> i64 {
+ value.as_i64().unwrap_or_else(|| cell_text(value).parse::<f64>().unwrap().round() as i64)
+}
+
+fn cell_text(value: &Value) -> String {
+ value.as_str().map(str::to_string).unwrap_or_else(|| value.to_string())
+}
+
+fn verify_export(path: &Path, format: &str) {
+ let parsed = if format == "csv" {
+ parse_csv_bytes(&std::fs::read(path).unwrap(), 20_001).unwrap()
+ } else {
+ parse_xlsx_file(path.to_str().unwrap(), 20_001).unwrap()
+ };
+ assert_eq!(parsed.columns, vec!["rowid", "项目名称", "结算账期", "金额"]);
+ let mut ids = BTreeSet::new();
+ let mut totals = BTreeMap::<String, i64>::new();
+ for row in &parsed.rows {
+ let id = cell_number(&row[0]);
+ assert!(ids.insert(id), "duplicate id {id} in {}", path.display());
+ assert_eq!(cell_text(&row[2]), "2026-09");
+ let cents = (cell_text(&row[3]).parse::<f64>().unwrap() * 100.0).round() as i64;
+ assert_eq!(cents, id + 1, "amount for id {id}");
+ *totals.entry(cell_text(&row[1])).or_default() += cents;
+ }
+ let expected_ids = (0..20_000).filter(|id| id % 5 != 0).collect::<BTreeSet<_>>();
+ let mut expected_totals = BTreeMap::<String, i64>::new();
+ for id in &expected_ids {
+ let project = if *id < 5_000 {
+ "Project A"
+ } else if *id < 10_000 {
+ "Project B"
+ } else {
+ "Project C"
+ };
+ *expected_totals.entry(project.into()).or_default() += id + 1;
+ }
+ assert_eq!(ids, expected_ids, "missing rows in {}", path.display());
+ assert_eq!(totals, expected_totals, "project totals in {}", path.display());
+}
+
+async fn parquet_exports_are_consistent(driver: PathBuf) {
+ let fixture = Fixture::new(driver).await;
+ fixture.sql("CREATE TEMP TABLE keep_me AS SELECT 42").await;
+ let parquet = fixture.dir.join("settlements.parquet").to_string_lossy().replace('\\', "/");
+ fixture.sql(&format!(
+ "COPY (SELECT i AS id, CASE WHEN i < 5000 THEN 'Project A' WHEN i < 10000 THEN 'Project B' ELSE 'Project C' END AS project, \
+ CASE WHEN i % 5 = 0 THEN '2025-09' ELSE '2026-09' END AS period, \
+ ((i + 1) / 100.0)::DECIMAL(18,2) AS amount FROM range(20000) t(i)) \
+ TO '{}' (FORMAT PARQUET, ROW_GROUP_SIZE 2048)", parquet.replace('\'', "''")
+ )).await;
+ fixture.sql(&format!(
+ "CREATE VIEW settlements AS SELECT id AS rowid, project AS \"项目名称\", period AS \"结算账期\", amount AS \"金额\" FROM read_parquet('{}')",
+ parquet.replace('\'', "''")
+ )).await;
+ let sql = "SELECT * FROM settlements WHERE \"结算账期\" LIKE '%2026%' ORDER BY \"结算账期\" DESC";
+ for preserve in [false, true] {
+ fixture.sql(&format!("SET preserve_insertion_order = {preserve}")).await;
+ fixture.state.configs.write().await.get_mut(&fixture.id).unwrap().read_only = preserve;
+ for format in ["csv", "xlsx"] {
+ for table_export in [true, false] {
+ let path = fixture.dir.join(format!("{preserve}-{table_export}.{format}"));
+ if table_export {
+ let request: TableExportRequest = serde_json::from_value(json!({
+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": fixture.id,
+ "database": "main", "schema": "main", "tableName": "settlements",
+ "filePath": path, "format": format, "numericColumnRightAlign": false,
+ "columns": ["rowid", "项目名称", "结算账期", "金额"],
+ "columnTypes": ["BIGINT", "VARCHAR", "VARCHAR", "DECIMAL(18,2)"],
+ "primaryKeys": [], "skipCount": true, "batchSize": 512,
+ "whereInput": "\"结算账期\" LIKE '%2026%'", "orderBy": "\"结算账期\" DESC"
+ }))
+ .unwrap();
+ export_table_data_core(&fixture.state, &request, |_| {}).await.unwrap();
+ } else {
+ let request = fixture.query_request(sql, &path, format);
+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();
+ }
+ verify_export(&path, format);
+ fixture.assert_clean().await;
+ }
+ }
+ }
+ fixture.finish().await;
+}
+
+async fn snapshots_preserve_query_values_and_cleanup(driver: PathBuf) {
+ let fixture = Fixture::new(driver).await;
+ fixture.sql("CREATE TEMP TABLE keep_me AS SELECT 42").await;
+ fixture.sql("CREATE SEQUENCE calls START 1").await;
+ let path = fixture.dir.join("query.csv");
+ let sql =
+ "SELECT nextval('calls') AS id, i AS dup, -i AS dup, i AS rowid FROM range(2050) t(i); -- trailing comment";
+ let request = fixture.query_request(sql, &path, "csv");
+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();
+ let bytes = std::fs::read(&path).unwrap();
+ let text = String::from_utf8(bytes.clone()).unwrap();
+ assert_eq!(text.trim_start_matches('\u{feff}').lines().next(), Some("\"id\",\"dup\",\"dup\",\"rowid\""));
+ let parsed = parse_csv_bytes(&bytes, 2051).unwrap();
+ // The importer assigns unique header names; the CSV itself keeps both labels.
+ assert_eq!(parsed.columns, vec!["id", "dup", "dup_1", "rowid"]);
+ assert_eq!(parsed.rows.len(), 2050);
+ for (i, row) in parsed.rows.iter().enumerate() {
+ assert_eq!(
+ row.iter().map(cell_number).collect::<Vec<_>>(),
+ vec![i as i64 + 1, i as i64, -(i as i64), i as i64]
+ );
+ }
+ assert_eq!(cell_number(&fixture.sql("SELECT nextval('calls')").await.rows[0][0]), 2051);
+ fixture.assert_clean().await;
+
+ let mut request = fixture.query_request("SELECT * FROM range(2050) t(id) ORDER BY id DESC", &path, "csv");
+ request.row_limit = Some(513);
+ export_query_result_core(&fixture.state, &request, None, |_| {}).await.unwrap();
+ let parsed = parse_csv_bytes(&std::fs::read(&path).unwrap(), 2051).unwrap();
+ assert_eq!(parsed.rows.len(), 513);
+ assert_eq!(cell_number(&parsed.rows[0][0]), 2049);
+ assert_eq!(cell_number(&parsed.rows[512][0]), 1537);
+ fixture.assert_clean().await;
+
+ let request = fixture.query_request("SELECT * FROM range(2050)", &fixture.dir, "xlsx");
+ assert!(export_query_result_core(&fixture.state, &request, None, |_| {}).await.is_err());
+ fixture.assert_clean().await;
+
+ fixture.sql("CREATE VIEW source AS SELECT * FROM range(2050) t(id)").await;
+ let table_request: TableExportRequest = serde_json::from_value(json!({
+ "exportId": uuid::Uuid::new_v4().to_string(), "connectionId": fixture.id,
+ "database": "main", "schema": "main", "tableName": "source",
+ "filePath": fixture.dir, "format": "xlsx", "numericColumnRightAlign": false,
+ "columns": ["id"], "columnTypes": ["BIGINT"], "primaryKeys": [], "skipCount": true
+ }))
+ .unwrap();
+ assert!(export_table_data_core(&fixture.state, &table_request, |_| {}).await.is_err());
+ fixture.assert_clean().await;
+
+ let request = fixture.query_request("SELECT * FROM range(2050)", &path, "csv");
+ let token = CancellationToken::new();
+ export_query_result_core(&fixture.state, &request, Some(token.clone()), |progress| {
+ if progress.rows_exported >= 512 {
+ token.cancel();
+ }
+ })
+ .await
+ .unwrap();
+ fixture.assert_clean().await;
+ fixture.finish().await;
+}
diff --git a/crates/dbx-drivers/src/db/duckdb_worker_process.rs b/crates/dbx-drivers/src/db/duckdb_worker_process.rs
index ee63d0b0fe..2665c21dee 100644
--- a/crates/dbx-drivers/src/db/duckdb_worker_process.rs
+++ b/crates/dbx-drivers/src/db/duckdb_worker_process.rs
@@ -33,9 +33,23 @@ const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
/// `connection.rs` so pool close cancellation cannot cut the checkpoint short.
const WORKER_SHUTDOWN_EXIT_WAIT: Duration = Duration::from_millis(2500);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
+/// Upper bound for the worker to acknowledge an interrupt-only cancel. The
+/// caller already reports its own timeout error, so waiting longer than this
+/// adds latency without improving the outcome.
+const WORKER_INTERRUPT_ACK_TIMEOUT: Duration = Duration::from_secs(2);
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
+/// How a request reacts to cancellation or a timeout. Interactive queries
+/// restart the worker because a DuckDB connection stuck inside a statement
+/// cannot be reused. Statements whose caller shares the session (`:memory:`
+/// contents, temp tables, views) must not: they only interrupt.
+#[derive(Clone, Copy)]
+enum AbortPolicy {
+ KillWorker,
+ InterruptOnly,
+}
+
struct PendingRequest {
generation: u64,
sender: oneshot::Sender<DuckDbWorkerResponse>,
@@ -210,7 +224,16 @@ impl DuckDbWorkerClient {
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, DuckDbWorkerError> {
- self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, false).await
+ self.execute_typed_with_options(
+ database,
+ sql,
+ max_rows,
+ cancel_token,
+ query_timeout,
+ false,
+ AbortPolicy::KillWorker,
+ )
+ .await
}
/// Executes a query with DuckDB insertion-order preservation enabled for
@@ -225,7 +248,32 @@ impl DuckDbWorkerClient {
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, String> {
- self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, true)
+ self.execute_typed_with_options(
+ database,
+ sql,
+ max_rows,
+ cancel_token,
+ query_timeout,
+ true,
+ AbortPolicy::KillWorker,
+ )
+ .await
+ .map_err(|error| error.message)
+ }
+
+ /// Executes a best-effort statement with a bounded wait that never restarts
+ /// the worker. On timeout the running statement is interrupted and a
+ /// timeout error is returned; the worker and its session stay alive. Used
+ /// where a SIGKILL would discard session state that outlives the statement
+ /// (e.g. dropping an export's temporary table on a shared connection).
+ pub async fn execute_interrupt_only(
+ &self,
+ database: Option<String>,
+ sql: String,
+ max_rows: Option<usize>,
+ timeout: Duration,
+ ) -> Result<db::QueryResult, String> {
+ self.execute_typed_with_options(database, sql, max_rows, None, Some(timeout), false, AbortPolicy::InterruptOnly)
.await
.map_err(|error| error.message)
}
@@ -238,10 +286,11 @@ impl DuckDbWorkerClient {
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
preserve_insertion_order: bool,
+ abort_policy: AbortPolicy,
) -> Result<db::QueryResult, DuckDbWorkerError> {
let _query_guard = self.inner.query_lock.lock().await;
let client = self.clone();
- // Cancellation and timeout restart the worker via cancel_or_kill below. An ordinary
+ // Cancellation and timeout apply `abort_policy` below. An ordinary
// query error (parser/binder/catalog/runtime) leaves the worker healthy, so we keep it
// alive to preserve session-local state (:memory: contents, temp tables, SET, raw ATTACH).
// The one exception is a poisoned connection: duckdb-rs can leave the connection unusable
@@ -277,33 +326,62 @@ impl DuckDbWorkerClient {
(Some(token), Some(duration)) => {
tokio::select! {
biased;
- _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await,
+ _ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,
result = &mut future => result,
- _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await,
+ _ = tokio::time::sleep(duration) => {
+ self.abort(crate::execution::timeout_error_for(duration), abort_policy).await
+ },
}
}
(Some(token), None) => {
tokio::select! {
biased;
- _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await,
+ _ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,
result = &mut future => result,
}
}
(None, Some(duration)) => {
tokio::select! {
result = &mut future => result,
- _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await,
+ _ = tokio::time::sleep(duration) => {
+ self.abort(crate::execution::timeout_error_for(duration), abort_policy).await
+ },
}
}
(None, None) => future.await,
}
}
- async fn cancel_or_kill(&self, final_error: String) -> Result<db::QueryResult, DuckDbWorkerError> {
- let _ = self.cancel().await;
+ /// Applies the request's abort policy and reports the cancellation/timeout
+ /// as the operation's final error. Interactive queries restart the worker,
+ /// because an unresponsive DuckDB session cannot be reused; best-effort
+ /// statements only interrupt so a shared session survives.
+ async fn abort(&self, final_error: String, policy: AbortPolicy) -> Result<db::QueryResult, DuckDbWorkerError> {
+ match policy {
+ AbortPolicy::KillWorker => {
+ let _ = self.cancel().await;
+ }
+ AbortPolicy::InterruptOnly => {
+ let _ = self.interrupt().await;
+ }
+ }
Err(DuckDbWorkerError::from(final_error))
}
+ /// Asks the worker to interrupt the statement it is running without
+ /// restarting it. Best effort: a worker that cannot acknowledge in time is
+ /// left running rather than escalated to SIGKILL.
+ async fn interrupt(&self) -> Result<(), String> {
+ self.send_request_structured::<serde_json::Value>(
+ DuckDbWorkerMethod::Cancel,
+ serde_json::json!({}),
+ Some(WORKER_INTERRUPT_ACK_TIMEOUT),
+ )
+ .await
+ .map(|_| ())
+ .map_err(|error| error.message)
+ }
+
pub async fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {
self.metadata_request(DuckDbWorkerMethod::ListDatabases, serde_json::json!({})).await
}