25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
#![cfg(feature = "duckdb-sidecar")]
use std::collections::HashMap;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex, OnceLock};
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, ChildStdin};
use tokio::sync::{oneshot, Mutex, OwnedSemaphorePermit, Semaphore};
use tokio_util::sync::CancellationToken;
use crate::db;
use crate::db::duckdb_worker_protocol::{
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams,
DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
};
use crate::models::connection::AttachedDatabaseConfig;
use crate::runtime_config::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
/// Error code the worker reports when a query error left the DuckDB connection poisoned
/// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the
/// next request starts a fresh one. Must match the code emitted by the standalone driver runtime.
const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
/// Grace period for a shutdown-acked worker to exit by itself (session drop
/// plus DuckDB shutdown checkpoint). Kept below the 3s pool close timeout in
/// `connection.rs` so pool close cancellation cannot cut the checkpoint short.
const WORKER_SHUTDOWN_EXIT_WAIT: Duration = Duration::from_millis(2500);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
/// Upper bound for the worker to acknowledge an interrupt-only cancel. The
/// caller already reports its own timeout error, so waiting longer than this
/// adds latency without improving the outcome.
const WORKER_INTERRUPT_ACK_TIMEOUT: Duration = Duration::from_secs(2);
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
#[path = "duckdb_query_sessions.rs"]
mod query_sessions;
use query_sessions::QuerySession;
/// How a request reacts to cancellation or a timeout. Interactive queries
/// restart the worker because a DuckDB connection stuck inside a statement
/// cannot be reused. Statements whose caller shares the session (`:memory:`
/// contents, temp tables, views) must not: they only interrupt.
#[derive(Clone, Copy)]
enum AbortPolicy {
KillWorker,
InterruptOnly,
}
struct PendingRequest {
generation: u64,
sender: oneshot::Sender<DuckDbWorkerResponse>,
}
#[derive(Clone)]
pub struct DuckDbWorkerClient {
inner: Arc<DuckDbWorkerClientInner>,
}
struct DuckDbWorkerClientInner {
state: Mutex<WorkerProcessState>,
pending: PendingRequests,
connect_lock: Mutex<()>,
query_lock: Mutex<()>,
query_sessions: Mutex<HashMap<String, QuerySession>>,
process_limiter: Arc<Semaphore>,
process_limit: usize,
executable: PathBuf,
executable_args: Vec<OsString>,
connect_params: DuckDbWorkerConnectParams,
request_timeout: Duration,
worker_start_timeout: Duration,
next_id: AtomicU64,
}
#[derive(Default)]
struct WorkerProcessState {
child: Option<Child>,
stdin: Option<ChildStdin>,
connected: bool,
generation: u64,
}
impl DuckDbWorkerClient {
pub async fn open(
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
) -> Result<Self, String> {
let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_command_and_process_limit(
executable,
executable_args,
path,
attached_databases,
init_script,
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
)
.await
}
pub async fn open_with_process_limit(
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_command_and_process_limit(
executable,
executable_args,
path,
attached_databases,
init_script,
process_limit,
)
.await
}
pub async fn open_with_executable(
executable: PathBuf,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
) -> Result<Self, String> {
Self::open_with_executable_and_process_limit(
executable,
path,
attached_databases,
init_script,
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
)
.await
}
pub async fn open_with_executable_and_process_limit(
executable: PathBuf,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
Self::open_with_command_and_process_limit(
executable,
Vec::new(),
path,
attached_databases,
init_script,
process_limit,
)
.await
}
async fn open_with_command_and_process_limit(
executable: PathBuf,
executable_args: Vec<OsString>,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
let client = Self::new_unconnected_with_timeouts(
executable,
executable_args,
path,
attached_databases,
init_script,
process_limit,
DEFAULT_WORKER_REQUEST_TIMEOUT,
DEFAULT_WORKER_START_WAIT,
);
client.ensure_connected().await?;
Ok(client)
}
#[doc(hidden)]
pub fn new_unconnected_with_timeouts(
executable: PathBuf,
executable_args: Vec<OsString>,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
request_timeout: Duration,
worker_start_timeout: Duration,
) -> Self {
let (process_limiter, process_limit) = duckdb_worker_process_limiter(process_limit);
Self {
inner: Arc::new(DuckDbWorkerClientInner {
state: Mutex::new(WorkerProcessState::default()),
pending: Arc::new(Mutex::new(HashMap::new())),
connect_lock: Mutex::new(()),
query_lock: Mutex::new(()),
query_sessions: Mutex::new(HashMap::new()),
process_limiter,
process_limit,
executable,
executable_args,
connect_params: DuckDbWorkerConnectParams { path, attached_databases, init_script },
request_timeout,
worker_start_timeout,
next_id: AtomicU64::new(1),
}),
}
}
pub async fn execute(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, String> {
self.execute_typed(database, sql, max_rows, cancel_token, query_timeout).await.map_err(|error| error.message)
}
pub async fn execute_typed(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, DuckDbWorkerError> {
self.execute_typed_with_options(
database,
sql,
max_rows,
cancel_token,
query_timeout,
false,
AbortPolicy::KillWorker,
)
.await
}
/// Executes a query with DuckDB insertion-order preservation enabled for
/// the duration of that request. This is used by Parquet pagination:
/// independent LIMIT/OFFSET scans must see the same source order even
/// when the connection's init script disabled `preserve_insertion_order`.
pub async fn execute_preserving_insertion_order(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, String> {
self.execute_typed_with_options(
database,
sql,
max_rows,
cancel_token,
query_timeout,
true,
AbortPolicy::KillWorker,
)
.await
.map_err(|error| error.message)
}
/// Executes a best-effort statement with a bounded wait that never restarts
/// the worker. On timeout the running statement is interrupted and a
/// timeout error is returned; the worker and its session stay alive. Used
/// where a SIGKILL would discard session state that outlives the statement
/// (e.g. dropping an export's temporary table on a shared connection).
pub async fn execute_interrupt_only(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
timeout: Duration,
) -> Result<db::QueryResult, String> {
self.execute_typed_with_options(database, sql, max_rows, None, Some(timeout), false, AbortPolicy::InterruptOnly)
.await
.map_err(|error| error.message)
}
async fn execute_typed_with_options(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
preserve_insertion_order: bool,
abort_policy: AbortPolicy,
) -> Result<db::QueryResult, DuckDbWorkerError> {
// A queued request must remain cancellable. In particular, cancelling
// a grid read waiting behind an export must not kill that export.
let lock = self.inner.query_lock.lock();
tokio::pin!(lock);
let _query_guard = match (cancel_token.as_ref(), query_timeout) {
(Some(token), Some(duration)) => tokio::select! {
biased;
_ = token.cancelled() => return Err(crate::execution::canceled_error().into()),
guard = &mut lock => guard,
_ = tokio::time::sleep(duration) => return Err(crate::execution::timeout_error_for(duration).into()),
},
(Some(token), None) => tokio::select! {
biased;
_ = token.cancelled() => return Err(crate::execution::canceled_error().into()),
guard = &mut lock => guard,
},
(None, Some(duration)) => tokio::select! {
guard = &mut lock => guard,
_ = tokio::time::sleep(duration) => return Err(crate::execution::timeout_error_for(duration).into()),
},
(None, None) => lock.await,
};
let client = self.clone();
// Cancellation and timeout apply `abort_policy` below. An ordinary
// query error (parser/binder/catalog/runtime) leaves the worker healthy, so we keep it
// alive to preserve session-local state (:memory: contents, temp tables, SET, raw ATTACH).
// The one exception is a poisoned connection: duckdb-rs can leave the connection unusable
// after a syntax Parser Error, and the worker reports that with `duckdb_worker_poisoned`.
// We kill on that code so the next request restarts a fresh worker. Killing here (rather
// than letting the worker self-exit) avoids racing our own next request against a dying
// worker, and OS-level kill never runs the destructor that would abort the process.
let future = async move {
client
.ensure_connected()
.await
.map_err(|message| DuckDbWorkerError::new("duckdb_worker_connect_failed", message))?;
match client
.send_request_structured::<db::QueryResult>(
DuckDbWorkerMethod::Execute,
DuckDbWorkerExecuteParams { sql, database, max_rows, preserve_insertion_order },
None,
)
.await
{
Ok(result) => Ok(result),
Err(error) => {
if error.code == DUCKDB_WORKER_POISONED_CODE {
client.kill().await;
}
Err(error)
}
}
};
tokio::pin!(future);
match (cancel_token, query_timeout) {
(Some(token), Some(duration)) => {
tokio::select! {
biased;
_ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,
result = &mut future => result,
_ = tokio::time::sleep(duration) => {
self.abort(crate::execution::timeout_error_for(duration), abort_policy).await
},
}
}
(Some(token), None) => {
tokio::select! {
biased;
_ = token.cancelled() => self.abort(crate::execution::canceled_error(), abort_policy).await,
result = &mut future => result,
}
}
(None, Some(duration)) => {
tokio::select! {
result = &mut future => result,
_ = tokio::time::sleep(duration) => {
self.abort(crate::execution::timeout_error_for(duration), abort_policy).await
},
}
}
(None, None) => future.await,
}
}
/// Applies the request's abort policy and reports the cancellation/timeout
/// as the operation's final error. Interactive queries restart the worker,
/// because an unresponsive DuckDB session cannot be reused; best-effort
/// statements only interrupt so a shared session survives.
async fn abort(&self, final_error: String, policy: AbortPolicy) -> Result<db::QueryResult, DuckDbWorkerError> {
match policy {
AbortPolicy::KillWorker => {
let _ = self.cancel().await;
}
AbortPolicy::InterruptOnly => {
let _ = self.interrupt().await;
}
}
Err(DuckDbWorkerError::from(final_error))
}
/// Asks the worker to interrupt the statement it is running without
/// restarting it. Best effort: a worker that cannot acknowledge in time is
/// left running rather than escalated to SIGKILL.
async fn interrupt(&self) -> Result<(), String> {
self.send_request_structured::<serde_json::Value>(
DuckDbWorkerMethod::Cancel,
serde_json::json!({}),
Some(WORKER_INTERRUPT_ACK_TIMEOUT),
)
.await
.map(|_| ())
.map_err(|error| error.message)
}
pub async fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {
self.metadata_request(DuckDbWorkerMethod::ListDatabases, serde_json::json!({})).await
}
pub async fn list_schemas(&self, database: String) -> Result<Vec<String>, String> {
self.metadata_request(DuckDbWorkerMethod::ListSchemas, serde_json::json!({ "database": database })).await
}
pub async fn list_tables(&self, database: String, schema: String) -> Result<Vec<db::TableInfo>, String> {
self.metadata_request(
DuckDbWorkerMethod::ListTables,
serde_json::json!({ "database": database, "schema": schema }),
)
.await
}
pub async fn list_columns(
&self,
database: String,
schema: String,
table: String,
) -> Result<Vec<db::ColumnInfo>, String> {
self.metadata_request(
DuckDbWorkerMethod::ListColumns,
serde_json::json!({ "database": database, "schema": schema, "table": table }),
)
.await
}
pub async fn get_table_ddl(&self, database: String, schema: String, table: String) -> Result<String, String> {
self.metadata_request(DuckDbWorkerMethod::GetTableDdl, DuckDbWorkerColumnParams { database, schema, table })
.await
}
pub async fn completion_assistant(
&self,
request: db::CompletionAssistantRequest,
) -> Result<db::CompletionAssistantResponse, String> {
self.metadata_request(DuckDbWorkerMethod::CompletionAssistant, request).await
}
pub async fn get_object_source(
&self,
database: String,
schema: String,
name: String,
object_type: db::ObjectSourceKind,
) -> Result<String, String> {
self.metadata_request(
DuckDbWorkerMethod::GetObjectSource,
DuckDbWorkerObjectSourceParams { database, schema, name, object_type },
)
.await
}
pub async fn attach_database(&self, attached: AttachedDatabaseConfig) -> Result<(), String> {
self.metadata_request::<serde_json::Value>(DuckDbWorkerMethod::AttachDatabase, attached).await?;
Ok(())
}
/// Runs a metadata request that executes synchronously inside the worker's stdin loop.
/// If it times out the worker is likely stuck inside DuckDB and can no longer read further
/// requests, so we kill it; the next call restarts a fresh worker via `ensure_connected`.
/// Ordinary errors (e.g. a missing table) leave the worker healthy and do not kill it.
async fn metadata_request<T>(&self, method: DuckDbWorkerMethod, params: impl serde::Serialize) -> Result<T, String>
where
T: serde::de::DeserializeOwned,
{
let timeout = self.inner.request_timeout;
match tokio::time::timeout(timeout, self.request::<T>(method, params, None)).await {
Ok(result) => result,
Err(_) => {
self.kill().await;
Err(format!("DuckDB worker request timed out after {}s", timeout.as_secs()))
}
}
}
pub async fn cancel(&self) -> Result<(), String> {
self.kill().await;
Ok(())
}
pub async fn shutdown(&self) {
let _ = self
.send_request::<serde_json::Value>(
DuckDbWorkerMethod::Shutdown,
serde_json::json!({}),
Some(self.inner.request_timeout),
)
.await;
self.wait_for_exit_or_kill().await;
}
/// Lets a shutdown-acked worker exit on its own so its session drop runs
/// the DuckDB shutdown checkpoint that removes the database WAL; SIGKILL
/// stays the fallback for stuck workers. Killing immediately after the
/// ack would truncate that checkpoint and leave the WAL file behind.
/// The wait must fit inside the caller's pool close timeout so the
/// detached close task is not cancelled mid-checkpoint.
async fn wait_for_exit_or_kill(&self) {
let (child, generation) = {
let mut state = self.inner.state.lock().await;
let child = state.child.take();
let generation = state.generation;
state.stdin = None;
state.connected = false;
(child, generation)
};
let Some(mut child) = child else { return };
match tokio::time::timeout(WORKER_SHUTDOWN_EXIT_WAIT, child.wait()).await {
Ok(Ok(status)) => {
log::info!("[duckdb-worker:shutdown:exit] status={status}");
self.fail_pending_for_generation(
generation,
"duckdb_worker_killed",
"DuckDB worker process was killed",
)
.await;
return;
}
Ok(Err(err)) => {
log::warn!("[duckdb-worker:shutdown:wait-failed] error={err}");
self.fail_pending_for_generation(
generation,
"duckdb_worker_killed",
"DuckDB worker process was killed",
)
.await;
return;
}
Err(_) => {
log::warn!("[duckdb-worker:shutdown:wait-timeout] wait_ms={}", WORKER_SHUTDOWN_EXIT_WAIT.as_millis());
}
}
let _ = child.start_kill();
match tokio::time::timeout(DEFAULT_WORKER_KILL_WAIT, child.wait()).await {
Ok(Ok(status)) => log::info!("[duckdb-worker:kill:exit] status={status}"),
Ok(Err(err)) => log::warn!("[duckdb-worker:kill:wait-failed] error={err}"),
Err(_) => {
log::warn!("[duckdb-worker:kill:wait-timeout] wait_ms={}", DEFAULT_WORKER_KILL_WAIT.as_millis())
}
}
self.fail_pending_for_generation(generation, "duckdb_worker_killed", "DuckDB worker process was killed").await;
}
pub async fn kill(&self) {
let (child, generation) = {
let mut state = self.inner.state.lock().await;
let child = state.child.take();
let generation = state.generation;
state.stdin = None;
state.connected = false;
(child, generation)
};
if let Some(mut child) = child {
let _ = child.start_kill();
match tokio::time::timeout(DEFAULT_WORKER_KILL_WAIT, child.wait()).await {
Ok(Ok(status)) => log::info!("[duckdb-worker:kill:exit] status={status}"),
Ok(Err(err)) => log::warn!("[duckdb-worker:kill:wait-failed] error={err}"),
Err(_) => {
log::warn!("[duckdb-worker:kill:wait-timeout] wait_ms={}", DEFAULT_WORKER_KILL_WAIT.as_millis())
}
}
}
self.fail_pending_for_generation(generation, "duckdb_worker_killed", "DuckDB worker process was killed").await;
}
async fn request<T>(
&self,
method: DuckDbWorkerMethod,
params: impl serde::Serialize,
timeout: Option<Duration>,
) -> Result<T, String>
where
T: serde::de::DeserializeOwned,
{
self.ensure_connected().await?;
self.send_request(method, params, timeout).await
}
async fn ensure_connected(&self) -> Result<(), String> {
let _guard = self.inner.connect_lock.lock().await;
{
let mut state = self.inner.state.lock().await;
self.ensure_started_locked(&mut state).await?;
if state.connected {
return Ok(());
}
}
let mut attempts = 0;
loop {
match self
.send_request_structured::<serde_json::Value>(
DuckDbWorkerMethod::Connect,
self.inner.connect_params.clone(),
Some(self.inner.request_timeout),
)
.await
{
Ok(_) => break,
Err(error) => {
let retry_delay = duckdb_connect_file_lock_retry_delay(attempts, &error.message);
// A failed Connect leaves this worker without a valid session. Keep no stale
// child around, especially after OS file-lock errors where the user may retry
// once the competing process exits.
self.kill().await;
if let Some(delay) = retry_delay {
attempts += 1;
log::warn!(
"[duckdb-worker:connect-retry] attempt={} delay_ms={} error={}",
attempts,
delay.as_millis(),
error.message
);
tokio::time::sleep(delay).await;
continue;
}
return Err(error.message);
}
}
}
let mut state = self.inner.state.lock().await;
state.connected = true;
Ok(())
}
async fn ensure_started_locked(&self, state: &mut WorkerProcessState) -> Result<(), String> {
let should_restart = match state.child.as_mut() {
Some(child) => match child.try_wait() {
Ok(Some(status)) => {
log::warn!("[duckdb-worker:exit] status={status}");
true
}
Ok(None) => return Ok(()),
Err(err) => {
log::warn!("[duckdb-worker:exit-check-failed] error={err}");
true
}
},
None => true,
};
if should_restart {
state.child = None;
state.stdin = None;
state.connected = false;
}
state.generation = state.generation.wrapping_add(1);
let generation = state.generation;
let permit =
tokio::time::timeout(self.inner.worker_start_timeout, self.inner.process_limiter.clone().acquire_owned())
.await
.map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))?
.map_err(|_| "DuckDB worker process limiter is closed".to_string())?;
log::info!(
"[duckdb-worker:start] executable={} args={:?}",
self.inner.executable.display(),
self.inner.executable_args
);
let mut command = crate::process::new_tokio_command(&self.inner.executable);
command.args(&self.inner.executable_args);
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.map_err(|e| format!("Failed to start DuckDB worker: {e}"))?;
let stdin = child.stdin.take().ok_or("DuckDB worker stdin unavailable")?;
let stdout = child.stdout.take().ok_or("DuckDB worker stdout unavailable")?;
spawn_stdout_reader(stdout, self.inner.pending.clone(), generation, permit);
state.child = Some(child);
state.stdin = Some(stdin);
state.connected = false;
Ok(())
}
async fn send_request<T>(
&self,
method: DuckDbWorkerMethod,
params: impl serde::Serialize,
timeout: Option<Duration>,
) -> Result<T, String>
where
T: serde::de::DeserializeOwned,
{
self.send_request_structured(method, params, timeout).await.map_err(|error| error.message)
}
/// Like `send_request` but preserves the worker error `code` so callers can react to
/// specific conditions (e.g. `duckdb_worker_poisoned`). A protocol/transport failure is
/// surfaced as an error with code `duckdb_worker_error`.
async fn send_request_structured<T>(
&self,
method: DuckDbWorkerMethod,
params: impl serde::Serialize,
timeout: Option<Duration>,
) -> Result<T, DuckDbWorkerError>
where
T: serde::de::DeserializeOwned,
{
let id = format!("duckdb-worker-{}", self.inner.next_id.fetch_add(1, Ordering::Relaxed));
let request = DuckDbWorkerRequest::new(&id, method, params)?;
let line = serde_json::to_string(&request).map_err(|e| e.to_string())?;
let (tx, rx) = oneshot::channel();
let write_result = async {
let mut state = self.inner.state.lock().await;
self.ensure_started_locked(&mut state).await?;
if method != DuckDbWorkerMethod::Connect && !state.connected {
return Err("DuckDB worker is not connected".to_string());
}
let generation = state.generation;
self.inner.pending.lock().await.insert(id.clone(), PendingRequest { generation, sender: tx });
let stdin = state.stdin.as_mut().ok_or("DuckDB worker stdin unavailable")?;
stdin.write_all(line.as_bytes()).await.map_err(|e| e.to_string())?;
stdin.write_all(b"\n").await.map_err(|e| e.to_string())?;
stdin.flush().await.map_err(|e| e.to_string())
}
.await;
if let Err(err) = write_result {
self.inner.pending.lock().await.remove(&id);
return Err(err.into());
}
let response = match timeout {
Some(timeout) => match tokio::time::timeout(timeout, rx).await {
Ok(Ok(response)) => response,
Ok(Err(_)) => return Err("DuckDB worker response channel closed".into()),
Err(_) => {
self.inner.pending.lock().await.remove(&id);
return Err(DuckDbWorkerError::new(
DUCKDB_WORKER_REQUEST_TIMEOUT_CODE,
format!("DuckDB worker request timed out after {}s", timeout.as_secs()),
));
}
},
None => rx.await.map_err(|_| DuckDbWorkerError::from("DuckDB worker response channel closed"))?,
};
if !response.ok {
let error = response
.error
.unwrap_or_else(|| DuckDbWorkerError::new("duckdb_worker_error", "DuckDB worker request failed"));
return Err(error);
}
let result = response.result.unwrap_or(serde_json::Value::Null);
serde_json::from_value(result).map_err(|e| DuckDbWorkerError::from(e.to_string()))
}
async fn fail_pending_for_generation(&self, generation: u64, code: &'static str, message: &'static str) {
let pending = {
let mut pending = self.inner.pending.lock().await;
let ids = pending
.iter()
.filter(|&(_id, request)| request.generation == generation)
.map(|(id, _request)| id.clone())
.collect::<Vec<_>>();
ids.into_iter().filter_map(|id| pending.remove(&id).map(|request| (id, request.sender))).collect::<Vec<_>>()
};
for (id, sender) in pending {
let _ = sender.send(DuckDbWorkerResponse::err(id, DuckDbWorkerError::new(code, message)));
}
}
}
fn resolve_duckdb_driver_command() -> Result<(PathBuf, Vec<OsString>), String> {
if let Some(executable) = std::env::var_os(DUCKDB_DRIVER_PATH_ENV).filter(|value| !value.is_empty()) {
let executable = PathBuf::from(executable);
ensure_duckdb_driver_exists(&executable)?;
return Ok((executable, Vec::new()));
}
Err(format!(
"DuckDB driver is not installed. Please install it from the Driver Manager or set {DUCKDB_DRIVER_PATH_ENV}."
))
}
fn ensure_duckdb_driver_exists(executable: &Path) -> Result<(), String> {
if executable.is_file() {
return Ok(());
}
Err(format!("DuckDB driver configured by {DUCKDB_DRIVER_PATH_ENV} was not found: {}", executable.display()))
}
struct WorkerProcessLimiterState {
limit: usize,
limiter: Arc<Semaphore>,
}
fn duckdb_worker_process_limiter(process_limit: usize) -> (Arc<Semaphore>, usize) {
static LIMITER: OnceLock<StdMutex<WorkerProcessLimiterState>> = OnceLock::new();
let process_limit = normalize_duckdb_worker_max_processes(process_limit);
let state = LIMITER.get_or_init(|| {
StdMutex::new(WorkerProcessLimiterState {
limit: process_limit,
limiter: Arc::new(Semaphore::new(process_limit)),
})
});
let mut state = state.lock().expect("DuckDB worker process limiter lock poisoned");
if state.limit != process_limit && state.limiter.available_permits() == state.limit {
*state = WorkerProcessLimiterState { limit: process_limit, limiter: Arc::new(Semaphore::new(process_limit)) };
}
(state.limiter.clone(), state.limit)
}
fn duckdb_worker_process_limit_error(process_limit: usize) -> String {
format!(
"DuckDB worker process limit reached (max {}). Close another DuckDB worker connection or retry later.",
process_limit
)
}
fn duckdb_connect_file_lock_retry_delay(attempts: usize, message: &str) -> Option<Duration> {
if !is_transient_duckdb_file_lock_error(message) {
return None;
}
match attempts {
0 => Some(Duration::from_millis(50)),
1 => Some(Duration::from_millis(100)),
2 => Some(Duration::from_millis(200)),
3 => Some(Duration::from_millis(400)),
4 => Some(Duration::from_millis(800)),
_ => None,
}
}
fn is_transient_duckdb_file_lock_error(message: &str) -> bool {
let lower = message.to_ascii_lowercase();
let mentions_file_open = lower.contains("cannot open file")
|| lower.contains("could not set lock")
|| lower.contains("file is already open");
let mentions_lock = lower.contains("file is already open")
|| lower.contains("being used by another process")
|| lower.contains("conflicting lock")
|| lower.contains("process cannot access the file")
|| lower.contains("sharing violation")
|| lower.contains("resource temporarily unavailable")
|| message.contains("另一个程序正在使用")
|| message.contains("进程无法访问");
mentions_file_open && mentions_lock
}
fn spawn_stdout_reader(
stdout: tokio::process::ChildStdout,
pending: PendingRequests,
generation: u64,
permit: OwnedSemaphorePermit,
) {
tokio::spawn(async move {
let _permit = permit;
let mut lines = BufReader::new(stdout).lines();
loop {
match lines.next_line().await {
Ok(Some(line)) => {
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<DuckDbWorkerResponse>(&line) {
Ok(response) => {
let sender = {
let mut pending = pending.lock().await;
let matches_generation = pending
.get(&response.id)
.map(|request| request.generation == generation)
.unwrap_or(false);
if matches_generation {
pending.remove(&response.id).map(|request| request.sender)
} else {
None
}
};
if let Some(sender) = sender {
let _ = sender.send(response);
}
}
Err(err) => {
log::warn!("[duckdb-worker:invalid-response] error={err} line={line}");
}
}
}
Ok(None) => break,
Err(err) => {
log::warn!("[duckdb-worker:stdout-error] error={err}");
break;
}
}
}
let pending = {
let mut pending = pending.lock().await;
let ids = pending
.iter()
.filter(|&(_id, request)| request.generation == generation)
.map(|(id, _request)| id.clone())
.collect::<Vec<_>>();
ids.into_iter().filter_map(|id| pending.remove(&id).map(|request| (id, request.sender))).collect::<Vec<_>>()
};
for (id, sender) in pending {
let _ = sender.send(DuckDbWorkerResponse::err(
id,
DuckDbWorkerError::new("duckdb_worker_exited", "DuckDB worker process exited"),
));
}
});
}