Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(import): keep doris import transactions alive against pool probing

zipg497d7c9e1ee5
diff --git a/crates/dbx-core/src/connection/mod.rs b/crates/dbx-core/src/connection/mod.rs
index 51ad92c411..b83afe393d 100644
--- a/crates/dbx-core/src/connection/mod.rs
+++ b/crates/dbx-core/src/connection/mod.rs
@@ -2178,6 +2178,12 @@ impl AppState {
match result {
Ok(Ok(())) => {}
Ok(Err(err)) => {
+ if !keepalive_failure_proves_pool_dead(&err.to_string()) {
+ log::debug!(
+ "Connection keepalive for '{key}' could not check out a connection; keeping the busy pool: {err}"
+ );
+ continue;
+ }
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
let replace_runtime =
err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime);
@@ -5936,6 +5942,18 @@ impl From<String> for KeepaliveError {
}
}
+/// Whether a failed keepalive probe is evidence that the pool is dead.
+///
+/// A probe that ran out of its checkout budget because every connection is in use reports
+/// pool saturation, which says nothing about the health of the pooled connections. That is
+/// the normal state of a session-scoped pool (a single connection) while a batch import or
+/// transfer holds a long transaction on it. Invalidating the pool there used to abort the
+/// running operation with "Connection not found for transaction" instead of letting it
+/// finish. Every other probe failure keeps the invalidate-and-reconnect behaviour.
+fn keepalive_failure_proves_pool_dead(error: &str) -> bool {
+ !crate::query::is_pool_saturation_error(error)
+}
+
impl KeepaliveTarget {
fn matches_pool(&self, pool: &PoolKind) -> bool {
match (self, pool) {
@@ -6913,8 +6931,8 @@ mod tests {
connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint,
database_connection_config, database_connection_config_with_catalog,
gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver,
- kafka_single_loopback_bootstrap_endpoint, metadata_connection_config, metadata_pool_database,
- mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
+ kafka_single_loopback_bootstrap_endpoint, keepalive_failure_proves_pool_dead, metadata_connection_config,
+ metadata_pool_database, mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id,
redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error,
sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh,
@@ -8345,6 +8363,23 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
+ #[test]
+ fn keepalive_probe_keeps_a_busy_pool_but_invalidates_a_dead_one() {
+ // Every connection is checked out: the probe learned nothing about pool health, so the
+ // pool must survive. Invalidating it here aborts whatever holds the connection — a
+ // truncate import reports "Connection not found for transaction" on its next chunk.
+ assert!(!keepalive_failure_proves_pool_dead(
+ "MySQL connection pool checkout timed out [stage=wait, timeout_ms=10000]"
+ ));
+ // A checkout failure while creating a connection is still evidence the pool is dead.
+ assert!(keepalive_failure_proves_pool_dead(
+ "MySQL connection pool checkout failed [stage=create]: connection refused"
+ ));
+ assert!(keepalive_failure_proves_pool_dead(
+ "MySQL connection pool checkout timed out [stage=create, timeout_ms=10000]"
+ ));
+ }
+
#[tokio::test]
async fn jdbc_plugin_env_uses_managed_jre_when_installed() {
let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4()));
diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs
index e07618764c..9ae6fdbcb7 100644
--- a/crates/dbx-core/src/data/table_import.rs
+++ b/crates/dbx-core/src/data/table_import.rs
@@ -78,6 +78,16 @@ pub fn table_import_client_session_id(import_id: &str) -> String {
task_client_session_id("table-import", import_id)
}
+/// Execution id registered with `RunningQueries` for the whole duration of an import.
+///
+/// Background pool maintenance (the keepalive probe, connection health refresh, idle
+/// reclamation) consults `is_pool_active` before touching a pool. An import keeps a
+/// connection checked out for the entire run — a session-scoped pool has exactly one
+/// connection — so without this registration the pool looks idle while it is in use.
+fn table_import_execution_id(import_id: &str) -> String {
+ format!("table-import-exec:{import_id}")
+}
+
#[derive(Debug, Clone)]
pub struct ParsedImportFile {
pub columns: Vec<String>,
@@ -7063,6 +7073,16 @@ where
let mut db_write_ms = 0u128;
let mut statement_count = 0usize;
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
+ // Mark the import's pool busy for the whole run. A session-scoped pool keeps a single
+ // connection, and the transactional batch path holds it for an entire chunk, so the
+ // keepalive probe cannot check out a connection on time. Without this registration the
+ // probe treats that healthy-but-busy pool as dead, invalidates it, and the next chunk
+ // fails with "Connection not found for transaction".
+ let import_execution_id = table_import_execution_id(&request.import_id);
+ let _import_activity = state.running_queries.register(import_execution_id.clone());
+ state.running_queries.set_pool_key(&import_execution_id, pool_key);
+ state.touch_pool_activity(pool_key).await;
+ let _activity_touch = state.pool_activity_touch(pool_key);
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
let conflict_policy = request.effective_conflict_policy();
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
@@ -13255,6 +13275,100 @@ mod tests {
);
}
+ // A session-scoped pool keeps a single connection, and the transactional batch path holds
+ // it for a whole chunk, so the keepalive probe cannot check out a connection while the
+ // import is running. The import must therefore advertise its pool as busy for its whole
+ // run; otherwise the probe invalidates the healthy pool and the next chunk fails with
+ // "Connection not found for transaction".
+ #[tokio::test]
+ async fn truncate_import_marks_its_pool_active_until_it_finishes() {
+ let dir = tempfile::tempdir().unwrap();
+ let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let connection_id = "sqlite-truncate-activity";
+ let pool_key = format!("{connection_id}:session:import");
+ let database_path = dir.path().join("target.db");
+ let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER, name TEXT)").await.unwrap();
+ state
+ .update_connection_pools(|connections| {
+ connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
+ })
+ .await;
+ let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+ "id": connection_id,
+ "name": "SQLite truncate activity test",
+ "db_type": "sqlite",
+ "host": "",
+ "port": 0,
+ "username": "",
+ "password": "",
+ "database": database_path.to_string_lossy()
+ }))
+ .unwrap();
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ let data_path = dir.path().join("rows.txt");
+ std::fs::write(&data_path, b"id%name\n1%Ada\n2%Grace\n3%Linus\n").unwrap();
+ let request = TableImportRequest {
+ import_id: "sqlite-truncate-activity".to_string(),
+ connection_id: connection_id.to_string(),
+ database: String::new(),
+ schema: String::new(),
+ table: "items".to_string(),
+ file_path: data_path.to_string_lossy().to_string(),
+ source_ref: None,
+ source_format: Some(TableImportSourceFormat::Delimited),
+ parse_options: TableImportParseOptions {
+ delimiter: Some("%".to_string()),
+ ..TableImportParseOptions::default()
+ },
+ mappings: vec![
+ TableImportColumnMapping {
+ source_column: "id".to_string(),
+ target_column: "id".to_string(),
+ target_data_type: None,
+ },
+ TableImportColumnMapping {
+ source_column: "name".to_string(),
+ target_column: "name".to_string(),
+ target_data_type: None,
+ },
+ ],
+ mode: TableImportMode::Truncate,
+ create_table: false,
+ batch_size: 2,
+ date_time_format: None,
+ prepared_source: None,
+ skip_duplicate_rows: false,
+ conflict_policy: None,
+ retain_source: false,
+ };
+
+ let mut active_during_import = Vec::new();
+ let summary = import_table_file_core(
+ &state,
+ &request,
+ &DatabaseType::Sqlite,
+ &pool_key,
+ |_| Box::pin(async { false }),
+ |progress| {
+ active_during_import.push((progress.status, state.running_queries.is_pool_active(&pool_key)));
+ },
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(summary.rows_imported, 3);
+ assert!(!active_during_import.is_empty(), "the import must report progress while it owns the pool");
+ for (status, active) in &active_during_import {
+ assert!(active, "progress event {status:?} happened while the import's pool was idle to sweeps");
+ }
+ assert!(
+ !state.running_queries.is_pool_active(&pool_key),
+ "the import must release its pool registration when it finishes"
+ );
+ }
+
#[tokio::test]
async fn sqlite_update_existing_import_counts_updated_rows_once() {
let dir = tempfile::tempdir().unwrap();