25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
fix(xugu): preserve empty CLOB values during transfer
Elias1daa65fd94ce
diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rsindex fedd16db28..741f435c36 100644--- a/crates/dbx-core/src/data/transfer.rs+++ b/crates/dbx-core/src/data/transfer.rs@@ -12096,8 +12096,10 @@ fn xugu_transfer_shares_agent_session(*source_db_type == DatabaseType::Xugu && source_pool_key == target_pool_key}+#[allow(clippy::too_many_arguments)]fn xugu_transfer_page_sql(columns: &[db::ColumnInfo],+ clob_indexes: &[usize],table: &str,schema: &str,catalog: Option<&str>,@@ -12110,17 +12112,45 @@ fn xugu_transfer_page_sql(let order = postgres_order_by_expression(primary_keys, &DatabaseType::Xugu).map(|expression| format!(" ORDER BY {expression}")).unwrap_or_default();- format!("{}{order} LIMIT {limit} OFFSET {offset}", xugu_transfer_cursor_sql(columns, table, schema, catalog))+ format!(+ "{}{order} LIMIT {limit} OFFSET {offset}",+ xugu_transfer_cursor_sql(columns, clob_indexes, table, schema, catalog)+ )+}++/// Only Xugu-to-Xugu transfers need these auxiliary values. A raw empty CLOB+/// currently arrives from the Go driver as a NUL byte, indistinguishable from+/// real content without asking the source database about its actual value.+fn xugu_transfer_clob_indexes(columns: &[db::ColumnInfo], source: &DatabaseType, target: &DatabaseType) -> Vec<usize> {+ if *source != DatabaseType::Xugu || *target != DatabaseType::Xugu {+ return Vec::new();+ }+ columns+ .iter()+ .enumerate()+ .filter_map(|(index, column)| (normalize_xugu_type_name(&column.data_type) == "CLOB").then_some(index))+ .collect()}/// Build a source-side projection for Xugu transfer rows whose wire values are/// not safely represented by the Go driver's generic `database/sql` values./// This stays on the Xugu path only; the returned strings are still formatted/// against the original column metadata when INSERT statements are generated.-fn xugu_transfer_cursor_sql(columns: &[db::ColumnInfo], table: &str, schema: &str, catalog: Option<&str>) -> String {+fn xugu_transfer_cursor_sql(+ columns: &[db::ColumnInfo],+ clob_indexes: &[usize],+ table: &str,+ schema: &str,+ catalog: Option<&str>,+) -> String {+ let full_table = qualified_table(table, schema, &DatabaseType::Xugu, catalog);+ let projection = xugu_transfer_projection(columns, clob_indexes);+ format!("SELECT {projection} FROM {full_table}")+}++fn xugu_transfer_projection(columns: &[db::ColumnInfo], clob_indexes: &[usize]) -> String {let db_type = DatabaseType::Xugu;- let full_table = qualified_table(table, schema, &db_type, catalog);- let projection = columns+ let mut projection = columns.iter().map(|column| {let identifier = quote_identifier(&column.name, &db_type);@@ -12164,9 +12194,62 @@ fn xugu_transfer_cursor_sql(columns: &[db::ColumnInfo], table: &str, schema: &st};format!("{expression} AS {identifier}")})- .collect::<Vec<_>>()- .join(", ");- format!("SELECT {projection} FROM {full_table}")+ .collect::<Vec<_>>();+ for index in clob_indexes {+ let identifier = quote_identifier(&columns[*index].name, &db_type);+ let mut alias = format!("__dbx_clob_state_{index}");+ while columns.iter().any(|column| column.name.eq_ignore_ascii_case(&alias)) {+ alias.push('_');+ }+ // No text cast/hex expansion: large CLOBs retain their original bytes.+ // NULL and an empty non-NULL CLOB must remain distinct on the target.+ projection.push(format!(+ "CASE WHEN {identifier} IS NULL THEN -1 WHEN LENGTH({identifier})=0 THEN 0 ELSE 1 END AS {}",+ quote_identifier(&alias, &db_type)+ ));+ }+ projection.join(", ")+}++fn normalize_xugu_transfer_clobs(+ rows: &mut [Vec<serde_json::Value>],+ columns: &[db::ColumnInfo],+ clob_indexes: &[usize],+) -> Result<(), String> {+ if clob_indexes.is_empty() {+ return Ok(());+ }+ let width = columns.len();+ // Validate the complete batch before modifying it or building any INSERT.+ for row in rows.iter() {+ if row.len() != width + clob_indexes.len() {+ return Err("Xugu CLOB transfer returned an unexpected number of columns".to_string());+ }+ for (marker, index) in clob_indexes.iter().enumerate() {+ let value = &row[*index];+ let valid = match row[width + marker].as_i64() {+ Some(-1) => value.is_null(),+ Some(0) => value.is_null() || matches!(value.as_str(), Some("" | "\0")),+ Some(1) => value.as_str().is_some_and(|text| !text.is_empty() && !text.contains('\0')),+ _ => false,+ };+ if !valid {+ return Err(format!(+ "Xugu CLOB column '{}' returned inconsistent content or a non-empty value containing NUL; refusing lossy transfer",+ columns[*index].name+ ));+ }+ }+ }+ for row in rows {+ for (marker, index) in clob_indexes.iter().enumerate() {+ if row[width + marker].as_i64() == Some(0) {+ row[*index] = serde_json::Value::String(String::new());+ }+ }+ row.truncate(width);+ }+ Ok(())}fn uses_agent_transfer_cursor(db_type: &DatabaseType) -> bool {@@ -13000,6 +13083,7 @@ where.await?;let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type);+ let xugu_clob_indexes = xugu_transfer_clob_indexes(&writable_columns, source_db_type, target_db_type);let default_rows_only = mysql_generated_only_transfer(&columns, source_db_type, target_db_type)|| xugu_rowversion_only_transfer(&columns, source_db_type, target_db_type);if writable_columns.is_empty() && !default_rows_only {@@ -13483,6 +13567,7 @@ whereif *source_db_type == DatabaseType::Xugu {xugu_transfer_cursor_sql(&writable_columns,+ &xugu_clob_indexes,table,&request.source_schema,request.source_catalog.as_deref(),@@ -13551,6 +13636,7 @@ where} else if xugu_same_session && !default_rows_only {let sql = xugu_transfer_page_sql(&writable_columns,+ &xugu_clob_indexes,table,&request.source_schema,request.source_catalog.as_deref(),@@ -13622,6 +13708,7 @@ where};let has_more = result.has_more;let row_count = result.rows.len();+ normalize_xugu_transfer_clobs(&mut result.rows, &writable_columns, &xugu_clob_indexes)?;if mysql_to_xugu {mysql_xugu::validate_rows(&writable_columns, &result.rows)?;}@@ -17853,7 +17940,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;assert!(xugu_transfer_shares_agent_session(&DatabaseType::Xugu, "conn:SYSTEM", "conn:SYSTEM"));assert!(!xugu_transfer_shares_agent_session(&DatabaseType::Xugu, "conn:SYSTEM", "conn:GIS"));assert!(!xugu_transfer_shares_agent_session(&DatabaseType::Postgres, "conn", "conn"));- let sql = xugu_transfer_page_sql(&[test_column("G", "GEOMETRY")], "SHAPES", "APP", None, &[], 1000, 1000);+ let sql = xugu_transfer_page_sql(&[test_column("G", "GEOMETRY")], &[], "SHAPES", "APP", None, &[], 1000, 1000);assert!(sql.contains("ST_AsEWKB(\"G\")"));assert!(sql.ends_with("LIMIT 1000 OFFSET 1000"));@@ -17861,6 +17948,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;// plain projection keeps the previous behavior.let keyed = xugu_transfer_page_sql(&[test_column("ID", "INTEGER"), test_column("G", "GEOMETRY")],+ &[],"SHAPES","APP",None,@@ -19114,6 +19202,94 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;assert!(!uses_agent_transfer_cursor(&DatabaseType::Mysql));}+ #[test]+ fn xugu_empty_clob_workaround_is_scoped_to_xugu_pairs_and_clob_columns() {+ let columns = vec![test_column("ID", "INTEGER"), test_column("BODY", "clob"), test_column("B", "BLOB")];+ assert_eq!(xugu_transfer_clob_indexes(&columns, &DatabaseType::Xugu, &DatabaseType::Xugu), vec![1]);+ for (source, target) in [+ (DatabaseType::Mysql, DatabaseType::Xugu),+ (DatabaseType::Xugu, DatabaseType::Mysql),+ (DatabaseType::Postgres, DatabaseType::Postgres),+ (DatabaseType::Dameng, DatabaseType::Dameng),+ ] {+ let indexes = xugu_transfer_clob_indexes(&columns, &source, &target);+ assert!(indexes.is_empty());+ let mut rows = vec![vec![json!(1), json!("\0"), json!("0x00")]];+ let original = rows.clone();+ normalize_xugu_transfer_clobs(&mut rows, &columns, &indexes).unwrap();+ assert_eq!(rows, original);+ }+ assert!(xugu_transfer_clob_indexes(&columns[..1], &DatabaseType::Xugu, &DatabaseType::Xugu).is_empty());+ // This fix must not enable the currently unsupported Xugu row-filter path.+ assert!(!transfer_table_filter_supported(&DatabaseType::Xugu));+ assert_eq!(+ xugu_transfer_cursor_sql(&columns, &[], "T", "APP", None),+ r#"SELECT "ID" AS "ID", "BODY" AS "BODY", CASE WHEN "B" IS NULL THEN NULL ELSE '0x'||RAWTOHEX("B") END AS "B" FROM "APP"."T""#+ );+ }++ #[test]+ fn xugu_empty_clob_markers_cover_cursor_and_pages_without_name_collisions() {+ let columns =+ vec![test_column("ID", "INTEGER"), test_column("BODY", "CLOB"), test_column("__DBX_CLOB_STATE_1", "CLOB")];+ let indexes = xugu_transfer_clob_indexes(&columns, &DatabaseType::Xugu, &DatabaseType::Xugu);+ let cursor = xugu_transfer_cursor_sql(&columns, &indexes, "T", "APP", None);+ assert!(cursor.contains(+ r#"CASE WHEN "BODY" IS NULL THEN -1 WHEN LENGTH("BODY")=0 THEN 0 ELSE 1 END AS "__dbx_clob_state_1_""#+ ));+ assert!(cursor.contains(r#"LENGTH("__DBX_CLOB_STATE_1")=0"#));+ assert!(!cursor.contains("CAST("));+ assert!(!cursor.contains("RAWTOHEX("));+ let page = xugu_transfer_page_sql(&columns, &indexes, "T", "APP", None, &["ID".into()], 2, 2);+ assert!(page.starts_with(&cursor));+ assert!(page.ends_with(r#" ORDER BY "ID" LIMIT 2 OFFSET 2"#));+ }++ #[test]+ fn xugu_empty_clob_normalization_preserves_null_spaces_unicode_and_large_text() {+ let columns = vec![test_column("ID", "INTEGER"), test_column("A", "CLOB"), test_column("B", "CLOB")];+ let large = "中文 it's a large CLOB\n".repeat(10_000);+ let mut rows = vec![+ vec![json!(1), json!(null), json!("\0"), json!(-1), json!(0)],+ vec![json!(2), json!(""), json!(" "), json!(0), json!(1)],+ vec![json!(3), json!(large), json!("中文 it's CLOB"), json!(1), json!(1)],+ vec![json!(4), json!(null), json!(null), json!(0), json!(-1)],+ ];+ normalize_xugu_transfer_clobs(&mut rows, &columns, &[1, 2]).unwrap();+ assert_eq!(rows[0], vec![json!(1), json!(null), json!("")]);+ assert_eq!(rows[1], vec![json!(2), json!(""), json!(" ")]);+ assert_eq!(rows[2], vec![json!(3), json!(large), json!("中文 it's CLOB")]);+ assert_eq!(rows[3], vec![json!(4), json!(""), json!(null)]);+ assert_eq!(escape_value_typed(&rows[0][1], &DatabaseType::Xugu, Some("CLOB")), "NULL");+ assert_eq!(escape_value_typed(&rows[0][2], &DatabaseType::Xugu, Some("CLOB")), "''");+ assert_eq!(escape_value_typed(&rows[1][2], &DatabaseType::Xugu, Some("CLOB")), "' '");+ assert_eq!(escape_value_typed(&rows[2][2], &DatabaseType::Xugu, Some("CLOB")), "'中文 it''s CLOB'");+ normalize_xugu_transfer_clobs(&mut [], &columns, &[1, 2]).unwrap();+ }++ #[test]+ fn xugu_empty_clob_normalization_rejects_ambiguous_or_malformed_batches_without_mutation() {+ let columns = vec![test_column("BODY", "CLOB")];+ for invalid in [+ vec![json!("\0")],+ vec![json!("\0"), json!(0), json!(0)],+ vec![json!("data"), json!(null)],+ vec![json!("data"), json!(2)],+ vec![json!("data"), json!("0")],+ vec![json!("data"), json!(0)],+ vec![json!("data"), json!(-1)],+ vec![json!(null), json!(1)],+ vec![json!(""), json!(1)],+ vec![json!("\0"), json!(1)],+ vec![json!("a\0b"), json!(1)],+ ] {+ let mut rows = vec![vec![json!("\0"), json!(0)], invalid];+ let original = rows.clone();+ assert!(normalize_xugu_transfer_clobs(&mut rows, &columns, &[0]).is_err());+ assert_eq!(rows, original);+ }+ }+#[test]fn xugu_transfer_cursor_projects_lossy_driver_types_as_text_or_hex() {let columns = vec![@@ -19128,7 +19304,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;];assert_eq!(- xugu_transfer_cursor_sql(&columns, "ITEMS", "APP", None),+ xugu_transfer_cursor_sql(&columns, &[], "ITEMS", "APP", None),r#"SELECT "ID" AS "ID", CASE WHEN "BLOB_C" IS NULL THEN NULL ELSE '0x'||RAWTOHEX("BLOB_C") END AS "BLOB_C", RTRIM(CAST("BIT_C" AS VARCHAR(8))) AS "BIT_C", RTRIM(CAST("VARBIT_C" AS VARCHAR(32767))) AS "VARBIT_C", CAST("TIME_C" AS VARCHAR(64)) AS "TIME_C", CAST("TIME_TZ_C" AS VARCHAR(256)) AS "TIME_TZ_C", CAST("IV_YM_C" AS VARCHAR(256)) AS "IV_YM_C", "NOTE" AS "NOTE" FROM "APP"."ITEMS""#);}@@ -19142,7 +19318,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;test_column("B3", "BOX3D"),];assert_eq!(- xugu_transfer_cursor_sql(&columns, "SHAPES", "APP", None),+ xugu_transfer_cursor_sql(&columns, &[], "SHAPES", "APP", None),r#"SELECT CASE WHEN "G" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB("G")) END AS "G", CASE WHEN "GG" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("GG" AS GEOMETRY))) END AS "GG", CASE WHEN "B2" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("B2" AS GEOMETRY))) END AS "B2", CASE WHEN "B3" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("B3" AS GEOMETRY))) END AS "B3" FROM "APP"."SHAPES""#);}