Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(xugu): preserve empty CLOB values during transfer

Elias1daa65fd94ce
diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rs
index fedd16db28..741f435c36 100644
--- a/crates/dbx-core/src/data/transfer.rs
+++ b/crates/dbx-core/src/data/transfer.rs
@@ -12096,8 +12096,10 @@ fn xugu_transfer_shares_agent_session(
*source_db_type == DatabaseType::Xugu && source_pool_key == target_pool_key
}
+#[allow(clippy::too_many_arguments)]
fn xugu_transfer_page_sql(
columns: &[db::ColumnInfo],
+ clob_indexes: &[usize],
table: &str,
schema: &str,
catalog: Option<&str>,
@@ -12110,17 +12112,45 @@ fn xugu_transfer_page_sql(
let order = postgres_order_by_expression(primary_keys, &DatabaseType::Xugu)
.map(|expression| format!(" ORDER BY {expression}"))
.unwrap_or_default();
- format!("{}{order} LIMIT {limit} OFFSET {offset}", xugu_transfer_cursor_sql(columns, table, schema, catalog))
+ format!(
+ "{}{order} LIMIT {limit} OFFSET {offset}",
+ xugu_transfer_cursor_sql(columns, clob_indexes, table, schema, catalog)
+ )
+}
+
+/// Only Xugu-to-Xugu transfers need these auxiliary values. A raw empty CLOB
+/// currently arrives from the Go driver as a NUL byte, indistinguishable from
+/// real content without asking the source database about its actual value.
+fn xugu_transfer_clob_indexes(columns: &[db::ColumnInfo], source: &DatabaseType, target: &DatabaseType) -> Vec<usize> {
+ if *source != DatabaseType::Xugu || *target != DatabaseType::Xugu {
+ return Vec::new();
+ }
+ columns
+ .iter()
+ .enumerate()
+ .filter_map(|(index, column)| (normalize_xugu_type_name(&column.data_type) == "CLOB").then_some(index))
+ .collect()
}
/// Build a source-side projection for Xugu transfer rows whose wire values are
/// not safely represented by the Go driver's generic `database/sql` values.
/// This stays on the Xugu path only; the returned strings are still formatted
/// against the original column metadata when INSERT statements are generated.
-fn xugu_transfer_cursor_sql(columns: &[db::ColumnInfo], table: &str, schema: &str, catalog: Option<&str>) -> String {
+fn xugu_transfer_cursor_sql(
+ columns: &[db::ColumnInfo],
+ clob_indexes: &[usize],
+ table: &str,
+ schema: &str,
+ catalog: Option<&str>,
+) -> String {
+ let full_table = qualified_table(table, schema, &DatabaseType::Xugu, catalog);
+ let projection = xugu_transfer_projection(columns, clob_indexes);
+ format!("SELECT {projection} FROM {full_table}")
+}
+
+fn xugu_transfer_projection(columns: &[db::ColumnInfo], clob_indexes: &[usize]) -> String {
let db_type = DatabaseType::Xugu;
- let full_table = qualified_table(table, schema, &db_type, catalog);
- let projection = columns
+ let mut projection = columns
.iter()
.map(|column| {
let identifier = quote_identifier(&column.name, &db_type);
@@ -12164,9 +12194,62 @@ fn xugu_transfer_cursor_sql(columns: &[db::ColumnInfo], table: &str, schema: &st
};
format!("{expression} AS {identifier}")
})
- .collect::<Vec<_>>()
- .join(", ");
- format!("SELECT {projection} FROM {full_table}")
+ .collect::<Vec<_>>();
+ for index in clob_indexes {
+ let identifier = quote_identifier(&columns[*index].name, &db_type);
+ let mut alias = format!("__dbx_clob_state_{index}");
+ while columns.iter().any(|column| column.name.eq_ignore_ascii_case(&alias)) {
+ alias.push('_');
+ }
+ // No text cast/hex expansion: large CLOBs retain their original bytes.
+ // NULL and an empty non-NULL CLOB must remain distinct on the target.
+ projection.push(format!(
+ "CASE WHEN {identifier} IS NULL THEN -1 WHEN LENGTH({identifier})=0 THEN 0 ELSE 1 END AS {}",
+ quote_identifier(&alias, &db_type)
+ ));
+ }
+ projection.join(", ")
+}
+
+fn normalize_xugu_transfer_clobs(
+ rows: &mut [Vec<serde_json::Value>],
+ columns: &[db::ColumnInfo],
+ clob_indexes: &[usize],
+) -> Result<(), String> {
+ if clob_indexes.is_empty() {
+ return Ok(());
+ }
+ let width = columns.len();
+ // Validate the complete batch before modifying it or building any INSERT.
+ for row in rows.iter() {
+ if row.len() != width + clob_indexes.len() {
+ return Err("Xugu CLOB transfer returned an unexpected number of columns".to_string());
+ }
+ for (marker, index) in clob_indexes.iter().enumerate() {
+ let value = &row[*index];
+ let valid = match row[width + marker].as_i64() {
+ Some(-1) => value.is_null(),
+ Some(0) => value.is_null() || matches!(value.as_str(), Some("" | "\0")),
+ Some(1) => value.as_str().is_some_and(|text| !text.is_empty() && !text.contains('\0')),
+ _ => false,
+ };
+ if !valid {
+ return Err(format!(
+ "Xugu CLOB column '{}' returned inconsistent content or a non-empty value containing NUL; refusing lossy transfer",
+ columns[*index].name
+ ));
+ }
+ }
+ }
+ for row in rows {
+ for (marker, index) in clob_indexes.iter().enumerate() {
+ if row[width + marker].as_i64() == Some(0) {
+ row[*index] = serde_json::Value::String(String::new());
+ }
+ }
+ row.truncate(width);
+ }
+ Ok(())
}
fn uses_agent_transfer_cursor(db_type: &DatabaseType) -> bool {
@@ -13000,6 +13083,7 @@ where
.await?;
let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type);
+ let xugu_clob_indexes = xugu_transfer_clob_indexes(&writable_columns, source_db_type, target_db_type);
let default_rows_only = mysql_generated_only_transfer(&columns, source_db_type, target_db_type)
|| xugu_rowversion_only_transfer(&columns, source_db_type, target_db_type);
if writable_columns.is_empty() && !default_rows_only {
@@ -13483,6 +13567,7 @@ where
if *source_db_type == DatabaseType::Xugu {
xugu_transfer_cursor_sql(
&writable_columns,
+ &xugu_clob_indexes,
table,
&request.source_schema,
request.source_catalog.as_deref(),
@@ -13551,6 +13636,7 @@ where
} else if xugu_same_session && !default_rows_only {
let sql = xugu_transfer_page_sql(
&writable_columns,
+ &xugu_clob_indexes,
table,
&request.source_schema,
request.source_catalog.as_deref(),
@@ -13622,6 +13708,7 @@ where
};
let has_more = result.has_more;
let row_count = result.rows.len();
+ normalize_xugu_transfer_clobs(&mut result.rows, &writable_columns, &xugu_clob_indexes)?;
if mysql_to_xugu {
mysql_xugu::validate_rows(&writable_columns, &result.rows)?;
}
@@ -17853,7 +17940,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
assert!(xugu_transfer_shares_agent_session(&DatabaseType::Xugu, "conn:SYSTEM", "conn:SYSTEM"));
assert!(!xugu_transfer_shares_agent_session(&DatabaseType::Xugu, "conn:SYSTEM", "conn:GIS"));
assert!(!xugu_transfer_shares_agent_session(&DatabaseType::Postgres, "conn", "conn"));
- let sql = xugu_transfer_page_sql(&[test_column("G", "GEOMETRY")], "SHAPES", "APP", None, &[], 1000, 1000);
+ let sql = xugu_transfer_page_sql(&[test_column("G", "GEOMETRY")], &[], "SHAPES", "APP", None, &[], 1000, 1000);
assert!(sql.contains("ST_AsEWKB(\"G\")"));
assert!(sql.ends_with("LIMIT 1000 OFFSET 1000"));
@@ -17861,6 +17948,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
// plain projection keeps the previous behavior.
let keyed = xugu_transfer_page_sql(
&[test_column("ID", "INTEGER"), test_column("G", "GEOMETRY")],
+ &[],
"SHAPES",
"APP",
None,
@@ -19114,6 +19202,94 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
assert!(!uses_agent_transfer_cursor(&DatabaseType::Mysql));
}
+ #[test]
+ fn xugu_empty_clob_workaround_is_scoped_to_xugu_pairs_and_clob_columns() {
+ let columns = vec![test_column("ID", "INTEGER"), test_column("BODY", "clob"), test_column("B", "BLOB")];
+ assert_eq!(xugu_transfer_clob_indexes(&columns, &DatabaseType::Xugu, &DatabaseType::Xugu), vec![1]);
+ for (source, target) in [
+ (DatabaseType::Mysql, DatabaseType::Xugu),
+ (DatabaseType::Xugu, DatabaseType::Mysql),
+ (DatabaseType::Postgres, DatabaseType::Postgres),
+ (DatabaseType::Dameng, DatabaseType::Dameng),
+ ] {
+ let indexes = xugu_transfer_clob_indexes(&columns, &source, &target);
+ assert!(indexes.is_empty());
+ let mut rows = vec![vec![json!(1), json!("\0"), json!("0x00")]];
+ let original = rows.clone();
+ normalize_xugu_transfer_clobs(&mut rows, &columns, &indexes).unwrap();
+ assert_eq!(rows, original);
+ }
+ assert!(xugu_transfer_clob_indexes(&columns[..1], &DatabaseType::Xugu, &DatabaseType::Xugu).is_empty());
+ // This fix must not enable the currently unsupported Xugu row-filter path.
+ assert!(!transfer_table_filter_supported(&DatabaseType::Xugu));
+ assert_eq!(
+ xugu_transfer_cursor_sql(&columns, &[], "T", "APP", None),
+ r#"SELECT "ID" AS "ID", "BODY" AS "BODY", CASE WHEN "B" IS NULL THEN NULL ELSE '0x'||RAWTOHEX("B") END AS "B" FROM "APP"."T""#
+ );
+ }
+
+ #[test]
+ fn xugu_empty_clob_markers_cover_cursor_and_pages_without_name_collisions() {
+ let columns =
+ vec![test_column("ID", "INTEGER"), test_column("BODY", "CLOB"), test_column("__DBX_CLOB_STATE_1", "CLOB")];
+ let indexes = xugu_transfer_clob_indexes(&columns, &DatabaseType::Xugu, &DatabaseType::Xugu);
+ let cursor = xugu_transfer_cursor_sql(&columns, &indexes, "T", "APP", None);
+ assert!(cursor.contains(
+ r#"CASE WHEN "BODY" IS NULL THEN -1 WHEN LENGTH("BODY")=0 THEN 0 ELSE 1 END AS "__dbx_clob_state_1_""#
+ ));
+ assert!(cursor.contains(r#"LENGTH("__DBX_CLOB_STATE_1")=0"#));
+ assert!(!cursor.contains("CAST("));
+ assert!(!cursor.contains("RAWTOHEX("));
+ let page = xugu_transfer_page_sql(&columns, &indexes, "T", "APP", None, &["ID".into()], 2, 2);
+ assert!(page.starts_with(&cursor));
+ assert!(page.ends_with(r#" ORDER BY "ID" LIMIT 2 OFFSET 2"#));
+ }
+
+ #[test]
+ fn xugu_empty_clob_normalization_preserves_null_spaces_unicode_and_large_text() {
+ let columns = vec![test_column("ID", "INTEGER"), test_column("A", "CLOB"), test_column("B", "CLOB")];
+ let large = "中文 it's a large CLOB\n".repeat(10_000);
+ let mut rows = vec![
+ vec![json!(1), json!(null), json!("\0"), json!(-1), json!(0)],
+ vec![json!(2), json!(""), json!(" "), json!(0), json!(1)],
+ vec![json!(3), json!(large), json!("中文 it's CLOB"), json!(1), json!(1)],
+ vec![json!(4), json!(null), json!(null), json!(0), json!(-1)],
+ ];
+ normalize_xugu_transfer_clobs(&mut rows, &columns, &[1, 2]).unwrap();
+ assert_eq!(rows[0], vec![json!(1), json!(null), json!("")]);
+ assert_eq!(rows[1], vec![json!(2), json!(""), json!(" ")]);
+ assert_eq!(rows[2], vec![json!(3), json!(large), json!("中文 it's CLOB")]);
+ assert_eq!(rows[3], vec![json!(4), json!(""), json!(null)]);
+ assert_eq!(escape_value_typed(&rows[0][1], &DatabaseType::Xugu, Some("CLOB")), "NULL");
+ assert_eq!(escape_value_typed(&rows[0][2], &DatabaseType::Xugu, Some("CLOB")), "''");
+ assert_eq!(escape_value_typed(&rows[1][2], &DatabaseType::Xugu, Some("CLOB")), "' '");
+ assert_eq!(escape_value_typed(&rows[2][2], &DatabaseType::Xugu, Some("CLOB")), "'中文 it''s CLOB'");
+ normalize_xugu_transfer_clobs(&mut [], &columns, &[1, 2]).unwrap();
+ }
+
+ #[test]
+ fn xugu_empty_clob_normalization_rejects_ambiguous_or_malformed_batches_without_mutation() {
+ let columns = vec![test_column("BODY", "CLOB")];
+ for invalid in [
+ vec![json!("\0")],
+ vec![json!("\0"), json!(0), json!(0)],
+ vec![json!("data"), json!(null)],
+ vec![json!("data"), json!(2)],
+ vec![json!("data"), json!("0")],
+ vec![json!("data"), json!(0)],
+ vec![json!("data"), json!(-1)],
+ vec![json!(null), json!(1)],
+ vec![json!(""), json!(1)],
+ vec![json!("\0"), json!(1)],
+ vec![json!("a\0b"), json!(1)],
+ ] {
+ let mut rows = vec![vec![json!("\0"), json!(0)], invalid];
+ let original = rows.clone();
+ assert!(normalize_xugu_transfer_clobs(&mut rows, &columns, &[0]).is_err());
+ assert_eq!(rows, original);
+ }
+ }
+
#[test]
fn xugu_transfer_cursor_projects_lossy_driver_types_as_text_or_hex() {
let columns = vec![
@@ -19128,7 +19304,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
];
assert_eq!(
- xugu_transfer_cursor_sql(&columns, "ITEMS", "APP", None),
+ xugu_transfer_cursor_sql(&columns, &[], "ITEMS", "APP", None),
r#"SELECT "ID" AS "ID", CASE WHEN "BLOB_C" IS NULL THEN NULL ELSE '0x'||RAWTOHEX("BLOB_C") END AS "BLOB_C", RTRIM(CAST("BIT_C" AS VARCHAR(8))) AS "BIT_C", RTRIM(CAST("VARBIT_C" AS VARCHAR(32767))) AS "VARBIT_C", CAST("TIME_C" AS VARCHAR(64)) AS "TIME_C", CAST("TIME_TZ_C" AS VARCHAR(256)) AS "TIME_TZ_C", CAST("IV_YM_C" AS VARCHAR(256)) AS "IV_YM_C", "NOTE" AS "NOTE" FROM "APP"."ITEMS""#
);
}
@@ -19142,7 +19318,7 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
test_column("B3", "BOX3D"),
];
assert_eq!(
- xugu_transfer_cursor_sql(&columns, "SHAPES", "APP", None),
+ xugu_transfer_cursor_sql(&columns, &[], "SHAPES", "APP", None),
r#"SELECT CASE WHEN "G" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB("G")) END AS "G", CASE WHEN "GG" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("GG" AS GEOMETRY))) END AS "GG", CASE WHEN "B2" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("B2" AS GEOMETRY))) END AS "B2", CASE WHEN "B3" IS NULL THEN NULL ELSE '0x'||RAWTOHEX(ST_AsEWKB(CAST("B3" AS GEOMETRY))) END AS "B3" FROM "APP"."SHAPES""#
);
}