25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
fix(sqlserver): keep rows readable on collation decode failure
Thanadon Kongkanun339c96cc81ba
diff --git a/Cargo.toml b/Cargo.tomlindex 6303e6325f..339d49660c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -37,7 +37,9 @@ postgres-protocol = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git"# rustls dangerous-mode connections to legacy certificates. Remove after upstream support lands.mysql_async = { git = "https://github.com/zipg/mysql_async.git", rev = "4a9e60a62f5a48e35ef9ccc9fc8277d5012b19dd" }# Preserve readable SQL Server result sets when a row contains an unpaired-# UTF-16 surrogate. Remove after the behavior is available in upstream tiberius.+# UTF-16 surrogate, or varchar/text bytes that are invalid in the column's+# collation (replaced with U+FFFD, no BOM sniffing). Remove after the behavior+# is available in upstream tiberius.tiberius = { path = "vendor/tiberius" }[profile.release]diff --git a/vendor/tiberius/DBX-PATCH.md b/vendor/tiberius/DBX-PATCH.mdindex b169911d81..301a0ca7db 100644--- a/vendor/tiberius/DBX-PATCH.md+++ b/vendor/tiberius/DBX-PATCH.md@@ -6,14 +6,19 @@ identified by the checksumupstream release tag points to commit`0e2897a276166503ba78fe3e1cee501e9a034021`.-DBX changes only SQL Server Unicode column-data decoding:+DBX changes only SQL Server column-data decoding:- NCHAR, NVARCHAR, and NTEXT row values replace unpaired UTF-16 surrogates withU+FFFD, matching the Microsoft JDBC driver's observable behavior.+- CHAR, VARCHAR, and TEXT row values under a non-Unicode collation decode+ lossily: byte sequences that are invalid in the column's collation become+ U+FFFD instead of failing the whole result set, following the JDBC driver's+ REPLACE policy. The lossy decoder does no BOM handling, so bytes that merely+ look like a UTF-8 BOM (e.g. under GB18030) stay ordinary collation data.- Odd byte lengths remain protocol errors, including an explicit NTEXT guardthat prevents truncating a trailing byte and desynchronizing the TDS stream.-- Metadata, environment tokens, other protocol strings, and non-Unicode- codepage decoding retain upstream's strict behavior.+- Metadata, environment tokens, and other protocol strings retain upstream's+ strict behavior.The regression tests in `src/tds/codec/column_data.rs` exercise the decoderwith raw TDS value frames. The upstream integration fixtures are omitted fromdiff --git a/vendor/tiberius/src/tds/codec/column_data.rs b/vendor/tiberius/src/tds/codec/column_data.rsindex b29f18c2da..03fa6c9a92 100644--- a/vendor/tiberius/src/tds/codec/column_data.rs+++ b/vendor/tiberius/src/tds/codec/column_data.rs@@ -758,6 +758,69 @@ mod tests {}}+ #[tokio::test]+ async fn varchar_replaces_invalid_collation_bytes() {+ // CP936/GB18030 (sort ID 198): 0x81 starts a two-byte sequence, and+ // 0x40..0x7e is a valid trail byte, so 0x81 0x20 is invalid. A legacy+ // varchar column can still hold those bytes when an application wrote+ // them under a different codepage.+ let value = decode_wire(+ VarLenType::BigVarChar,+ 20,+ Some(Collation::new(0x804, 198)),+ &short_string_wire(b"ok\x81\x20ok"),+ )+ .await+ .expect("undecodable varchar bytes must not fail the whole result set");++ let decoded = decoded_string(value).expect("varchar row value");+ assert!(decoded.starts_with("ok"), "readable bytes are preserved: {decoded:?}");+ assert!(decoded.ends_with("ok"), "bytes after the bad sequence are preserved: {decoded:?}");+ assert!(decoded.contains('\u{fffd}'), "the bad sequence is replaced: {decoded:?}");+ }++ #[tokio::test]+ async fn text_replaces_invalid_collation_bytes() {+ let value = decode_wire(+ VarLenType::Text,+ 0,+ Some(Collation::new(0x804, 198)),+ &ntext_wire(b"ok\x81\x20ok"),+ )+ .await+ .expect("undecodable text bytes must not fail the whole result set");++ let decoded = decoded_string(value).expect("text row value");+ assert!(decoded.starts_with("ok"), "readable bytes are preserved: {decoded:?}");+ assert!(decoded.ends_with("ok"), "bytes after the bad sequence are preserved: {decoded:?}");+ assert!(decoded.contains('\u{fffd}'), "the bad sequence is replaced: {decoded:?}");+ }++ #[tokio::test]+ async fn varchar_gb18030_does_not_sniff_a_utf8_bom() {+ // EF BB BF is fully valid GB18030 data: EF BB and BF 61 form two+ // two-byte sequences (锘縜) followed by ASCII "bc". The collation+ // already names the encoding, so the lossy decoder must not let+ // encoding_rs sniff those bytes as a UTF-8 BOM and return "abc".+ let (text, had_errors) = Collation::new(0x804, 198)+ .codec()+ .unwrap()+ .decode_lossy(b"\xef\xbb\xbfabc");+ assert_eq!(text, "锘縜bc");+ assert!(!had_errors, "the bytes are valid GB18030");++ let value = decode_wire(+ VarLenType::BigVarChar,+ 20,+ Some(Collation::new(0x804, 198)),+ &short_string_wire(b"\xef\xbb\xbfabc"),+ )+ .await+ .expect("BOM-like varchar bytes must decode as ordinary GB18030 data");++ assert_eq!(decoded_string(value).as_deref(), Some("锘縜bc"));+ }+#[tokio::test]async fn varchar_cp850_collation_decodes_french_text() {// SQL_1xCompat_CP850_CI_AS: LCID 0x409, sort ID 49. The DOS codepagesdiff --git a/vendor/tiberius/src/tds/codec/column_data/string.rs b/vendor/tiberius/src/tds/codec/column_data/string.rsindex 42528ecc56..eb80cfa9ed 100644--- a/vendor/tiberius/src/tds/codec/column_data/string.rs+++ b/vendor/tiberius/src/tds/codec/column_data/string.rs@@ -21,10 +21,9 @@ where// Codepages other than UTF(Some(buf), BigChar) | (Some(buf), BigVarChar) => {let collation = collation.as_ref().unwrap();- let s = collation- .codec()?- .decode(buf.as_ref())- .ok_or_else(|| Error::Encoding("invalid sequence".into()))?;+ // Lossy: a row holding bytes that are invalid in its declared+ // collation must stay readable instead of failing the result set.+ let (s, _) = collation.codec()?.decode_lossy(buf.as_ref());Ok(Some(s.into()))}diff --git a/vendor/tiberius/src/tds/codec/column_data/text.rs b/vendor/tiberius/src/tds/codec/column_data/text.rsindex 0bab38d843..736f260ae7 100644--- a/vendor/tiberius/src/tds/codec/column_data/text.rs+++ b/vendor/tiberius/src/tds/codec/column_data/text.rs@@ -31,9 +31,10 @@ wherebuf.push(src.read_u8().await?);}- codec- .decode(buf.as_ref())- .ok_or_else(|| Error::Encoding("invalid sequence".into()))?+ // Lossy: a row holding bytes that are invalid in its declared+ // collation must stay readable instead of failing the result set.+ let (text, _) = codec.decode_lossy(buf.as_ref());+ text}// NTEXTNone => {diff --git a/vendor/tiberius/src/tds/collation.rs b/vendor/tiberius/src/tds/collation.rsindex 6a45608f67..c13b759fdd 100644--- a/vendor/tiberius/src/tds/collation.rs+++ b/vendor/tiberius/src/tds/collation.rs@@ -103,6 +103,34 @@ impl CollationCodec {}}+ /// Decode legacy varchar/text data, substituting U+FFFD for byte sequences+ /// that are invalid in the target encoding.+ ///+ /// Legacy `varchar`/`text` columns can hold bytes that do not form a valid+ /// sequence in their declared collation, usually from an application that+ /// wrote data under a different codepage. Decoding those strictly fails the+ /// whole result set, so a single bad row makes an otherwise readable table+ /// impossible to browse, while other clients render the row and replace the+ /// undecodable bytes. This mirrors the lossy UTF-16 handling that `nchar`,+ /// `nvarchar`, and `ntext` already use for unpaired surrogates.+ ///+ /// Returns the decoded string and whether any byte had to be replaced.+ pub fn decode_lossy(&self, bytes: &[u8]) -> (String, bool) {+ match self {+ // No BOM handling: the collation already names the encoding, so a+ // leading byte sequence that looks like a UTF-8 BOM is ordinary+ // data (the NVARCHAR path likewise keeps a U+FEFF character).+ CollationCodec::BuiltIn(encoding) => {+ let (text, had_errors) = encoding.decode_without_bom_handling(bytes);+ (text.into_owned(), had_errors)+ }+ // Single-byte codecs map every one of the 256 byte values, so they+ // can never fail and never need a replacement character.+ CollationCodec::Cp437 => (decode_single_byte(bytes, &CP437_HIGH), false),+ CollationCodec::Cp850 => (decode_single_byte(bytes, &CP850_HIGH), false),+ }+ }+/// Encode a string for a legacy varchar parameter. Returns `None` when the/// input contains a character that the target encoding cannot represent.pub fn encode_from_utf8(&self, text: &str) -> Option<Vec<u8>> {