Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(sqlserver): keep rows readable on collation decode failure

Thanadon Kongkanun339c96cc81ba
diff --git a/Cargo.toml b/Cargo.toml
index 6303e6325f..339d49660c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,7 +37,9 @@ postgres-protocol = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git"
# rustls dangerous-mode connections to legacy certificates. Remove after upstream support lands.
mysql_async = { git = "https://github.com/zipg/mysql_async.git", rev = "4a9e60a62f5a48e35ef9ccc9fc8277d5012b19dd" }
# Preserve readable SQL Server result sets when a row contains an unpaired
-# UTF-16 surrogate. Remove after the behavior is available in upstream tiberius.
+# UTF-16 surrogate, or varchar/text bytes that are invalid in the column's
+# collation (replaced with U+FFFD, no BOM sniffing). Remove after the behavior
+# is available in upstream tiberius.
tiberius = { path = "vendor/tiberius" }
[profile.release]
diff --git a/vendor/tiberius/DBX-PATCH.md b/vendor/tiberius/DBX-PATCH.md
index b169911d81..301a0ca7db 100644
--- a/vendor/tiberius/DBX-PATCH.md
+++ b/vendor/tiberius/DBX-PATCH.md
@@ -6,14 +6,19 @@ identified by the checksum
upstream release tag points to commit
`0e2897a276166503ba78fe3e1cee501e9a034021`.
-DBX changes only SQL Server Unicode column-data decoding:
+DBX changes only SQL Server column-data decoding:
- NCHAR, NVARCHAR, and NTEXT row values replace unpaired UTF-16 surrogates with
U+FFFD, matching the Microsoft JDBC driver's observable behavior.
+- CHAR, VARCHAR, and TEXT row values under a non-Unicode collation decode
+ lossily: byte sequences that are invalid in the column's collation become
+ U+FFFD instead of failing the whole result set, following the JDBC driver's
+ REPLACE policy. The lossy decoder does no BOM handling, so bytes that merely
+ look like a UTF-8 BOM (e.g. under GB18030) stay ordinary collation data.
- Odd byte lengths remain protocol errors, including an explicit NTEXT guard
that prevents truncating a trailing byte and desynchronizing the TDS stream.
-- Metadata, environment tokens, other protocol strings, and non-Unicode
- codepage decoding retain upstream's strict behavior.
+- Metadata, environment tokens, and other protocol strings retain upstream's
+ strict behavior.
The regression tests in `src/tds/codec/column_data.rs` exercise the decoder
with raw TDS value frames. The upstream integration fixtures are omitted from
diff --git a/vendor/tiberius/src/tds/codec/column_data.rs b/vendor/tiberius/src/tds/codec/column_data.rs
index b29f18c2da..03fa6c9a92 100644
--- a/vendor/tiberius/src/tds/codec/column_data.rs
+++ b/vendor/tiberius/src/tds/codec/column_data.rs
@@ -758,6 +758,69 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn varchar_replaces_invalid_collation_bytes() {
+ // CP936/GB18030 (sort ID 198): 0x81 starts a two-byte sequence, and
+ // 0x40..0x7e is a valid trail byte, so 0x81 0x20 is invalid. A legacy
+ // varchar column can still hold those bytes when an application wrote
+ // them under a different codepage.
+ let value = decode_wire(
+ VarLenType::BigVarChar,
+ 20,
+ Some(Collation::new(0x804, 198)),
+ &short_string_wire(b"ok\x81\x20ok"),
+ )
+ .await
+ .expect("undecodable varchar bytes must not fail the whole result set");
+
+ let decoded = decoded_string(value).expect("varchar row value");
+ assert!(decoded.starts_with("ok"), "readable bytes are preserved: {decoded:?}");
+ assert!(decoded.ends_with("ok"), "bytes after the bad sequence are preserved: {decoded:?}");
+ assert!(decoded.contains('\u{fffd}'), "the bad sequence is replaced: {decoded:?}");
+ }
+
+ #[tokio::test]
+ async fn text_replaces_invalid_collation_bytes() {
+ let value = decode_wire(
+ VarLenType::Text,
+ 0,
+ Some(Collation::new(0x804, 198)),
+ &ntext_wire(b"ok\x81\x20ok"),
+ )
+ .await
+ .expect("undecodable text bytes must not fail the whole result set");
+
+ let decoded = decoded_string(value).expect("text row value");
+ assert!(decoded.starts_with("ok"), "readable bytes are preserved: {decoded:?}");
+ assert!(decoded.ends_with("ok"), "bytes after the bad sequence are preserved: {decoded:?}");
+ assert!(decoded.contains('\u{fffd}'), "the bad sequence is replaced: {decoded:?}");
+ }
+
+ #[tokio::test]
+ async fn varchar_gb18030_does_not_sniff_a_utf8_bom() {
+ // EF BB BF is fully valid GB18030 data: EF BB and BF 61 form two
+ // two-byte sequences (锘縜) followed by ASCII "bc". The collation
+ // already names the encoding, so the lossy decoder must not let
+ // encoding_rs sniff those bytes as a UTF-8 BOM and return "abc".
+ let (text, had_errors) = Collation::new(0x804, 198)
+ .codec()
+ .unwrap()
+ .decode_lossy(b"\xef\xbb\xbfabc");
+ assert_eq!(text, "锘縜bc");
+ assert!(!had_errors, "the bytes are valid GB18030");
+
+ let value = decode_wire(
+ VarLenType::BigVarChar,
+ 20,
+ Some(Collation::new(0x804, 198)),
+ &short_string_wire(b"\xef\xbb\xbfabc"),
+ )
+ .await
+ .expect("BOM-like varchar bytes must decode as ordinary GB18030 data");
+
+ assert_eq!(decoded_string(value).as_deref(), Some("锘縜bc"));
+ }
+
#[tokio::test]
async fn varchar_cp850_collation_decodes_french_text() {
// SQL_1xCompat_CP850_CI_AS: LCID 0x409, sort ID 49. The DOS codepages
diff --git a/vendor/tiberius/src/tds/codec/column_data/string.rs b/vendor/tiberius/src/tds/codec/column_data/string.rs
index 42528ecc56..eb80cfa9ed 100644
--- a/vendor/tiberius/src/tds/codec/column_data/string.rs
+++ b/vendor/tiberius/src/tds/codec/column_data/string.rs
@@ -21,10 +21,9 @@ where
// Codepages other than UTF
(Some(buf), BigChar) | (Some(buf), BigVarChar) => {
let collation = collation.as_ref().unwrap();
- let s = collation
- .codec()?
- .decode(buf.as_ref())
- .ok_or_else(|| Error::Encoding("invalid sequence".into()))?;
+ // Lossy: a row holding bytes that are invalid in its declared
+ // collation must stay readable instead of failing the result set.
+ let (s, _) = collation.codec()?.decode_lossy(buf.as_ref());
Ok(Some(s.into()))
}
diff --git a/vendor/tiberius/src/tds/codec/column_data/text.rs b/vendor/tiberius/src/tds/codec/column_data/text.rs
index 0bab38d843..736f260ae7 100644
--- a/vendor/tiberius/src/tds/codec/column_data/text.rs
+++ b/vendor/tiberius/src/tds/codec/column_data/text.rs
@@ -31,9 +31,10 @@ where
buf.push(src.read_u8().await?);
}
- codec
- .decode(buf.as_ref())
- .ok_or_else(|| Error::Encoding("invalid sequence".into()))?
+ // Lossy: a row holding bytes that are invalid in its declared
+ // collation must stay readable instead of failing the result set.
+ let (text, _) = codec.decode_lossy(buf.as_ref());
+ text
}
// NTEXT
None => {
diff --git a/vendor/tiberius/src/tds/collation.rs b/vendor/tiberius/src/tds/collation.rs
index 6a45608f67..c13b759fdd 100644
--- a/vendor/tiberius/src/tds/collation.rs
+++ b/vendor/tiberius/src/tds/collation.rs
@@ -103,6 +103,34 @@ impl CollationCodec {
}
}
+ /// Decode legacy varchar/text data, substituting U+FFFD for byte sequences
+ /// that are invalid in the target encoding.
+ ///
+ /// Legacy `varchar`/`text` columns can hold bytes that do not form a valid
+ /// sequence in their declared collation, usually from an application that
+ /// wrote data under a different codepage. Decoding those strictly fails the
+ /// whole result set, so a single bad row makes an otherwise readable table
+ /// impossible to browse, while other clients render the row and replace the
+ /// undecodable bytes. This mirrors the lossy UTF-16 handling that `nchar`,
+ /// `nvarchar`, and `ntext` already use for unpaired surrogates.
+ ///
+ /// Returns the decoded string and whether any byte had to be replaced.
+ pub fn decode_lossy(&self, bytes: &[u8]) -> (String, bool) {
+ match self {
+ // No BOM handling: the collation already names the encoding, so a
+ // leading byte sequence that looks like a UTF-8 BOM is ordinary
+ // data (the NVARCHAR path likewise keeps a U+FEFF character).
+ CollationCodec::BuiltIn(encoding) => {
+ let (text, had_errors) = encoding.decode_without_bom_handling(bytes);
+ (text.into_owned(), had_errors)
+ }
+ // Single-byte codecs map every one of the 256 byte values, so they
+ // can never fail and never need a replacement character.
+ CollationCodec::Cp437 => (decode_single_byte(bytes, &CP437_HIGH), false),
+ CollationCodec::Cp850 => (decode_single_byte(bytes, &CP850_HIGH), false),
+ }
+ }
+
/// Encode a string for a legacy varchar parameter. Returns `None` when the
/// input contains a character that the target encoding cannot represent.
pub fn encode_from_utf8(&self, text: &str) -> Option<Vec<u8>> {