Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
language: Avoid UTF-16 false positive with embedded ASCII (#61250)
Sarah Weskerd2779c344350
diff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rsindex bb543478e5e..e02748257ba 100644--- a/crates/language/src/file_content.rs+++ b/crates/language/src/file_content.rs@@ -111,6 +111,7 @@ fn is_known_binary_header(bytes: &[u8]) -> bool {// ranges or form unpaired surrogates, which real text almost never contains.fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {let mut suspicious_count = 0usize;+ let mut word_like_count = 0usize;let mut total = 0usize;let mut i = 0;@@ -119,6 +120,9 @@ fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {match code_unit {0x0009 | 0x000A | 0x000C | 0x000D => {}+ 0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {+ word_like_count += 1;+ }// C0/C1 control characters and non-characters0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,0xD800..=0xDBFF => {@@ -127,6 +131,7 @@ fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {.is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));if has_low_surrogate {total += 1;+ word_like_count += 2;i += 2;} else {suspicious_count += 1;@@ -134,6 +139,7 @@ fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {}// Lone low surrogate without a preceding high surrogate0xDC00..=0xDFFF => suspicious_count += 1,+ 0x0100.. => word_like_count += 1,_ => {}}@@ -146,7 +152,21 @@ fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {// Real UTF-16 text has near-zero control characters; binary data with// small 16-bit values typically exceeds 5%. 2% provides a safe margin.- suspicious_count * 100 < total * 2+ let low_control_ratio = suspicious_count * 100 < total * 2;++ // Binary formats that interleave short ASCII fragments with small+ // length/type fields (e.g. game asset formats) can dodge the control+ // character check above while barely containing any real words: their+ // code units land on ASCII punctuation and Latin-1 symbol values rather+ // than letters, digits, or spaces. Real text is overwhelmingly made of+ // word characters, so require a minimum share of them. Code units above+ // the Latin-1 range (and surrogate pairs) also count as word-like so+ // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,+ // are still recognized -- tag bytes paired with a zero byte can never+ // land there.+ let enough_word_chars = word_like_count * 100 >= total * 30;++ low_control_ratio && enough_word_chars}fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rsindex 820e0e59021..15c665ec2eb 100644--- a/crates/worktree/src/worktree.rs+++ b/crates/worktree/src/worktree.rs@@ -7299,6 +7299,38 @@ mod tests {);}+ // Mimics binary formats that interleave short ASCII fragments with small+ // length/type fields (as seen in some game/asset binary formats, e.g.+ // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's+ // null-byte pattern for ASCII, but the low bytes are mostly non-word+ // "tag" values rather than real letters/digits/spaces.+ fn build_tag_interleaved_binary_bytes() -> Vec<u8> {+ let mut bytes = Vec::new();+ let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];+ let mut i = 0;+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.push(tags[i % tags.len()]);+ bytes.push(0x00);+ i += 1;+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);+ bytes+ }++ #[test]+ fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {+ let bytes = build_tag_interleaved_binary_bytes();+ assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);++ let result = analyze_byte_content(&bytes);+ assert_eq!(+ result,+ ByteContent::Binary,+ "binary data with sparse non-word low bytes and null high bytes \+ should not be misdetected as UTF-16LE text"+ );+ }+#[test]fn test_utf16le_text_detected_as_utf16le() {let text = "Hello, world! This is a UTF-16 test string. ";@@ -7323,6 +7355,30 @@ mod tests {assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);}+ #[test]+ fn test_utf16le_cyrillic_text_detected_as_utf16le() {+ let text = "Привет, мир! Это тестовая строка в UTF-16. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);+ }++ #[test]+ fn test_utf16be_greek_text_detected_as_utf16be() {+ let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);+ }+#[test]fn test_known_binary_headers() {let cases: &[(&[u8], &str)] = &[