Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Speed up project search over misclassified binary files (#64549)
Kirill Bulatov8bbcd8396863
diff --git a/Cargo.lock b/Cargo.lockindex 4cd4be6cbad..da2a5543fe0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6421,6 +6421,16 @@ dependencies = ["subtle",]+[[package]]+name = "file_content"+version = "0.1.0"+dependencies = [+ "anyhow",+ "chardetng",+ "encoding_rs",+ "futures-lite 2.6.1",+]+[[package]]name = "file_finder"version = "0.1.0"@@ -7146,6 +7156,7 @@ dependencies = ["async-trait","collections","derive_more",+ "file_content","futures 0.3.34","gpui","http_client",@@ -9575,7 +9586,6 @@ version = "0.1.0"dependencies = ["anyhow","async-trait",- "chardetng","clock","collections","criterion",@@ -9583,6 +9593,7 @@ dependencies = ["diffy","ec4rs","encoding_rs",+ "file_content","fs","futures 0.3.34","futures-lite 2.6.1",@@ -14116,6 +14127,7 @@ dependencies = ["encoding_rs","extension","fancy-regex",+ "file_content","fs","futures 0.3.34","fuzzy",@@ -22327,6 +22339,7 @@ dependencies = ["clock","collections","encoding_rs",+ "file_content","fs","futures 0.3.34","futures-lite 2.6.1",diff --git a/Cargo.toml b/Cargo.tomlindex 1a61f86114d..c52733570bc 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -79,6 +79,7 @@ members = ["crates/feature_flags","crates/feature_flags_macros","crates/feedback",+ "crates/file_content","crates/file_finder","crates/file_icons","crates/fs",@@ -351,6 +352,7 @@ extensions_ui = { path = "crates/extensions_ui" }feature_flags = { path = "crates/feature_flags" }feature_flags_macros = { path = "crates/feature_flags_macros" }feedback = { path = "crates/feedback" }+file_content = { path = "crates/file_content" }file_finder = { path = "crates/file_finder" }file_icons = { path = "crates/file_icons" }fs = { path = "crates/fs" }@@ -1087,6 +1089,7 @@ command_palette = { codegen-units = 1 }command_palette_hooks = { codegen-units = 1 }edit_prediction_ui = { codegen-units = 1 }feature_flags = { codegen-units = 1 }+file_content = { codegen-units = 1 }file_icons = { codegen-units = 1 }image_viewer = { codegen-units = 1 }install_cli = { codegen-units = 1 }diff --git a/crates/file_content/Cargo.toml b/crates/file_content/Cargo.tomlnew file mode 100644index 00000000000..7f5f268fcbd--- /dev/null+++ b/crates/file_content/Cargo.toml@@ -0,0 +1,19 @@+[package]+name = "file_content"+version = "0.1.0"+edition.workspace = true+publish.workspace = true+license = "GPL-3.0-or-later"++[lints]+workspace = true++[lib]+path = "src/file_content.rs"+doctest = false++[dependencies]+anyhow.workspace = true+chardetng.workspace = true+encoding_rs.workspace = true+futures-lite.workspace = truediff --git a/crates/file_content/LICENSE-GPL b/crates/file_content/LICENSE-GPLnew file mode 120000index 00000000000..89e542f750c--- /dev/null+++ b/crates/file_content/LICENSE-GPL@@ -0,0 +1 @@+../../LICENSE-GPL\ No newline at end of filediff --git a/crates/file_content/src/file_content.rs b/crates/file_content/src/file_content.rsnew file mode 100644index 00000000000..b978f4aa40e--- /dev/null+++ b/crates/file_content/src/file_content.rs@@ -0,0 +1,1072 @@+//! Classifies file contents as text or binary and decodes text of any+//! supported encoding into UTF-8, both whole and as a stream.++use std::io::{self, Read};++use anyhow::Result;+use chardetng::EncodingDetector;+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE};+use futures_lite::future::yield_now;++/// Number of leading bytes that [`analyze_byte_content`] inspects.+pub const FILE_ANALYSIS_BYTES: usize = 1024;++const DECODING_BLOCK_BYTES: usize = 64 * 1024;++/// Text produced by [`decode_text`], with what is needed to write it back via [`encode_text`].+pub struct DecodedText {+ /// The decoded UTF-8 text.+ pub text: String,+ /// The encoding the bytes were decoded from.+ pub encoding: &'static Encoding,+ /// Whether the bytes started with a byte order mark.+ pub has_bom: bool,+}++pub fn read_byte_header(reader: &mut dyn Read) -> io::Result<(Vec<u8>, bool)> {+ let mut header = vec![0; FILE_ANALYSIS_BYTES];+ let mut length = 0;+ while length < header.len() {+ let read = match reader.read(&mut header[length..]) {+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,+ result => result?,+ };+ if read == 0 {+ header.truncate(length);+ return Ok((header, true));+ }+ length += read;+ }+ Ok((header, false))+}++/// Inspects the start of a file.+/// Returns the BOM encoding, if any, and otherwise the [`ByteContent`] classification of the prefix.+pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {+ if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {+ return (Some(encoding), ByteContent::Unknown);+ }+ (None, analyze_byte_content(prefix))+}++/// Decodes a whole file into UTF-8, detecting its encoding.+/// Fails when the bytes are classified as [`ByteContent::Binary`].+pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {+ let (bom_encoding, byte_content) = decode_byte_header(&bytes);+ if let Some(encoding) = bom_encoding {+ let (text, _) = encoding.decode_with_bom_removal(&bytes);+ return Ok(DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: true,+ });+ }++ if let Some(encoding) = byte_content.encoding() {+ let (text, _, _) = encoding.decode(&bytes);+ return Ok(DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: false,+ });+ }+ anyhow::ensure!(+ byte_content != ByteContent::Binary,+ "Binary files cannot be decoded into text"+ );++ match String::from_utf8(bytes) {+ Ok(text) if !text.contains('\x1b') => Ok(DecodedText {+ text,+ encoding: UTF_8,+ has_bom: false,+ }),+ Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),+ Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),+ }+}++pub async fn detect_encoding(reader: &mut (dyn Read + Send)) -> io::Result<&'static Encoding> {+ let mut detector = EncodingDetector::new();+ let mut block = [0; DECODING_BLOCK_BYTES];+ let mut length = 0;+ loop {+ let read = match reader.read(&mut block[length..]) {+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,+ result => result?,+ };+ if read == 0 {+ detector.feed(&block[..length], true);+ return Ok(detector.guess(None, true));+ }+ length += read;+ if length == block.len() {+ detector.feed(&block, false);+ yield_now().await;+ length = 0;+ }+ }+}++/// Inverse of [`decode_text`]: encodes UTF-8 text into `encoding`, prepending a BOM when asked.+pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {+ if encoding == UTF_8 && !has_bom {+ return text.into_bytes();+ }++ // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.+ if encoding == UTF_16BE {+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);+ if has_bom {+ bytes.extend_from_slice(&[0xFE, 0xFF]);+ }+ bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));+ return bytes;+ }++ if encoding == UTF_16LE {+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);+ if has_bom {+ bytes.extend_from_slice(&[0xFF, 0xFE]);+ }+ bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));+ return bytes;+ }++ let (encoded, _, _) = encoding.encode(&text);+ if has_bom && encoding == UTF_8 {+ let mut bytes = Vec::with_capacity(encoded.len() + 3);+ bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);+ bytes.extend_from_slice(&encoded);+ bytes+ } else {+ encoded.into_owned()+ }+}++/// A [`Read`] adapter that transcodes its source into UTF-8 block by block, with bounded memory.+pub struct DecodingReader<'a> {+ inner: &'a mut (dyn Read + Send),+ decoder: Decoder,+ input: Vec<u8>,+ output: Vec<u8>,+ output_position: usize,+ finished: bool,+}++impl<'a> DecodingReader<'a> {+ pub fn new(inner: &'a mut (dyn Read + Send), encoding: &'static Encoding) -> Self {+ Self {+ inner,+ decoder: encoding.new_decoder(),+ input: vec![0; DECODING_BLOCK_BYTES],+ output: Vec::new(),+ output_position: 0,+ finished: false,+ }+ }++ pub fn encoding(&self) -> &'static Encoding {+ self.decoder.encoding()+ }++ fn fill_output(&mut self) -> io::Result<()> {+ self.output.clear();+ self.output_position = 0;+ let read = self.inner.read(&mut self.input)?;+ self.finished = read == 0;+ decode_block(+ &mut self.decoder,+ &self.input[..read],+ self.finished,+ &mut self.output,+ )+ }+}++impl Read for DecodingReader<'_> {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ if buffer.is_empty() {+ return Ok(0);+ }+ while self.output_position == self.output.len() {+ if self.finished {+ return Ok(0);+ }+ match self.fill_output() {+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,+ result => result?,+ }+ }++ let available = &self.output[self.output_position..];+ let count = available.len().min(buffer.len());+ buffer[..count].copy_from_slice(&available[..count]);+ self.output_position += count;+ Ok(count)+ }+}++/// Verdict of [`analyze_byte_content`] on a file prefix.+#[derive(Debug, PartialEq)]+pub enum ByteContent {+ /// UTF-16 little-endian text without a BOM.+ Utf16Le,+ /// UTF-16 big-endian text without a BOM.+ Utf16Be,+ /// Not text; the file must not be decoded or searched.+ Binary,+ /// Text in UTF-8 or some yet unknown encoding.+ Unknown,+}++impl ByteContent {+ /// The encoding this classification pins down, if any.+ pub fn encoding(&self) -> Option<&'static Encoding> {+ match self {+ Self::Utf16Le => Some(UTF_16LE),+ Self::Utf16Be => Some(UTF_16BE),+ Self::Binary | Self::Unknown => None,+ }+ }+}++/// Classifies the first [`FILE_ANALYSIS_BYTES`] of a file.+///+/// Heuristic check using null byte distribution plus a generic text-likeness+/// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise+/// distinguishes between text-like and binary-like content.+pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent {+ if bytes.len() < 2 {+ return ByteContent::Unknown;+ }++ if has_binary_header(bytes, KNOWN_BINARY_HEADERS) {+ return ByteContent::Binary;+ }++ let limit = bytes.len().min(FILE_ANALYSIS_BYTES);+ let mut even_null_count = 0usize;+ let mut odd_null_count = 0usize;+ let mut non_text_like_count = 0usize;++ for (i, &byte) in bytes[..limit].iter().enumerate() {+ if byte == 0 {+ if i % 2 == 0 {+ even_null_count += 1;+ } else {+ odd_null_count += 1;+ }+ non_text_like_count += 1;+ continue;+ }++ let is_text_like = match byte {+ b'\t' | b'\n' | b'\r' | 0x0C => true,+ 0x20..=0x7E | 0x80..=0xBF | 0xC2..=0xF4 => true,+ _ => false,+ };++ if !is_text_like {+ non_text_like_count += 1;+ }+ }++ let total_null_count = even_null_count + odd_null_count;+ let has_significant_nulls = total_null_count > 0 && total_null_count >= limit / 16;+ let nulls_skew_to_even = even_null_count > odd_null_count * 4;+ let nulls_skew_to_odd = odd_null_count > even_null_count * 4;++ if has_significant_nulls {+ let sample = &bytes[..limit];++ // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first)+ // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first)++ if nulls_skew_to_even && is_plausible_utf16_text(sample, false) {+ return ByteContent::Utf16Be;+ }++ if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) {+ return ByteContent::Utf16Le;+ }++ return ByteContent::Binary;+ }++ if has_binary_header(bytes, ADDITIONAL_BINARY_HEADERS)+ || is_bitmap_header(bytes)+ || is_gguf_header(bytes)+ || is_safetensors_header(bytes)+ {+ return ByteContent::Binary;+ }++ if total_null_count == 0 || non_text_like_count * 100 < limit * 8 {+ ByteContent::Unknown+ } else {+ ByteContent::Binary+ }+}++fn decode_block(+ decoder: &mut Decoder,+ block: &[u8],+ last: bool,+ output: &mut Vec<u8>,+) -> io::Result<()> {+ let capacity = decoder+ .max_utf8_buffer_length(block.len())+ .ok_or_else(|| io::Error::new(io::ErrorKind::OutOfMemory, "decoded block is too large"))?;+ let start = output.len();+ output.resize(start + capacity, 0);+ let (result, consumed, written, _) = decoder.decode_to_utf8(block, &mut output[start..], last);+ output.truncate(start + written);+ if result != CoderResult::InputEmpty || consumed != block.len() {+ return Err(io::Error::other("decoder did not consume the whole block"));+ }+ Ok(())+}++fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {+ let mut detector = EncodingDetector::new();+ detector.feed(&bytes, true);+ let encoding = detector.guess(None, true);+ let (text, _, _) = encoding.decode(&bytes);+ DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: false,+ }+}++const KNOWN_BINARY_HEADERS: &[&[u8]] = &[+ b"%PDF-", // PDF+ b"GIF87a", // GIF87a+ b"GIF89a", // GIF89a+ b"ID3", // MP3 with ID3v2 tag+ b"IWAD", // Doom IWAD archive+ b"OggS", // OGG (Vorbis, Opus, FLAC)+ b"PK\x03\x04", // ZIP local header+ b"PK\x05\x06", // ZIP end of central directory+ b"PK\x07\x08", // ZIP spanning/splitting+ b"PWAD", // Doom PWAD archive+ b"RIFF", // WAV, AVI, WebP+ b"fLaC", // FLAC+ b"\x89PNG\r\n\x1a\n", // PNG+ b"\xFF\xD8\xFF", // JPEG+ b"\xFF\xF2", // MP3 frame sync (MPEG2 Layer3)+ b"\xFF\xF3", // MP3 frame sync (MPEG2 Layer3)+ b"\xFF\xFA", // MP3 frame sync (MPEG1 Layer3)+ b"\xFF\xFB", // MP3 frame sync (MPEG1 Layer3)+];++const ADDITIONAL_BINARY_HEADERS: &[&[u8]] = &[+ b"\x00\x00\x00\x0CJXL \r\n\x87\n", // JPEG XL container+ b"\x00\x00\x00\x0CjP ", // JPEG 2000+ b"\x00\x00\x01\x00", // Windows ICO+ b"\x00\x00\x02\x00", // Windows CUR+ b"\x00\x01\x00\x00\x00", // TrueType font+ b"\x00asm", // WebAssembly+ b"\x04\x22\x4D\x18", // LZ4 frame+ b"\x1A\x45\xDF\xA3", // Matroska, WebM+ b"\x1F\x8B\x08", // gzip+ b"7z\xBC\xAF\x27\x1C", // 7-Zip+ b"8BPS\x00", // Photoshop PSD+ b"DDS |\x00\x00\x00", // DirectDraw Surface+ b"II*\x00", // TIFF little-endian+ b"Kaydara FBX Binary \x00", // FBX binary+ b"MM\x00*", // TIFF big-endian+ b"MSCF\x00\x00\x00\x00", // Microsoft Cabinet+ b"OTTO\x00", // OpenType font with CFF outlines+ b"Rar!\x1A\x07", // RAR+ b"SQLite format 3\x00", // SQLite database+ b"glTF\x01\x00\x00\x00", // Binary glTF 1+ b"glTF\x02\x00\x00\x00", // Binary glTF 2+ b"\x7FELF", // ELF+ b"\x89HDF\r\n\x1a\n", // HDF5+ b"\x93NUMPY\x01\x00", // NumPy array+ b"\x93NUMPY\x02\x00",+ b"\x93NUMPY\x03\x00",+ b"\x95\x04\x12\xDE", // gettext MO big-endian+ b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04", // pcap big-endian+ b"\xABKTX 11\xBB\r\n\x1A\n", // KTX texture+ b"\xABKTX 20\xBB\r\n\x1A\n",+ b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1", // OLE2 compound file (legacy MS Office)+ b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00", // pcap little-endian+ b"\xDE\x12\x04\x95", // gettext MO little-endian+ b"\xFD7zXZ\x00", // xz+];++fn has_binary_header(bytes: &[u8], headers: &[&[u8]]) -> bool {+ let index = headers.partition_point(|header| *header <= bytes);+ headers[..index]+ .last()+ .is_some_and(|header| bytes.starts_with(header))+}++fn is_gguf_header(bytes: &[u8]) -> bool {+ bytes.starts_with(b"GGUF") && bytes.get(5..8) == Some(&[0, 0, 0])+}++fn is_bitmap_header(bytes: &[u8]) -> bool {+ bytes.starts_with(b"BM") && bytes.get(6..10) == Some(&[0, 0, 0, 0])+}++fn is_safetensors_header(bytes: &[u8]) -> bool {+ bytes.get(3..8) == Some(&[0, 0, 0, 0, 0]) && bytes.get(8) == Some(&b'{')+}++// Null byte skew alone is not enough to identify UTF-16 -- binary formats with+// small 16-bit values (like PCM audio) produce the same pattern. Decode the+// bytes as UTF-16 and reject if too many code units land in control character+// ranges or form unpaired surrogates, which real text almost never contains.+fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {+ let mut suspicious_count = 0usize;+ let mut word_like_count = 0usize;+ let mut total = 0usize;++ let mut i = 0;+ while let Some(code_unit) = read_u16(bytes, i, little_endian) {+ total += 1;++ match code_unit {+ 0x0009 | 0x000A | 0x000C | 0x000D => {}+ 0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {+ word_like_count += 1;+ }+ // C0/C1 control characters and non-characters+ 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,+ 0xD800..=0xDBFF => {+ let next_offset = i + 2;+ let has_low_surrogate = read_u16(bytes, next_offset, little_endian)+ .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));+ if has_low_surrogate {+ total += 1;+ word_like_count += 2;+ i += 2;+ } else {+ suspicious_count += 1;+ }+ }+ // Lone low surrogate without a preceding high surrogate+ 0xDC00..=0xDFFF => suspicious_count += 1,+ 0x0100.. => word_like_count += 1,+ _ => {}+ }++ i += 2;+ }++ if total == 0 {+ return false;+ }++ // Real UTF-16 text has near-zero control characters; binary data with+ // small 16-bit values typically exceeds 5%. 2% provides a safe margin.+ let low_control_ratio = suspicious_count * 100 < total * 2;++ // Binary formats that interleave short ASCII fragments with small+ // length/type fields (e.g. game asset formats) can dodge the control+ // character check above while barely containing any real words: their+ // code units land on ASCII punctuation and Latin-1 symbol values rather+ // than letters, digits, or spaces. Real text is overwhelmingly made of+ // word characters, so require a minimum share of them. Code units above+ // the Latin-1 range (and surrogate pairs) also count as word-like so+ // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,+ // are still recognized -- tag bytes paired with a zero byte can never+ // land there.+ let enough_word_chars = word_like_count * 100 >= total * 30;++ low_control_ratio && enough_word_chars+}++fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {+ let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?];+ if little_endian {+ return Some(u16::from_le_bytes(pair));+ }+ Some(u16::from_be_bytes(pair))+}++#[cfg(test)]+mod tests {+ use super::*;+ use futures_lite::future::{block_on, poll_once};++ #[test]+ fn decodes_and_encodes_legacy_text() {+ for (expected, encoding) in [+ (+ String::from("“NUMPY is useful for array processing.”\n"),+ encoding_rs::WINDOWS_1252,+ ),+ (+ String::from(+ "«KTX est un format de texture déjà utilisé à côté des images françaises.»\n",+ ),+ encoding_rs::WINDOWS_1252,+ ),+ (+ String::from(+ "〔迷宫〕这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n",+ ),+ encoding_rs::GBK,+ ),+ (+ String::from(+ "€\x02£\x02Symboles monétaires déjà utilisés à côté des factures françaises.\n",+ ),+ encoding_rs::WINDOWS_1252,+ ),+ (+ String::from(+ "\x1fќерка\x1fОва е обичен македонски текст за проверка на кодирањето.\n",+ ),+ encoding_rs::WINDOWS_1251,+ ),+ (+ format!(+ "я{}{}",+ "x".repeat(1024 * 1024),+ "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(1000)+ ),+ encoding_rs::WINDOWS_1251,+ ),+ (+ String::from("ясно, что это обычный русский текст. Сегодня хорошая погода.\n"),+ encoding_rs::WINDOWS_1251,+ ),+ (+ String::from("ящик содержит обычный русский текст. Сегодня хорошая погода.\n"),+ encoding_rs::WINDOWS_1251,+ ),+ (+ format!(+ "\x1b[0m{}{}",+ "x".repeat(1024 * 1024),+ " déjà été à côté français ".repeat(1000)+ ),+ encoding_rs::WINDOWS_1252,+ ),+ ] {+ let (bytes, _, _) = encoding.encode(&expected);+ let decoded = decode_text(bytes.clone().into_owned()).unwrap();++ assert_eq!(decoded.encoding, encoding);+ assert_eq!(decoded.text, expected);+ assert!(!decoded.has_bom);+ assert_eq!(+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),+ bytes.as_ref()+ );+ }+ }++ #[test]+ fn preserves_unicode_boms() {+ let expected = "七个人上山,下个月再来。";+ for encoding in [UTF_8, UTF_16LE, UTF_16BE] {+ let bytes = encode_text(expected.to_owned(), encoding, true);+ assert_eq!(+ decode_byte_header(&bytes),+ (Some(encoding), ByteContent::Unknown)+ );+ let decoded = decode_text(bytes.clone()).unwrap();++ assert_eq!(decoded.text, expected);+ assert_eq!(decoded.encoding, encoding);+ assert!(decoded.has_bom);+ assert_eq!(+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),+ bytes+ );+ }+ }++ #[test]+ fn detects_known_binary_headers() {+ let cases: &[(&[u8], &str)] = &[+ (b"II*\x00\x08\x00\x00\x00", "TIFF little endian"),+ (b"MM\x00*\x00\x00\x00\x08", "TIFF big endian"),+ (b"glTF\x02\x00\x00\x00\x10\x00\x00\x00", "glTF binary"),+ (b"\x7FELF\x02\x01\x01\x00", "ELF"),+ (b"\x1F\x8B\x08\x00", "gzip"),+ (b"SQLite format 3\x00", "SQLite"),+ (b"\x93NUMPY\x01\x00\x76\x00{'descr': '|u1'", "NumPy 1"),+ (b"\x93NUMPY\x02\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 2"),+ (b"\x93NUMPY\x03\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 3"),+ (b"\xABKTX 11\xBB\r\n\x1A\n\x04\x03\x02\x01", "KTX 1"),+ (b"\xABKTX 20\xBB\r\n\x1A\n\x00\x00\x00\x00", "KTX 2"),+ (+ b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x00\x01",+ "pcap big endian",+ ),+ (+ b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x01\x00\x00\x00",+ "pcap little endian",+ ),+ (b"GGUF\x03\x00\x00\x00", "GGUF"),+ (b"BM\x36\x10\x0E\x00\x00\x00\x00\x00\x36\x00\x00\x00", "BMP"),+ (+ b"\x90\x01\x00\x00\x00\x00\x00\x00{\"__metadata__\"",+ "safetensors",+ ),+ ];+ for (header, label) in cases {+ let mut bytes = header.to_vec();+ bytes.resize(FILE_ANALYSIS_BYTES, b'A');+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Binary, "{label}");+ assert_eq!(+ decode_text(bytes).err().map(|error| error.to_string()),+ Some(String::from("Binary files cannot be decoded into text")),+ "{label}"+ );+ }+ assert_eq!(+ analyze_byte_content(b"\x28\xB5\x2F\xFD\x20\x00\x01\x00\x00"),+ ByteContent::Binary+ );+ assert_eq!(+ analyze_byte_content(+ b"\x28\xB5\x2F\xFD\x20\x20\x01\x01\x00needle-abcdefghijklmnopqrstuvwxy"+ ),+ ByteContent::Binary+ );+ for text in [+ "FLV\x01video/x-flv\x01Flash video\n",+ "v/1\x01api/users\x01version one route\n",+ "!<arch>\nThis documents the archive identifier.\n",+ "glTF Sample Models\n",+ "OTTO GmbH\n",+ "GGUF is a format\n",+ "wOFF = 1\n",+ "wOF2 = 1\n",+ ] {+ assert_eq!(+ analyze_byte_content(text.as_bytes()),+ ByteContent::Unknown,+ "{text:?}"+ );+ }+ }++ #[test]+ fn binary_headers_are_sorted_and_prefix_free() {+ for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {+ for pair in headers.windows(2) {+ assert!(pair[0] < pair[1], "{pair:?}");+ assert_eq!(pair[1].strip_prefix(pair[0]), None, "{pair:?}");+ }+ }+ }++ #[test]+ fn binary_header_lookup_matches_linear_scan() {+ for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {+ let check = |bytes: &[u8]| {+ let expected = headers.iter().any(|header| bytes.starts_with(header));+ assert_eq!(has_binary_header(bytes, headers), expected, "{bytes:?}");+ };++ for header in headers {+ for length in 0..=header.len() {+ check(&header[..length]);+ }+ for suffix in [0, u8::MAX] {+ let mut bytes = header.to_vec();+ bytes.push(suffix);+ check(&bytes);+ }+ for index in 0..header.len() {+ let mut bytes = header.to_vec();+ for byte in 0..=u8::MAX {+ bytes[index] = byte;+ check(&bytes);+ }+ }+ }+ }+ }++ #[test]+ fn classifies_nul_free_content() {+ let mut protobuf_like = Vec::new();+ while protobuf_like.len() < FILE_ANALYSIS_BYTES {+ protobuf_like.extend_from_slice(b"\x0a\x0c/conv1/Conv_0\x12\x02\x08\x01\x1a\x04Conv\x22\x06\x08\x03\x10\x05\x18\x07");+ }+ let random_like = pseudo_random_bytes(FILE_ANALYSIS_BYTES)+ .into_iter()+ .map(|byte| if byte == 0 { 1 } else { byte })+ .collect::<Vec<_>>();+ let russian_text = "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(20);+ let (russian, _, _) = encoding_rs::WINDOWS_1251.encode(&russian_text);+ let chinese_text =+ "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20);+ let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);+ let ansi_log =+ "\x1b[32mINFO\x1b[0m \x1b[1m[server]\x1b[0m ok id=\x1b[33m42\x1b[0m\n".repeat(40);+ let control_delimited = "alpha\x01beta\x01gamma\x01delta\x02\n".repeat(60);+ let overstrike =+ "N\x08NA\x08AM\x08ME\x08E\n ls - list directory contents\n".repeat(30);++ let cases: [(&str, &[u8], ByteContent); 7] = [+ ("protobuf-like", &protobuf_like, ByteContent::Unknown),+ ("random without NUL", &random_like, ByteContent::Unknown),+ ("windows-1251", &russian, ByteContent::Unknown),+ ("gbk", &chinese, ByteContent::Unknown),+ ("ansi log", ansi_log.as_bytes(), ByteContent::Unknown),+ (+ "control delimited",+ control_delimited.as_bytes(),+ ByteContent::Unknown,+ ),+ (+ "nroff overstrike",+ overstrike.as_bytes(),+ ByteContent::Unknown,+ ),+ ];+ for (label, bytes, expected) in cases {+ assert_eq!(analyze_byte_content(bytes), expected, "{label}");+ }++ let thai = "สวัสดีชาวโลกนี่คือข้อความทดสอบภาษาไทย";+ for encoding in [UTF_16LE, UTF_16BE] {+ let bytes = encode_text(String::from(thai), encoding, false);+ assert_eq!(decode_byte_header(&bytes), (None, ByteContent::Unknown));+ assert_eq!(decode_text(bytes.clone()).unwrap().encoding, UTF_8);+ assert_eq!(encoding.decode(&bytes).0, thai);+ }+ }++ #[test]+ fn streaming_decoder_matches_whole_file_decoding() {+ let chinese_text =+ "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20_000);+ let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);+ assert!(chinese.len() > 1024 * 1024);+ let (russian, _, _) =+ encoding_rs::WINDOWS_1251.encode("Съешь же ещё этих мягких французских булок\n");+ let mut ascii_then_russian = "plain ascii line\n".repeat(5_000).into_bytes();+ ascii_then_russian.extend_from_slice(&russian);+ let mut utf16 = vec![0xFF, 0xFE];+ utf16.extend(+ "utf-16 text\n"+ .repeat(100)+ .encode_utf16()+ .flat_map(u16::to_le_bytes),+ );++ let interior_bom = "prefix  déjà été à côté français ".repeat(10);+ let (interior_bom_bytes, _, _) = encoding_rs::WINDOWS_1252.encode(&interior_bom);+ assert_eq!(+ decode_text(interior_bom_bytes.to_vec()).unwrap().text,+ interior_bom+ );++ for (label, bytes) in [+ ("large gbk", chinese.into_owned()),+ ("ascii prefix then windows-1251", ascii_then_russian),+ ("utf-16 with bom", utf16),+ ("pure ascii", b"just ascii\n".repeat(3)),+ ("empty", Vec::new()),+ ("truncated utf-16", b"\xFF\xFEa\0b".to_vec()),+ (+ "short utf-16le",+ encode_text(String::from("needleé"), UTF_16LE, false),+ ),+ (+ "short utf-16be",+ encode_text(String::from("needleé"), UTF_16BE, false),+ ),+ (+ "utf-16be fullwidth asterisk",+ encode_text(+ String::from("* This is a fullwidth asterisk followed by ordinary text.\n"),+ UTF_16BE,+ false,+ ),+ ),+ (+ "utf-16be pickle prefix",+ encode_text(+ String::from("考试 instructions for tomorrow\n"),+ UTF_16BE,+ false,+ ),+ ),+ (+ "utf-16be ftyp",+ encode_text(+ String::from("AB晴祰 ordinary UTF-16 text\n"),+ UTF_16BE,+ false,+ ),+ ),+ (+ "interior BOM in windows-1252",+ interior_bom_bytes.into_owned(),+ ),+ (+ "interior UTF-8 BOM",+ "prefix \u{feff}suffix\x1b[0m".as_bytes().to_vec(),+ ),+ ] {+ let expected = decode_text(bytes.clone()).unwrap();+ for read_limit in [usize::MAX, 8 * 1024, 7] {+ let mut source = ShortReads {+ inner: io::Cursor::new(bytes.clone()),+ limit: 1,+ };+ let (header, _) = read_byte_header(&mut source).unwrap();+ let (bom_encoding, byte_content) = decode_byte_header(&header);+ assert_ne!(byte_content, ByteContent::Binary, "{label}");+ source.inner.set_position(0);+ source.limit = read_limit;+ let encoding = bom_encoding+ .or(byte_content.encoding())+ .unwrap_or_else(|| block_on(detect_encoding(&mut source)).unwrap());+ source.inner.set_position(0);+ let mut reader = DecodingReader::new(&mut source, encoding);+ let mut streamed = String::new();+ reader.read_to_string(&mut streamed).unwrap();+ assert_eq!(+ streamed, expected.text,+ "{label}, read_limit = {read_limit}"+ );+ assert_eq!(+ reader.encoding(),+ expected.encoding,+ "{label}, read_limit = {read_limit}"+ );+ }+ }+ }++ #[test]+ fn encoding_detection_yields_between_blocks() {+ let mut source = io::Cursor::new(vec![b'a'; 2 * DECODING_BLOCK_BYTES]);+ assert!(block_on(poll_once(detect_encoding(&mut source))).is_none());+ assert_eq!(source.position(), DECODING_BLOCK_BYTES as u64);+ }++ /// reproduction of issue #50785+ fn build_pcm16_wav_bytes() -> Vec<u8> {+ let header: Vec<u8> = vec![+ /* RIFF header */+ 0x52, 0x49, 0x46, 0x46, // "RIFF"+ 0xc6, 0xcf, 0x00, 0x00, // file size: 8+ 0x57, 0x41, 0x56, 0x45, // "WAVE"+ /* fmt chunk */+ 0x66, 0x6d, 0x74, 0x20, // "fmt "+ 0x10, 0x00, 0x00, 0x00, // chunk size: 16+ 0x01, 0x00, // format: PCM (1)+ 0x01, 0x00, // channels: 1 (mono)+ 0x80, 0x3e, 0x00, 0x00, // sample rate: 16000+ 0x00, 0x7d, 0x00, 0x00, // byte rate: 32000+ 0x02, 0x00, // block align: 2+ 0x10, 0x00, // bits per sample: 16+ /* LIST chunk */+ 0x4c, 0x49, 0x53, 0x54, // "LIST"+ 0x1a, 0x00, 0x00, 0x00, // chunk size: 26+ 0x49, 0x4e, 0x46, 0x4f, // "INFO"+ 0x49, 0x53, 0x46, 0x54, // "ISFT"+ 0x0d, 0x00, 0x00, 0x00, // sub-chunk size: 13+ 0x4c, 0x61, 0x76, 0x66, 0x36, 0x32, 0x2e, 0x33, // "Lavf62.3"+ 0x2e, 0x31, 0x30, 0x30, 0x00, // ".100\0"+ /* padding byte for word alignment */+ 0x00, // data chunk header+ 0x64, 0x61, 0x74, 0x61, // "data"+ 0x80, 0xcf, 0x00, 0x00, // chunk size+ ];++ let mut bytes = header;++ // fill remaining space up to `FILE_ANALYSIS_BYTES` with synthetic PCM+ let audio_bytes_needed = FILE_ANALYSIS_BYTES - bytes.len();+ for i in 0..(audio_bytes_needed / 2) {+ let sample = (i & 0xFF) as u8;+ bytes.push(sample); // low byte: varies+ bytes.push(0x00); // high byte: zero for small values+ }++ bytes+ }++ #[test]+ fn test_pcm16_wav_detected_as_binary() {+ let wav_bytes = build_pcm16_wav_bytes();+ assert_eq!(wav_bytes.len(), FILE_ANALYSIS_BYTES);++ let result = analyze_byte_content(&wav_bytes);+ assert_eq!(+ result,+ ByteContent::Binary,+ "PCM 16-bit WAV should be detected as Binary via RIFF header"+ );+ }++ #[test]+ fn test_le16_binary_not_misdetected_as_utf16le() {+ let mut bytes = b"FAKE".to_vec();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ let sample = (bytes.len() & 0xFF) as u8;+ bytes.push(sample);+ bytes.push(0x00);+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ let result = analyze_byte_content(&bytes);+ assert_eq!(+ result,+ ByteContent::Binary,+ "LE 16-bit binary with control characters should be detected as Binary"+ );+ }++ #[test]+ fn test_be16_binary_not_misdetected_as_utf16be() {+ let mut bytes = b"FAKE".to_vec();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.push(0x00);+ let sample = (bytes.len() & 0xFF) as u8;+ bytes.push(sample);+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ let result = analyze_byte_content(&bytes);+ assert_eq!(+ result,+ ByteContent::Binary,+ "BE 16-bit binary with control characters should be detected as Binary"+ );+ }++ // Mimics binary formats that interleave short ASCII fragments with small+ // length/type fields (as seen in some game/asset binary formats, e.g.+ // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's+ // null-byte pattern for ASCII, but the low bytes are mostly non-word+ // "tag" values rather than real letters/digits/spaces.+ fn build_tag_interleaved_binary_bytes() -> Vec<u8> {+ let mut bytes = Vec::new();+ let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];+ let mut i = 0;+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.push(tags[i % tags.len()]);+ bytes.push(0x00);+ i += 1;+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);+ bytes+ }++ #[test]+ fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {+ let bytes = build_tag_interleaved_binary_bytes();+ assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);++ let result = analyze_byte_content(&bytes);+ assert_eq!(+ result,+ ByteContent::Binary,+ "binary data with sparse non-word low bytes and null high bytes \+ should not be misdetected as UTF-16LE text"+ );+ }++ #[test]+ fn test_utf16le_text_detected_as_utf16le() {+ let text = "Hello, world! This is a UTF-16 test string. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);+ }++ #[test]+ fn test_utf16be_text_detected_as_utf16be() {+ let text = "Hello, world! This is a UTF-16 test string. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);+ }++ #[test]+ fn test_utf16le_cyrillic_text_detected_as_utf16le() {+ let text = "Привет, мир! Это тестовая строка в UTF-16. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);+ }++ #[test]+ fn test_utf16be_greek_text_detected_as_utf16be() {+ let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";+ let mut bytes = Vec::new();+ while bytes.len() < FILE_ANALYSIS_BYTES {+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));+ }+ bytes.truncate(FILE_ANALYSIS_BYTES);++ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);+ }++ #[test]+ fn test_known_binary_headers() {+ let cases: &[(&[u8], &str)] = &[+ (b"RIFF\x00\x00\x00\x00WAVE", "WAV"),+ (b"RIFF\x00\x00\x00\x00AVI ", "AVI"),+ (b"OggS\x00\x02", "OGG"),+ (b"fLaC\x00\x00", "FLAC"),+ (b"ID3\x03\x00", "MP3 ID3v2"),+ (b"\xFF\xFB\x90\x00", "MP3 MPEG1 Layer3"),+ (b"\xFF\xF3\x90\x00", "MP3 MPEG2 Layer3"),+ ];++ for (header, label) in cases {+ let mut bytes = header.to_vec();+ bytes.resize(FILE_ANALYSIS_BYTES, 0x41); // pad with 'A'+ assert_eq!(+ analyze_byte_content(&bytes),+ ByteContent::Binary,+ "{label} should be detected as Binary"+ );+ }+ }++ struct ShortReads<R> {+ inner: R,+ limit: usize,+ }++ impl<R: Read> Read for ShortReads<R> {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ let len = buffer.len().min(self.limit);+ self.inner.read(&mut buffer[..len])+ }+ }++ fn pseudo_random_bytes(len: usize) -> Vec<u8> {+ let mut state = 0x2545_F491_4F6C_DD1Du64;+ (0..len)+ .map(|_| {+ state = state+ .wrapping_mul(6364136223846793005)+ .wrapping_add(1442695040888963407);+ (state >> 56) as u8+ })+ .collect()+ }+}diff --git a/crates/fs/src/fs.rs b/crates/fs/src/fs.rsindex 7d37384cd2a..f56c08e63f7 100644--- a/crates/fs/src/fs.rs+++ b/crates/fs/src/fs.rs@@ -76,6 +76,10 @@ pub trait Watcher: Send + Sync {fn remove(&self, path: &Path) -> Result<()>;}+pub trait ReadSeek: io::Read + io::Seek {}++impl<T: io::Read + io::Seek + ?Sized> ReadSeek for T {}+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]pub enum PathEventKind {Removed,@@ -130,7 +134,7 @@ pub trait Fs: Send + Sync {async fn remove_file(&self, path: &Path, options: RemoveOptions) -> Result<()>;async fn open_handle(&self, path: &Path) -> Result<Arc<dyn FileHandle>>;- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>>;+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>>;async fn load(&self, path: &Path) -> Result<String> {Ok(String::from_utf8(self.load_bytes(path).await?)?)}@@ -936,7 +940,7 @@ impl Fs for RealFs {Ok(self.trash.lock().insert(entry))}- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>> {+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>> {Ok(Box::new(std::fs::File::open(path)?))}@@ -3211,7 +3215,7 @@ impl Fs for FakeFs {self.remove_file_inner(path, options).await.map(|_| ())}- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>> {+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>> {let bytes = self.load_internal(path).await?;Ok(Box::new(io::Cursor::new(bytes)))}diff --git a/crates/git/Cargo.toml b/crates/git/Cargo.tomlindex 72db89537e5..551a9253182 100644--- a/crates/git/Cargo.toml+++ b/crates/git/Cargo.toml@@ -21,6 +21,7 @@ askpass.workspace = trueasync-trait.workspace = truecollections.workspace = truederive_more.workspace = true+file_content.workspace = truegpui.workspace = truehttp_client.workspace = trueitertools.workspace = truediff --git a/crates/git/src/repository.rs b/crates/git/src/repository.rsindex 7962bd6cc9a..b467990e20d 100644--- a/crates/git/src/repository.rs+++ b/crates/git/src/repository.rs@@ -7,6 +7,7 @@ use crate::{Oid, RunHook, SHA256_HEX_LENGTH, SHORT_SHA_LENGTH};use anyhow::{Context as _, Result, anyhow, bail};use async_channel::Sender;use collections::HashMap;+use file_content::{ByteContent, decode_byte_header};use futures::channel::oneshot;use futures::future::BoxFuture;use futures::io::BufWriter;@@ -569,8 +570,6 @@ impl CommitDetails {}}-/// Detects if content is binary by checking for NUL bytes in the first 8000 bytes.-/// This matches git's binary detection heuristic.pub fn is_binary_content(content: &[u8]) -> bool {let check_len = content.len().min(8000);content[..check_len].contains(&0)@@ -598,7 +597,8 @@ async fn read_commit_blob<R: smol::io::AsyncBufRead + Unpin>(stdout.read_exact(&mut bytes).await?;stdout.read_exact(newline).await?;- let is_binary = is_binary_content(&bytes);+ let is_binary =+ is_binary_content(&bytes) || decode_byte_header(&bytes).1 == ByteContent::Binary;Ok(LoadedCommitObject {content: bytes,is_binary,@@ -4633,6 +4633,43 @@ mod tests {);}+ #[test]+ fn test_read_commit_blob_binary_detection() {+ let mut pickle = b"\x80\x02X\0\x04\0\0".to_vec();+ pickle.extend_from_slice(&b"A\0".repeat(512));+ pickle.extend_from_slice(b"q\0.");+ let mut cases = vec![+ (pickle, true),+ (b"hello\n".to_vec(), false),+ (b"\xFF\xFEH\0e\0l\0l\0o\0\n\0".to_vec(), true),+ (b"\xFE\xFF\0H\0e\0l\0l\0o\0\n".to_vec(), true),+ (b"H\0e\0l\0l\0o\0\n\0".to_vec(), true),+ (b"\0H\0e\0l\0l\0o\0\n".to_vec(), true),+ (b"\xEF\xBB\xBFhello\0".to_vec(), true),+ (b"\x89PNG\r\n\x1a\n".to_vec(), true),+ ];+ for offset in [4096, 7999, 8000] {+ let mut bytes = vec![b'a'; offset];+ bytes.push(0);+ cases.push((bytes, offset < 8000));+ }++ for (bytes, expected_binary) in cases {+ let mut output = format!("{}\n", bytes.len()).into_bytes();+ output.extend_from_slice(&bytes);+ output.push(b'\n');+ let object = smol::block_on(read_commit_blob(+ &mut smol::io::Cursor::new(output),+ &mut String::new(),+ &mut [0],+ ))+ .unwrap();++ assert_eq!(object.is_binary, expected_binary);+ assert_eq!(object.content, bytes);+ }+ }+#[gpui::test]async fn test_load_commit_with_type_changed_file(cx: &mut TestAppContext) {disable_git_global_config();diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.tomlindex bfc9c5ac24e..1e4772ff64c 100644--- a/crates/language/Cargo.toml+++ b/crates/language/Cargo.toml@@ -32,11 +32,11 @@ test-support = [anyhow.workspace = trueasync-trait.workspace = trueclock.workspace = true-chardetng.workspace = truecollections.workspace = truediffy.workspace = trueec4rs.workspace = trueencoding_rs.workspace = true+file_content.workspace = truefs.workspace = truefutures.workspace = truefutures-lite.workspace = truediff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rsindex e54228e6e7e..32b633d3e0d 100644--- a/crates/language/src/buffer.rs+++ b/crates/language/src/buffer.rs@@ -3,9 +3,13 @@ pub mod row_chunk;pub use bracket_ranges::BracketMatch;+pub use crate::{+ CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry,+ diagnostic_set::DiagnosticSet, proto,+};use crate::{- ByteContent, DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig,- PLAIN_TEXT, RunnableTag, TextObject, TreeSitterOptions, analyze_byte_content,+ DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig, PLAIN_TEXT,+ RunnableTag, TextObject, TreeSitterOptions,diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup},language_settings::{AutoIndentMode, LanguageSettings},outline::OutlineItem,@@ -19,16 +23,13 @@ use crate::{text_diff::text_diff,unified_diff_with_offsets,};-pub use crate::{- CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry,- diagnostic_set::DiagnosticSet, proto,-};use anyhow::{Context as _, Result};use clock::Lamport;pub use clock::ReplicaId;use collections::HashMap;use encoding_rs::Encoding;+use file_content::{ByteContent, decode_byte_header};use fs::MTime;use futures::channel::oneshot;use futures_lite::future::yield_now;@@ -1696,7 +1697,7 @@ impl Buffer {let bytes = load_bytes_task.await?;anyhow::ensure!(- analyze_byte_content(&bytes) != ByteContent::Binary,+ decode_byte_header(&bytes).1 != ByteContent::Binary,"Binary files are not supported");diff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rsdeleted file mode 100644index 4acfd5d04a8..00000000000--- a/crates/language/src/file_content.rs+++ /dev/null@@ -1,308 +0,0 @@-use anyhow::{Result, bail};-use chardetng::EncodingDetector;-use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};--pub const FILE_ANALYSIS_BYTES: usize = 1024;--pub struct DecodedText {- pub text: String,- pub encoding: &'static Encoding,- pub has_bom: bool,-}--pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {- if let Some((encoding, _bom_len)) = Encoding::for_bom(&bytes) {- let (text, _) = encoding.decode_with_bom_removal(&bytes);- return Ok(DecodedText {- text: text.into_owned(),- encoding,- has_bom: true,- });- }-- let encoding = match analyze_byte_content(&bytes) {- ByteContent::Utf16Le => UTF_16LE,- ByteContent::Utf16Be => UTF_16BE,- ByteContent::Binary => bail!("Binary files are not supported"),- ByteContent::Unknown => {- return match String::from_utf8(bytes) {- Ok(text) if !text.contains('\x1b') => Ok(DecodedText {- text,- encoding: UTF_8,- has_bom: false,- }),- Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),- Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),- };- }- };-- let (text, _, _) = encoding.decode(&bytes);- Ok(DecodedText {- text: text.into_owned(),- encoding,- has_bom: false,- })-}--pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {- if encoding == UTF_8 && !has_bom {- return text.into_bytes();- }-- // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.- if encoding == UTF_16BE {- let mut bytes = Vec::with_capacity(text.len() * 2 + 2);- if has_bom {- bytes.extend_from_slice(&[0xFE, 0xFF]);- }- bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));- return bytes;- }-- if encoding == UTF_16LE {- let mut bytes = Vec::with_capacity(text.len() * 2 + 2);- if has_bom {- bytes.extend_from_slice(&[0xFF, 0xFE]);- }- bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));- return bytes;- }-- let (encoded, _, _) = encoding.encode(&text);- if has_bom && encoding == UTF_8 {- let mut bytes = Vec::with_capacity(encoded.len() + 3);- bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);- bytes.extend_from_slice(&encoded);- bytes- } else {- encoded.into_owned()- }-}--fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {- let mut detector = EncodingDetector::new();- detector.feed(&bytes, true);- let encoding = detector.guess(None, true);- let (text, _, _) = encoding.decode(&bytes);- DecodedText {- text: text.into_owned(),- encoding,- has_bom: false,- }-}--#[derive(Debug, PartialEq)]-pub enum ByteContent {- Utf16Le,- Utf16Be,- Binary,- Unknown,-}--// Heuristic check using null byte distribution plus a generic text-likeness-// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise-// distinguishes between text-like and binary-like content.-pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent {- if bytes.len() < 2 {- return ByteContent::Unknown;- }-- if is_known_binary_header(bytes) {- return ByteContent::Binary;- }-- let limit = bytes.len().min(FILE_ANALYSIS_BYTES);- let mut even_null_count = 0usize;- let mut odd_null_count = 0usize;- let mut non_text_like_count = 0usize;-- for (i, &byte) in bytes[..limit].iter().enumerate() {- if byte == 0 {- if i % 2 == 0 {- even_null_count += 1;- } else {- odd_null_count += 1;- }- non_text_like_count += 1;- continue;- }-- let is_text_like = match byte {- b'\t' | b'\n' | b'\r' | 0x0C => true,- 0x20..=0x7E => true,- // Treat bytes that are likely part of UTF-8 or single-byte encodings as text-like.- 0x80..=0xBF | 0xC2..=0xF4 => true,- _ => false,- };-- if !is_text_like {- non_text_like_count += 1;- }- }-- let total_null_count = even_null_count + odd_null_count;-- // If there are no NUL bytes at all, this is overwhelmingly likely to be text.- if total_null_count == 0 {- return ByteContent::Unknown;- }-- let has_significant_nulls = total_null_count >= limit / 16;- let nulls_skew_to_even = even_null_count > odd_null_count * 4;- let nulls_skew_to_odd = odd_null_count > even_null_count * 4;-- if has_significant_nulls {- let sample = &bytes[..limit];-- // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first)- // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first)-- if nulls_skew_to_even && is_plausible_utf16_text(sample, false) {- return ByteContent::Utf16Be;- }-- if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) {- return ByteContent::Utf16Le;- }-- return ByteContent::Binary;- }-- if non_text_like_count * 100 < limit * 8 {- ByteContent::Unknown- } else {- ByteContent::Binary- }-}--fn is_known_binary_header(bytes: &[u8]) -> bool {- bytes.starts_with(b"%PDF-") // PDF- || bytes.starts_with(b"PK\x03\x04") // ZIP local header- || bytes.starts_with(b"PK\x05\x06") // ZIP end of central directory- || bytes.starts_with(b"PK\x07\x08") // ZIP spanning/splitting- || bytes.starts_with(b"\x89PNG\r\n\x1a\n") // PNG- || bytes.starts_with(b"\xFF\xD8\xFF") // JPEG- || bytes.starts_with(b"GIF87a") // GIF87a- || bytes.starts_with(b"GIF89a") // GIF89a- || bytes.starts_with(b"IWAD") // Doom IWAD archive- || bytes.starts_with(b"PWAD") // Doom PWAD archive- || bytes.starts_with(b"RIFF") // WAV, AVI, WebP- || bytes.starts_with(b"OggS") // OGG (Vorbis, Opus, FLAC)- || bytes.starts_with(b"fLaC") // FLAC- || bytes.starts_with(b"ID3") // MP3 with ID3v2 tag- || bytes.starts_with(b"\xFF\xFB") // MP3 frame sync (MPEG1 Layer3)- || bytes.starts_with(b"\xFF\xFA") // MP3 frame sync (MPEG1 Layer3)- || bytes.starts_with(b"\xFF\xF3") // MP3 frame sync (MPEG2 Layer3)- || bytes.starts_with(b"\xFF\xF2") // MP3 frame sync (MPEG2 Layer3)-}--// Null byte skew alone is not enough to identify UTF-16 -- binary formats with-// small 16-bit values (like PCM audio) produce the same pattern. Decode the-// bytes as UTF-16 and reject if too many code units land in control character-// ranges or form unpaired surrogates, which real text almost never contains.-fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {- let mut suspicious_count = 0usize;- let mut word_like_count = 0usize;- let mut total = 0usize;-- let mut i = 0;- while let Some(code_unit) = read_u16(bytes, i, little_endian) {- total += 1;-- match code_unit {- 0x0009 | 0x000A | 0x000C | 0x000D => {}- 0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {- word_like_count += 1;- }- // C0/C1 control characters and non-characters- 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,- 0xD800..=0xDBFF => {- let next_offset = i + 2;- let has_low_surrogate = read_u16(bytes, next_offset, little_endian)- .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));- if has_low_surrogate {- total += 1;- word_like_count += 2;- i += 2;- } else {- suspicious_count += 1;- }- }- // Lone low surrogate without a preceding high surrogate- 0xDC00..=0xDFFF => suspicious_count += 1,- 0x0100.. => word_like_count += 1,- _ => {}- }-- i += 2;- }-- if total == 0 {- return false;- }-- // Real UTF-16 text has near-zero control characters; binary data with- // small 16-bit values typically exceeds 5%. 2% provides a safe margin.- let low_control_ratio = suspicious_count * 100 < total * 2;-- // Binary formats that interleave short ASCII fragments with small- // length/type fields (e.g. game asset formats) can dodge the control- // character check above while barely containing any real words: their- // code units land on ASCII punctuation and Latin-1 symbol values rather- // than letters, digits, or spaces. Real text is overwhelmingly made of- // word characters, so require a minimum share of them. Code units above- // the Latin-1 range (and surrogate pairs) also count as word-like so- // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,- // are still recognized -- tag bytes paired with a zero byte can never- // land there.- let enough_word_chars = word_like_count * 100 >= total * 30;-- low_control_ratio && enough_word_chars-}--fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {- let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?];- if little_endian {- return Some(u16::from_le_bytes(pair));- }- Some(u16::from_be_bytes(pair))-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn decodes_and_encodes_windows_1251() {- let expected = "строка один\nстрока два\n";- let (bytes, _, _) = encoding_rs::WINDOWS_1251.encode(expected);-- let decoded = decode_text(bytes.clone().into_owned()).unwrap();-- assert_eq!(decoded.text, expected);- assert_eq!(decoded.encoding, encoding_rs::WINDOWS_1251);- assert!(!decoded.has_bom);- assert_eq!(- encode_text(decoded.text, decoded.encoding, decoded.has_bom),- bytes.as_ref()- );- }-- #[test]- fn preserves_unicode_boms() {- let expected = "Hello, мир\n";- for encoding in [UTF_8, UTF_16LE, UTF_16BE] {- let bytes = encode_text(expected.to_owned(), encoding, true);- let decoded = decode_text(bytes.clone()).unwrap();-- assert_eq!(decoded.text, expected);- assert_eq!(decoded.encoding, encoding);- assert!(decoded.has_bom);- assert_eq!(- encode_text(decoded.text, decoded.encoding, decoded.has_bom),- bytes- );- }- }-}diff --git a/crates/language/src/language.rs b/crates/language/src/language.rsindex 6dcb998500d..76de9695f33 100644--- a/crates/language/src/language.rs+++ b/crates/language/src/language.rs@@ -10,7 +10,6 @@ mod available_languages;mod buffer;mod diagnostic;mod diagnostic_set;-mod file_content;mod language_registry;pub mod language_settings;@@ -108,9 +107,6 @@ pub use diagnostic::{Diagnostic, DiagnosticMessage, DiagnosticSourceKind, RelatedInformation, RelatedLocation,};pub use diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup};-pub use file_content::{- ByteContent, DecodedText, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,-};pub use language_registry::{BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QueryFile,QueryFileContents, QueryFiles,diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.tomlindex 20f6cd6abc2..45626f1b33f 100644--- a/crates/project/Cargo.toml+++ b/crates/project/Cargo.toml@@ -51,6 +51,7 @@ dap.workspace = trueencoding_rs.workspace = trueextension.workspace = truefancy-regex.workspace = true+file_content.workspace = truefs.workspace = truefutures.workspace = truefuzzy.workspace = truediff --git a/crates/project/src/git_store.rs b/crates/project/src/git_store.rsindex d02a4c5bfb7..5468e825009 100644--- a/crates/project/src/git_store.rs+++ b/crates/project/src/git_store.rs@@ -22,6 +22,7 @@ use buffer_diff::{use client::ProjectId;use collections::HashMap;pub use conflict_set::{ConflictRegion, ConflictSet, ConflictSetSnapshot, ConflictSetUpdate};+use file_content::{decode_text, encode_text};use fs::{Fs, RemoveOptions};use futures::{FutureExt, SinkExt, Stream, StreamExt,@@ -55,7 +56,7 @@ use gpui::{Subscription, Task, TaskExt, WeakEntity,};use language::{- Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry, decode_text, encode_text,+ Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry,proto::{deserialize_version, serialize_version},};use parking_lot::Mutex;@@ -216,7 +217,12 @@ fn pending_hunks(}fn decode_git_text(bytes: Vec<u8>) -> Result<String> {- Ok(decode_text(bytes)?.text)+ let text = decode_text(bytes)?.text;+ anyhow::ensure!(+ !is_binary_content(text.as_bytes()),+ "Binary files are not supported"+ );+ Ok(text)}#[derive(Debug)]@@ -10414,7 +10420,7 @@ impl Repository {}}});- cx.spawn(move |_: &mut AsyncApp| async move {+ cx.background_spawn(async move {let (content, blame) = rx.await??;anyhow::ensure!(!is_binary_content(content.as_bytes()),@@ -11691,28 +11697,45 @@ mod tests {}#[gpui::test]- fn test_decode_git_text_windows_1251_one_line_change(cx: &mut TestAppContext) {+ fn test_decode_git_text(cx: &mut TestAppContext) {let old_text = "строка один\nстрока два\n";let new_text = "строка один\nстрока три\n";- let (old_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(old_text);- let (new_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(new_text);-- let decoded_old = decode_git_text(old_bytes.into_owned()).unwrap();- let decoded_new = decode_git_text(new_bytes.into_owned()).unwrap();- let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));- let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());- let diff = cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));- let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));- let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();- let [hunk] = hunks.as_slice() else {- panic!("expected one modified hunk, got {hunks:?}");- };+ let sparse_nul_text = format!("{}\0", "a".repeat(4096));+ for (encoding, has_bom) in [+ (encoding_rs::WINDOWS_1251, false),+ (encoding_rs::UTF_16LE, false),+ (encoding_rs::UTF_16LE, true),+ (encoding_rs::UTF_16BE, false),+ (encoding_rs::UTF_16BE, true),+ ] {+ let old_bytes = encode_text(old_text.to_owned(), encoding, has_bom);+ let new_bytes = encode_text(new_text.to_owned(), encoding, has_bom);+ let decoded_old = decode_git_text(old_bytes).unwrap();+ let decoded_new = decode_git_text(new_bytes).unwrap();+ assert_eq!(decoded_old, old_text);+ assert_eq!(decoded_new, new_text);+ let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));+ let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());+ let diff =+ cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));+ let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));+ let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();+ let [hunk] = hunks.as_slice() else {+ panic!("expected one modified hunk, got {hunks:?}");+ };- assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));- assert_eq!(- hunk.diff_base_byte_range,- old_text.find("строка два").unwrap()..old_text.len()- );+ assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));+ assert_eq!(+ hunk.diff_base_byte_range,+ old_text.find("строка два").unwrap()..old_text.len()+ );+ assert_eq!(+ decode_git_text(encode_text(sparse_nul_text.clone(), encoding, has_bom))+ .unwrap_err()+ .to_string(),+ "Binary files are not supported"+ );+ }}#[gpui::test]diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rsindex 3abcde1cf1c..edb4b0adf59 100644--- a/crates/project/src/project_search.rs+++ b/crates/project/src/project_search.rs@@ -1,7 +1,7 @@use std::{cell::LazyCell,collections::BTreeSet,- io::{BufRead, BufReader, Cursor, ErrorKind, Read},+ io::{Cursor, ErrorKind, Read},ops::Range,path::{Path, PathBuf},pin::pin,@@ -12,7 +12,7 @@ use std::{use anyhow::Context;use async_channel::{Receiver, Sender, bounded, unbounded};use collections::HashSet;-use fs::Fs;+use fs::{Fs, ReadSeek};use futures::FutureExt as _;use futures::{SinkExt, StreamExt, select_biased, stream::FuturesOrdered};use gpui::{App, AppContext, AsyncApp, BackgroundExecutor, Entity, Priority, Task};@@ -21,12 +21,12 @@ use parking_lot::Mutex;use postage::oneshot;use rpc::{AnyProtoClient, proto};-use language::ByteContent;-use util::{ResultExt, maybe, rel_path::RelPath};-use worktree::{- Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header,- decode_file_text,+use encoding_rs::Encoding;+use file_content::{+ ByteContent, DecodingReader, FILE_ANALYSIS_BYTES, decode_byte_header, detect_encoding,};+use util::{ResultExt, maybe, rel_path::RelPath};+use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings};use crate::{Project, ProjectItem, ProjectPath, RemotelyCreatedModels,@@ -614,7 +614,7 @@ impl Search {) {_ = maybe!(async move {while let Ok((buffer, line_hint)) = rx.recv().await {- let snapshot = buffer.read_with(&mut cx, |this, _| this.snapshot());+ let snapshot = buffer.read_with(&mut cx, |buffer, _| buffer.snapshot());let (tx, rx) = oneshot::channel();find_all_matches_tx.send(FindAllMatchesRequest {@@ -834,13 +834,14 @@ impl RequestHandler<'_> {let fs = self.fs.context("Trying to query filesystem in remote project search")?;- let Some(file) = fs.open_sync(&abs_path).await.log_err() else {+ let Some(mut file) = fs.open_sync(&abs_path).await.log_err() else {return anyhow::Ok(());};- let mut file = BufReader::new(file);- let file_start = file.fill_buf()?;- let (bom_encoding, byte_content) = decode_byte_header(file_start);+ let mut file_start = Vec::with_capacity(8 * 1024);+ (&mut *file).take(8 * 1024).read_to_end(&mut file_start)?;+ let (bom_encoding, byte_content) =+ decode_byte_header(&file_start[..file_start.len().min(FILE_ANALYSIS_BYTES)]);if byte_content == ByteContent::Binary {log::debug!("Skipping binary file {abs_path:?}");return Ok(());@@ -848,27 +849,35 @@ impl RequestHandler<'_> {let is_plain_utf8 = bom_encoding.is_none()&& byte_content == ByteContent::Unknown- && is_utf8_prefix(file_start);-- let line_hint = if is_plain_utf8 {- match self.query.detect(file).await {+ && is_utf8_prefix(&file_start);+ let encoding = bom_encoding.or(byte_content.encoding());++ let first_match = if is_plain_utf8 {+ match self+ .query+ .detect(&mut Cursor::new(file_start).chain(&mut *file))+ .await+ {Ok(line_hint) => line_hint,Err(error)if error.downcast_ref::<std::io::Error>().is_some_and(|error| error.kind() == ErrorKind::InvalidData) =>{- self.detect_in_decoded_file(fs, &abs_path).await?+ self.detect_in_decoded_file(&mut *file, encoding).await?}Err(error) => return Err(error),}} else {- self.detect_in_decoded_file(fs, &abs_path).await?+ self.detect_in_decoded_file(&mut *file, encoding).await?};- if let Some(line_hint) = line_hint {+ if first_match.is_some() {// Yes, we should scan the whole file.- entry.should_scan_tx.send((entry.path, line_hint)).await?;+ entry+ .should_scan_tx+ .send((entry.path, MatchPositionHint::default()))+ .await?;}Ok(())}@@ -878,12 +887,21 @@ impl RequestHandler<'_> {async fn detect_in_decoded_file(&self,- fs: &dyn Fs,- abs_path: &Path,+ file: &mut (dyn ReadSeek + Send),+ encoding: Option<&'static Encoding>,) -> anyhow::Result<Option<MatchPositionHint>> {- let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?;- let reader: Box<dyn Read + Send + Sync> = Box::new(Cursor::new(text.into_bytes()));- self.query.detect(BufReader::new(reader)).await+ file.rewind()?;+ let encoding = match encoding {+ Some(encoding) => encoding,+ None => {+ let encoding = detect_encoding(file).await?;+ file.rewind()?;+ encoding+ }+ };+ self.query+ .detect(&mut DecodingReader::new(file, encoding))+ .await}async fn handle_scan_path(&self, req: InputPath) {diff --git a/crates/project/src/search.rs b/crates/project/src/search.rsindex fb2e88a1240..49c7f931142 100644--- a/crates/project/src/search.rs+++ b/crates/project/src/search.rs@@ -9,7 +9,7 @@ use smol::future::yield_now;use std::{borrow::Cow,collections::BTreeSet,- io::{BufRead, BufReader, Read},+ io::{self, Read},ops::Range,sync::{Arc, LazyLock},};@@ -46,7 +46,7 @@ pub struct SearchInputs {buffers: Option<Vec<Entity<Buffer>>>,}-#[derive(Clone, Copy, Debug)]+#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub enum MatchPositionHint {Line(u32),ByteOffset(usize),@@ -397,49 +397,28 @@ impl SearchQuery {pub async fn detect(&self,- mut reader: BufReader<Box<dyn Read + Send + Sync>>,+ reader: &mut (dyn Read + Send),) -> Result<Option<MatchPositionHint>> {let query_str = self.as_str();if query_str.is_empty() {return Ok(None);}- // Yield from this function every 20KB scanned.- const YIELD_THRESHOLD: usize = 20 * 1024;-match self {+ Self::Text { search, .. } if !query_str.contains('\n') => {+ detect_single_line(search, reader).await+ }Self::Text { search, .. } => {- let mut text = String::new();- if query_str.contains('\n') {- reader.read_to_string(&mut text)?;- text::LineEnding::normalize(&mut text);- if search.is_match(&text) {- Ok(Some(MatchPositionHint::default()))- } else {- Ok(None)- }+ let mut text = read_to_string(reader).await?;+ text::LineEnding::normalize(&mut text);+ if search.is_match(&text) {+ Ok(Some(MatchPositionHint::default()))} else {- let mut bytes_read = 0;- let mut line_number = u32::default();- while reader.read_line(&mut text)? > 0 {- if search.is_match(&text) {- return Ok(Some(MatchPositionHint::Line(line_number)));- }- bytes_read += text.len();- if bytes_read >= YIELD_THRESHOLD {- bytes_read = 0;- smol::future::yield_now().await;- }- text.clear();- line_number += 1;- }Ok(None)}}Self::Regex { regex, .. } => {- let mut text = String::new();-- reader.read_to_string(&mut text)?;+ let mut text = read_to_string(reader).await?;text::LineEnding::normalize(&mut text);if let Some(m) = regex.find(&text)? {Ok(Some(MatchPositionHint::ByteOffset(m.start())))@@ -713,3 +692,145 @@ impl SearchQuery {matches}}++async fn detect_single_line(+ search: &AhoCorasick,+ reader: &mut (dyn Read + Send),+) -> Result<Option<MatchPositionHint>> {+ const BLOCK_BYTES: usize = 64 * 1024;+ let batch_size = search.max_pattern_len().max(BLOCK_BYTES);+ let carry_len = search.max_pattern_len().saturating_sub(1).max(3);+ let mut block = vec![0; BLOCK_BYTES];+ let mut window = TextSearchWindow::default();+ loop {+ let batch_end = window.batch_start+ + if window.first_match.is_some() {+ BLOCK_BYTES+ } else {+ batch_size+ };+ let limit = (batch_end - window.bytes.len()).min(BLOCK_BYTES);+ let read_result = match reader.read(&mut block[..limit]) {+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,+ result => result,+ };+ if let std::result::Result::Ok(read) = read_result {+ window.extend(&block[..read], batch_size + carry_len)?;+ if read > 0 && window.bytes.len() < batch_end {+ yield_now().await;+ continue;+ }+ }+ let line_end = window.find_matching_line_end(search);+ window.validate_prefix(line_end, read_result.is_err())?;+ if line_end.is_some() {+ return Ok(window.first_match);+ }+ if read_result? == 0 {+ if window.validated_len < window.bytes.len() {+ return Err(invalid_data());+ }+ return Ok(window.first_match);+ }+ window.retain_suffix(if window.first_match.is_some() {+ 3+ } else {+ carry_len+ });+ yield_now().await;+ }+}++#[derive(Default)]+struct TextSearchWindow {+ bytes: Vec<u8>,+ batch_start: usize,+ validated_len: usize,+ lines_before_window: usize,+ first_match: Option<MatchPositionHint>,+}++impl TextSearchWindow {+ fn extend(&mut self, bytes: &[u8], max_capacity: usize) -> Result<()> {+ if self.bytes.len() + bytes.len() > self.bytes.capacity() {+ let capacity = self+ .bytes+ .capacity()+ .saturating_mul(2)+ .max(self.bytes.len() + bytes.len())+ .min(max_capacity);+ self.bytes.try_reserve_exact(capacity - self.bytes.len())?;+ }+ self.bytes.extend_from_slice(bytes);+ Ok(())+ }++ fn find_matching_line_end(&mut self, search: &AhoCorasick) -> Option<usize> {+ let start = if self.first_match.is_some() {+ self.batch_start+ } else {+ let found = search.find(&self.bytes)?;+ let line = self.lines_before_window + count_newlines(&self.bytes[..found.start()]);+ self.first_match = Some(MatchPositionHint::Line(+ u32::try_from(line).unwrap_or(u32::MAX),+ ));+ found.end()+ };+ self.bytes[start..]+ .iter()+ .position(|&byte| byte == b'\n')+ .map(|offset| start + offset + 1)+ }++ fn validate_prefix(&mut self, line_end: Option<usize>, read_failed: bool) -> Result<()> {+ let validation_end = line_end.unwrap_or_else(|| {+ if read_failed {+ self.bytes+ .iter()+ .rposition(|&byte| byte == b'\n')+ .map_or(self.validated_len, |offset| {+ (offset + 1).max(self.validated_len)+ })+ } else {+ self.bytes.len()+ }+ });+ self.validated_len +=+ match std::str::from_utf8(&self.bytes[self.validated_len..validation_end]) {+ Err(error) if error.error_len().is_some() => return Err(invalid_data()),+ Err(error) => error.valid_up_to(),+ _ => validation_end - self.validated_len,+ };+ Ok(())+ }++ fn retain_suffix(&mut self, length: usize) {+ let consumed = self.bytes.len().saturating_sub(length);+ self.lines_before_window += count_newlines(&self.bytes[..consumed]);+ self.bytes.drain(..consumed);+ self.batch_start = self.bytes.len();+ self.validated_len -= consumed;+ }+}++async fn read_to_string(reader: &mut (dyn Read + Send)) -> Result<String> {+ const BLOCK_BYTES: usize = 64 * 1024;+ let mut bytes = Vec::new();+ loop {+ let bytes_read = (&mut *reader)+ .take(BLOCK_BYTES as u64)+ .read_to_end(&mut bytes)?;+ if bytes_read < BLOCK_BYTES {+ return String::from_utf8(bytes).map_err(|_| invalid_data());+ }+ yield_now().await;+ }+}++fn count_newlines(bytes: &[u8]) -> usize {+ bytes.iter().filter(|&&byte| byte == b'\n').count()+}++fn invalid_data() -> anyhow::Error {+ anyhow::Error::from(io::Error::from(io::ErrorKind::InvalidData))+}diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rsindex ee488e76342..9149042ce07 100644--- a/crates/project/tests/integration/project_tests.rs+++ b/crates/project/tests/integration/project_tests.rs@@ -19738,6 +19738,22 @@ async fn test_undo_encoding_change(cx: &mut gpui::TestAppContext) {assert_ne!(buffer.text(), "Hi");assert!(!buffer.is_dirty());});++ let thai = "สวัสดีชาวโลกนี่คือข้อความทดสอบภาษาไทย";+ fs.insert_file(+ path!("/dir/test.txt"),+ thai.encode_utf16().flat_map(u16::to_le_bytes).collect(),+ )+ .await;+ cx.run_until_parked();+ assert_eq!(buffer.read_with(cx, |buffer, _| buffer.text()), thai);+ buffer+ .update(cx, |buffer, cx| {+ buffer.reload_with_encoding(encoding_rs::UTF_16LE, cx)+ })+ .await+ .unwrap();+ assert_eq!(buffer.read_with(cx, |buffer, _| buffer.text()), thai);}#[gpui::test]diff --git a/crates/project/tests/integration/search.rs b/crates/project/tests/integration/search.rsindex d961c87bceb..5b94509425a 100644--- a/crates/project/tests/integration/search.rs+++ b/crates/project/tests/integration/search.rs@@ -1,13 +1,23 @@-use std::io::BufReader;+use std::io::{self, BufReader, Cursor};+use collections::HashMap;+use fs::FakeFs;+use futures::FutureExt as _;use language::Buffer;-use project::search::SearchQuery;+use project::{+ Project,+ search::{MatchPositionHint, SearchQuery},+};+use serde_json::json;use text::Rope;use util::{+ path,paths::{PathMatcher, PathStyle},rel_path::RelPath,};+use crate::init_test;+#[test]fn path_matcher_creation_for_valid_paths() {for valid_path in [@@ -197,7 +207,7 @@ async fn regex_with_eol_detects_lines() {.unwrap();let input = " Bool\nsomething else";let result = re- .detect(BufReader::new(Box::new(input.as_bytes())))+ .detect(&mut BufReader::new(input.as_bytes())).await.unwrap();assert!(result.is_some());@@ -219,8 +229,335 @@ async fn multi_line_regex_detects_matches() {.unwrap();let input = " Bool\nbool";let result = re- .detect(BufReader::new(Box::new(input.as_bytes())))+ .detect(&mut BufReader::new(input.as_bytes())).await.unwrap();assert!(result.is_some());}++#[test]+fn detect_reports_line_across_block_boundaries() {+ let line_len = 1000;+ let filler = format!("{}\n", "x".repeat(line_len - 1));+ let mut text = filler.repeat(70);+ let needle_line = text.lines().count();+ text.push_str("prefix needle suffix\n");+ text.push_str(&filler.repeat(3));++ for (query, case_sensitive) in [("needle", true), ("NEEDLE", false), ("x\nx", true)] {+ let query = SearchQuery::text(+ query,+ false,+ case_sensitive,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ let mut input = reader(text.as_bytes());+ assert!(query.detect(&mut input).now_or_never().is_none());+ assert_eq!(input.inner.position(), 64 * 1024);+ let hint = smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap();+ let expected = if query.as_str().contains('\n') {+ MatchPositionHint::default()+ } else {+ MatchPositionHint::Line(needle_line as u32)+ };+ assert_eq!(hint, Some(expected), "{:?}", query.as_str());++ if !query.as_str().contains('\n') {+ let mut invalid = format!("needle{}", "x".repeat(128 * 1024)).into_bytes();+ invalid.push(0xff);+ let error = smol::block_on(query.detect(&mut reader(&invalid))).unwrap_err();+ assert_eq!(+ error.downcast_ref::<io::Error>().map(|error| error.kind()),+ Some(io::ErrorKind::InvalidData)+ );+ }+ }++ let query = SearchQuery::text(+ "absent",+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ assert_eq!(+ smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap(),+ None+ );++ let mut invalid = text.into_bytes();+ invalid.extend_from_slice(b"\xff\xfe tail\n");+ let error = smol::block_on(query.detect(&mut reader(&invalid))).unwrap_err();+ assert_eq!(+ error.downcast_ref::<io::Error>().map(|error| error.kind()),+ Some(io::ErrorKind::InvalidData)+ );+}++#[test]+fn detect_handles_block_boundary_splits() {+ const BLOCK_BYTES: usize = 64 * 1024;+ let needle = "néédle";+ for split in 1..needle.len() {+ let mut text = "a".repeat(BLOCK_BYTES - split);+ text.push_str(needle);+ text.push_str("\ntail\n");+ for case_sensitive in [true, false] {+ let query = SearchQuery::text(+ needle,+ false,+ case_sensitive,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ let expected = if query.is_regex() {+ MatchPositionHint::ByteOffset(BLOCK_BYTES - split)+ } else {+ MatchPositionHint::Line(0)+ };+ let mut input = reader(text.as_bytes());+ assert!(query.detect(&mut input).now_or_never().is_none());+ assert_eq!(input.inner.position(), BLOCK_BYTES as u64);+ assert_eq!(+ smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap(),+ Some(expected),+ "split = {split}, case_sensitive = {case_sensitive}"+ );+ }+ }++ let needle = (0..9000)+ .map(|index| format!("{index:08x}"))+ .collect::<String>();+ let query = SearchQuery::text(+ &needle,+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ let text = format!("ignored\n{}x\n{needle}", &needle[..BLOCK_BYTES - 1]);+ for max_read in [257, BLOCK_BYTES] {+ let mut input = reader(text.as_bytes());+ input.max_read = max_read;+ assert!(query.detect(&mut input).now_or_never().is_none());+ assert_eq!(input.inner.position(), max_read as u64);+ let mut input = reader(text.as_bytes());+ input.max_read = max_read;+ assert_eq!(+ smol::block_on(query.detect(&mut input)).unwrap(),+ Some(MatchPositionHint::Line(2)),+ );+ }++ let query = SearchQuery::text(+ "absent",+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ let emoji = "\u{1f600}";+ for split in 1..emoji.len() {+ let mut text = "a".repeat(BLOCK_BYTES - split).into_bytes();+ text.extend_from_slice(emoji.as_bytes());+ text.extend_from_slice(b"\nline\n");+ assert_eq!(+ smol::block_on(query.detect(&mut reader(&text))).unwrap(),+ None,+ "split = {split}"+ );+ let mut matched_text = text.clone();+ matched_text[..query.as_str().len()].copy_from_slice(query.as_str().as_bytes());+ matched_text.push(0xff);+ assert_eq!(+ smol::block_on(query.detect(&mut reader(&matched_text))).unwrap(),+ Some(MatchPositionHint::Line(0)),+ "matched split = {split}"+ );+ text.truncate(BLOCK_BYTES - split + 2);+ let error = smol::block_on(query.detect(&mut reader(&text))).unwrap_err();+ assert_eq!(+ error.downcast_ref::<io::Error>().map(|error| error.kind()),+ Some(io::ErrorKind::InvalidData),+ "truncated split = {split}"+ );+ }+}++#[test]+fn detect_validates_matching_line_before_io_errors() {+ let query = SearchQuery::text(+ "n",+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ let found = Ok(Some(MatchPositionHint::Line(0)));+ let invalid = Err(io::ErrorKind::InvalidData);+ let failed = Err(io::ErrorKind::Other);+ for (bytes, at_eof, at_error) in [+ (b"".as_slice(), Ok(None), failed),+ (b"n", found, failed),+ (b"n\xff", invalid, failed),+ (b"\xff\nn", invalid, invalid),+ (b"n\xff\n", invalid, invalid),+ (b"n\n\xff", found, found),+ (b"n\xf0\x9f", invalid, failed),+ (b"n\n\xf0\x9f", found, found),+ (+ "x\n😀n".as_bytes(),+ Ok(Some(MatchPositionHint::Line(1))),+ failed,+ ),+ ] {+ for terminal_error in [None, Some(io::ErrorKind::Other)] {+ for max_read in [1, 64 * 1024] {+ let mut input = reader(bytes);+ input.max_read = max_read;+ input.terminal_error = terminal_error;+ let actual = smol::block_on(query.detect(&mut input))+ .map_err(|error| error.downcast_ref::<io::Error>().unwrap().kind());+ let expected = if terminal_error.is_some() {+ at_error+ } else {+ at_eof+ };+ assert_eq!(+ actual, expected,+ "{bytes:?}, {terminal_error:?}, {max_read}"+ );+ }+ }+ }+}++#[gpui::test]+async fn searches_legacy_text_after_utf8_prefixes(cx: &mut gpui::TestAppContext) {+ init_test(cx);++ for (text, encoding, needle) in [+ (+ format!("é\n{} où voilà un garçon à la maison ", "x".repeat(1021)),+ encoding_rs::WINDOWS_1252,+ "é",+ ),+ (+ format!(+ "é\n©{}{}",+ "x".repeat(70000),+ " déjà été à côté français ".repeat(1000)+ ),+ encoding_rs::WINDOWS_1252,+ "©",+ ),+ (+ format!(+ "\x1b[0m{}{}©",+ "x".repeat(1024 * 1024),+ " déjà été à côté français ".repeat(1000)+ ),+ encoding_rs::WINDOWS_1252,+ "©",+ ),+ (+ format!(+ "я{}{}ЖЕТОН",+ "x".repeat(1024 * 1024),+ "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(1000)+ ),+ encoding_rs::WINDOWS_1251,+ "ЖЕТОН",+ ),+ ] {+ let (bytes, _, had_errors) = encoding.encode(&text);+ assert!(!had_errors);+ let expected = text+ .match_indices(needle)+ .map(|(offset, matched)| offset..offset + matched.len())+ .collect::<Vec<_>>();+ let fs = FakeFs::new(cx.executor());+ fs.insert_tree(path!("/dir"), json!({})).await;+ fs.insert_file(path!("/dir/legacy.txt"), bytes.into_owned())+ .await;+ let project = Project::test(fs, [path!("/dir").as_ref()], cx).await;+ let query = SearchQuery::text(+ needle,+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap();+ assert_eq!(+ super::search(&project, query, cx).await.unwrap(),+ HashMap::from_iter(+ (!expected.is_empty()).then(|| (path!("dir/legacy.txt").to_string(), expected))+ ),+ );+ }+}++fn reader(bytes: &[u8]) -> InterruptingReader {+ InterruptingReader {+ inner: Cursor::new(bytes.to_vec()),+ interrupt_next_read: false,+ max_read: usize::MAX,+ terminal_error: None,+ }+}++struct InterruptingReader {+ inner: Cursor<Vec<u8>>,+ interrupt_next_read: bool,+ max_read: usize,+ terminal_error: Option<io::ErrorKind>,+}++impl io::Read for InterruptingReader {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ self.interrupt_next_read = !self.interrupt_next_read;+ if self.interrupt_next_read {+ Err(io::Error::from(io::ErrorKind::Interrupted))+ } else {+ let length = buffer.len().min(self.max_read);+ let read = self.inner.read(&mut buffer[..length])?;+ if read == 0+ && !buffer.is_empty()+ && let Some(error) = self.terminal_error+ {+ return Err(io::Error::from(error));+ }+ Ok(read)+ }+ }+}diff --git a/crates/worktree/Cargo.toml b/crates/worktree/Cargo.tomlindex fb0ef41338a..983c8b4ff91 100644--- a/crates/worktree/Cargo.toml+++ b/crates/worktree/Cargo.toml@@ -35,6 +35,7 @@ async-lock.workspace = trueclock.workspace = truecollections.workspace = trueencoding_rs.workspace = true+file_content.workspace = truefs.workspace = truefutures.workspace = truefutures-lite.workspace = truediff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rsindex 8037416e1ef..1f1809a8e91 100644--- a/crates/worktree/src/worktree.rs+++ b/crates/worktree/src/worktree.rs@@ -6,6 +6,9 @@ use anyhow::{Context as _, Result, anyhow};use clock::ReplicaId;use collections::{BTreeMap, HashMap, HashSet, VecDeque};use encoding_rs::Encoding;+use file_content::{+ ByteContent, DecodingReader, decode_byte_header, detect_encoding, encode_text, read_byte_header,+};use fs::{Fs, MTime, PathEvent, PathEventKind, RemoveOptions, TrashId, Watcher, copy_recursive,read_dir_items,@@ -32,9 +35,7 @@ use gpui::{Task,};pub use ignore::{IgnoreKind, IgnoreStack};-use language::{- ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,-};+use language::DiskState;use async_channel::{self, Sender};use parking_lot::Mutex;@@ -7217,84 +7218,8 @@ impl fs::Watcher for NullWatcher {}}-/// Reads the beginning of `file` to determine its kind and encoding, returning-/// the bytes consumed and whether the file ended within them.-fn read_file_header(file: &mut dyn Read, abs_path: &Path) -> Result<(Vec<u8>, bool)> {- let mut header = Vec::with_capacity(FILE_ANALYSIS_BYTES);- let mut buf = [0u8; FILE_ANALYSIS_BYTES];- let mut reached_eof = false;- while header.len() < FILE_ANALYSIS_BYTES {- let n = file- .read(&mut buf)- .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;- if n == 0 {- reached_eof = true;- break;- }- header.extend_from_slice(&buf[..n]);- }- Ok((header, reached_eof))-}-const STREAM_BLOCK_BYTES: usize = 1024 * 1024;-async fn read_file_to_end(- file: &mut (dyn Read + Send),- content: &mut Vec<u8>,- abs_path: &Path,-) -> Result<()> {- let mut buf = vec![0u8; STREAM_BLOCK_BYTES];- loop {- let mut block_len = 0;- while block_len < buf.len() {- let n = file- .read(&mut buf[block_len..])- .with_context(|| format!("reading remaining bytes of the file {abs_path:?}"))?;- if n == 0 {- break;- }- block_len += n;- }-- if block_len == 0 {- break;- }-- content.extend_from_slice(&buf[..block_len]);- if block_len < buf.len() {- break;- }-- yield_now().await;- }- Ok(())-}--pub async fn decode_file_text(- fs: &dyn Fs,- abs_path: &Path,-) -> Result<(String, &'static Encoding, bool)> {- let mut file = fs- .open_sync(&abs_path)- .await- .with_context(|| format!("opening file {abs_path:?}"))?;-- let (file_first_bytes, reached_eof) = read_file_header(&mut *file, abs_path)?;- let (_, byte_content) = decode_byte_header(&file_first_bytes);- anyhow::ensure!(- byte_content != ByteContent::Binary,- "Binary files are not supported"- );-- // If the file is eligible for opening, read the rest of the file.- let mut content = file_first_bytes;- if !reached_eof {- read_file_to_end(&mut *file, &mut content, abs_path).await?;- }- let decoded = decode_text(content)?;- Ok((decoded.text, decoded.encoding, decoded.has_bom))-}-/// Reads and decodes a file straight into a [`Rope`]./// The returned rope has already had its line endings normalized, the/// [`LineEnding`] detected before normalizing is returned alongside it.@@ -7307,28 +7232,37 @@ pub async fn decode_file_text_to_rope(.await.with_context(|| format!("opening file {abs_path:?}"))?;- let (prefix, reached_eof) = read_file_header(&mut *file, abs_path)?;+ let (prefix, reached_eof) = read_byte_header(&mut *file)+ .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;let (bom_encoding, byte_content) = decode_byte_header(&prefix);anyhow::ensure!(byte_content != ByteContent::Binary,"Binary files are not supported");- // Only BOM-less, non-UTF-16 files are candidates for streaming: everything- // else needs the whole byte buffer in hand to decode or to detect encoding.if bom_encoding.is_none()&& byte_content == ByteContent::Unknown&& let Some((rope, line_ending)) =- stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path).await?+ stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path, true).await?{return Ok((rope, line_ending, encoding_rs::UTF_8, false));}- // Not plain UTF-8 after all. Re-read the file and decode it all at once.- let (mut text, encoding, has_bom) = decode_file_text(fs, abs_path).await?;- let line_ending = LineEnding::detect(&text);- LineEnding::normalize(&mut text);- Ok((Rope::from(text), line_ending, encoding, has_bom))+ file.rewind()?;+ let encoding = match bom_encoding.or(byte_content.encoding()) {+ Some(encoding) => encoding,+ None => {+ let encoding = detect_encoding(&mut *file).await?;+ file.rewind()?;+ encoding+ }+ };+ let mut reader = DecodingReader::new(&mut *file, encoding);+ let (rope, line_ending) =+ stream_utf8_into_rope(&mut reader, Vec::new(), false, abs_path, false)+ .await?+ .with_context(|| format!("decoding the file {abs_path:?}"))?;+ Ok((rope, line_ending, reader.encoding(), bom_encoding.is_some()))}/// Streams a presumed-UTF-8 file into a [`Rope`], normalizing line endings as it@@ -7342,6 +7276,7 @@ async fn stream_utf8_into_rope(prefix: Vec<u8>,reached_eof: bool,abs_path: &Path,+ reject_escape_sequences: bool,) -> Result<Option<(Rope, LineEnding)>> {let mut rope = Rope::new();let mut line_ending = None;@@ -7386,7 +7321,7 @@ async fn stream_utf8_into_rope(// ISO-2022-JP and friends are valid UTF-8 but carry escape sequences, so// they need the full-file encoding detector rather than this fast path.- if text.contains('\x1b') {+ if reject_escape_sequences && text.contains('\x1b') {return Ok(None);}@@ -7446,13 +7381,6 @@ fn push_normalized(rope: &mut Rope, text: &str, scratch: &mut String) {rope.push(scratch);}-pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {- if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {- return (Some(encoding), ByteContent::Unknown);- }- (None, analyze_byte_content(prefix))-}-#[cfg(test)]mod tests {use super::*;@@ -7461,7 +7389,7 @@ mod tests {/// decoded text and detected line ending, or `None` if the fast path bailed.async fn stream(bytes: &[u8]) -> Option<(String, LineEnding)> {let mut reader = std::io::Cursor::new(bytes.to_vec());- stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"), true).await.unwrap().map(|(rope, line_ending)| (rope.to_string(), line_ending))@@ -7472,27 +7400,13 @@ mod tests {let mut reader = std::io::Cursor::new(vec![b'a'; STREAM_BLOCK_BYTES * 2]);assert!(- stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"), true).now_or_never().is_none());assert_eq!(reader.position(), STREAM_BLOCK_BYTES as u64);}- #[test]- fn test_file_reading_yields_between_blocks() {- let mut reader = std::io::Cursor::new(vec![b'a'; STREAM_BLOCK_BYTES * 2]);- let mut content = Vec::new();-- assert!(- read_file_to_end(&mut reader, &mut content, Path::new("test"))- .now_or_never()- .is_none()- );- assert_eq!(reader.position(), STREAM_BLOCK_BYTES as u64);- assert_eq!(content.len(), STREAM_BLOCK_BYTES);- }-#[gpui::test]async fn test_stream_utf8_normalizes_line_endings() {let crlf = "one\r\ntwo\r\nthree\r\n".repeat(40);@@ -7543,199 +7457,4 @@ mod tests {assert_eq!(stream(b"hello \xe2\x82").await, None, "truncated at eof");assert_eq!(stream(b"plain \x1b$B text").await, None, "iso-2022 escape");}-- /// reproduction of issue #50785- fn build_pcm16_wav_bytes() -> Vec<u8> {- let header: Vec<u8> = vec![- /* RIFF header */- 0x52, 0x49, 0x46, 0x46, // "RIFF"- 0xc6, 0xcf, 0x00, 0x00, // file size: 8- 0x57, 0x41, 0x56, 0x45, // "WAVE"- /* fmt chunk */- 0x66, 0x6d, 0x74, 0x20, // "fmt "- 0x10, 0x00, 0x00, 0x00, // chunk size: 16- 0x01, 0x00, // format: PCM (1)- 0x01, 0x00, // channels: 1 (mono)- 0x80, 0x3e, 0x00, 0x00, // sample rate: 16000- 0x00, 0x7d, 0x00, 0x00, // byte rate: 32000- 0x02, 0x00, // block align: 2- 0x10, 0x00, // bits per sample: 16- /* LIST chunk */- 0x4c, 0x49, 0x53, 0x54, // "LIST"- 0x1a, 0x00, 0x00, 0x00, // chunk size: 26- 0x49, 0x4e, 0x46, 0x4f, // "INFO"- 0x49, 0x53, 0x46, 0x54, // "ISFT"- 0x0d, 0x00, 0x00, 0x00, // sub-chunk size: 13- 0x4c, 0x61, 0x76, 0x66, 0x36, 0x32, 0x2e, 0x33, // "Lavf62.3"- 0x2e, 0x31, 0x30, 0x30, 0x00, // ".100\0"- /* padding byte for word alignment */- 0x00, // data chunk header- 0x64, 0x61, 0x74, 0x61, // "data"- 0x80, 0xcf, 0x00, 0x00, // chunk size- ];-- let mut bytes = header;-- // fill remaining space up to `FILE_ANALYSIS_BYTES` with synthetic PCM- let audio_bytes_needed = FILE_ANALYSIS_BYTES - bytes.len();- for i in 0..(audio_bytes_needed / 2) {- let sample = (i & 0xFF) as u8;- bytes.push(sample); // low byte: varies- bytes.push(0x00); // high byte: zero for small values- }-- bytes- }-- #[test]- fn test_pcm16_wav_detected_as_binary() {- let wav_bytes = build_pcm16_wav_bytes();- assert_eq!(wav_bytes.len(), FILE_ANALYSIS_BYTES);-- let result = analyze_byte_content(&wav_bytes);- assert_eq!(- result,- ByteContent::Binary,- "PCM 16-bit WAV should be detected as Binary via RIFF header"- );- }-- #[test]- fn test_le16_binary_not_misdetected_as_utf16le() {- let mut bytes = b"FAKE".to_vec();- while bytes.len() < FILE_ANALYSIS_BYTES {- let sample = (bytes.len() & 0xFF) as u8;- bytes.push(sample);- bytes.push(0x00);- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- let result = analyze_byte_content(&bytes);- assert_eq!(- result,- ByteContent::Binary,- "LE 16-bit binary with control characters should be detected as Binary"- );- }-- #[test]- fn test_be16_binary_not_misdetected_as_utf16be() {- let mut bytes = b"FAKE".to_vec();- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.push(0x00);- let sample = (bytes.len() & 0xFF) as u8;- bytes.push(sample);- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- let result = analyze_byte_content(&bytes);- assert_eq!(- result,- ByteContent::Binary,- "BE 16-bit binary with control characters should be detected as Binary"- );- }-- // Mimics binary formats that interleave short ASCII fragments with small- // length/type fields (as seen in some game/asset binary formats, e.g.- // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's- // null-byte pattern for ASCII, but the low bytes are mostly non-word- // "tag" values rather than real letters/digits/spaces.- fn build_tag_interleaved_binary_bytes() -> Vec<u8> {- let mut bytes = Vec::new();- let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];- let mut i = 0;- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.push(tags[i % tags.len()]);- bytes.push(0x00);- i += 1;- }- bytes.truncate(FILE_ANALYSIS_BYTES);- bytes- }-- #[test]- fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {- let bytes = build_tag_interleaved_binary_bytes();- assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);-- let result = analyze_byte_content(&bytes);- assert_eq!(- result,- ByteContent::Binary,- "binary data with sparse non-word low bytes and null high bytes \- should not be misdetected as UTF-16LE text"- );- }-- #[test]- fn test_utf16le_text_detected_as_utf16le() {- let text = "Hello, world! This is a UTF-16 test string. ";- let mut bytes = Vec::new();- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);- }-- #[test]- fn test_utf16be_text_detected_as_utf16be() {- let text = "Hello, world! This is a UTF-16 test string. ";- let mut bytes = Vec::new();- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);- }-- #[test]- fn test_utf16le_cyrillic_text_detected_as_utf16le() {- let text = "Привет, мир! Это тестовая строка в UTF-16. ";- let mut bytes = Vec::new();- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);- }-- #[test]- fn test_utf16be_greek_text_detected_as_utf16be() {- let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";- let mut bytes = Vec::new();- while bytes.len() < FILE_ANALYSIS_BYTES {- bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));- }- bytes.truncate(FILE_ANALYSIS_BYTES);-- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);- }-- #[test]- fn test_known_binary_headers() {- let cases: &[(&[u8], &str)] = &[- (b"RIFF\x00\x00\x00\x00WAVE", "WAV"),- (b"RIFF\x00\x00\x00\x00AVI ", "AVI"),- (b"OggS\x00\x02", "OGG"),- (b"fLaC\x00\x00", "FLAC"),- (b"ID3\x03\x00", "MP3 ID3v2"),- (b"\xFF\xFB\x90\x00", "MP3 MPEG1 Layer3"),- (b"\xFF\xF3\x90\x00", "MP3 MPEG2 Layer3"),- ];-- for (header, label) in cases {- let mut bytes = header.to_vec();- bytes.resize(FILE_ANALYSIS_BYTES, 0x41); // pad with 'A'- assert_eq!(- analyze_byte_content(&bytes),- ByteContent::Binary,- "{label} should be detected as Binary"- );- }- }}