Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Speed up project search over misclassified binary files (#64549)

Kirill Bulatov8bbcd8396863
diff --git a/Cargo.lock b/Cargo.lock
index 4cd4be6cbad..da2a5543fe0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6421,6 +6421,16 @@ dependencies = [
"subtle",
]
+[[package]]
+name = "file_content"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chardetng",
+ "encoding_rs",
+ "futures-lite 2.6.1",
+]
+
[[package]]
name = "file_finder"
version = "0.1.0"
@@ -7146,6 +7156,7 @@ dependencies = [
"async-trait",
"collections",
"derive_more",
+ "file_content",
"futures 0.3.34",
"gpui",
"http_client",
@@ -9575,7 +9586,6 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
- "chardetng",
"clock",
"collections",
"criterion",
@@ -9583,6 +9593,7 @@ dependencies = [
"diffy",
"ec4rs",
"encoding_rs",
+ "file_content",
"fs",
"futures 0.3.34",
"futures-lite 2.6.1",
@@ -14116,6 +14127,7 @@ dependencies = [
"encoding_rs",
"extension",
"fancy-regex",
+ "file_content",
"fs",
"futures 0.3.34",
"fuzzy",
@@ -22327,6 +22339,7 @@ dependencies = [
"clock",
"collections",
"encoding_rs",
+ "file_content",
"fs",
"futures 0.3.34",
"futures-lite 2.6.1",
diff --git a/Cargo.toml b/Cargo.toml
index 1a61f86114d..c52733570bc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -79,6 +79,7 @@ members = [
"crates/feature_flags",
"crates/feature_flags_macros",
"crates/feedback",
+ "crates/file_content",
"crates/file_finder",
"crates/file_icons",
"crates/fs",
@@ -351,6 +352,7 @@ extensions_ui = { path = "crates/extensions_ui" }
feature_flags = { path = "crates/feature_flags" }
feature_flags_macros = { path = "crates/feature_flags_macros" }
feedback = { path = "crates/feedback" }
+file_content = { path = "crates/file_content" }
file_finder = { path = "crates/file_finder" }
file_icons = { path = "crates/file_icons" }
fs = { path = "crates/fs" }
@@ -1087,6 +1089,7 @@ command_palette = { codegen-units = 1 }
command_palette_hooks = { codegen-units = 1 }
edit_prediction_ui = { codegen-units = 1 }
feature_flags = { codegen-units = 1 }
+file_content = { codegen-units = 1 }
file_icons = { codegen-units = 1 }
image_viewer = { codegen-units = 1 }
install_cli = { codegen-units = 1 }
diff --git a/crates/file_content/Cargo.toml b/crates/file_content/Cargo.toml
new file mode 100644
index 00000000000..7f5f268fcbd
--- /dev/null
+++ b/crates/file_content/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "file_content"
+version = "0.1.0"
+edition.workspace = true
+publish.workspace = true
+license = "GPL-3.0-or-later"
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/file_content.rs"
+doctest = false
+
+[dependencies]
+anyhow.workspace = true
+chardetng.workspace = true
+encoding_rs.workspace = true
+futures-lite.workspace = true
diff --git a/crates/file_content/LICENSE-GPL b/crates/file_content/LICENSE-GPL
new file mode 120000
index 00000000000..89e542f750c
--- /dev/null
+++ b/crates/file_content/LICENSE-GPL
@@ -0,0 +1 @@
+../../LICENSE-GPL
\ No newline at end of file
diff --git a/crates/file_content/src/file_content.rs b/crates/file_content/src/file_content.rs
new file mode 100644
index 00000000000..b978f4aa40e
--- /dev/null
+++ b/crates/file_content/src/file_content.rs
@@ -0,0 +1,1072 @@
+//! Classifies file contents as text or binary and decodes text of any
+//! supported encoding into UTF-8, both whole and as a stream.
+
+use std::io::{self, Read};
+
+use anyhow::Result;
+use chardetng::EncodingDetector;
+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE};
+use futures_lite::future::yield_now;
+
+/// Number of leading bytes that [`analyze_byte_content`] inspects.
+pub const FILE_ANALYSIS_BYTES: usize = 1024;
+
+const DECODING_BLOCK_BYTES: usize = 64 * 1024;
+
+/// Text produced by [`decode_text`], with what is needed to write it back via [`encode_text`].
+pub struct DecodedText {
+ /// The decoded UTF-8 text.
+ pub text: String,
+ /// The encoding the bytes were decoded from.
+ pub encoding: &'static Encoding,
+ /// Whether the bytes started with a byte order mark.
+ pub has_bom: bool,
+}
+
+pub fn read_byte_header(reader: &mut dyn Read) -> io::Result<(Vec<u8>, bool)> {
+ let mut header = vec![0; FILE_ANALYSIS_BYTES];
+ let mut length = 0;
+ while length < header.len() {
+ let read = match reader.read(&mut header[length..]) {
+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+ result => result?,
+ };
+ if read == 0 {
+ header.truncate(length);
+ return Ok((header, true));
+ }
+ length += read;
+ }
+ Ok((header, false))
+}
+
+/// Inspects the start of a file.
+/// Returns the BOM encoding, if any, and otherwise the [`ByteContent`] classification of the prefix.
+pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
+ if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
+ return (Some(encoding), ByteContent::Unknown);
+ }
+ (None, analyze_byte_content(prefix))
+}
+
+/// Decodes a whole file into UTF-8, detecting its encoding.
+/// Fails when the bytes are classified as [`ByteContent::Binary`].
+pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {
+ let (bom_encoding, byte_content) = decode_byte_header(&bytes);
+ if let Some(encoding) = bom_encoding {
+ let (text, _) = encoding.decode_with_bom_removal(&bytes);
+ return Ok(DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: true,
+ });
+ }
+
+ if let Some(encoding) = byte_content.encoding() {
+ let (text, _, _) = encoding.decode(&bytes);
+ return Ok(DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: false,
+ });
+ }
+ anyhow::ensure!(
+ byte_content != ByteContent::Binary,
+ "Binary files cannot be decoded into text"
+ );
+
+ match String::from_utf8(bytes) {
+ Ok(text) if !text.contains('\x1b') => Ok(DecodedText {
+ text,
+ encoding: UTF_8,
+ has_bom: false,
+ }),
+ Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),
+ Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),
+ }
+}
+
+pub async fn detect_encoding(reader: &mut (dyn Read + Send)) -> io::Result<&'static Encoding> {
+ let mut detector = EncodingDetector::new();
+ let mut block = [0; DECODING_BLOCK_BYTES];
+ let mut length = 0;
+ loop {
+ let read = match reader.read(&mut block[length..]) {
+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+ result => result?,
+ };
+ if read == 0 {
+ detector.feed(&block[..length], true);
+ return Ok(detector.guess(None, true));
+ }
+ length += read;
+ if length == block.len() {
+ detector.feed(&block, false);
+ yield_now().await;
+ length = 0;
+ }
+ }
+}
+
+/// Inverse of [`decode_text`]: encodes UTF-8 text into `encoding`, prepending a BOM when asked.
+pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {
+ if encoding == UTF_8 && !has_bom {
+ return text.into_bytes();
+ }
+
+ // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.
+ if encoding == UTF_16BE {
+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
+ if has_bom {
+ bytes.extend_from_slice(&[0xFE, 0xFF]);
+ }
+ bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));
+ return bytes;
+ }
+
+ if encoding == UTF_16LE {
+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
+ if has_bom {
+ bytes.extend_from_slice(&[0xFF, 0xFE]);
+ }
+ bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));
+ return bytes;
+ }
+
+ let (encoded, _, _) = encoding.encode(&text);
+ if has_bom && encoding == UTF_8 {
+ let mut bytes = Vec::with_capacity(encoded.len() + 3);
+ bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);
+ bytes.extend_from_slice(&encoded);
+ bytes
+ } else {
+ encoded.into_owned()
+ }
+}
+
+/// A [`Read`] adapter that transcodes its source into UTF-8 block by block, with bounded memory.
+pub struct DecodingReader<'a> {
+ inner: &'a mut (dyn Read + Send),
+ decoder: Decoder,
+ input: Vec<u8>,
+ output: Vec<u8>,
+ output_position: usize,
+ finished: bool,
+}
+
+impl<'a> DecodingReader<'a> {
+ pub fn new(inner: &'a mut (dyn Read + Send), encoding: &'static Encoding) -> Self {
+ Self {
+ inner,
+ decoder: encoding.new_decoder(),
+ input: vec![0; DECODING_BLOCK_BYTES],
+ output: Vec::new(),
+ output_position: 0,
+ finished: false,
+ }
+ }
+
+ pub fn encoding(&self) -> &'static Encoding {
+ self.decoder.encoding()
+ }
+
+ fn fill_output(&mut self) -> io::Result<()> {
+ self.output.clear();
+ self.output_position = 0;
+ let read = self.inner.read(&mut self.input)?;
+ self.finished = read == 0;
+ decode_block(
+ &mut self.decoder,
+ &self.input[..read],
+ self.finished,
+ &mut self.output,
+ )
+ }
+}
+
+impl Read for DecodingReader<'_> {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ if buffer.is_empty() {
+ return Ok(0);
+ }
+ while self.output_position == self.output.len() {
+ if self.finished {
+ return Ok(0);
+ }
+ match self.fill_output() {
+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+ result => result?,
+ }
+ }
+
+ let available = &self.output[self.output_position..];
+ let count = available.len().min(buffer.len());
+ buffer[..count].copy_from_slice(&available[..count]);
+ self.output_position += count;
+ Ok(count)
+ }
+}
+
+/// Verdict of [`analyze_byte_content`] on a file prefix.
+#[derive(Debug, PartialEq)]
+pub enum ByteContent {
+ /// UTF-16 little-endian text without a BOM.
+ Utf16Le,
+ /// UTF-16 big-endian text without a BOM.
+ Utf16Be,
+ /// Not text; the file must not be decoded or searched.
+ Binary,
+ /// Text in UTF-8 or some yet unknown encoding.
+ Unknown,
+}
+
+impl ByteContent {
+ /// The encoding this classification pins down, if any.
+ pub fn encoding(&self) -> Option<&'static Encoding> {
+ match self {
+ Self::Utf16Le => Some(UTF_16LE),
+ Self::Utf16Be => Some(UTF_16BE),
+ Self::Binary | Self::Unknown => None,
+ }
+ }
+}
+
+/// Classifies the first [`FILE_ANALYSIS_BYTES`] of a file.
+///
+/// Heuristic check using null byte distribution plus a generic text-likeness
+/// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise
+/// distinguishes between text-like and binary-like content.
+pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent {
+ if bytes.len() < 2 {
+ return ByteContent::Unknown;
+ }
+
+ if has_binary_header(bytes, KNOWN_BINARY_HEADERS) {
+ return ByteContent::Binary;
+ }
+
+ let limit = bytes.len().min(FILE_ANALYSIS_BYTES);
+ let mut even_null_count = 0usize;
+ let mut odd_null_count = 0usize;
+ let mut non_text_like_count = 0usize;
+
+ for (i, &byte) in bytes[..limit].iter().enumerate() {
+ if byte == 0 {
+ if i % 2 == 0 {
+ even_null_count += 1;
+ } else {
+ odd_null_count += 1;
+ }
+ non_text_like_count += 1;
+ continue;
+ }
+
+ let is_text_like = match byte {
+ b'\t' | b'\n' | b'\r' | 0x0C => true,
+ 0x20..=0x7E | 0x80..=0xBF | 0xC2..=0xF4 => true,
+ _ => false,
+ };
+
+ if !is_text_like {
+ non_text_like_count += 1;
+ }
+ }
+
+ let total_null_count = even_null_count + odd_null_count;
+ let has_significant_nulls = total_null_count > 0 && total_null_count >= limit / 16;
+ let nulls_skew_to_even = even_null_count > odd_null_count * 4;
+ let nulls_skew_to_odd = odd_null_count > even_null_count * 4;
+
+ if has_significant_nulls {
+ let sample = &bytes[..limit];
+
+ // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first)
+ // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first)
+
+ if nulls_skew_to_even && is_plausible_utf16_text(sample, false) {
+ return ByteContent::Utf16Be;
+ }
+
+ if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) {
+ return ByteContent::Utf16Le;
+ }
+
+ return ByteContent::Binary;
+ }
+
+ if has_binary_header(bytes, ADDITIONAL_BINARY_HEADERS)
+ || is_bitmap_header(bytes)
+ || is_gguf_header(bytes)
+ || is_safetensors_header(bytes)
+ {
+ return ByteContent::Binary;
+ }
+
+ if total_null_count == 0 || non_text_like_count * 100 < limit * 8 {
+ ByteContent::Unknown
+ } else {
+ ByteContent::Binary
+ }
+}
+
+fn decode_block(
+ decoder: &mut Decoder,
+ block: &[u8],
+ last: bool,
+ output: &mut Vec<u8>,
+) -> io::Result<()> {
+ let capacity = decoder
+ .max_utf8_buffer_length(block.len())
+ .ok_or_else(|| io::Error::new(io::ErrorKind::OutOfMemory, "decoded block is too large"))?;
+ let start = output.len();
+ output.resize(start + capacity, 0);
+ let (result, consumed, written, _) = decoder.decode_to_utf8(block, &mut output[start..], last);
+ output.truncate(start + written);
+ if result != CoderResult::InputEmpty || consumed != block.len() {
+ return Err(io::Error::other("decoder did not consume the whole block"));
+ }
+ Ok(())
+}
+
+fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {
+ let mut detector = EncodingDetector::new();
+ detector.feed(&bytes, true);
+ let encoding = detector.guess(None, true);
+ let (text, _, _) = encoding.decode(&bytes);
+ DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: false,
+ }
+}
+
+const KNOWN_BINARY_HEADERS: &[&[u8]] = &[
+ b"%PDF-", // PDF
+ b"GIF87a", // GIF87a
+ b"GIF89a", // GIF89a
+ b"ID3", // MP3 with ID3v2 tag
+ b"IWAD", // Doom IWAD archive
+ b"OggS", // OGG (Vorbis, Opus, FLAC)
+ b"PK\x03\x04", // ZIP local header
+ b"PK\x05\x06", // ZIP end of central directory
+ b"PK\x07\x08", // ZIP spanning/splitting
+ b"PWAD", // Doom PWAD archive
+ b"RIFF", // WAV, AVI, WebP
+ b"fLaC", // FLAC
+ b"\x89PNG\r\n\x1a\n", // PNG
+ b"\xFF\xD8\xFF", // JPEG
+ b"\xFF\xF2", // MP3 frame sync (MPEG2 Layer3)
+ b"\xFF\xF3", // MP3 frame sync (MPEG2 Layer3)
+ b"\xFF\xFA", // MP3 frame sync (MPEG1 Layer3)
+ b"\xFF\xFB", // MP3 frame sync (MPEG1 Layer3)
+];
+
+const ADDITIONAL_BINARY_HEADERS: &[&[u8]] = &[
+ b"\x00\x00\x00\x0CJXL \r\n\x87\n", // JPEG XL container
+ b"\x00\x00\x00\x0CjP ", // JPEG 2000
+ b"\x00\x00\x01\x00", // Windows ICO
+ b"\x00\x00\x02\x00", // Windows CUR
+ b"\x00\x01\x00\x00\x00", // TrueType font
+ b"\x00asm", // WebAssembly
+ b"\x04\x22\x4D\x18", // LZ4 frame
+ b"\x1A\x45\xDF\xA3", // Matroska, WebM
+ b"\x1F\x8B\x08", // gzip
+ b"7z\xBC\xAF\x27\x1C", // 7-Zip
+ b"8BPS\x00", // Photoshop PSD
+ b"DDS |\x00\x00\x00", // DirectDraw Surface
+ b"II*\x00", // TIFF little-endian
+ b"Kaydara FBX Binary \x00", // FBX binary
+ b"MM\x00*", // TIFF big-endian
+ b"MSCF\x00\x00\x00\x00", // Microsoft Cabinet
+ b"OTTO\x00", // OpenType font with CFF outlines
+ b"Rar!\x1A\x07", // RAR
+ b"SQLite format 3\x00", // SQLite database
+ b"glTF\x01\x00\x00\x00", // Binary glTF 1
+ b"glTF\x02\x00\x00\x00", // Binary glTF 2
+ b"\x7FELF", // ELF
+ b"\x89HDF\r\n\x1a\n", // HDF5
+ b"\x93NUMPY\x01\x00", // NumPy array
+ b"\x93NUMPY\x02\x00",
+ b"\x93NUMPY\x03\x00",
+ b"\x95\x04\x12\xDE", // gettext MO big-endian
+ b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04", // pcap big-endian
+ b"\xABKTX 11\xBB\r\n\x1A\n", // KTX texture
+ b"\xABKTX 20\xBB\r\n\x1A\n",
+ b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1", // OLE2 compound file (legacy MS Office)
+ b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00", // pcap little-endian
+ b"\xDE\x12\x04\x95", // gettext MO little-endian
+ b"\xFD7zXZ\x00", // xz
+];
+
+fn has_binary_header(bytes: &[u8], headers: &[&[u8]]) -> bool {
+ let index = headers.partition_point(|header| *header <= bytes);
+ headers[..index]
+ .last()
+ .is_some_and(|header| bytes.starts_with(header))
+}
+
+fn is_gguf_header(bytes: &[u8]) -> bool {
+ bytes.starts_with(b"GGUF") && bytes.get(5..8) == Some(&[0, 0, 0])
+}
+
+fn is_bitmap_header(bytes: &[u8]) -> bool {
+ bytes.starts_with(b"BM") && bytes.get(6..10) == Some(&[0, 0, 0, 0])
+}
+
+fn is_safetensors_header(bytes: &[u8]) -> bool {
+ bytes.get(3..8) == Some(&[0, 0, 0, 0, 0]) && bytes.get(8) == Some(&b'{')
+}
+
+// Null byte skew alone is not enough to identify UTF-16 -- binary formats with
+// small 16-bit values (like PCM audio) produce the same pattern. Decode the
+// bytes as UTF-16 and reject if too many code units land in control character
+// ranges or form unpaired surrogates, which real text almost never contains.
+fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {
+ let mut suspicious_count = 0usize;
+ let mut word_like_count = 0usize;
+ let mut total = 0usize;
+
+ let mut i = 0;
+ while let Some(code_unit) = read_u16(bytes, i, little_endian) {
+ total += 1;
+
+ match code_unit {
+ 0x0009 | 0x000A | 0x000C | 0x000D => {}
+ 0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {
+ word_like_count += 1;
+ }
+ // C0/C1 control characters and non-characters
+ 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,
+ 0xD800..=0xDBFF => {
+ let next_offset = i + 2;
+ let has_low_surrogate = read_u16(bytes, next_offset, little_endian)
+ .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));
+ if has_low_surrogate {
+ total += 1;
+ word_like_count += 2;
+ i += 2;
+ } else {
+ suspicious_count += 1;
+ }
+ }
+ // Lone low surrogate without a preceding high surrogate
+ 0xDC00..=0xDFFF => suspicious_count += 1,
+ 0x0100.. => word_like_count += 1,
+ _ => {}
+ }
+
+ i += 2;
+ }
+
+ if total == 0 {
+ return false;
+ }
+
+ // Real UTF-16 text has near-zero control characters; binary data with
+ // small 16-bit values typically exceeds 5%. 2% provides a safe margin.
+ let low_control_ratio = suspicious_count * 100 < total * 2;
+
+ // Binary formats that interleave short ASCII fragments with small
+ // length/type fields (e.g. game asset formats) can dodge the control
+ // character check above while barely containing any real words: their
+ // code units land on ASCII punctuation and Latin-1 symbol values rather
+ // than letters, digits, or spaces. Real text is overwhelmingly made of
+ // word characters, so require a minimum share of them. Code units above
+ // the Latin-1 range (and surrogate pairs) also count as word-like so
+ // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,
+ // are still recognized -- tag bytes paired with a zero byte can never
+ // land there.
+ let enough_word_chars = word_like_count * 100 >= total * 30;
+
+ low_control_ratio && enough_word_chars
+}
+
+fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {
+ let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?];
+ if little_endian {
+ return Some(u16::from_le_bytes(pair));
+ }
+ Some(u16::from_be_bytes(pair))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use futures_lite::future::{block_on, poll_once};
+
+ #[test]
+ fn decodes_and_encodes_legacy_text() {
+ for (expected, encoding) in [
+ (
+ String::from("“NUMPY is useful for array processing.”\n"),
+ encoding_rs::WINDOWS_1252,
+ ),
+ (
+ String::from(
+ "«KTX est un format de texture déjà utilisé à côté des images françaises.»\n",
+ ),
+ encoding_rs::WINDOWS_1252,
+ ),
+ (
+ String::from(
+ "〔迷宫〕这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n",
+ ),
+ encoding_rs::GBK,
+ ),
+ (
+ String::from(
+ "€\x02£\x02Symboles monétaires déjà utilisés à côté des factures françaises.\n",
+ ),
+ encoding_rs::WINDOWS_1252,
+ ),
+ (
+ String::from(
+ "\x1fќерка\x1fОва е обичен македонски текст за проверка на кодирањето.\n",
+ ),
+ encoding_rs::WINDOWS_1251,
+ ),
+ (
+ format!(
+ "я{}{}",
+ "x".repeat(1024 * 1024),
+ "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(1000)
+ ),
+ encoding_rs::WINDOWS_1251,
+ ),
+ (
+ String::from("ясно, что это обычный русский текст. Сегодня хорошая погода.\n"),
+ encoding_rs::WINDOWS_1251,
+ ),
+ (
+ String::from("ящик содержит обычный русский текст. Сегодня хорошая погода.\n"),
+ encoding_rs::WINDOWS_1251,
+ ),
+ (
+ format!(
+ "\x1b[0m{}{}",
+ "x".repeat(1024 * 1024),
+ " déjà été à côté français ".repeat(1000)
+ ),
+ encoding_rs::WINDOWS_1252,
+ ),
+ ] {
+ let (bytes, _, _) = encoding.encode(&expected);
+ let decoded = decode_text(bytes.clone().into_owned()).unwrap();
+
+ assert_eq!(decoded.encoding, encoding);
+ assert_eq!(decoded.text, expected);
+ assert!(!decoded.has_bom);
+ assert_eq!(
+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),
+ bytes.as_ref()
+ );
+ }
+ }
+
+ #[test]
+ fn preserves_unicode_boms() {
+ let expected = "七个人上山,下个月再来。";
+ for encoding in [UTF_8, UTF_16LE, UTF_16BE] {
+ let bytes = encode_text(expected.to_owned(), encoding, true);
+ assert_eq!(
+ decode_byte_header(&bytes),
+ (Some(encoding), ByteContent::Unknown)
+ );
+ let decoded = decode_text(bytes.clone()).unwrap();
+
+ assert_eq!(decoded.text, expected);
+ assert_eq!(decoded.encoding, encoding);
+ assert!(decoded.has_bom);
+ assert_eq!(
+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),
+ bytes
+ );
+ }
+ }
+
+ #[test]
+ fn detects_known_binary_headers() {
+ let cases: &[(&[u8], &str)] = &[
+ (b"II*\x00\x08\x00\x00\x00", "TIFF little endian"),
+ (b"MM\x00*\x00\x00\x00\x08", "TIFF big endian"),
+ (b"glTF\x02\x00\x00\x00\x10\x00\x00\x00", "glTF binary"),
+ (b"\x7FELF\x02\x01\x01\x00", "ELF"),
+ (b"\x1F\x8B\x08\x00", "gzip"),
+ (b"SQLite format 3\x00", "SQLite"),
+ (b"\x93NUMPY\x01\x00\x76\x00{'descr': '|u1'", "NumPy 1"),
+ (b"\x93NUMPY\x02\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 2"),
+ (b"\x93NUMPY\x03\x00\x74\x00\x00\x00{'descr': '|u1'", "NumPy 3"),
+ (b"\xABKTX 11\xBB\r\n\x1A\n\x04\x03\x02\x01", "KTX 1"),
+ (b"\xABKTX 20\xBB\r\n\x1A\n\x00\x00\x00\x00", "KTX 2"),
+ (
+ b"\xA1\xB2\xC3\xD4\x00\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x00\x01",
+ "pcap big endian",
+ ),
+ (
+ b"\xD4\xC3\xB2\xA1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\x00\x00\x01\x00\x00\x00",
+ "pcap little endian",
+ ),
+ (b"GGUF\x03\x00\x00\x00", "GGUF"),
+ (b"BM\x36\x10\x0E\x00\x00\x00\x00\x00\x36\x00\x00\x00", "BMP"),
+ (
+ b"\x90\x01\x00\x00\x00\x00\x00\x00{\"__metadata__\"",
+ "safetensors",
+ ),
+ ];
+ for (header, label) in cases {
+ let mut bytes = header.to_vec();
+ bytes.resize(FILE_ANALYSIS_BYTES, b'A');
+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Binary, "{label}");
+ assert_eq!(
+ decode_text(bytes).err().map(|error| error.to_string()),
+ Some(String::from("Binary files cannot be decoded into text")),
+ "{label}"
+ );
+ }
+ assert_eq!(
+ analyze_byte_content(b"\x28\xB5\x2F\xFD\x20\x00\x01\x00\x00"),
+ ByteContent::Binary
+ );
+ assert_eq!(
+ analyze_byte_content(
+ b"\x28\xB5\x2F\xFD\x20\x20\x01\x01\x00needle-abcdefghijklmnopqrstuvwxy"
+ ),
+ ByteContent::Binary
+ );
+ for text in [
+ "FLV\x01video/x-flv\x01Flash video\n",
+ "v/1\x01api/users\x01version one route\n",
+ "!<arch>\nThis documents the archive identifier.\n",
+ "glTF Sample Models\n",
+ "OTTO GmbH\n",
+ "GGUF is a format\n",
+ "wOFF = 1\n",
+ "wOF2 = 1\n",
+ ] {
+ assert_eq!(
+ analyze_byte_content(text.as_bytes()),
+ ByteContent::Unknown,
+ "{text:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn binary_headers_are_sorted_and_prefix_free() {
+ for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {
+ for pair in headers.windows(2) {
+ assert!(pair[0] < pair[1], "{pair:?}");
+ assert_eq!(pair[1].strip_prefix(pair[0]), None, "{pair:?}");
+ }
+ }
+ }
+
+ #[test]
+ fn binary_header_lookup_matches_linear_scan() {
+ for headers in [KNOWN_BINARY_HEADERS, ADDITIONAL_BINARY_HEADERS] {
+ let check = |bytes: &[u8]| {
+ let expected = headers.iter().any(|header| bytes.starts_with(header));
+ assert_eq!(has_binary_header(bytes, headers), expected, "{bytes:?}");
+ };
+
+ for header in headers {
+ for length in 0..=header.len() {
+ check(&header[..length]);
+ }
+ for suffix in [0, u8::MAX] {
+ let mut bytes = header.to_vec();
+ bytes.push(suffix);
+ check(&bytes);
+ }
+ for index in 0..header.len() {
+ let mut bytes = header.to_vec();
+ for byte in 0..=u8::MAX {
+ bytes[index] = byte;
+ check(&bytes);
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn classifies_nul_free_content() {
+ let mut protobuf_like = Vec::new();
+ while protobuf_like.len() < FILE_ANALYSIS_BYTES {
+ protobuf_like.extend_from_slice(b"\x0a\x0c/conv1/Conv_0\x12\x02\x08\x01\x1a\x04Conv\x22\x06\x08\x03\x10\x05\x18\x07");
+ }
+ let random_like = pseudo_random_bytes(FILE_ANALYSIS_BYTES)
+ .into_iter()
+ .map(|byte| if byte == 0 { 1 } else { byte })
+ .collect::<Vec<_>>();
+ let russian_text = "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(20);
+ let (russian, _, _) = encoding_rs::WINDOWS_1251.encode(&russian_text);
+ let chinese_text =
+ "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20);
+ let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);
+ let ansi_log =
+ "\x1b[32mINFO\x1b[0m \x1b[1m[server]\x1b[0m ok id=\x1b[33m42\x1b[0m\n".repeat(40);
+ let control_delimited = "alpha\x01beta\x01gamma\x01delta\x02\n".repeat(60);
+ let overstrike =
+ "N\x08NA\x08AM\x08ME\x08E\n ls - list directory contents\n".repeat(30);
+
+ let cases: [(&str, &[u8], ByteContent); 7] = [
+ ("protobuf-like", &protobuf_like, ByteContent::Unknown),
+ ("random without NUL", &random_like, ByteContent::Unknown),
+ ("windows-1251", &russian, ByteContent::Unknown),
+ ("gbk", &chinese, ByteContent::Unknown),
+ ("ansi log", ansi_log.as_bytes(), ByteContent::Unknown),
+ (
+ "control delimited",
+ control_delimited.as_bytes(),
+ ByteContent::Unknown,
+ ),
+ (
+ "nroff overstrike",
+ overstrike.as_bytes(),
+ ByteContent::Unknown,
+ ),
+ ];
+ for (label, bytes, expected) in cases {
+ assert_eq!(analyze_byte_content(bytes), expected, "{label}");
+ }
+
+ let thai = "สวัสดีชาวโลกนี่คือข้อความทดสอบภาษาไทย";
+ for encoding in [UTF_16LE, UTF_16BE] {
+ let bytes = encode_text(String::from(thai), encoding, false);
+ assert_eq!(decode_byte_header(&bytes), (None, ByteContent::Unknown));
+ assert_eq!(decode_text(bytes.clone()).unwrap().encoding, UTF_8);
+ assert_eq!(encoding.decode(&bytes).0, thai);
+ }
+ }
+
+ #[test]
+ fn streaming_decoder_matches_whole_file_decoding() {
+ let chinese_text =
+ "这是一个用于测试编码检测的中文句子,其中不包含任何西文字符。\n".repeat(20_000);
+ let (chinese, _, _) = encoding_rs::GBK.encode(&chinese_text);
+ assert!(chinese.len() > 1024 * 1024);
+ let (russian, _, _) =
+ encoding_rs::WINDOWS_1251.encode("Съешь же ещё этих мягких французских булок\n");
+ let mut ascii_then_russian = "plain ascii line\n".repeat(5_000).into_bytes();
+ ascii_then_russian.extend_from_slice(&russian);
+ let mut utf16 = vec![0xFF, 0xFE];
+ utf16.extend(
+ "utf-16 text\n"
+ .repeat(100)
+ .encode_utf16()
+ .flat_map(u16::to_le_bytes),
+ );
+
+ let interior_bom = "prefix  déjà été à côté français ".repeat(10);
+ let (interior_bom_bytes, _, _) = encoding_rs::WINDOWS_1252.encode(&interior_bom);
+ assert_eq!(
+ decode_text(interior_bom_bytes.to_vec()).unwrap().text,
+ interior_bom
+ );
+
+ for (label, bytes) in [
+ ("large gbk", chinese.into_owned()),
+ ("ascii prefix then windows-1251", ascii_then_russian),
+ ("utf-16 with bom", utf16),
+ ("pure ascii", b"just ascii\n".repeat(3)),
+ ("empty", Vec::new()),
+ ("truncated utf-16", b"\xFF\xFEa\0b".to_vec()),
+ (
+ "short utf-16le",
+ encode_text(String::from("needleé"), UTF_16LE, false),
+ ),
+ (
+ "short utf-16be",
+ encode_text(String::from("needleé"), UTF_16BE, false),
+ ),
+ (
+ "utf-16be fullwidth asterisk",
+ encode_text(
+ String::from("* This is a fullwidth asterisk followed by ordinary text.\n"),
+ UTF_16BE,
+ false,
+ ),
+ ),
+ (
+ "utf-16be pickle prefix",
+ encode_text(
+ String::from("考试 instructions for tomorrow\n"),
+ UTF_16BE,
+ false,
+ ),
+ ),
+ (
+ "utf-16be ftyp",
+ encode_text(
+ String::from("AB晴祰 ordinary UTF-16 text\n"),
+ UTF_16BE,
+ false,
+ ),
+ ),
+ (
+ "interior BOM in windows-1252",
+ interior_bom_bytes.into_owned(),
+ ),
+ (
+ "interior UTF-8 BOM",
+ "prefix \u{feff}suffix\x1b[0m".as_bytes().to_vec(),
+ ),
+ ] {
+ let expected = decode_text(bytes.clone()).unwrap();
+ for read_limit in [usize::MAX, 8 * 1024, 7] {
+ let mut source = ShortReads {
+ inner: io::Cursor::new(bytes.clone()),
+ limit: 1,
+ };
+ let (header, _) = read_byte_header(&mut source).unwrap();
+ let (bom_encoding, byte_content) = decode_byte_header(&header);
+ assert_ne!(byte_content, ByteContent::Binary, "{label}");
+ source.inner.set_position(0);
+ source.limit = read_limit;
+ let encoding = bom_encoding
+ .or(byte_content.encoding())
+ .unwrap_or_else(|| block_on(detect_encoding(&mut source)).unwrap());
+ source.inner.set_position(0);
+ let mut reader = DecodingReader::new(&mut source, encoding);
+ let mut streamed = String::new();
+ reader.read_to_string(&mut streamed).unwrap();
+ assert_eq!(
+ streamed, expected.text,
+ "{label}, read_limit = {read_limit}"
+ );
+ assert_eq!(
+ reader.encoding(),
+ expected.encoding,
+ "{label}, read_limit = {read_limit}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn encoding_detection_yields_between_blocks() {
+ let mut source = io::Cursor::new(vec![b'a'; 2 * DECODING_BLOCK_BYTES]);
+ assert!(block_on(poll_once(detect_encoding(&mut source))).is_none());
+ assert_eq!(source.position(), DECODING_BLOCK_BYTES as u64);
+ }
+
+ /// reproduction of issue #50785
+ fn build_pcm16_wav_bytes() -> Vec<u8> {
+ let header: Vec<u8> = vec![
+ /* RIFF header */
+ 0x52, 0x49, 0x46, 0x46, // "RIFF"
+ 0xc6, 0xcf, 0x00, 0x00, // file size: 8
+ 0x57, 0x41, 0x56, 0x45, // "WAVE"
+ /* fmt chunk */
+ 0x66, 0x6d, 0x74, 0x20, // "fmt "
+ 0x10, 0x00, 0x00, 0x00, // chunk size: 16
+ 0x01, 0x00, // format: PCM (1)
+ 0x01, 0x00, // channels: 1 (mono)
+ 0x80, 0x3e, 0x00, 0x00, // sample rate: 16000
+ 0x00, 0x7d, 0x00, 0x00, // byte rate: 32000
+ 0x02, 0x00, // block align: 2
+ 0x10, 0x00, // bits per sample: 16
+ /* LIST chunk */
+ 0x4c, 0x49, 0x53, 0x54, // "LIST"
+ 0x1a, 0x00, 0x00, 0x00, // chunk size: 26
+ 0x49, 0x4e, 0x46, 0x4f, // "INFO"
+ 0x49, 0x53, 0x46, 0x54, // "ISFT"
+ 0x0d, 0x00, 0x00, 0x00, // sub-chunk size: 13
+ 0x4c, 0x61, 0x76, 0x66, 0x36, 0x32, 0x2e, 0x33, // "Lavf62.3"
+ 0x2e, 0x31, 0x30, 0x30, 0x00, // ".100\0"
+ /* padding byte for word alignment */
+ 0x00, // data chunk header
+ 0x64, 0x61, 0x74, 0x61, // "data"
+ 0x80, 0xcf, 0x00, 0x00, // chunk size
+ ];
+
+ let mut bytes = header;
+
+ // fill remaining space up to `FILE_ANALYSIS_BYTES` with synthetic PCM
+ let audio_bytes_needed = FILE_ANALYSIS_BYTES - bytes.len();
+ for i in 0..(audio_bytes_needed / 2) {
+ let sample = (i & 0xFF) as u8;
+ bytes.push(sample); // low byte: varies
+ bytes.push(0x00); // high byte: zero for small values
+ }
+
+ bytes
+ }
+
+ #[test]
+ fn test_pcm16_wav_detected_as_binary() {
+ let wav_bytes = build_pcm16_wav_bytes();
+ assert_eq!(wav_bytes.len(), FILE_ANALYSIS_BYTES);
+
+ let result = analyze_byte_content(&wav_bytes);
+ assert_eq!(
+ result,
+ ByteContent::Binary,
+ "PCM 16-bit WAV should be detected as Binary via RIFF header"
+ );
+ }
+
+ #[test]
+ fn test_le16_binary_not_misdetected_as_utf16le() {
+ let mut bytes = b"FAKE".to_vec();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ let sample = (bytes.len() & 0xFF) as u8;
+ bytes.push(sample);
+ bytes.push(0x00);
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ let result = analyze_byte_content(&bytes);
+ assert_eq!(
+ result,
+ ByteContent::Binary,
+ "LE 16-bit binary with control characters should be detected as Binary"
+ );
+ }
+
+ #[test]
+ fn test_be16_binary_not_misdetected_as_utf16be() {
+ let mut bytes = b"FAKE".to_vec();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.push(0x00);
+ let sample = (bytes.len() & 0xFF) as u8;
+ bytes.push(sample);
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ let result = analyze_byte_content(&bytes);
+ assert_eq!(
+ result,
+ ByteContent::Binary,
+ "BE 16-bit binary with control characters should be detected as Binary"
+ );
+ }
+
+ // Mimics binary formats that interleave short ASCII fragments with small
+ // length/type fields (as seen in some game/asset binary formats, e.g.
+ // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's
+ // null-byte pattern for ASCII, but the low bytes are mostly non-word
+ // "tag" values rather than real letters/digits/spaces.
+ fn build_tag_interleaved_binary_bytes() -> Vec<u8> {
+ let mut bytes = Vec::new();
+ let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];
+ let mut i = 0;
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.push(tags[i % tags.len()]);
+ bytes.push(0x00);
+ i += 1;
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+ bytes
+ }
+
+ #[test]
+ fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {
+ let bytes = build_tag_interleaved_binary_bytes();
+ assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);
+
+ let result = analyze_byte_content(&bytes);
+ assert_eq!(
+ result,
+ ByteContent::Binary,
+ "binary data with sparse non-word low bytes and null high bytes \
+ should not be misdetected as UTF-16LE text"
+ );
+ }
+
+ #[test]
+ fn test_utf16le_text_detected_as_utf16le() {
+ let text = "Hello, world! This is a UTF-16 test string. ";
+ let mut bytes = Vec::new();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
+ }
+
+ #[test]
+ fn test_utf16be_text_detected_as_utf16be() {
+ let text = "Hello, world! This is a UTF-16 test string. ";
+ let mut bytes = Vec::new();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
+ }
+
+ #[test]
+ fn test_utf16le_cyrillic_text_detected_as_utf16le() {
+ let text = "Привет, мир! Это тестовая строка в UTF-16. ";
+ let mut bytes = Vec::new();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
+ }
+
+ #[test]
+ fn test_utf16be_greek_text_detected_as_utf16be() {
+ let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";
+ let mut bytes = Vec::new();
+ while bytes.len() < FILE_ANALYSIS_BYTES {
+ bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
+ }
+ bytes.truncate(FILE_ANALYSIS_BYTES);
+
+ assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
+ }
+
+ #[test]
+ fn test_known_binary_headers() {
+ let cases: &[(&[u8], &str)] = &[
+ (b"RIFF\x00\x00\x00\x00WAVE", "WAV"),
+ (b"RIFF\x00\x00\x00\x00AVI ", "AVI"),
+ (b"OggS\x00\x02", "OGG"),
+ (b"fLaC\x00\x00", "FLAC"),
+ (b"ID3\x03\x00", "MP3 ID3v2"),
+ (b"\xFF\xFB\x90\x00", "MP3 MPEG1 Layer3"),
+ (b"\xFF\xF3\x90\x00", "MP3 MPEG2 Layer3"),
+ ];
+
+ for (header, label) in cases {
+ let mut bytes = header.to_vec();
+ bytes.resize(FILE_ANALYSIS_BYTES, 0x41); // pad with 'A'
+ assert_eq!(
+ analyze_byte_content(&bytes),
+ ByteContent::Binary,
+ "{label} should be detected as Binary"
+ );
+ }
+ }
+
+ struct ShortReads<R> {
+ inner: R,
+ limit: usize,
+ }
+
+ impl<R: Read> Read for ShortReads<R> {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ let len = buffer.len().min(self.limit);
+ self.inner.read(&mut buffer[..len])
+ }
+ }
+
+ fn pseudo_random_bytes(len: usize) -> Vec<u8> {
+ let mut state = 0x2545_F491_4F6C_DD1Du64;
+ (0..len)
+ .map(|_| {
+ state = state
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (state >> 56) as u8
+ })
+ .collect()
+ }
+}
diff --git a/crates/fs/src/fs.rs b/crates/fs/src/fs.rs
index 7d37384cd2a..f56c08e63f7 100644
--- a/crates/fs/src/fs.rs
+++ b/crates/fs/src/fs.rs
@@ -76,6 +76,10 @@ pub trait Watcher: Send + Sync {
fn remove(&self, path: &Path) -> Result<()>;
}
+pub trait ReadSeek: io::Read + io::Seek {}
+
+impl<T: io::Read + io::Seek + ?Sized> ReadSeek for T {}
+
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
pub enum PathEventKind {
Removed,
@@ -130,7 +134,7 @@ pub trait Fs: Send + Sync {
async fn remove_file(&self, path: &Path, options: RemoveOptions) -> Result<()>;
async fn open_handle(&self, path: &Path) -> Result<Arc<dyn FileHandle>>;
- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>>;
+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>>;
async fn load(&self, path: &Path) -> Result<String> {
Ok(String::from_utf8(self.load_bytes(path).await?)?)
}
@@ -936,7 +940,7 @@ impl Fs for RealFs {
Ok(self.trash.lock().insert(entry))
}
- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>> {
+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>> {
Ok(Box::new(std::fs::File::open(path)?))
}
@@ -3211,7 +3215,7 @@ impl Fs for FakeFs {
self.remove_file_inner(path, options).await.map(|_| ())
}
- async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read + Send + Sync>> {
+ async fn open_sync(&self, path: &Path) -> Result<Box<dyn ReadSeek + Send + Sync>> {
let bytes = self.load_internal(path).await?;
Ok(Box::new(io::Cursor::new(bytes)))
}
diff --git a/crates/git/Cargo.toml b/crates/git/Cargo.toml
index 72db89537e5..551a9253182 100644
--- a/crates/git/Cargo.toml
+++ b/crates/git/Cargo.toml
@@ -21,6 +21,7 @@ askpass.workspace = true
async-trait.workspace = true
collections.workspace = true
derive_more.workspace = true
+file_content.workspace = true
gpui.workspace = true
http_client.workspace = true
itertools.workspace = true
diff --git a/crates/git/src/repository.rs b/crates/git/src/repository.rs
index 7962bd6cc9a..b467990e20d 100644
--- a/crates/git/src/repository.rs
+++ b/crates/git/src/repository.rs
@@ -7,6 +7,7 @@ use crate::{Oid, RunHook, SHA256_HEX_LENGTH, SHORT_SHA_LENGTH};
use anyhow::{Context as _, Result, anyhow, bail};
use async_channel::Sender;
use collections::HashMap;
+use file_content::{ByteContent, decode_byte_header};
use futures::channel::oneshot;
use futures::future::BoxFuture;
use futures::io::BufWriter;
@@ -569,8 +570,6 @@ impl CommitDetails {
}
}
-/// Detects if content is binary by checking for NUL bytes in the first 8000 bytes.
-/// This matches git's binary detection heuristic.
pub fn is_binary_content(content: &[u8]) -> bool {
let check_len = content.len().min(8000);
content[..check_len].contains(&0)
@@ -598,7 +597,8 @@ async fn read_commit_blob<R: smol::io::AsyncBufRead + Unpin>(
stdout.read_exact(&mut bytes).await?;
stdout.read_exact(newline).await?;
- let is_binary = is_binary_content(&bytes);
+ let is_binary =
+ is_binary_content(&bytes) || decode_byte_header(&bytes).1 == ByteContent::Binary;
Ok(LoadedCommitObject {
content: bytes,
is_binary,
@@ -4633,6 +4633,43 @@ mod tests {
);
}
+ #[test]
+ fn test_read_commit_blob_binary_detection() {
+ let mut pickle = b"\x80\x02X\0\x04\0\0".to_vec();
+ pickle.extend_from_slice(&b"A\0".repeat(512));
+ pickle.extend_from_slice(b"q\0.");
+ let mut cases = vec![
+ (pickle, true),
+ (b"hello\n".to_vec(), false),
+ (b"\xFF\xFEH\0e\0l\0l\0o\0\n\0".to_vec(), true),
+ (b"\xFE\xFF\0H\0e\0l\0l\0o\0\n".to_vec(), true),
+ (b"H\0e\0l\0l\0o\0\n\0".to_vec(), true),
+ (b"\0H\0e\0l\0l\0o\0\n".to_vec(), true),
+ (b"\xEF\xBB\xBFhello\0".to_vec(), true),
+ (b"\x89PNG\r\n\x1a\n".to_vec(), true),
+ ];
+ for offset in [4096, 7999, 8000] {
+ let mut bytes = vec![b'a'; offset];
+ bytes.push(0);
+ cases.push((bytes, offset < 8000));
+ }
+
+ for (bytes, expected_binary) in cases {
+ let mut output = format!("{}\n", bytes.len()).into_bytes();
+ output.extend_from_slice(&bytes);
+ output.push(b'\n');
+ let object = smol::block_on(read_commit_blob(
+ &mut smol::io::Cursor::new(output),
+ &mut String::new(),
+ &mut [0],
+ ))
+ .unwrap();
+
+ assert_eq!(object.is_binary, expected_binary);
+ assert_eq!(object.content, bytes);
+ }
+ }
+
#[gpui::test]
async fn test_load_commit_with_type_changed_file(cx: &mut TestAppContext) {
disable_git_global_config();
diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml
index bfc9c5ac24e..1e4772ff64c 100644
--- a/crates/language/Cargo.toml
+++ b/crates/language/Cargo.toml
@@ -32,11 +32,11 @@ test-support = [
anyhow.workspace = true
async-trait.workspace = true
clock.workspace = true
-chardetng.workspace = true
collections.workspace = true
diffy.workspace = true
ec4rs.workspace = true
encoding_rs.workspace = true
+file_content.workspace = true
fs.workspace = true
futures.workspace = true
futures-lite.workspace = true
diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs
index e54228e6e7e..32b633d3e0d 100644
--- a/crates/language/src/buffer.rs
+++ b/crates/language/src/buffer.rs
@@ -3,9 +3,13 @@ pub mod row_chunk;
pub use bracket_ranges::BracketMatch;
+pub use crate::{
+ CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry,
+ diagnostic_set::DiagnosticSet, proto,
+};
use crate::{
- ByteContent, DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig,
- PLAIN_TEXT, RunnableTag, TextObject, TreeSitterOptions, analyze_byte_content,
+ DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig, PLAIN_TEXT,
+ RunnableTag, TextObject, TreeSitterOptions,
diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup},
language_settings::{AutoIndentMode, LanguageSettings},
outline::OutlineItem,
@@ -19,16 +23,13 @@ use crate::{
text_diff::text_diff,
unified_diff_with_offsets,
};
-pub use crate::{
- CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry,
- diagnostic_set::DiagnosticSet, proto,
-};
use anyhow::{Context as _, Result};
use clock::Lamport;
pub use clock::ReplicaId;
use collections::HashMap;
use encoding_rs::Encoding;
+use file_content::{ByteContent, decode_byte_header};
use fs::MTime;
use futures::channel::oneshot;
use futures_lite::future::yield_now;
@@ -1696,7 +1697,7 @@ impl Buffer {
let bytes = load_bytes_task.await?;
anyhow::ensure!(
- analyze_byte_content(&bytes) != ByteContent::Binary,
+ decode_byte_header(&bytes).1 != ByteContent::Binary,
"Binary files are not supported"
);
diff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rs
deleted file mode 100644
index 4acfd5d04a8..00000000000
--- a/crates/language/src/file_content.rs
+++ /dev/null
@@ -1,308 +0,0 @@
-use anyhow::{Result, bail};
-use chardetng::EncodingDetector;
-use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};
-
-pub const FILE_ANALYSIS_BYTES: usize = 1024;
-
-pub struct DecodedText {
- pub text: String,
- pub encoding: &'static Encoding,
- pub has_bom: bool,
-}
-
-pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {
- if let Some((encoding, _bom_len)) = Encoding::for_bom(&bytes) {
- let (text, _) = encoding.decode_with_bom_removal(&bytes);
- return Ok(DecodedText {
- text: text.into_owned(),
- encoding,
- has_bom: true,
- });
- }
-
- let encoding = match analyze_byte_content(&bytes) {
- ByteContent::Utf16Le => UTF_16LE,
- ByteContent::Utf16Be => UTF_16BE,
- ByteContent::Binary => bail!("Binary files are not supported"),
- ByteContent::Unknown => {
- return match String::from_utf8(bytes) {
- Ok(text) if !text.contains('\x1b') => Ok(DecodedText {
- text,
- encoding: UTF_8,
- has_bom: false,
- }),
- Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),
- Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),
- };
- }
- };
-
- let (text, _, _) = encoding.decode(&bytes);
- Ok(DecodedText {
- text: text.into_owned(),
- encoding,
- has_bom: false,
- })
-}
-
-pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {
- if encoding == UTF_8 && !has_bom {
- return text.into_bytes();
- }
-
- // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.
- if encoding == UTF_16BE {
- let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
- if has_bom {
- bytes.extend_from_slice(&[0xFE, 0xFF]);
- }
- bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));
- return bytes;
- }
-
- if encoding == UTF_16LE {
- let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
- if has_bom {
- bytes.extend_from_slice(&[0xFF, 0xFE]);
- }
- bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));
- return bytes;
- }
-
- let (encoded, _, _) = encoding.encode(&text);
- if has_bom && encoding == UTF_8 {
- let mut bytes = Vec::with_capacity(encoded.len() + 3);
- bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);
- bytes.extend_from_slice(&encoded);
- bytes
- } else {
- encoded.into_owned()
- }
-}
-
-fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {
- let mut detector = EncodingDetector::new();
- detector.feed(&bytes, true);
- let encoding = detector.guess(None, true);
- let (text, _, _) = encoding.decode(&bytes);
- DecodedText {
- text: text.into_owned(),
- encoding,
- has_bom: false,
- }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum ByteContent {
- Utf16Le,
- Utf16Be,
- Binary,
- Unknown,
-}
-
-// Heuristic check using null byte distribution plus a generic text-likeness
-// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise
-// distinguishes between text-like and binary-like content.
-pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent {
- if bytes.len() < 2 {
- return ByteContent::Unknown;
- }
-
- if is_known_binary_header(bytes) {
- return ByteContent::Binary;
- }
-
- let limit = bytes.len().min(FILE_ANALYSIS_BYTES);
- let mut even_null_count = 0usize;
- let mut odd_null_count = 0usize;
- let mut non_text_like_count = 0usize;
-
- for (i, &byte) in bytes[..limit].iter().enumerate() {
- if byte == 0 {
- if i % 2 == 0 {
- even_null_count += 1;
- } else {
- odd_null_count += 1;
- }
- non_text_like_count += 1;
- continue;
- }
-
- let is_text_like = match byte {
- b'\t' | b'\n' | b'\r' | 0x0C => true,
- 0x20..=0x7E => true,
- // Treat bytes that are likely part of UTF-8 or single-byte encodings as text-like.
- 0x80..=0xBF | 0xC2..=0xF4 => true,
- _ => false,
- };
-
- if !is_text_like {
- non_text_like_count += 1;
- }
- }
-
- let total_null_count = even_null_count + odd_null_count;
-
- // If there are no NUL bytes at all, this is overwhelmingly likely to be text.
- if total_null_count == 0 {
- return ByteContent::Unknown;
- }
-
- let has_significant_nulls = total_null_count >= limit / 16;
- let nulls_skew_to_even = even_null_count > odd_null_count * 4;
- let nulls_skew_to_odd = odd_null_count > even_null_count * 4;
-
- if has_significant_nulls {
- let sample = &bytes[..limit];
-
- // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first)
- // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first)
-
- if nulls_skew_to_even && is_plausible_utf16_text(sample, false) {
- return ByteContent::Utf16Be;
- }
-
- if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) {
- return ByteContent::Utf16Le;
- }
-
- return ByteContent::Binary;
- }
-
- if non_text_like_count * 100 < limit * 8 {
- ByteContent::Unknown
- } else {
- ByteContent::Binary
- }
-}
-
-fn is_known_binary_header(bytes: &[u8]) -> bool {
- bytes.starts_with(b"%PDF-") // PDF
- || bytes.starts_with(b"PK\x03\x04") // ZIP local header
- || bytes.starts_with(b"PK\x05\x06") // ZIP end of central directory
- || bytes.starts_with(b"PK\x07\x08") // ZIP spanning/splitting
- || bytes.starts_with(b"\x89PNG\r\n\x1a\n") // PNG
- || bytes.starts_with(b"\xFF\xD8\xFF") // JPEG
- || bytes.starts_with(b"GIF87a") // GIF87a
- || bytes.starts_with(b"GIF89a") // GIF89a
- || bytes.starts_with(b"IWAD") // Doom IWAD archive
- || bytes.starts_with(b"PWAD") // Doom PWAD archive
- || bytes.starts_with(b"RIFF") // WAV, AVI, WebP
- || bytes.starts_with(b"OggS") // OGG (Vorbis, Opus, FLAC)
- || bytes.starts_with(b"fLaC") // FLAC
- || bytes.starts_with(b"ID3") // MP3 with ID3v2 tag
- || bytes.starts_with(b"\xFF\xFB") // MP3 frame sync (MPEG1 Layer3)
- || bytes.starts_with(b"\xFF\xFA") // MP3 frame sync (MPEG1 Layer3)
- || bytes.starts_with(b"\xFF\xF3") // MP3 frame sync (MPEG2 Layer3)
- || bytes.starts_with(b"\xFF\xF2") // MP3 frame sync (MPEG2 Layer3)
-}
-
-// Null byte skew alone is not enough to identify UTF-16 -- binary formats with
-// small 16-bit values (like PCM audio) produce the same pattern. Decode the
-// bytes as UTF-16 and reject if too many code units land in control character
-// ranges or form unpaired surrogates, which real text almost never contains.
-fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool {
- let mut suspicious_count = 0usize;
- let mut word_like_count = 0usize;
- let mut total = 0usize;
-
- let mut i = 0;
- while let Some(code_unit) = read_u16(bytes, i, little_endian) {
- total += 1;
-
- match code_unit {
- 0x0009 | 0x000A | 0x000C | 0x000D => {}
- 0x0020 | 0x0030..=0x0039 | 0x0041..=0x005A | 0x0061..=0x007A => {
- word_like_count += 1;
- }
- // C0/C1 control characters and non-characters
- 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1,
- 0xD800..=0xDBFF => {
- let next_offset = i + 2;
- let has_low_surrogate = read_u16(bytes, next_offset, little_endian)
- .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next));
- if has_low_surrogate {
- total += 1;
- word_like_count += 2;
- i += 2;
- } else {
- suspicious_count += 1;
- }
- }
- // Lone low surrogate without a preceding high surrogate
- 0xDC00..=0xDFFF => suspicious_count += 1,
- 0x0100.. => word_like_count += 1,
- _ => {}
- }
-
- i += 2;
- }
-
- if total == 0 {
- return false;
- }
-
- // Real UTF-16 text has near-zero control characters; binary data with
- // small 16-bit values typically exceeds 5%. 2% provides a safe margin.
- let low_control_ratio = suspicious_count * 100 < total * 2;
-
- // Binary formats that interleave short ASCII fragments with small
- // length/type fields (e.g. game asset formats) can dodge the control
- // character check above while barely containing any real words: their
- // code units land on ASCII punctuation and Latin-1 symbol values rather
- // than letters, digits, or spaces. Real text is overwhelmingly made of
- // word characters, so require a minimum share of them. Code units above
- // the Latin-1 range (and surrogate pairs) also count as word-like so
- // that scripts such as Cyrillic or Greek, whose letters are non-ASCII,
- // are still recognized -- tag bytes paired with a zero byte can never
- // land there.
- let enough_word_chars = word_like_count * 100 >= total * 30;
-
- low_control_ratio && enough_word_chars
-}
-
-fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {
- let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?];
- if little_endian {
- return Some(u16::from_le_bytes(pair));
- }
- Some(u16::from_be_bytes(pair))
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn decodes_and_encodes_windows_1251() {
- let expected = "строка один\nстрока два\n";
- let (bytes, _, _) = encoding_rs::WINDOWS_1251.encode(expected);
-
- let decoded = decode_text(bytes.clone().into_owned()).unwrap();
-
- assert_eq!(decoded.text, expected);
- assert_eq!(decoded.encoding, encoding_rs::WINDOWS_1251);
- assert!(!decoded.has_bom);
- assert_eq!(
- encode_text(decoded.text, decoded.encoding, decoded.has_bom),
- bytes.as_ref()
- );
- }
-
- #[test]
- fn preserves_unicode_boms() {
- let expected = "Hello, мир\n";
- for encoding in [UTF_8, UTF_16LE, UTF_16BE] {
- let bytes = encode_text(expected.to_owned(), encoding, true);
- let decoded = decode_text(bytes.clone()).unwrap();
-
- assert_eq!(decoded.text, expected);
- assert_eq!(decoded.encoding, encoding);
- assert!(decoded.has_bom);
- assert_eq!(
- encode_text(decoded.text, decoded.encoding, decoded.has_bom),
- bytes
- );
- }
- }
-}
diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs
index 6dcb998500d..76de9695f33 100644
--- a/crates/language/src/language.rs
+++ b/crates/language/src/language.rs
@@ -10,7 +10,6 @@ mod available_languages;
mod buffer;
mod diagnostic;
mod diagnostic_set;
-mod file_content;
mod language_registry;
pub mod language_settings;
@@ -108,9 +107,6 @@ pub use diagnostic::{
Diagnostic, DiagnosticMessage, DiagnosticSourceKind, RelatedInformation, RelatedLocation,
};
pub use diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup};
-pub use file_content::{
- ByteContent, DecodedText, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,
-};
pub use language_registry::{
BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QueryFile,
QueryFileContents, QueryFiles,
diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml
index 20f6cd6abc2..45626f1b33f 100644
--- a/crates/project/Cargo.toml
+++ b/crates/project/Cargo.toml
@@ -51,6 +51,7 @@ dap.workspace = true
encoding_rs.workspace = true
extension.workspace = true
fancy-regex.workspace = true
+file_content.workspace = true
fs.workspace = true
futures.workspace = true
fuzzy.workspace = true
diff --git a/crates/project/src/git_store.rs b/crates/project/src/git_store.rs
index d02a4c5bfb7..5468e825009 100644
--- a/crates/project/src/git_store.rs
+++ b/crates/project/src/git_store.rs
@@ -22,6 +22,7 @@ use buffer_diff::{
use client::ProjectId;
use collections::HashMap;
pub use conflict_set::{ConflictRegion, ConflictSet, ConflictSetSnapshot, ConflictSetUpdate};
+use file_content::{decode_text, encode_text};
use fs::{Fs, RemoveOptions};
use futures::{
FutureExt, SinkExt, Stream, StreamExt,
@@ -55,7 +56,7 @@ use gpui::{
Subscription, Task, TaskExt, WeakEntity,
};
use language::{
- Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry, decode_text, encode_text,
+ Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry,
proto::{deserialize_version, serialize_version},
};
use parking_lot::Mutex;
@@ -216,7 +217,12 @@ fn pending_hunks(
}
fn decode_git_text(bytes: Vec<u8>) -> Result<String> {
- Ok(decode_text(bytes)?.text)
+ let text = decode_text(bytes)?.text;
+ anyhow::ensure!(
+ !is_binary_content(text.as_bytes()),
+ "Binary files are not supported"
+ );
+ Ok(text)
}
#[derive(Debug)]
@@ -10414,7 +10420,7 @@ impl Repository {
}
}
});
- cx.spawn(move |_: &mut AsyncApp| async move {
+ cx.background_spawn(async move {
let (content, blame) = rx.await??;
anyhow::ensure!(
!is_binary_content(content.as_bytes()),
@@ -11691,28 +11697,45 @@ mod tests {
}
#[gpui::test]
- fn test_decode_git_text_windows_1251_one_line_change(cx: &mut TestAppContext) {
+ fn test_decode_git_text(cx: &mut TestAppContext) {
let old_text = "строка один\nстрока два\n";
let new_text = "строка один\nстрока три\n";
- let (old_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(old_text);
- let (new_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(new_text);
-
- let decoded_old = decode_git_text(old_bytes.into_owned()).unwrap();
- let decoded_new = decode_git_text(new_bytes.into_owned()).unwrap();
- let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));
- let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
- let diff = cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));
- let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));
- let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();
- let [hunk] = hunks.as_slice() else {
- panic!("expected one modified hunk, got {hunks:?}");
- };
+ let sparse_nul_text = format!("{}\0", "a".repeat(4096));
+ for (encoding, has_bom) in [
+ (encoding_rs::WINDOWS_1251, false),
+ (encoding_rs::UTF_16LE, false),
+ (encoding_rs::UTF_16LE, true),
+ (encoding_rs::UTF_16BE, false),
+ (encoding_rs::UTF_16BE, true),
+ ] {
+ let old_bytes = encode_text(old_text.to_owned(), encoding, has_bom);
+ let new_bytes = encode_text(new_text.to_owned(), encoding, has_bom);
+ let decoded_old = decode_git_text(old_bytes).unwrap();
+ let decoded_new = decode_git_text(new_bytes).unwrap();
+ assert_eq!(decoded_old, old_text);
+ assert_eq!(decoded_new, new_text);
+ let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));
+ let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ let diff =
+ cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));
+ let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));
+ let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();
+ let [hunk] = hunks.as_slice() else {
+ panic!("expected one modified hunk, got {hunks:?}");
+ };
- assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));
- assert_eq!(
- hunk.diff_base_byte_range,
- old_text.find("строка два").unwrap()..old_text.len()
- );
+ assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));
+ assert_eq!(
+ hunk.diff_base_byte_range,
+ old_text.find("строка два").unwrap()..old_text.len()
+ );
+ assert_eq!(
+ decode_git_text(encode_text(sparse_nul_text.clone(), encoding, has_bom))
+ .unwrap_err()
+ .to_string(),
+ "Binary files are not supported"
+ );
+ }
}
#[gpui::test]
diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rs
index 3abcde1cf1c..edb4b0adf59 100644
--- a/crates/project/src/project_search.rs
+++ b/crates/project/src/project_search.rs
@@ -1,7 +1,7 @@
use std::{
cell::LazyCell,
collections::BTreeSet,
- io::{BufRead, BufReader, Cursor, ErrorKind, Read},
+ io::{Cursor, ErrorKind, Read},
ops::Range,
path::{Path, PathBuf},
pin::pin,
@@ -12,7 +12,7 @@ use std::{
use anyhow::Context;
use async_channel::{Receiver, Sender, bounded, unbounded};
use collections::HashSet;
-use fs::Fs;
+use fs::{Fs, ReadSeek};
use futures::FutureExt as _;
use futures::{SinkExt, StreamExt, select_biased, stream::FuturesOrdered};
use gpui::{App, AppContext, AsyncApp, BackgroundExecutor, Entity, Priority, Task};
@@ -21,12 +21,12 @@ use parking_lot::Mutex;
use postage::oneshot;
use rpc::{AnyProtoClient, proto};
-use language::ByteContent;
-use util::{ResultExt, maybe, rel_path::RelPath};
-use worktree::{
- Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header,
- decode_file_text,
+use encoding_rs::Encoding;
+use file_content::{
+ ByteContent, DecodingReader, FILE_ANALYSIS_BYTES, decode_byte_header, detect_encoding,
};
+use util::{ResultExt, maybe, rel_path::RelPath};
+use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings};
use crate::{
Project, ProjectItem, ProjectPath, RemotelyCreatedModels,
@@ -614,7 +614,7 @@ impl Search {
) {
_ = maybe!(async move {
while let Ok((buffer, line_hint)) = rx.recv().await {
- let snapshot = buffer.read_with(&mut cx, |this, _| this.snapshot());
+ let snapshot = buffer.read_with(&mut cx, |buffer, _| buffer.snapshot());
let (tx, rx) = oneshot::channel();
find_all_matches_tx
.send(FindAllMatchesRequest {
@@ -834,13 +834,14 @@ impl RequestHandler<'_> {
let fs = self
.fs
.context("Trying to query filesystem in remote project search")?;
- let Some(file) = fs.open_sync(&abs_path).await.log_err() else {
+ let Some(mut file) = fs.open_sync(&abs_path).await.log_err() else {
return anyhow::Ok(());
};
- let mut file = BufReader::new(file);
- let file_start = file.fill_buf()?;
- let (bom_encoding, byte_content) = decode_byte_header(file_start);
+ let mut file_start = Vec::with_capacity(8 * 1024);
+ (&mut *file).take(8 * 1024).read_to_end(&mut file_start)?;
+ let (bom_encoding, byte_content) =
+ decode_byte_header(&file_start[..file_start.len().min(FILE_ANALYSIS_BYTES)]);
if byte_content == ByteContent::Binary {
log::debug!("Skipping binary file {abs_path:?}");
return Ok(());
@@ -848,27 +849,35 @@ impl RequestHandler<'_> {
let is_plain_utf8 = bom_encoding.is_none()
&& byte_content == ByteContent::Unknown
- && is_utf8_prefix(file_start);
-
- let line_hint = if is_plain_utf8 {
- match self.query.detect(file).await {
+ && is_utf8_prefix(&file_start);
+ let encoding = bom_encoding.or(byte_content.encoding());
+
+ let first_match = if is_plain_utf8 {
+ match self
+ .query
+ .detect(&mut Cursor::new(file_start).chain(&mut *file))
+ .await
+ {
Ok(line_hint) => line_hint,
Err(error)
if error
.downcast_ref::<std::io::Error>()
.is_some_and(|error| error.kind() == ErrorKind::InvalidData) =>
{
- self.detect_in_decoded_file(fs, &abs_path).await?
+ self.detect_in_decoded_file(&mut *file, encoding).await?
}
Err(error) => return Err(error),
}
} else {
- self.detect_in_decoded_file(fs, &abs_path).await?
+ self.detect_in_decoded_file(&mut *file, encoding).await?
};
- if let Some(line_hint) = line_hint {
+ if first_match.is_some() {
// Yes, we should scan the whole file.
- entry.should_scan_tx.send((entry.path, line_hint)).await?;
+ entry
+ .should_scan_tx
+ .send((entry.path, MatchPositionHint::default()))
+ .await?;
}
Ok(())
}
@@ -878,12 +887,21 @@ impl RequestHandler<'_> {
async fn detect_in_decoded_file(
&self,
- fs: &dyn Fs,
- abs_path: &Path,
+ file: &mut (dyn ReadSeek + Send),
+ encoding: Option<&'static Encoding>,
) -> anyhow::Result<Option<MatchPositionHint>> {
- let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?;
- let reader: Box<dyn Read + Send + Sync> = Box::new(Cursor::new(text.into_bytes()));
- self.query.detect(BufReader::new(reader)).await
+ file.rewind()?;
+ let encoding = match encoding {
+ Some(encoding) => encoding,
+ None => {
+ let encoding = detect_encoding(file).await?;
+ file.rewind()?;
+ encoding
+ }
+ };
+ self.query
+ .detect(&mut DecodingReader::new(file, encoding))
+ .await
}
async fn handle_scan_path(&self, req: InputPath) {
diff --git a/crates/project/src/search.rs b/crates/project/src/search.rs
index fb2e88a1240..49c7f931142 100644
--- a/crates/project/src/search.rs
+++ b/crates/project/src/search.rs
@@ -9,7 +9,7 @@ use smol::future::yield_now;
use std::{
borrow::Cow,
collections::BTreeSet,
- io::{BufRead, BufReader, Read},
+ io::{self, Read},
ops::Range,
sync::{Arc, LazyLock},
};
@@ -46,7 +46,7 @@ pub struct SearchInputs {
buffers: Option<Vec<Entity<Buffer>>>,
}
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MatchPositionHint {
Line(u32),
ByteOffset(usize),
@@ -397,49 +397,28 @@ impl SearchQuery {
pub async fn detect(
&self,
- mut reader: BufReader<Box<dyn Read + Send + Sync>>,
+ reader: &mut (dyn Read + Send),
) -> Result<Option<MatchPositionHint>> {
let query_str = self.as_str();
if query_str.is_empty() {
return Ok(None);
}
- // Yield from this function every 20KB scanned.
- const YIELD_THRESHOLD: usize = 20 * 1024;
-
match self {
+ Self::Text { search, .. } if !query_str.contains('\n') => {
+ detect_single_line(search, reader).await
+ }
Self::Text { search, .. } => {
- let mut text = String::new();
- if query_str.contains('\n') {
- reader.read_to_string(&mut text)?;
- text::LineEnding::normalize(&mut text);
- if search.is_match(&text) {
- Ok(Some(MatchPositionHint::default()))
- } else {
- Ok(None)
- }
+ let mut text = read_to_string(reader).await?;
+ text::LineEnding::normalize(&mut text);
+ if search.is_match(&text) {
+ Ok(Some(MatchPositionHint::default()))
} else {
- let mut bytes_read = 0;
- let mut line_number = u32::default();
- while reader.read_line(&mut text)? > 0 {
- if search.is_match(&text) {
- return Ok(Some(MatchPositionHint::Line(line_number)));
- }
- bytes_read += text.len();
- if bytes_read >= YIELD_THRESHOLD {
- bytes_read = 0;
- smol::future::yield_now().await;
- }
- text.clear();
- line_number += 1;
- }
Ok(None)
}
}
Self::Regex { regex, .. } => {
- let mut text = String::new();
-
- reader.read_to_string(&mut text)?;
+ let mut text = read_to_string(reader).await?;
text::LineEnding::normalize(&mut text);
if let Some(m) = regex.find(&text)? {
Ok(Some(MatchPositionHint::ByteOffset(m.start())))
@@ -713,3 +692,145 @@ impl SearchQuery {
matches
}
}
+
+async fn detect_single_line(
+ search: &AhoCorasick,
+ reader: &mut (dyn Read + Send),
+) -> Result<Option<MatchPositionHint>> {
+ const BLOCK_BYTES: usize = 64 * 1024;
+ let batch_size = search.max_pattern_len().max(BLOCK_BYTES);
+ let carry_len = search.max_pattern_len().saturating_sub(1).max(3);
+ let mut block = vec![0; BLOCK_BYTES];
+ let mut window = TextSearchWindow::default();
+ loop {
+ let batch_end = window.batch_start
+ + if window.first_match.is_some() {
+ BLOCK_BYTES
+ } else {
+ batch_size
+ };
+ let limit = (batch_end - window.bytes.len()).min(BLOCK_BYTES);
+ let read_result = match reader.read(&mut block[..limit]) {
+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+ result => result,
+ };
+ if let std::result::Result::Ok(read) = read_result {
+ window.extend(&block[..read], batch_size + carry_len)?;
+ if read > 0 && window.bytes.len() < batch_end {
+ yield_now().await;
+ continue;
+ }
+ }
+ let line_end = window.find_matching_line_end(search);
+ window.validate_prefix(line_end, read_result.is_err())?;
+ if line_end.is_some() {
+ return Ok(window.first_match);
+ }
+ if read_result? == 0 {
+ if window.validated_len < window.bytes.len() {
+ return Err(invalid_data());
+ }
+ return Ok(window.first_match);
+ }
+ window.retain_suffix(if window.first_match.is_some() {
+ 3
+ } else {
+ carry_len
+ });
+ yield_now().await;
+ }
+}
+
+#[derive(Default)]
+struct TextSearchWindow {
+ bytes: Vec<u8>,
+ batch_start: usize,
+ validated_len: usize,
+ lines_before_window: usize,
+ first_match: Option<MatchPositionHint>,
+}
+
+impl TextSearchWindow {
+ fn extend(&mut self, bytes: &[u8], max_capacity: usize) -> Result<()> {
+ if self.bytes.len() + bytes.len() > self.bytes.capacity() {
+ let capacity = self
+ .bytes
+ .capacity()
+ .saturating_mul(2)
+ .max(self.bytes.len() + bytes.len())
+ .min(max_capacity);
+ self.bytes.try_reserve_exact(capacity - self.bytes.len())?;
+ }
+ self.bytes.extend_from_slice(bytes);
+ Ok(())
+ }
+
+ fn find_matching_line_end(&mut self, search: &AhoCorasick) -> Option<usize> {
+ let start = if self.first_match.is_some() {
+ self.batch_start
+ } else {
+ let found = search.find(&self.bytes)?;
+ let line = self.lines_before_window + count_newlines(&self.bytes[..found.start()]);
+ self.first_match = Some(MatchPositionHint::Line(
+ u32::try_from(line).unwrap_or(u32::MAX),
+ ));
+ found.end()
+ };
+ self.bytes[start..]
+ .iter()
+ .position(|&byte| byte == b'\n')
+ .map(|offset| start + offset + 1)
+ }
+
+ fn validate_prefix(&mut self, line_end: Option<usize>, read_failed: bool) -> Result<()> {
+ let validation_end = line_end.unwrap_or_else(|| {
+ if read_failed {
+ self.bytes
+ .iter()
+ .rposition(|&byte| byte == b'\n')
+ .map_or(self.validated_len, |offset| {
+ (offset + 1).max(self.validated_len)
+ })
+ } else {
+ self.bytes.len()
+ }
+ });
+ self.validated_len +=
+ match std::str::from_utf8(&self.bytes[self.validated_len..validation_end]) {
+ Err(error) if error.error_len().is_some() => return Err(invalid_data()),
+ Err(error) => error.valid_up_to(),
+ _ => validation_end - self.validated_len,
+ };
+ Ok(())
+ }
+
+ fn retain_suffix(&mut self, length: usize) {
+ let consumed = self.bytes.len().saturating_sub(length);
+ self.lines_before_window += count_newlines(&self.bytes[..consumed]);
+ self.bytes.drain(..consumed);
+ self.batch_start = self.bytes.len();
+ self.validated_len -= consumed;
+ }
+}
+
+async fn read_to_string(reader: &mut (dyn Read + Send)) -> Result<String> {
+ const BLOCK_BYTES: usize = 64 * 1024;
+ let mut bytes = Vec::new();
+ loop {
+ let bytes_read = (&mut *reader)
+ .take(BLOCK_BYTES as u64)
+ .read_to_end(&mut bytes)?;
+ if bytes_read < BLOCK_BYTES {
+ return String::from_utf8(bytes).map_err(|_| invalid_data());
+ }
+ yield_now().await;
+ }
+}
+
+fn count_newlines(bytes: &[u8]) -> usize {
+ bytes.iter().filter(|&&byte| byte == b'\n').count()
+}
+
+fn invalid_data() -> anyhow::Error {
+ anyhow::Error::from(io::Error::from(io::ErrorKind::InvalidData))
+}
diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs
index ee488e76342..9149042ce07 100644
--- a/crates/project/tests/integration/project_tests.rs
+++ b/crates/project/tests/integration/project_tests.rs
@@ -19738,6 +19738,22 @@ async fn test_undo_encoding_change(cx: &mut gpui::TestAppContext) {
assert_ne!(buffer.text(), "Hi");
assert!(!buffer.is_dirty());
});
+
+ let thai = "สวัสดีชาวโลกนี่คือข้อความทดสอบภาษาไทย";
+ fs.insert_file(
+ path!("/dir/test.txt"),
+ thai.encode_utf16().flat_map(u16::to_le_bytes).collect(),
+ )
+ .await;
+ cx.run_until_parked();
+ assert_eq!(buffer.read_with(cx, |buffer, _| buffer.text()), thai);
+ buffer
+ .update(cx, |buffer, cx| {
+ buffer.reload_with_encoding(encoding_rs::UTF_16LE, cx)
+ })
+ .await
+ .unwrap();
+ assert_eq!(buffer.read_with(cx, |buffer, _| buffer.text()), thai);
}
#[gpui::test]
diff --git a/crates/project/tests/integration/search.rs b/crates/project/tests/integration/search.rs
index d961c87bceb..5b94509425a 100644
--- a/crates/project/tests/integration/search.rs
+++ b/crates/project/tests/integration/search.rs
@@ -1,13 +1,23 @@
-use std::io::BufReader;
+use std::io::{self, BufReader, Cursor};
+use collections::HashMap;
+use fs::FakeFs;
+use futures::FutureExt as _;
use language::Buffer;
-use project::search::SearchQuery;
+use project::{
+ Project,
+ search::{MatchPositionHint, SearchQuery},
+};
+use serde_json::json;
use text::Rope;
use util::{
+ path,
paths::{PathMatcher, PathStyle},
rel_path::RelPath,
};
+use crate::init_test;
+
#[test]
fn path_matcher_creation_for_valid_paths() {
for valid_path in [
@@ -197,7 +207,7 @@ async fn regex_with_eol_detects_lines() {
.unwrap();
let input = " Bool\nsomething else";
let result = re
- .detect(BufReader::new(Box::new(input.as_bytes())))
+ .detect(&mut BufReader::new(input.as_bytes()))
.await
.unwrap();
assert!(result.is_some());
@@ -219,8 +229,335 @@ async fn multi_line_regex_detects_matches() {
.unwrap();
let input = " Bool\nbool";
let result = re
- .detect(BufReader::new(Box::new(input.as_bytes())))
+ .detect(&mut BufReader::new(input.as_bytes()))
.await
.unwrap();
assert!(result.is_some());
}
+
+#[test]
+fn detect_reports_line_across_block_boundaries() {
+ let line_len = 1000;
+ let filler = format!("{}\n", "x".repeat(line_len - 1));
+ let mut text = filler.repeat(70);
+ let needle_line = text.lines().count();
+ text.push_str("prefix needle suffix\n");
+ text.push_str(&filler.repeat(3));
+
+ for (query, case_sensitive) in [("needle", true), ("NEEDLE", false), ("x\nx", true)] {
+ let query = SearchQuery::text(
+ query,
+ false,
+ case_sensitive,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ let mut input = reader(text.as_bytes());
+ assert!(query.detect(&mut input).now_or_never().is_none());
+ assert_eq!(input.inner.position(), 64 * 1024);
+ let hint = smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap();
+ let expected = if query.as_str().contains('\n') {
+ MatchPositionHint::default()
+ } else {
+ MatchPositionHint::Line(needle_line as u32)
+ };
+ assert_eq!(hint, Some(expected), "{:?}", query.as_str());
+
+ if !query.as_str().contains('\n') {
+ let mut invalid = format!("needle{}", "x".repeat(128 * 1024)).into_bytes();
+ invalid.push(0xff);
+ let error = smol::block_on(query.detect(&mut reader(&invalid))).unwrap_err();
+ assert_eq!(
+ error.downcast_ref::<io::Error>().map(|error| error.kind()),
+ Some(io::ErrorKind::InvalidData)
+ );
+ }
+ }
+
+ let query = SearchQuery::text(
+ "absent",
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ assert_eq!(
+ smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap(),
+ None
+ );
+
+ let mut invalid = text.into_bytes();
+ invalid.extend_from_slice(b"\xff\xfe tail\n");
+ let error = smol::block_on(query.detect(&mut reader(&invalid))).unwrap_err();
+ assert_eq!(
+ error.downcast_ref::<io::Error>().map(|error| error.kind()),
+ Some(io::ErrorKind::InvalidData)
+ );
+}
+
+#[test]
+fn detect_handles_block_boundary_splits() {
+ const BLOCK_BYTES: usize = 64 * 1024;
+ let needle = "néédle";
+ for split in 1..needle.len() {
+ let mut text = "a".repeat(BLOCK_BYTES - split);
+ text.push_str(needle);
+ text.push_str("\ntail\n");
+ for case_sensitive in [true, false] {
+ let query = SearchQuery::text(
+ needle,
+ false,
+ case_sensitive,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ let expected = if query.is_regex() {
+ MatchPositionHint::ByteOffset(BLOCK_BYTES - split)
+ } else {
+ MatchPositionHint::Line(0)
+ };
+ let mut input = reader(text.as_bytes());
+ assert!(query.detect(&mut input).now_or_never().is_none());
+ assert_eq!(input.inner.position(), BLOCK_BYTES as u64);
+ assert_eq!(
+ smol::block_on(query.detect(&mut reader(text.as_bytes()))).unwrap(),
+ Some(expected),
+ "split = {split}, case_sensitive = {case_sensitive}"
+ );
+ }
+ }
+
+ let needle = (0..9000)
+ .map(|index| format!("{index:08x}"))
+ .collect::<String>();
+ let query = SearchQuery::text(
+ &needle,
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ let text = format!("ignored\n{}x\n{needle}", &needle[..BLOCK_BYTES - 1]);
+ for max_read in [257, BLOCK_BYTES] {
+ let mut input = reader(text.as_bytes());
+ input.max_read = max_read;
+ assert!(query.detect(&mut input).now_or_never().is_none());
+ assert_eq!(input.inner.position(), max_read as u64);
+ let mut input = reader(text.as_bytes());
+ input.max_read = max_read;
+ assert_eq!(
+ smol::block_on(query.detect(&mut input)).unwrap(),
+ Some(MatchPositionHint::Line(2)),
+ );
+ }
+
+ let query = SearchQuery::text(
+ "absent",
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ let emoji = "\u{1f600}";
+ for split in 1..emoji.len() {
+ let mut text = "a".repeat(BLOCK_BYTES - split).into_bytes();
+ text.extend_from_slice(emoji.as_bytes());
+ text.extend_from_slice(b"\nline\n");
+ assert_eq!(
+ smol::block_on(query.detect(&mut reader(&text))).unwrap(),
+ None,
+ "split = {split}"
+ );
+ let mut matched_text = text.clone();
+ matched_text[..query.as_str().len()].copy_from_slice(query.as_str().as_bytes());
+ matched_text.push(0xff);
+ assert_eq!(
+ smol::block_on(query.detect(&mut reader(&matched_text))).unwrap(),
+ Some(MatchPositionHint::Line(0)),
+ "matched split = {split}"
+ );
+ text.truncate(BLOCK_BYTES - split + 2);
+ let error = smol::block_on(query.detect(&mut reader(&text))).unwrap_err();
+ assert_eq!(
+ error.downcast_ref::<io::Error>().map(|error| error.kind()),
+ Some(io::ErrorKind::InvalidData),
+ "truncated split = {split}"
+ );
+ }
+}
+
+#[test]
+fn detect_validates_matching_line_before_io_errors() {
+ let query = SearchQuery::text(
+ "n",
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ let found = Ok(Some(MatchPositionHint::Line(0)));
+ let invalid = Err(io::ErrorKind::InvalidData);
+ let failed = Err(io::ErrorKind::Other);
+ for (bytes, at_eof, at_error) in [
+ (b"".as_slice(), Ok(None), failed),
+ (b"n", found, failed),
+ (b"n\xff", invalid, failed),
+ (b"\xff\nn", invalid, invalid),
+ (b"n\xff\n", invalid, invalid),
+ (b"n\n\xff", found, found),
+ (b"n\xf0\x9f", invalid, failed),
+ (b"n\n\xf0\x9f", found, found),
+ (
+ "x\n😀n".as_bytes(),
+ Ok(Some(MatchPositionHint::Line(1))),
+ failed,
+ ),
+ ] {
+ for terminal_error in [None, Some(io::ErrorKind::Other)] {
+ for max_read in [1, 64 * 1024] {
+ let mut input = reader(bytes);
+ input.max_read = max_read;
+ input.terminal_error = terminal_error;
+ let actual = smol::block_on(query.detect(&mut input))
+ .map_err(|error| error.downcast_ref::<io::Error>().unwrap().kind());
+ let expected = if terminal_error.is_some() {
+ at_error
+ } else {
+ at_eof
+ };
+ assert_eq!(
+ actual, expected,
+ "{bytes:?}, {terminal_error:?}, {max_read}"
+ );
+ }
+ }
+ }
+}
+
+#[gpui::test]
+async fn searches_legacy_text_after_utf8_prefixes(cx: &mut gpui::TestAppContext) {
+ init_test(cx);
+
+ for (text, encoding, needle) in [
+ (
+ format!("é\n{} où voilà un garçon à la maison ", "x".repeat(1021)),
+ encoding_rs::WINDOWS_1252,
+ "é",
+ ),
+ (
+ format!(
+ "é\n©{}{}",
+ "x".repeat(70000),
+ " déjà été à côté français ".repeat(1000)
+ ),
+ encoding_rs::WINDOWS_1252,
+ "©",
+ ),
+ (
+ format!(
+ "\x1b[0m{}{}©",
+ "x".repeat(1024 * 1024),
+ " déjà été à côté français ".repeat(1000)
+ ),
+ encoding_rs::WINDOWS_1252,
+ "©",
+ ),
+ (
+ format!(
+ "я{}{}ЖЕТОН",
+ "x".repeat(1024 * 1024),
+ "Съешь же ещё этих мягких французских булок, да выпей чаю.\n".repeat(1000)
+ ),
+ encoding_rs::WINDOWS_1251,
+ "ЖЕТОН",
+ ),
+ ] {
+ let (bytes, _, had_errors) = encoding.encode(&text);
+ assert!(!had_errors);
+ let expected = text
+ .match_indices(needle)
+ .map(|(offset, matched)| offset..offset + matched.len())
+ .collect::<Vec<_>>();
+ let fs = FakeFs::new(cx.executor());
+ fs.insert_tree(path!("/dir"), json!({})).await;
+ fs.insert_file(path!("/dir/legacy.txt"), bytes.into_owned())
+ .await;
+ let project = Project::test(fs, [path!("/dir").as_ref()], cx).await;
+ let query = SearchQuery::text(
+ needle,
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap();
+ assert_eq!(
+ super::search(&project, query, cx).await.unwrap(),
+ HashMap::from_iter(
+ (!expected.is_empty()).then(|| (path!("dir/legacy.txt").to_string(), expected))
+ ),
+ );
+ }
+}
+
+fn reader(bytes: &[u8]) -> InterruptingReader {
+ InterruptingReader {
+ inner: Cursor::new(bytes.to_vec()),
+ interrupt_next_read: false,
+ max_read: usize::MAX,
+ terminal_error: None,
+ }
+}
+
+struct InterruptingReader {
+ inner: Cursor<Vec<u8>>,
+ interrupt_next_read: bool,
+ max_read: usize,
+ terminal_error: Option<io::ErrorKind>,
+}
+
+impl io::Read for InterruptingReader {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ self.interrupt_next_read = !self.interrupt_next_read;
+ if self.interrupt_next_read {
+ Err(io::Error::from(io::ErrorKind::Interrupted))
+ } else {
+ let length = buffer.len().min(self.max_read);
+ let read = self.inner.read(&mut buffer[..length])?;
+ if read == 0
+ && !buffer.is_empty()
+ && let Some(error) = self.terminal_error
+ {
+ return Err(io::Error::from(error));
+ }
+ Ok(read)
+ }
+ }
+}
diff --git a/crates/worktree/Cargo.toml b/crates/worktree/Cargo.toml
index fb0ef41338a..983c8b4ff91 100644
--- a/crates/worktree/Cargo.toml
+++ b/crates/worktree/Cargo.toml
@@ -35,6 +35,7 @@ async-lock.workspace = true
clock.workspace = true
collections.workspace = true
encoding_rs.workspace = true
+file_content.workspace = true
fs.workspace = true
futures.workspace = true
futures-lite.workspace = true
diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs
index 8037416e1ef..1f1809a8e91 100644
--- a/crates/worktree/src/worktree.rs
+++ b/crates/worktree/src/worktree.rs
@@ -6,6 +6,9 @@ use anyhow::{Context as _, Result, anyhow};
use clock::ReplicaId;
use collections::{BTreeMap, HashMap, HashSet, VecDeque};
use encoding_rs::Encoding;
+use file_content::{
+ ByteContent, DecodingReader, decode_byte_header, detect_encoding, encode_text, read_byte_header,
+};
use fs::{
Fs, MTime, PathEvent, PathEventKind, RemoveOptions, TrashId, Watcher, copy_recursive,
read_dir_items,
@@ -32,9 +35,7 @@ use gpui::{
Task,
};
pub use ignore::{IgnoreKind, IgnoreStack};
-use language::{
- ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,
-};
+use language::DiskState;
use async_channel::{self, Sender};
use parking_lot::Mutex;
@@ -7217,84 +7218,8 @@ impl fs::Watcher for NullWatcher {
}
}
-/// Reads the beginning of `file` to determine its kind and encoding, returning
-/// the bytes consumed and whether the file ended within them.
-fn read_file_header(file: &mut dyn Read, abs_path: &Path) -> Result<(Vec<u8>, bool)> {
- let mut header = Vec::with_capacity(FILE_ANALYSIS_BYTES);
- let mut buf = [0u8; FILE_ANALYSIS_BYTES];
- let mut reached_eof = false;
- while header.len() < FILE_ANALYSIS_BYTES {
- let n = file
- .read(&mut buf)
- .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;
- if n == 0 {
- reached_eof = true;
- break;
- }
- header.extend_from_slice(&buf[..n]);
- }
- Ok((header, reached_eof))
-}
-
const STREAM_BLOCK_BYTES: usize = 1024 * 1024;
-async fn read_file_to_end(
- file: &mut (dyn Read + Send),
- content: &mut Vec<u8>,
- abs_path: &Path,
-) -> Result<()> {
- let mut buf = vec![0u8; STREAM_BLOCK_BYTES];
- loop {
- let mut block_len = 0;
- while block_len < buf.len() {
- let n = file
- .read(&mut buf[block_len..])
- .with_context(|| format!("reading remaining bytes of the file {abs_path:?}"))?;
- if n == 0 {
- break;
- }
- block_len += n;
- }
-
- if block_len == 0 {
- break;
- }
-
- content.extend_from_slice(&buf[..block_len]);
- if block_len < buf.len() {
- break;
- }
-
- yield_now().await;
- }
- Ok(())
-}
-
-pub async fn decode_file_text(
- fs: &dyn Fs,
- abs_path: &Path,
-) -> Result<(String, &'static Encoding, bool)> {
- let mut file = fs
- .open_sync(&abs_path)
- .await
- .with_context(|| format!("opening file {abs_path:?}"))?;
-
- let (file_first_bytes, reached_eof) = read_file_header(&mut *file, abs_path)?;
- let (_, byte_content) = decode_byte_header(&file_first_bytes);
- anyhow::ensure!(
- byte_content != ByteContent::Binary,
- "Binary files are not supported"
- );
-
- // If the file is eligible for opening, read the rest of the file.
- let mut content = file_first_bytes;
- if !reached_eof {
- read_file_to_end(&mut *file, &mut content, abs_path).await?;
- }
- let decoded = decode_text(content)?;
- Ok((decoded.text, decoded.encoding, decoded.has_bom))
-}
-
/// Reads and decodes a file straight into a [`Rope`].
/// The returned rope has already had its line endings normalized, the
/// [`LineEnding`] detected before normalizing is returned alongside it.
@@ -7307,28 +7232,37 @@ pub async fn decode_file_text_to_rope(
.await
.with_context(|| format!("opening file {abs_path:?}"))?;
- let (prefix, reached_eof) = read_file_header(&mut *file, abs_path)?;
+ let (prefix, reached_eof) = read_byte_header(&mut *file)
+ .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;
let (bom_encoding, byte_content) = decode_byte_header(&prefix);
anyhow::ensure!(
byte_content != ByteContent::Binary,
"Binary files are not supported"
);
- // Only BOM-less, non-UTF-16 files are candidates for streaming: everything
- // else needs the whole byte buffer in hand to decode or to detect encoding.
if bom_encoding.is_none()
&& byte_content == ByteContent::Unknown
&& let Some((rope, line_ending)) =
- stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path).await?
+ stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path, true).await?
{
return Ok((rope, line_ending, encoding_rs::UTF_8, false));
}
- // Not plain UTF-8 after all. Re-read the file and decode it all at once.
- let (mut text, encoding, has_bom) = decode_file_text(fs, abs_path).await?;
- let line_ending = LineEnding::detect(&text);
- LineEnding::normalize(&mut text);
- Ok((Rope::from(text), line_ending, encoding, has_bom))
+ file.rewind()?;
+ let encoding = match bom_encoding.or(byte_content.encoding()) {
+ Some(encoding) => encoding,
+ None => {
+ let encoding = detect_encoding(&mut *file).await?;
+ file.rewind()?;
+ encoding
+ }
+ };
+ let mut reader = DecodingReader::new(&mut *file, encoding);
+ let (rope, line_ending) =
+ stream_utf8_into_rope(&mut reader, Vec::new(), false, abs_path, false)
+ .await?
+ .with_context(|| format!("decoding the file {abs_path:?}"))?;
+ Ok((rope, line_ending, reader.encoding(), bom_encoding.is_some()))
}
/// Streams a presumed-UTF-8 file into a [`Rope`], normalizing line endings as it
@@ -7342,6 +7276,7 @@ async fn stream_utf8_into_rope(
prefix: Vec<u8>,
reached_eof: bool,
abs_path: &Path,
+ reject_escape_sequences: bool,
) -> Result<Option<(Rope, LineEnding)>> {
let mut rope = Rope::new();
let mut line_ending = None;
@@ -7386,7 +7321,7 @@ async fn stream_utf8_into_rope(
// ISO-2022-JP and friends are valid UTF-8 but carry escape sequences, so
// they need the full-file encoding detector rather than this fast path.
- if text.contains('\x1b') {
+ if reject_escape_sequences && text.contains('\x1b') {
return Ok(None);
}
@@ -7446,13 +7381,6 @@ fn push_normalized(rope: &mut Rope, text: &str, scratch: &mut String) {
rope.push(scratch);
}
-pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
- if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
- return (Some(encoding), ByteContent::Unknown);
- }
- (None, analyze_byte_content(prefix))
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -7461,7 +7389,7 @@ mod tests {
/// decoded text and detected line ending, or `None` if the fast path bailed.
async fn stream(bytes: &[u8]) -> Option<(String, LineEnding)> {
let mut reader = std::io::Cursor::new(bytes.to_vec());
- stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))
+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"), true)
.await
.unwrap()
.map(|(rope, line_ending)| (rope.to_string(), line_ending))
@@ -7472,27 +7400,13 @@ mod tests {
let mut reader = std::io::Cursor::new(vec![b'a'; STREAM_BLOCK_BYTES * 2]);
assert!(
- stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))
+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"), true)
.now_or_never()
.is_none()
);
assert_eq!(reader.position(), STREAM_BLOCK_BYTES as u64);
}
- #[test]
- fn test_file_reading_yields_between_blocks() {
- let mut reader = std::io::Cursor::new(vec![b'a'; STREAM_BLOCK_BYTES * 2]);
- let mut content = Vec::new();
-
- assert!(
- read_file_to_end(&mut reader, &mut content, Path::new("test"))
- .now_or_never()
- .is_none()
- );
- assert_eq!(reader.position(), STREAM_BLOCK_BYTES as u64);
- assert_eq!(content.len(), STREAM_BLOCK_BYTES);
- }
-
#[gpui::test]
async fn test_stream_utf8_normalizes_line_endings() {
let crlf = "one\r\ntwo\r\nthree\r\n".repeat(40);
@@ -7543,199 +7457,4 @@ mod tests {
assert_eq!(stream(b"hello \xe2\x82").await, None, "truncated at eof");
assert_eq!(stream(b"plain \x1b$B text").await, None, "iso-2022 escape");
}
-
- /// reproduction of issue #50785
- fn build_pcm16_wav_bytes() -> Vec<u8> {
- let header: Vec<u8> = vec![
- /* RIFF header */
- 0x52, 0x49, 0x46, 0x46, // "RIFF"
- 0xc6, 0xcf, 0x00, 0x00, // file size: 8
- 0x57, 0x41, 0x56, 0x45, // "WAVE"
- /* fmt chunk */
- 0x66, 0x6d, 0x74, 0x20, // "fmt "
- 0x10, 0x00, 0x00, 0x00, // chunk size: 16
- 0x01, 0x00, // format: PCM (1)
- 0x01, 0x00, // channels: 1 (mono)
- 0x80, 0x3e, 0x00, 0x00, // sample rate: 16000
- 0x00, 0x7d, 0x00, 0x00, // byte rate: 32000
- 0x02, 0x00, // block align: 2
- 0x10, 0x00, // bits per sample: 16
- /* LIST chunk */
- 0x4c, 0x49, 0x53, 0x54, // "LIST"
- 0x1a, 0x00, 0x00, 0x00, // chunk size: 26
- 0x49, 0x4e, 0x46, 0x4f, // "INFO"
- 0x49, 0x53, 0x46, 0x54, // "ISFT"
- 0x0d, 0x00, 0x00, 0x00, // sub-chunk size: 13
- 0x4c, 0x61, 0x76, 0x66, 0x36, 0x32, 0x2e, 0x33, // "Lavf62.3"
- 0x2e, 0x31, 0x30, 0x30, 0x00, // ".100\0"
- /* padding byte for word alignment */
- 0x00, // data chunk header
- 0x64, 0x61, 0x74, 0x61, // "data"
- 0x80, 0xcf, 0x00, 0x00, // chunk size
- ];
-
- let mut bytes = header;
-
- // fill remaining space up to `FILE_ANALYSIS_BYTES` with synthetic PCM
- let audio_bytes_needed = FILE_ANALYSIS_BYTES - bytes.len();
- for i in 0..(audio_bytes_needed / 2) {
- let sample = (i & 0xFF) as u8;
- bytes.push(sample); // low byte: varies
- bytes.push(0x00); // high byte: zero for small values
- }
-
- bytes
- }
-
- #[test]
- fn test_pcm16_wav_detected_as_binary() {
- let wav_bytes = build_pcm16_wav_bytes();
- assert_eq!(wav_bytes.len(), FILE_ANALYSIS_BYTES);
-
- let result = analyze_byte_content(&wav_bytes);
- assert_eq!(
- result,
- ByteContent::Binary,
- "PCM 16-bit WAV should be detected as Binary via RIFF header"
- );
- }
-
- #[test]
- fn test_le16_binary_not_misdetected_as_utf16le() {
- let mut bytes = b"FAKE".to_vec();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- let sample = (bytes.len() & 0xFF) as u8;
- bytes.push(sample);
- bytes.push(0x00);
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- let result = analyze_byte_content(&bytes);
- assert_eq!(
- result,
- ByteContent::Binary,
- "LE 16-bit binary with control characters should be detected as Binary"
- );
- }
-
- #[test]
- fn test_be16_binary_not_misdetected_as_utf16be() {
- let mut bytes = b"FAKE".to_vec();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.push(0x00);
- let sample = (bytes.len() & 0xFF) as u8;
- bytes.push(sample);
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- let result = analyze_byte_content(&bytes);
- assert_eq!(
- result,
- ByteContent::Binary,
- "BE 16-bit binary with control characters should be detected as Binary"
- );
- }
-
- // Mimics binary formats that interleave short ASCII fragments with small
- // length/type fields (as seen in some game/asset binary formats, e.g.
- // Tibia-style OTBM maps): most high bytes are zero, matching UTF-16LE's
- // null-byte pattern for ASCII, but the low bytes are mostly non-word
- // "tag" values rather than real letters/digits/spaces.
- fn build_tag_interleaved_binary_bytes() -> Vec<u8> {
- let mut bytes = Vec::new();
- let tags: [u8; 6] = [0xFE, 0xFF, 0x25, 0x2B, 0xA3, 0xC5];
- let mut i = 0;
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.push(tags[i % tags.len()]);
- bytes.push(0x00);
- i += 1;
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
- bytes
- }
-
- #[test]
- fn test_tag_interleaved_binary_not_misdetected_as_utf16le() {
- let bytes = build_tag_interleaved_binary_bytes();
- assert_eq!(bytes.len(), FILE_ANALYSIS_BYTES);
-
- let result = analyze_byte_content(&bytes);
- assert_eq!(
- result,
- ByteContent::Binary,
- "binary data with sparse non-word low bytes and null high bytes \
- should not be misdetected as UTF-16LE text"
- );
- }
-
- #[test]
- fn test_utf16le_text_detected_as_utf16le() {
- let text = "Hello, world! This is a UTF-16 test string. ";
- let mut bytes = Vec::new();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
- }
-
- #[test]
- fn test_utf16be_text_detected_as_utf16be() {
- let text = "Hello, world! This is a UTF-16 test string. ";
- let mut bytes = Vec::new();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
- }
-
- #[test]
- fn test_utf16le_cyrillic_text_detected_as_utf16le() {
- let text = "Привет, мир! Это тестовая строка в UTF-16. ";
- let mut bytes = Vec::new();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Le);
- }
-
- #[test]
- fn test_utf16be_greek_text_detected_as_utf16be() {
- let text = "Γεια σου κόσμε! Αυτή είναι μια δοκιμαστική συμβολοσειρά. ";
- let mut bytes = Vec::new();
- while bytes.len() < FILE_ANALYSIS_BYTES {
- bytes.extend(text.encode_utf16().flat_map(|u| u.to_be_bytes()));
- }
- bytes.truncate(FILE_ANALYSIS_BYTES);
-
- assert_eq!(analyze_byte_content(&bytes), ByteContent::Utf16Be);
- }
-
- #[test]
- fn test_known_binary_headers() {
- let cases: &[(&[u8], &str)] = &[
- (b"RIFF\x00\x00\x00\x00WAVE", "WAV"),
- (b"RIFF\x00\x00\x00\x00AVI ", "AVI"),
- (b"OggS\x00\x02", "OGG"),
- (b"fLaC\x00\x00", "FLAC"),
- (b"ID3\x03\x00", "MP3 ID3v2"),
- (b"\xFF\xFB\x90\x00", "MP3 MPEG1 Layer3"),
- (b"\xFF\xF3\x90\x00", "MP3 MPEG2 Layer3"),
- ];
-
- for (header, label) in cases {
- let mut bytes = header.to_vec();
- bytes.resize(FILE_ANALYSIS_BYTES, 0x41); // pad with 'A'
- assert_eq!(
- analyze_byte_content(&bytes),
- ByteContent::Binary,
- "{label} should be detected as Binary"
- );
- }
- }
}