Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Unify non-Unicode file detection code (#62581)
Kirill Bulatov4efba7161f84
diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rsindex 8b8bdac3010..438b2201880 100644--- a/crates/project/src/project_search.rs+++ b/crates/project/src/project_search.rs@@ -1,7 +1,7 @@use std::{cell::LazyCell,collections::BTreeSet,- io::{BufRead, BufReader},+ io::{BufRead, BufReader, Cursor, ErrorKind, Read},ops::Range,path::{Path, PathBuf},pin::pin,@@ -21,8 +21,12 @@ use parking_lot::Mutex;use postage::oneshot;use rpc::{AnyProtoClient, proto};+use language::ByteContent;use util::{ResultExt, maybe, paths::compare_rel_paths, rel_path::RelPath};-use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings};+use worktree::{+ Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header,+ decode_file_text,+};use crate::{Project, ProjectItem, ProjectPath, RemotelyCreatedModels,@@ -779,32 +783,42 @@ impl RequestHandler<'_> {async fn handle_find_first_match(&self, mut entry: MatchingEntry) {async move {let abs_path = entry.worktree_root.join(entry.path.path.as_std_path());- let Some(file) = self+ let fs = self.fs- .context("Trying to query filesystem in remote project search")?- .open_sync(&abs_path)- .await- .log_err()- else {+ .context("Trying to query filesystem in remote project search")?;+ let Some(file) = fs.open_sync(&abs_path).await.log_err() else {return anyhow::Ok(());};let mut file = BufReader::new(file);let file_start = file.fill_buf()?;-- if let Err(Some(starting_position)) =- std::str::from_utf8(file_start).map_err(|e| e.error_len())- {- // Before attempting to match the file content, throw away files that have invalid UTF-8 sequences early on;- // That way we can still match files in a streaming fashion without having look at "obviously binary" files.- log::debug!(- "Invalid UTF-8 sequence in file {abs_path:?} \- at byte position {starting_position}"- );+ let (bom_encoding, byte_content) = decode_byte_header(file_start);+ if byte_content == ByteContent::Binary {+ log::debug!("Skipping binary file {abs_path:?}");return Ok(());}- if let Some(line_hint) = self.query.detect(file).await.ok().flatten() {+ let is_plain_utf8 = bom_encoding.is_none()+ && byte_content == ByteContent::Unknown+ && is_utf8_prefix(file_start);++ let line_hint = if is_plain_utf8 {+ match self.query.detect(file).await {+ Ok(line_hint) => line_hint,+ Err(error)+ if error+ .downcast_ref::<std::io::Error>()+ .is_some_and(|error| error.kind() == ErrorKind::InvalidData) =>+ {+ self.detect_in_decoded_file(fs, &abs_path).await?+ }+ Err(error) => return Err(error),+ }+ } else {+ self.detect_in_decoded_file(fs, &abs_path).await?+ };++ if let Some(line_hint) = line_hint {// Yes, we should scan the whole file.entry.should_scan_tx.send((entry.path, line_hint)).await?;}@@ -814,6 +828,16 @@ impl RequestHandler<'_> {.ok();}+ async fn detect_in_decoded_file(+ &self,+ fs: &dyn Fs,+ abs_path: &Path,+ ) -> anyhow::Result<Option<MatchPositionHint>> {+ let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?;+ let reader: Box<dyn Read + Send + Sync> = Box::new(Cursor::new(text.into_bytes()));+ self.query.detect(BufReader::new(reader)).await+ }+async fn handle_scan_path(&self, req: InputPath) {_ = maybe!(async move {let InputPath {@@ -870,6 +894,13 @@ impl RequestHandler<'_> {}}+fn is_utf8_prefix(bytes: &[u8]) -> bool {+ match std::str::from_utf8(bytes) {+ Ok(_) => true,+ Err(error) => error.error_len().is_none(),+ }+}+struct InputPath {entry: Entry,snapshot: Snapshot,diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rsindex fa168b2fa4b..7734ed38166 100644--- a/crates/project/tests/integration/project_tests.rs+++ b/crates/project/tests/integration/project_tests.rs@@ -9235,6 +9235,117 @@ async fn test_search_with_unicode(cx: &mut gpui::TestAppContext) {);}+#[gpui::test]+async fn test_search_in_unopened_non_utf8_files(cx: &mut gpui::TestAppContext) {+ init_test(cx);++ let text = "// 你好世界 hello\n// 这是一个中文注释,包含很多汉字,用来帮助编码检测器正确判断文件编码。\n// 编码检测需要足够多的中文内容才能可靠工作。\n";++ let gb2312 = encoding_rs::Encoding::for_label(b"gb2312").unwrap();+ assert_eq!(gb2312, encoding_rs::GBK);+ let (gbk_bytes, _, had_errors) = gb2312.encode(text);+ assert!(!had_errors);++ let korean_text = "// 안녕하세요 세계 hello\n// 이것은 한국어 주석입니다. 인코딩 감지기가 파일 인코딩을 올바르게 판단할 수 있도록 충분히 많은 한글 내용을 담고 있습니다.\n// 인코딩 감지는 충분한 한국어 내용이 있어야 안정적으로 작동합니다.\n";+ let (euc_kr_bytes, _, had_errors) = encoding_rs::EUC_KR.encode(korean_text);+ assert!(!had_errors);++ let mut utf16_bytes = vec![0xFF, 0xFE];+ utf16_bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));++ let mut binary_bytes = b"\x89PNG\r\n\x1a\n".to_vec();+ binary_bytes.extend_from_slice(text.as_bytes());++ let fs = FakeFs::new(cx.executor());+ fs.insert_tree(+ path!("/dir"),+ json!({+ "utf8.rs": text,+ }),+ )+ .await;+ fs.insert_file(path!("/dir/gbk.rs"), gbk_bytes.into_owned())+ .await;+ fs.insert_file(path!("/dir/euc_kr.rs"), euc_kr_bytes.into_owned())+ .await;+ fs.insert_file(path!("/dir/utf16.rs"), utf16_bytes).await;+ fs.insert_file(path!("/dir/binary.dat"), binary_bytes).await;++ let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await;++ assert_eq!(+ search(+ &project,+ SearchQuery::text(+ "世界",+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap(),+ cx+ )+ .await+ .unwrap(),+ HashMap::from_iter([+ (path!("dir/utf8.rs").to_string(), vec![9..15]),+ (path!("dir/gbk.rs").to_string(), vec![9..15]),+ (path!("dir/utf16.rs").to_string(), vec![9..15]),+ ])+ );++ assert_eq!(+ search(+ &project,+ SearchQuery::text(+ "HELLO",+ false,+ false,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap(),+ cx+ )+ .await+ .unwrap(),+ HashMap::from_iter([+ (path!("dir/utf8.rs").to_string(), vec![16..21]),+ (path!("dir/gbk.rs").to_string(), vec![16..21]),+ (path!("dir/euc_kr.rs").to_string(), vec![26..31]),+ (path!("dir/utf16.rs").to_string(), vec![16..21]),+ ])+ );++ assert_eq!(+ search(+ &project,+ SearchQuery::text(+ "세계",+ false,+ true,+ false,+ PathMatcher::default(),+ PathMatcher::default(),+ false,+ None,+ )+ .unwrap(),+ cx+ )+ .await+ .unwrap(),+ HashMap::from_iter([(path!("dir/euc_kr.rs").to_string(), vec![19..25])])+ );+}+#[gpui::test]async fn test_create_entry(cx: &mut gpui::TestAppContext) {init_test(cx);diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rsindex 123debb3483..701f5e869bd 100644--- a/crates/worktree/src/worktree.rs+++ b/crates/worktree/src/worktree.rs@@ -7143,7 +7143,7 @@ impl fs::Watcher for NullWatcher {}}-async fn decode_file_text(+pub async fn decode_file_text(fs: &dyn Fs,abs_path: &Path,) -> Result<(String, &'static Encoding, bool)> {@@ -7193,7 +7193,7 @@ async fn decode_file_text(decode_byte_full(content, bom_encoding, byte_content)}-fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {+pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {return (Some(encoding), ByteContent::Unknown);}