Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Unify non-Unicode file detection code (#62581)

Kirill Bulatov4efba7161f84
diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rs
index 8b8bdac3010..438b2201880 100644
--- a/crates/project/src/project_search.rs
+++ b/crates/project/src/project_search.rs
@@ -1,7 +1,7 @@
use std::{
cell::LazyCell,
collections::BTreeSet,
- io::{BufRead, BufReader},
+ io::{BufRead, BufReader, Cursor, ErrorKind, Read},
ops::Range,
path::{Path, PathBuf},
pin::pin,
@@ -21,8 +21,12 @@ use parking_lot::Mutex;
use postage::oneshot;
use rpc::{AnyProtoClient, proto};
+use language::ByteContent;
use util::{ResultExt, maybe, paths::compare_rel_paths, rel_path::RelPath};
-use worktree::{Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings};
+use worktree::{
+ Entry, ProjectEntryId, Snapshot, Worktree, WorktreeSettings, decode_byte_header,
+ decode_file_text,
+};
use crate::{
Project, ProjectItem, ProjectPath, RemotelyCreatedModels,
@@ -779,32 +783,42 @@ impl RequestHandler<'_> {
async fn handle_find_first_match(&self, mut entry: MatchingEntry) {
async move {
let abs_path = entry.worktree_root.join(entry.path.path.as_std_path());
- let Some(file) = self
+ let fs = self
.fs
- .context("Trying to query filesystem in remote project search")?
- .open_sync(&abs_path)
- .await
- .log_err()
- else {
+ .context("Trying to query filesystem in remote project search")?;
+ let Some(file) = fs.open_sync(&abs_path).await.log_err() else {
return anyhow::Ok(());
};
let mut file = BufReader::new(file);
let file_start = file.fill_buf()?;
-
- if let Err(Some(starting_position)) =
- std::str::from_utf8(file_start).map_err(|e| e.error_len())
- {
- // Before attempting to match the file content, throw away files that have invalid UTF-8 sequences early on;
- // That way we can still match files in a streaming fashion without having look at "obviously binary" files.
- log::debug!(
- "Invalid UTF-8 sequence in file {abs_path:?} \
- at byte position {starting_position}"
- );
+ let (bom_encoding, byte_content) = decode_byte_header(file_start);
+ if byte_content == ByteContent::Binary {
+ log::debug!("Skipping binary file {abs_path:?}");
return Ok(());
}
- if let Some(line_hint) = self.query.detect(file).await.ok().flatten() {
+ let is_plain_utf8 = bom_encoding.is_none()
+ && byte_content == ByteContent::Unknown
+ && is_utf8_prefix(file_start);
+
+ let line_hint = if is_plain_utf8 {
+ match self.query.detect(file).await {
+ Ok(line_hint) => line_hint,
+ Err(error)
+ if error
+ .downcast_ref::<std::io::Error>()
+ .is_some_and(|error| error.kind() == ErrorKind::InvalidData) =>
+ {
+ self.detect_in_decoded_file(fs, &abs_path).await?
+ }
+ Err(error) => return Err(error),
+ }
+ } else {
+ self.detect_in_decoded_file(fs, &abs_path).await?
+ };
+
+ if let Some(line_hint) = line_hint {
// Yes, we should scan the whole file.
entry.should_scan_tx.send((entry.path, line_hint)).await?;
}
@@ -814,6 +828,16 @@ impl RequestHandler<'_> {
.ok();
}
+ async fn detect_in_decoded_file(
+ &self,
+ fs: &dyn Fs,
+ abs_path: &Path,
+ ) -> anyhow::Result<Option<MatchPositionHint>> {
+ let (text, _encoding, _has_bom) = decode_file_text(fs, abs_path).await?;
+ let reader: Box<dyn Read + Send + Sync> = Box::new(Cursor::new(text.into_bytes()));
+ self.query.detect(BufReader::new(reader)).await
+ }
+
async fn handle_scan_path(&self, req: InputPath) {
_ = maybe!(async move {
let InputPath {
@@ -870,6 +894,13 @@ impl RequestHandler<'_> {
}
}
+fn is_utf8_prefix(bytes: &[u8]) -> bool {
+ match std::str::from_utf8(bytes) {
+ Ok(_) => true,
+ Err(error) => error.error_len().is_none(),
+ }
+}
+
struct InputPath {
entry: Entry,
snapshot: Snapshot,
diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs
index fa168b2fa4b..7734ed38166 100644
--- a/crates/project/tests/integration/project_tests.rs
+++ b/crates/project/tests/integration/project_tests.rs
@@ -9235,6 +9235,117 @@ async fn test_search_with_unicode(cx: &mut gpui::TestAppContext) {
);
}
+#[gpui::test]
+async fn test_search_in_unopened_non_utf8_files(cx: &mut gpui::TestAppContext) {
+ init_test(cx);
+
+ let text = "// 你好世界 hello\n// 这是一个中文注释,包含很多汉字,用来帮助编码检测器正确判断文件编码。\n// 编码检测需要足够多的中文内容才能可靠工作。\n";
+
+ let gb2312 = encoding_rs::Encoding::for_label(b"gb2312").unwrap();
+ assert_eq!(gb2312, encoding_rs::GBK);
+ let (gbk_bytes, _, had_errors) = gb2312.encode(text);
+ assert!(!had_errors);
+
+ let korean_text = "// 안녕하세요 세계 hello\n// 이것은 한국어 주석입니다. 인코딩 감지기가 파일 인코딩을 올바르게 판단할 수 있도록 충분히 많은 한글 내용을 담고 있습니다.\n// 인코딩 감지는 충분한 한국어 내용이 있어야 안정적으로 작동합니다.\n";
+ let (euc_kr_bytes, _, had_errors) = encoding_rs::EUC_KR.encode(korean_text);
+ assert!(!had_errors);
+
+ let mut utf16_bytes = vec![0xFF, 0xFE];
+ utf16_bytes.extend(text.encode_utf16().flat_map(|u| u.to_le_bytes()));
+
+ let mut binary_bytes = b"\x89PNG\r\n\x1a\n".to_vec();
+ binary_bytes.extend_from_slice(text.as_bytes());
+
+ let fs = FakeFs::new(cx.executor());
+ fs.insert_tree(
+ path!("/dir"),
+ json!({
+ "utf8.rs": text,
+ }),
+ )
+ .await;
+ fs.insert_file(path!("/dir/gbk.rs"), gbk_bytes.into_owned())
+ .await;
+ fs.insert_file(path!("/dir/euc_kr.rs"), euc_kr_bytes.into_owned())
+ .await;
+ fs.insert_file(path!("/dir/utf16.rs"), utf16_bytes).await;
+ fs.insert_file(path!("/dir/binary.dat"), binary_bytes).await;
+
+ let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await;
+
+ assert_eq!(
+ search(
+ &project,
+ SearchQuery::text(
+ "世界",
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap(),
+ cx
+ )
+ .await
+ .unwrap(),
+ HashMap::from_iter([
+ (path!("dir/utf8.rs").to_string(), vec![9..15]),
+ (path!("dir/gbk.rs").to_string(), vec![9..15]),
+ (path!("dir/utf16.rs").to_string(), vec![9..15]),
+ ])
+ );
+
+ assert_eq!(
+ search(
+ &project,
+ SearchQuery::text(
+ "HELLO",
+ false,
+ false,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap(),
+ cx
+ )
+ .await
+ .unwrap(),
+ HashMap::from_iter([
+ (path!("dir/utf8.rs").to_string(), vec![16..21]),
+ (path!("dir/gbk.rs").to_string(), vec![16..21]),
+ (path!("dir/euc_kr.rs").to_string(), vec![26..31]),
+ (path!("dir/utf16.rs").to_string(), vec![16..21]),
+ ])
+ );
+
+ assert_eq!(
+ search(
+ &project,
+ SearchQuery::text(
+ "세계",
+ false,
+ true,
+ false,
+ PathMatcher::default(),
+ PathMatcher::default(),
+ false,
+ None,
+ )
+ .unwrap(),
+ cx
+ )
+ .await
+ .unwrap(),
+ HashMap::from_iter([(path!("dir/euc_kr.rs").to_string(), vec![19..25])])
+ );
+}
+
#[gpui::test]
async fn test_create_entry(cx: &mut gpui::TestAppContext) {
init_test(cx);
diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs
index 123debb3483..701f5e869bd 100644
--- a/crates/worktree/src/worktree.rs
+++ b/crates/worktree/src/worktree.rs
@@ -7143,7 +7143,7 @@ impl fs::Watcher for NullWatcher {
}
}
-async fn decode_file_text(
+pub async fn decode_file_text(
fs: &dyn Fs,
abs_path: &Path,
) -> Result<(String, &'static Encoding, bool)> {
@@ -7193,7 +7193,7 @@ async fn decode_file_text(
decode_byte_full(content, bom_encoding, byte_content)
}
-fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
+pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
return (Some(encoding), ByteContent::Unknown);
}