Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
git: Decode non-UTF-8 blobs for project diffs (#60821)
hvck8968bf78084f
diff --git a/Cargo.lock b/Cargo.lockindex 5eda1ece25f..795928677a4 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -9760,6 +9760,7 @@ version = "0.1.0"dependencies = ["anyhow","async-trait",+ "chardetng","clock","collections","criterion",@@ -22852,7 +22853,6 @@ dependencies = ["anyhow","async-channel 2.5.0","async-lock",- "chardetng","clock","collections","encoding_rs",diff --git a/crates/collab/tests/integration/editor_tests.rs b/crates/collab/tests/integration/editor_tests.rsindex 3b16f57d04a..37fcd5150ff 100644--- a/crates/collab/tests/integration/editor_tests.rs+++ b/crates/collab/tests/integration/editor_tests.rs@@ -4405,9 +4405,10 @@ async fn test_git_blame_is_forwarded(cx_a: &mut TestAppContext, cx_b: &mut TestA.fs().with_git_state(Path::new(path!("/my-repo/.git")), true, |state| {state.refs.insert("HEAD".into(), revision.to_string());- state- .head_contents- .insert(repo_path("file.txt"), content_at_revision.to_string());+ state.head_contents.insert(+ repo_path("file.txt"),+ content_at_revision.as_bytes().to_vec(),+ );state.blames_at_revision.insert((repo_path("file.txt"), revision), blame_at_revision.clone());diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rsindex ad84ff1dc52..6739552bf21 100644--- a/crates/editor/src/editor_tests.rs+++ b/crates/editor/src/editor_tests.rs@@ -27071,12 +27071,9 @@ async fn test_merge_base_diff_hunks_are_read_only(cx: &mut TestAppContext) {.unwrap();assert_eq!(index_contents,- [(- ::git::repository::repo_path("file.rs"),- "head\n".to_string(),- )]- .into_iter()- .collect::<HashMap<_, _>>()+ [(::git::repository::repo_path("file.rs"), b"head\n".to_vec(),)]+ .into_iter()+ .collect::<HashMap<_, _>>());}diff --git a/crates/editor/src/test/editor_test_context.rs b/crates/editor/src/test/editor_test_context.rsindex 84b03d91ca1..517e3fa3b35 100644--- a/crates/editor/src/test/editor_test_context.rs+++ b/crates/editor/src/test/editor_test_context.rs@@ -374,7 +374,7 @@ impl EditorTestContext {.cloned();}).unwrap();- assert_eq!(expected, found.as_deref());+ assert_eq!(expected.map(str::as_bytes), found.as_deref());}/// Change the editor's text and selections using a string containingdiff --git a/crates/fs/src/fake_git_repo.rs b/crates/fs/src/fake_git_repo.rsindex 543a11a9c2f..95209c5a8f3 100644--- a/crates/fs/src/fake_git_repo.rs+++ b/crates/fs/src/fake_git_repo.rs@@ -44,8 +44,8 @@ pub struct FakeGitRepository {#[derive(Debug, Clone)]pub struct FakeCommitSnapshot {- pub head_contents: HashMap<RepoPath, String>,- pub index_contents: HashMap<RepoPath, String>,+ pub head_contents: HashMap<RepoPath, Vec<u8>>,+ pub index_contents: HashMap<RepoPath, Vec<u8>>,pub sha: String,}@@ -60,11 +60,11 @@ pub struct FakeGitRepositoryState {pub commit_history: Vec<FakeCommitSnapshot>,pub event_emitter: async_channel::Sender<PathBuf>,pub unmerged_paths: HashMap<RepoPath, UnmergedStatus>,- pub head_contents: HashMap<RepoPath, String>,- pub index_contents: HashMap<RepoPath, String>,+ pub head_contents: HashMap<RepoPath, Vec<u8>>,+ pub index_contents: HashMap<RepoPath, Vec<u8>>,// everything in commit contents is in oidspub merge_base_contents: HashMap<RepoPath, Oid>,- pub oids: HashMap<Oid, String>,+ pub oids: HashMap<Oid, Vec<u8>>,pub blames: HashMap<RepoPath, Blame>,pub blames_at_revision: HashMap<(RepoPath, Oid), Blame>,pub current_branch_name: Option<String>,@@ -170,7 +170,7 @@ impl GitRepository for FakeGitRepository {self.with_state_async(false, |state| Ok(state.commit_template.clone()))}- fn load_blob_content(&self, oid: git::Oid) -> BoxFuture<'_, Result<String>> {+ fn load_blob_content(&self, oid: git::Oid) -> BoxFuture<'_, Result<Vec<u8>>> {self.with_state_async(false, move |state| {state.oids.get(&oid).cloned().context("oid does not exist")})@@ -188,7 +188,7 @@ impl GitRepository for FakeGitRepository {fn set_index_text(&self,path: RepoPath,- content: Option<String>,+ content: Option<Vec<u8>>,_env: Arc<HashMap<String, String>>,_is_executable: bool,) -> BoxFuture<'_, anyhow::Result<()>> {@@ -219,7 +219,7 @@ impl GitRepository for FakeGitRepository {.filter_map(|path| {let path_in_repo = path.strip_prefix(workdir_path).ok()?;let path_in_repo = RelPath::new(path_in_repo, PathStyle::local()).ok()?;- let content = String::from_utf8(self.fs.read_file_sync(path).ok()?).ok()?;+ let content = self.fs.read_file_sync(path).ok()?;Some((RepoPath::from_rel_path(&path_in_repo), content))}).collect::<HashMap<_, _>>()@@ -261,7 +261,10 @@ impl GitRepository for FakeGitRepository {})}- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>> {+ fn load_revisions(+ &self,+ revisions: Vec<String>,+ ) -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>> {let fut = self.with_state_async(false, move |state| {Ok(revisions.into_iter()@@ -398,7 +401,7 @@ impl GitRepository for FakeGitRepository {.collect::<Vec<_>>();// Load working copy files.- let git_files: HashMap<RepoPath, (String, bool)> = self+ let git_files: HashMap<RepoPath, (Vec<u8>, bool)> = self.fs.files().iter()@@ -413,11 +416,7 @@ impl GitRepository for FakeGitRepository {ignore::Match::Whitelist(_) => break,}}- let content = self- .fs- .read_file_sync(path)- .ok()- .map(|content| String::from_utf8(content).unwrap())?;+ let content = self.fs.read_file_sync(path).ok()?;let repo_path = RelPath::new(repo_path, PathStyle::local()).ok()?;Some((RepoPath::from_rel_path(&repo_path), (content, is_ignored)))})@@ -1026,7 +1025,9 @@ impl GitRepository for FakeGitRepository {.parent().unwrap().join(&path.as_std_path());- Box::pin(async move { (path.clone(), self.fs.load(&abs_path).await.ok()) })+ Box::pin(+ async move { (path.clone(), self.fs.load_bytes(&abs_path).await.ok()) },+ )}).collect::<Vec<_>>();let contents = join_all(contents).await;@@ -1200,11 +1201,11 @@ impl GitRepository for FakeGitRepository {diff: git::repository::DiffStatType,path_prefixes: &[RepoPath],) -> BoxFuture<'static, Result<git::status::GitDiffStat>> {- fn count_lines(s: &str) -> u32 {- if s.is_empty() {+ fn count_lines(bytes: &[u8]) -> u32 {+ if bytes.is_empty() {0} else {- s.lines().count() as u32+ String::from_utf8_lossy(bytes).lines().count() as u32}}@@ -1224,7 +1225,7 @@ impl GitRepository for FakeGitRepository {let path_prefixes = path_prefixes.to_vec();let workdir_path = self.dot_git_path.parent().unwrap().to_path_buf();- let worktree_files: HashMap<RepoPath, String> = self+ let worktree_files: HashMap<RepoPath, Vec<u8>> = self.fs.files().iter()@@ -1233,11 +1234,7 @@ impl GitRepository for FakeGitRepository {if repo_path.starts_with(".git") {return None;}- let content = self- .fs- .read_file_sync(path)- .ok()- .and_then(|bytes| String::from_utf8(bytes).ok())?;+ let content = self.fs.read_file_sync(path).ok()?;let repo_path = RelPath::new(repo_path, PathStyle::local()).ok()?;Some((RepoPath::from_rel_path(&repo_path), content))})diff --git a/crates/fs/src/fs.rs b/crates/fs/src/fs.rsindex 711aa53f279..de8ceaa4060 100644--- a/crates/fs/src/fs.rs+++ b/crates/fs/src/fs.rs@@ -2294,7 +2294,7 @@ impl FakeFs {state.index_contents.extend(index_state.iter()- .map(|(path, content)| (repo_path(path), content.clone())),+ .map(|(path, content)| (repo_path(path), content.as_bytes().to_vec())),);}).unwrap();@@ -2311,7 +2311,7 @@ impl FakeFs {state.head_contents.extend(head_state.iter()- .map(|(path, content)| (repo_path(path), content.clone())),+ .map(|(path, content)| (repo_path(path), content.as_bytes().to_vec())),);state.refs.insert("HEAD".into(), sha.into());})@@ -2324,7 +2324,7 @@ impl FakeFs {state.head_contents.extend(contents_by_path.iter()- .map(|(path, contents)| (repo_path(path), contents.clone())),+ .map(|(path, contents)| (repo_path(path), contents.as_bytes().to_vec())),);state.index_contents = state.head_contents.clone();})@@ -2345,7 +2345,7 @@ impl FakeFs {.map(|n| Oid::from_bytes(n.repeat(20).as_bytes()).unwrap());for ((path, content), oid) in contents_by_path.iter().zip(oids) {state.merge_base_contents.insert(repo_path(path), oid);- state.oids.insert(oid, content.clone());+ state.oids.insert(oid, content.as_bytes().to_vec());}}).unwrap();@@ -2472,10 +2472,14 @@ impl FakeFs {};if let Some(content) = index_content {- state.index_contents.insert(repo_path.clone(), content);+ state+ .index_contents+ .insert(repo_path.clone(), content.into_bytes());}if let Some(content) = head_content {- state.head_contents.insert(repo_path.clone(), content);+ state+ .head_contents+ .insert(repo_path.clone(), content.into_bytes());}}}).unwrap();diff --git a/crates/git/src/repository.rs b/crates/git/src/repository.rsindex 7cddc70b1a5..4b52142d56e 100644--- a/crates/git/src/repository.rs+++ b/crates/git/src/repository.rs@@ -547,14 +547,14 @@ pub enum CommitFileStatus {#[derive(Debug)]pub struct CommitFile {pub path: RepoPath,- pub old_text: Option<String>,- pub new_text: Option<String>,+ pub old_content: Option<Vec<u8>>,+ pub new_content: Option<Vec<u8>>,pub is_binary: bool,}impl CommitFile {pub fn status(&self) -> CommitFileStatus {- match (&self.old_text, &self.new_text) {+ match (&self.old_content, &self.new_content) {(None, Some(_)) => CommitFileStatus::Added,(Some(_), None) => CommitFileStatus::Deleted,_ => CommitFileStatus::Modified,@@ -576,7 +576,7 @@ pub fn is_binary_content(content: &[u8]) -> bool {}struct LoadedCommitObject {- text: String,+ content: Vec<u8>,is_binary: bool,}@@ -599,11 +599,7 @@ async fn read_commit_blob<R: smol::io::AsyncBufRead + Unpin>(let is_binary = is_binary_content(&bytes);Ok(LoadedCommitObject {- text: if is_binary {- String::new()- } else {- String::from_utf8_lossy(&bytes).to_string()- },+ content: bytes,is_binary,})}@@ -617,7 +613,7 @@ async fn load_commit_object<R: smol::io::AsyncBufRead + Unpin>(match object {Some(object) if object.kind == CommitDiffObjectKind::Gitlink => {Ok(Some(LoadedCommitObject {- text: format!("Subproject commit {}\n", object.oid),+ content: format!("Subproject commit {}\n", object.oid).into_bytes(),is_binary: false,}))}@@ -754,7 +750,7 @@ pub trait GitRepository: Send + Sync {/// Returns the contents of an entry in the repository's index, or None if there is no entry for the given path.////// Also returns `None` for symlinks.- fn load_index_text(&self, path: RepoPath) -> BoxFuture<'_, Option<String>> {+ fn load_index_text(&self, path: RepoPath) -> BoxFuture<'_, Option<Vec<u8>>> {let future = self.load_revisions(vec![format!(":{}", path.as_unix_str())]);async move { future.await.ok()?.pop()? }.boxed()}@@ -762,16 +758,16 @@ pub trait GitRepository: Send + Sync {/// Returns the contents of an entry in the repository's HEAD, or None if HEAD does not exist or has no entry for the given path.////// Also returns `None` for symlinks.- fn load_committed_text(&self, path: RepoPath) -> BoxFuture<'_, Option<String>> {+ fn load_committed_text(&self, path: RepoPath) -> BoxFuture<'_, Option<Vec<u8>>> {let future = self.load_revisions(vec![format!("HEAD:{}", path.as_unix_str())]);async move { future.await.ok()?.pop()? }.boxed()}- fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<String>>;+ fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<Vec<u8>>>;fn set_index_text(&self,path: RepoPath,- content: Option<String>,+ content: Option<Vec<u8>>,env: Arc<HashMap<String, String>>,is_executable: bool,) -> BoxFuture<'_, anyhow::Result<()>>;@@ -789,7 +785,8 @@ pub trait GitRepository: Send + Sync {/// Resolve a list of refs to SHAs.fn revparse_batch(&self, revs: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>>;- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>>;+ fn load_revisions(&self, revisions: Vec<String>)+ -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>>;fn head_sha(&self) -> BoxFuture<'_, Option<String>> {async move {@@ -1461,25 +1458,13 @@ impl GitRepository for RealGitRepository {.await?;let is_binary = new_object.as_ref().is_some_and(|object| object.is_binary)|| old_object.as_ref().is_some_and(|object| object.is_binary);- let new_text = new_object.map(|object| {- if is_binary {- String::new()- } else {- object.text- }- });- let old_text = old_object.map(|object| {- if is_binary {- String::new()- } else {- object.text- }- });+ let new_content = new_object.map(|object| object.content);+ let old_content = old_object.map(|object| object.content);files.push(CommitFile {path: RepoPath(Arc::from(rel_path)),- old_text,- new_text,+ old_content,+ new_content,is_binary,})}@@ -1547,11 +1532,23 @@ impl GitRepository for RealGitRepository {.boxed()}- fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<String>> {+ fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<Vec<u8>>> {let git_binary = self.git_binary();let oid_str = oid.to_string();self.executor- .spawn(async move { git_binary.run_raw(&["cat-file", "blob", &oid_str]).await })+ .spawn(async move {+ let mut command = git_binary.build_command(&["cat-file", "blob", &oid_str]);+ let output = command.output().await?;+ anyhow::ensure!(+ output.status.success(),+ GitBinaryCommandError {+ stdout: String::from_utf8_lossy(&output.stdout).to_string(),+ stderr: String::from_utf8_lossy(&output.stderr).to_string(),+ status: output.status,+ }+ );+ Ok(output.stdout)+ }).boxed()}@@ -1600,7 +1597,7 @@ impl GitRepository for RealGitRepository {fn set_index_text(&self,path: RepoPath,- content: Option<String>,+ content: Option<Vec<u8>>,env: Arc<HashMap<String, String>>,is_executable: bool,) -> BoxFuture<'_, anyhow::Result<()>> {@@ -1611,17 +1608,32 @@ impl GitRepository for RealGitRepository {if let Some(content) = content {let mut child = git- .build_command(&["hash-object", "-w", "--stdin"])+ .build_command(&[+ "hash-object",+ "-w",+ "--stdin",+ "--path",+ path.as_unix_str(),+ ]).envs(env.iter()).stdin(Stdio::piped()).stdout(Stdio::piped())+ .stderr(Stdio::piped()).spawn()?;- let mut stdin = child.stdin.take().unwrap();- stdin.write_all(content.as_bytes()).await?;+ let mut stdin = child.stdin.take().context("hash-object has no stdin")?;+ stdin.write_all(&content).await?;stdin.flush().await?;drop(stdin);- let output = child.output().await?.stdout;- let sha = str::from_utf8(&output)?.trim();+ let output = child.output().await?;+ anyhow::ensure!(+ output.status.success(),+ GitBinaryCommandError {+ stdout: String::from_utf8_lossy(&output.stdout).to_string(),+ stderr: String::from_utf8_lossy(&output.stderr).to_string(),+ status: output.status,+ }+ );+ let sha = str::from_utf8(&output.stdout)?.trim();log::debug!("indexing SHA: {sha}, path {path:?}");@@ -1722,7 +1734,10 @@ impl GitRepository for RealGitRepository {.boxed()}- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>> {+ fn load_revisions(+ &self,+ revisions: Vec<String>,+ ) -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>> {let git = self.git_binary();self.executor.spawn(async move {@@ -1773,7 +1788,7 @@ impl GitRepository for RealGitRepository {stdout.read_exact(&mut newline).await?;if object_type == "blob" {- results.push(String::from_utf8(content).ok());+ results.push(Some(content));} else {results.push(None);}@@ -4510,8 +4525,11 @@ mod tests {.first().expect("type-changed file should be present");assert_eq!(file.path.as_unix_str(), "file.txt");- assert_eq!(file.old_text.as_deref(), Some("regular contents\n"));- assert_eq!(file.new_text.as_deref(), Some("target"));+ assert_eq!(+ file.old_content.as_deref(),+ Some(b"regular contents\n".as_slice())+ );+ assert_eq!(file.new_content.as_deref(), Some(b"target".as_slice()));assert_eq!(file.status(), CommitFileStatus::Modified);}@@ -4560,10 +4578,10 @@ mod tests {.find(|file| file.path.as_unix_str() == "modules/example").expect("gitlink should be present alongside the regular file");assert_eq!(gitlink.status(), CommitFileStatus::Added);- assert_eq!(gitlink.old_text, None);+ assert_eq!(gitlink.old_content, None);assert_eq!(- gitlink.new_text.as_deref(),- Some("Subproject commit 1111111111111111111111111111111111111111\n")+ gitlink.new_content.as_deref(),+ Some(b"Subproject commit 1111111111111111111111111111111111111111\n".as_slice()));assert!(!gitlink.is_binary);@@ -4588,12 +4606,12 @@ mod tests {};assert_eq!(gitlink.status(), CommitFileStatus::Modified);assert_eq!(- gitlink.old_text.as_deref(),- Some("Subproject commit 1111111111111111111111111111111111111111\n")+ gitlink.old_content.as_deref(),+ Some(b"Subproject commit 1111111111111111111111111111111111111111\n".as_slice()));assert_eq!(- gitlink.new_text.as_deref(),- Some("Subproject commit 2222222222222222222222222222222222222222\n")+ gitlink.new_content.as_deref(),+ Some(b"Subproject commit 2222222222222222222222222222222222222222\n".as_slice()));assert!(!gitlink.is_binary);@@ -4609,10 +4627,10 @@ mod tests {};assert_eq!(gitlink.status(), CommitFileStatus::Deleted);assert_eq!(- gitlink.old_text.as_deref(),- Some("Subproject commit 2222222222222222222222222222222222222222\n")+ gitlink.old_content.as_deref(),+ Some(b"Subproject commit 2222222222222222222222222222222222222222\n".as_slice()));- assert_eq!(gitlink.new_text, None);+ assert_eq!(gitlink.new_content, None);assert!(!gitlink.is_binary);}@@ -5464,12 +5482,12 @@ mod tests {assert_eq!(results,vec![- Some("file1 committed contents".into()),- Some("file1 index contents".into()),- Some("file2 committed contents".into()),- Some("file2 committed contents".into()), // untouched in index, should match HEAD+ Some(b"file1 committed contents".to_vec()),+ Some(b"file1 index contents".to_vec()),+ Some(b"file2 committed contents".to_vec()),+ Some(b"file2 committed contents".to_vec()), // untouched in index, should match HEADNone,- Some("space file committed contents".into()),+ Some(b"space file committed contents".to_vec()),None,]);diff --git a/crates/git_ui/src/commit_view.rs b/crates/git_ui/src/commit_view.rsindex 009aa368add..7d720e2f7e2 100644--- a/crates/git_ui/src/commit_view.rs+++ b/crates/git_ui/src/commit_view.rs@@ -6,7 +6,7 @@ use editor::{SplittableEditor, hover_markdown_style, multibuffer_context_lines,};use futures_lite::future::yield_now;-use git::repository::{CommitDetails, CommitDiff, RepoPath, is_binary_content};+use git::repository::{CommitDetails, RepoPath};use git::status::{FileStatus, StatusCode, TrackedStatus};use git::{BuildCommitPermalinkParams, GitHostingProviderRegistry, GitRemote, ParsedGitRemote,@@ -24,7 +24,10 @@ use language::{};use markdown::{Markdown, MarkdownElement};use multi_buffer::PathKey;-use project::{Project, ProjectPath, WorktreeId, git_store::Repository};+use project::{+ Project, ProjectPath, WorktreeId,+ git_store::{CommitDiff, Repository},+};use settings::{DiffViewStyle, Settings};use std::{any::{Any, TypeId},@@ -322,11 +325,7 @@ impl CommitView {let raw_new_text = file.new_text.unwrap_or_default();let raw_old_text = file.old_text;- let is_binary = file.is_binary- || is_binary_content(raw_new_text.as_bytes())- || raw_old_text- .as_ref()- .is_some_and(|text| is_binary_content(text.as_bytes()));+ let is_binary = file.is_binary;let new_text = if is_binary {"(binary file not shown)".to_string()diff --git a/crates/git_ui/src/git_graph.rs b/crates/git_ui/src/git_graph.rsindex 1003541824b..4e67356feca 100644--- a/crates/git_ui/src/git_graph.rs+++ b/crates/git_ui/src/git_graph.rs@@ -11,10 +11,7 @@ use file_icons::FileIcons;use git::{BuildCommitPermalinkParams, GitHostingProviderRegistry, GitRemote, Oid, ParsedGitRemote,parse_git_remote_url,- repository::{- CommitDiff, CommitFile, InitialGraphCommitData, LogOrder, LogSource, RepoPath,- SearchCommitArgs,- },+ repository::{InitialGraphCommitData, LogOrder, LogSource, RepoPath, SearchCommitArgs},status::{FileStatus, StatusCode, TrackedStatus},};use gpui::{@@ -32,8 +29,8 @@ use picker::{Picker, PickerDelegate};use project::{ProjectPath,git_store::{- CommitDataState, GitGraphEvent, GitStore, GitStoreEvent, GraphDataResponse, Repository,- RepositoryEvent, RepositoryId,+ CommitDataState, CommitDiff, CommitFile, GitGraphEvent, GitStore, GitStoreEvent,+ GraphDataResponse, Repository, RepositoryEvent, RepositoryId,},};use smallvec::{SmallVec, smallvec};diff --git a/crates/git_ui/src/staged_diff.rs b/crates/git_ui/src/staged_diff.rsindex c99ed2a58ce..648b1980eb5 100644--- a/crates/git_ui/src/staged_diff.rs+++ b/crates/git_ui/src/staged_diff.rs@@ -976,7 +976,7 @@ mod tests {repo.load_index_text(RepoPath::from_rel_path(rel_path("src/main.rs"))).await.unwrap(),- committed_contents+ committed_contents.as_bytes());fs.unpause_events_and_flush();diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.tomlindex 69789da21ec..7fcef180cee 100644--- a/crates/language/Cargo.toml+++ b/crates/language/Cargo.toml@@ -32,6 +32,7 @@ test-support = [anyhow.workspace = trueasync-trait.workspace = trueclock.workspace = true+chardetng.workspace = truecollections.workspace = trueec4rs.workspace = trueencoding_rs.workspace = truediff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rsindex e02748257ba..4acfd5d04a8 100644--- a/crates/language/src/file_content.rs+++ b/crates/language/src/file_content.rs@@ -1,5 +1,97 @@+use anyhow::{Result, bail};+use chardetng::EncodingDetector;+use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};+pub const FILE_ANALYSIS_BYTES: usize = 1024;+pub struct DecodedText {+ pub text: String,+ pub encoding: &'static Encoding,+ pub has_bom: bool,+}++pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {+ if let Some((encoding, _bom_len)) = Encoding::for_bom(&bytes) {+ let (text, _) = encoding.decode_with_bom_removal(&bytes);+ return Ok(DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: true,+ });+ }++ let encoding = match analyze_byte_content(&bytes) {+ ByteContent::Utf16Le => UTF_16LE,+ ByteContent::Utf16Be => UTF_16BE,+ ByteContent::Binary => bail!("Binary files are not supported"),+ ByteContent::Unknown => {+ return match String::from_utf8(bytes) {+ Ok(text) if !text.contains('\x1b') => Ok(DecodedText {+ text,+ encoding: UTF_8,+ has_bom: false,+ }),+ Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),+ Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),+ };+ }+ };++ let (text, _, _) = encoding.decode(&bytes);+ Ok(DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: false,+ })+}++pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {+ if encoding == UTF_8 && !has_bom {+ return text.into_bytes();+ }++ // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.+ if encoding == UTF_16BE {+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);+ if has_bom {+ bytes.extend_from_slice(&[0xFE, 0xFF]);+ }+ bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));+ return bytes;+ }++ if encoding == UTF_16LE {+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);+ if has_bom {+ bytes.extend_from_slice(&[0xFF, 0xFE]);+ }+ bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));+ return bytes;+ }++ let (encoded, _, _) = encoding.encode(&text);+ if has_bom && encoding == UTF_8 {+ let mut bytes = Vec::with_capacity(encoded.len() + 3);+ bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);+ bytes.extend_from_slice(&encoded);+ bytes+ } else {+ encoded.into_owned()+ }+}++fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {+ let mut detector = EncodingDetector::new();+ detector.feed(&bytes, true);+ let encoding = detector.guess(None, true);+ let (text, _, _) = encoding.decode(&bytes);+ DecodedText {+ text: text.into_owned(),+ encoding,+ has_bom: false,+ }+}+#[derive(Debug, PartialEq)]pub enum ByteContent {Utf16Le,@@ -176,3 +268,41 @@ fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {}Some(u16::from_be_bytes(pair))}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn decodes_and_encodes_windows_1251() {+ let expected = "строка один\nстрока два\n";+ let (bytes, _, _) = encoding_rs::WINDOWS_1251.encode(expected);++ let decoded = decode_text(bytes.clone().into_owned()).unwrap();++ assert_eq!(decoded.text, expected);+ assert_eq!(decoded.encoding, encoding_rs::WINDOWS_1251);+ assert!(!decoded.has_bom);+ assert_eq!(+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),+ bytes.as_ref()+ );+ }++ #[test]+ fn preserves_unicode_boms() {+ let expected = "Hello, мир\n";+ for encoding in [UTF_8, UTF_16LE, UTF_16BE] {+ let bytes = encode_text(expected.to_owned(), encoding, true);+ let decoded = decode_text(bytes.clone()).unwrap();++ assert_eq!(decoded.text, expected);+ assert_eq!(decoded.encoding, encoding);+ assert!(decoded.has_bom);+ assert_eq!(+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),+ bytes+ );+ }+ }+}diff --git a/crates/language/src/language.rs b/crates/language/src/language.rsindex 6ba6a834898..bf794425e2a 100644--- a/crates/language/src/language.rs+++ b/crates/language/src/language.rs@@ -102,7 +102,9 @@ pub use buffer::Operation;pub use buffer::*;pub use diagnostic::{Diagnostic, DiagnosticSourceKind};pub use diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup};-pub use file_content::{ByteContent, FILE_ANALYSIS_BYTES, analyze_byte_content};+pub use file_content::{+ ByteContent, DecodedText, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,+};pub use language_registry::{BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QUERY_FILENAME_PREFIXES,};diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.tomlindex b3ee5a88a22..b42ddfcc950 100644--- a/crates/project/Cargo.toml+++ b/crates/project/Cargo.toml@@ -48,6 +48,7 @@ collections.workspace = truecontext_server.workspace = truecredentials_provider.workspace = truedap.workspace = true+encoding_rs.workspace = trueextension.workspace = truefancy-regex.workspace = truefs.workspace = true@@ -109,7 +110,6 @@ ztracing.workspace = true[dev-dependencies]client = { workspace = true, features = ["test-support"] }-encoding_rs.workspace = truecollections = { workspace = true, features = ["test-support"] }context_server = { workspace = true, features = ["test-support"] }buffer_diff = { workspace = true, features = ["test-support"] }diff --git a/crates/project/src/git_store.rs b/crates/project/src/git_store.rsindex c4f77c78784..1df75d5192c 100644--- a/crates/project/src/git_store.rs+++ b/crates/project/src/git_store.rs@@ -34,12 +34,12 @@ use git::{blame::Blame,parse_git_remote_url,repository::{- Branch, BranchesScanResult, CommitData, CommitDetails, CommitDiff, CommitFile,- CommitOptions, CreateWorktreeTarget, DiffStatType, DiffType, FetchOptions,- FileHistoryChangedFileSets, GitCommitTemplate, GitRepository, GitRepositoryCheckpoint,- InitialGraphCommitData, LogOrder, LogSource, PushOptions, Remote, RemoteCommandOutput,- RepoPath, ResetMode, SearchCommitArgs, UpstreamTrackingStatus, Worktree as GitWorktree,- delete_branch_flag, is_binary_content,+ Branch, BranchesScanResult, CommitData, CommitDetails, CommitFileStatus, CommitOptions,+ CreateWorktreeTarget, DiffStatType, DiffType, FetchOptions, FileHistoryChangedFileSets,+ GitCommitTemplate, GitRepository, GitRepositoryCheckpoint, InitialGraphCommitData,+ LogOrder, LogSource, PushOptions, Remote, RemoteCommandOutput, RepoPath, ResetMode,+ SearchCommitArgs, UpstreamTrackingStatus, Worktree as GitWorktree, delete_branch_flag,+ is_binary_content,},stash::{GitStash, StashEntry},status::{@@ -52,7 +52,7 @@ use gpui::{Subscription, Task, TaskExt, WeakEntity,};use language::{- Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry,+ Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry, decode_text, encode_text,proto::{deserialize_version, serialize_version},};use parking_lot::Mutex;@@ -208,6 +208,85 @@ fn pending_hunks(.collect()}+fn decode_git_text(bytes: Vec<u8>) -> Result<String> {+ Ok(decode_text(bytes)?.text)+}++#[derive(Debug)]+pub struct CommitDiff {+ pub files: Vec<CommitFile>,+}++#[derive(Debug)]+pub struct CommitFile {+ pub path: RepoPath,+ pub old_text: Option<String>,+ pub new_text: Option<String>,+ pub is_binary: bool,+}++impl CommitFile {+ pub fn status(&self) -> CommitFileStatus {+ match (&self.old_text, &self.new_text) {+ (None, Some(_)) => CommitFileStatus::Added,+ (Some(_), None) => CommitFileStatus::Deleted,+ _ => CommitFileStatus::Modified,+ }+ }+}++fn decode_commit_diff(diff: git::repository::CommitDiff) -> CommitDiff {+ let files = diff+ .files+ .into_iter()+ .map(|file| {+ let git::repository::CommitFile {+ path,+ old_content,+ new_content,+ mut is_binary,+ } = file;++ if is_binary {+ return CommitFile {+ path,+ old_text: old_content.map(|_| String::new()),+ new_text: new_content.map(|_| String::new()),+ is_binary,+ };+ }++ let mut decode_content = |content: Option<Vec<u8>>| {+ content.map(|content| match decode_git_text(content) {+ Ok(text) => text,+ Err(error) => {+ log::debug!(+ "treating commit content for {} as binary after decoding failed: {error:#}",+ path.as_unix_str()+ );+ is_binary = true;+ String::new()+ }+ })+ };+ let mut old_text = decode_content(old_content);+ let mut new_text = decode_content(new_content);+ if is_binary {+ old_text = old_text.map(|_| String::new());+ new_text = new_text.map(|_| String::new());+ }++ CommitFile {+ path,+ old_text,+ new_text,+ is_binary,+ }+ })+ .collect();+ CommitDiff { files }+}+#[derive(Clone, Debug)]enum DiffBasesChange {SetIndex(Option<String>),@@ -2864,11 +2943,22 @@ impl GitStore {let Some((repo, path)) = self.repository_and_path_for_buffer_id(buffer_id, cx) else {return;};+ let (encoding, has_bom) = self+ .buffer_store+ .read(cx)+ .get(buffer_id)+ .map(|buffer| {+ let buffer = buffer.read(cx);+ (buffer.encoding(), buffer.has_bom())+ })+ .unwrap_or((encoding_rs::UTF_8, false));let recv = repo.update(cx, |repo, cx| {log::debug!("hunks changed for {}", path.as_unix_str());repo.spawn_set_index_text_job(path,new_index_text,+ encoding,+ has_bom,Some(hunk_staging_operation_count),cx,)@@ -3435,11 +3525,30 @@ impl GitStore {let repository_handle = Self::repository_for_request(&this, repository_id, &mut cx)?;let repo_path = RepoPath::from_proto(&envelope.payload.path)?;+ let (encoding, has_bom) = cx.update(|cx| {+ repository_handle+ .read(cx)+ .repo_path_to_project_path(&repo_path, cx)+ .and_then(|project_path| {+ this.read(cx)+ .buffer_store+ .read(cx)+ .get_by_path(&project_path)+ })+ .map(|buffer| {+ let buffer = buffer.read(cx);+ (buffer.encoding(), buffer.has_bom())+ })+ .unwrap_or((encoding_rs::UTF_8, false))+ });+repository_handle.update(&mut cx, |repository_handle, cx| {repository_handle.spawn_set_index_text_job(repo_path,envelope.payload.text,+ encoding,+ has_bom,None,cx,)@@ -6346,11 +6455,13 @@ impl Repository {{let index_text = (current_index_text.is_some() && !*is_symlink).then(|| loaded_revisions.next().flatten())- .flatten();+ .flatten()+ .and_then(|bytes| decode_git_text(bytes).log_err());let head_text = (current_head_text.is_some() && !*is_symlink).then(|| loaded_revisions.next().flatten())- .flatten();+ .flatten()+ .and_then(|bytes| decode_git_text(bytes).log_err());let change =match (current_index_text.as_ref(), current_head_text.as_ref()) {@@ -6800,9 +6911,10 @@ impl Repository {let id = self.id;self.send_job("load_commit_diff", None, move |git_repo, cx| async move {match git_repo {- RepositoryState::Local(LocalRepositoryState { backend, .. }) => {- backend.load_commit(commit, cx).await- }+ RepositoryState::Local(LocalRepositoryState { backend, .. }) => backend+ .load_commit(commit, cx)+ .await+ .map(decode_commit_diff),RepositoryState::Remote(RemoteRepositoryState {client, project_id, ..}) => {@@ -8414,6 +8526,8 @@ impl Repository {&mut self,path: RepoPath,content: Option<String>,+ encoding: &'static encoding_rs::Encoding,+ has_bom: bool,hunk_staging_operation_count: Option<usize>,cx: &mut Context<Self>,) -> oneshot::Receiver<anyhow::Result<()>> {@@ -8443,6 +8557,8 @@ impl Repository {Ok(None) => false,Err(_err) => false,};+ let content =+ content.map(|content| encode_text(content, encoding, has_bom));backend.set_index_text(path.clone(), content, environment.clone(), executable).await?;@@ -9748,9 +9864,11 @@ impl Repository {) -> Task<Result<Option<String>>> {let rx = self.send_job("load_staged_text", None, move |state, _| async move {match state {- RepositoryState::Local(LocalRepositoryState { backend, .. }) => {- anyhow::Ok(backend.load_index_text(repo_path).await)- }+ RepositoryState::Local(LocalRepositoryState { backend, .. }) => backend+ .load_index_text(repo_path)+ .await+ .map(decode_git_text)+ .transpose(),RepositoryState::Remote(RemoteRepositoryState { project_id, client }) => {let response = client.request(proto::OpenUnstagedDiff {@@ -9779,8 +9897,16 @@ impl Repository {format!(":{}", repo_path.as_unix_str()),];let mut loaded_revisions = backend.load_revisions(revisions).await?.into_iter();- let committed_text = loaded_revisions.next().flatten();- let staged_text = loaded_revisions.next().flatten();+ let committed_text = loaded_revisions+ .next()+ .flatten()+ .map(decode_git_text)+ .transpose()?;+ let staged_text = loaded_revisions+ .next()+ .flatten()+ .map(decode_git_text)+ .transpose()?;let diff_bases_change = if committed_text == staged_text {DiffBasesChange::SetBoth(committed_text)} else {@@ -9866,6 +9992,7 @@ impl Repository {path.as_ref())})?;+ let content = decode_git_text(content)?;anyhow::Ok((content, blame))}RepositoryState::Remote(RemoteRepositoryState { client, project_id }) => {@@ -9903,7 +10030,7 @@ impl Repository {let rx = self.send_job("load_blob_content", None, move |state, _| async move {match state {RepositoryState::Local(LocalRepositoryState { backend, .. }) => {- backend.load_blob_content(oid).await+ decode_git_text(backend.load_blob_content(oid).await?)}RepositoryState::Remote(RemoteRepositoryState { client, project_id }) => {let response = client@@ -10931,6 +11058,31 @@ mod tests {assert_eq!(resolved, None);}+ #[gpui::test]+ fn test_decode_git_text_windows_1251_one_line_change(cx: &mut TestAppContext) {+ let old_text = "строка один\nстрока два\n";+ let new_text = "строка один\nстрока три\n";+ let (old_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(old_text);+ let (new_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(new_text);++ let decoded_old = decode_git_text(old_bytes.into_owned()).unwrap();+ let decoded_new = decode_git_text(new_bytes.into_owned()).unwrap();+ let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));+ let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());+ let diff = cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));+ let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));+ let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();+ let [hunk] = hunks.as_slice() else {+ panic!("expected one modified hunk, got {hunks:?}");+ };++ assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));+ assert_eq!(+ hunk.diff_base_byte_range,+ old_text.find("строка два").unwrap()..old_text.len()+ );+ }+#[gpui::test]async fn test_open_uncommitted_diff_skips_symlinks(cx: &mut TestAppContext) {use util::rel_path::rel_path;diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rsindex 73acdc013d9..cba0f6cb8e4 100644--- a/crates/project/tests/integration/project_tests.rs+++ b/crates/project/tests/integration/project_tests.rs@@ -11580,7 +11580,7 @@ async fn test_restaging_hunk_after_optimistic_unstage(cx: &mut gpui::TestAppContrepo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt"))).await.unwrap(),- file_contents,+ file_contents.as_bytes(),"the re-stage should have overridden the in-flight unstage");@@ -11605,7 +11605,73 @@ async fn test_restaging_hunk_after_optimistic_unstage(cx: &mut gpui::TestAppContrepo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt"))).await.unwrap(),- file_contents+ file_contents.as_bytes()+ );+}++#[gpui::test]+async fn test_staging_non_utf8_hunk_preserves_index_encoding(cx: &mut gpui::TestAppContext) {+ init_test(cx);++ let committed_text = "строка один\nстрока два\n";+ let file_text = "строка один\nстрока три\n";+ let (committed_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(committed_text);+ let (file_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(file_text);+ let committed_bytes = committed_bytes.into_owned();+ let file_bytes = file_bytes.into_owned();++ let fs = FakeFs::new(cx.background_executor.clone());+ fs.insert_tree(path!("/dir"), json!({ ".git": {} })).await;+ fs.insert_file(path!("/dir/file.txt"), file_bytes.clone())+ .await;+ fs.with_git_state(path!("/dir/.git").as_ref(), true, |state| {+ state+ .head_contents+ .insert(repo_path("file.txt"), committed_bytes.clone());+ state+ .index_contents+ .insert(repo_path("file.txt"), committed_bytes.clone());+ state.refs.insert("HEAD".into(), "deadbeef".into());+ })+ .unwrap();+ let repo = fs+ .open_repo(path!("/dir/.git").as_ref(), Some("git".as_ref()))+ .unwrap();++ let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await;+ let buffer = project+ .update(cx, |project, cx| {+ project.open_local_buffer(path!("/dir/file.txt"), cx)+ })+ .await+ .unwrap();+ assert_eq!(+ buffer.read_with(cx, |buffer, _| buffer.encoding()),+ encoding_rs::WINDOWS_1251+ );++ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());+ let uncommitted_diff = project+ .update(cx, |project, cx| {+ project.open_uncommitted_diff(buffer.clone(), cx)+ })+ .await+ .unwrap();+ let range = snapshot.anchor_before(Point::new(1, 0))..snapshot.anchor_before(Point::new(2, 0));++ project+ .update(cx, |project, cx| {+ let unstaged_diff = uncommitted_diff.read(cx).secondary_diff().unwrap();+ project.stage_hunks(buffer.clone(), unstaged_diff, vec![range], cx)+ })+ .unwrap();+ cx.run_until_parked();++ assert_eq!(+ repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))+ .await+ .unwrap(),+ file_bytes);}@@ -11705,7 +11771,7 @@ async fn test_staging_random_hunks(repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt"))).await.unwrap(),- index_text+ index_text.as_bytes());fs.unpause_events_and_flush();cx.run_until_parked();@@ -11762,9 +11828,12 @@ async fn test_staging_random_hunks(log::info!("index text:\n{}",- repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))- .await- .unwrap()+ String::from_utf8_lossy(+ &repo+ .load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))+ .await+ .unwrap()+ ));uncommitted_diff.update(cx, |diff, cx| {@@ -11794,14 +11863,16 @@ async fn test_staging_random_hunks_with_edits(init_test(cx);fn disk_index_text(fs: &FakeFs) -> String {- fs.with_git_state(path!("/dir/.git").as_ref(), false, |state| {- state- .index_contents- .get(&repo_path("file.txt"))- .cloned()- .expect("file is always present in the index")- })- .unwrap()+ let bytes = fs+ .with_git_state(path!("/dir/.git").as_ref(), false, |state| {+ state+ .index_contents+ .get(&repo_path("file.txt"))+ .cloned()+ .expect("file is always present in the index")+ })+ .unwrap();+ String::from_utf8(bytes).expect("test index contents are valid UTF-8")}fn random_row_range(@@ -12190,10 +12261,12 @@ async fn test_staging_random_hunks_with_edits(let old_staged_state = capture_diff_state(&staged_diff, cx);let old_uncommitted_state = capture_diff_state(&uncommitted_diff, cx);- let disk_index_text = repo- .load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))- .await- .unwrap();+ let disk_index_text = String::from_utf8(+ repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))+ .await+ .unwrap(),+ )+ .unwrap();assert_eq!(old_unstaged_state.1.as_deref(),Some(disk_index_text.as_str()),@@ -14948,10 +15021,10 @@ async fn test_git_worktrees_and_submodules(cx: &mut gpui::TestAppContext) {|state| {state.head_contents- .insert(repo_path("src/b.txt"), "b".to_owned());+ .insert(repo_path("src/b.txt"), b"b".to_vec());state.index_contents- .insert(repo_path("src/b.txt"), "b".to_owned());+ .insert(repo_path("src/b.txt"), b"b".to_vec());},).unwrap();@@ -15001,10 +15074,10 @@ async fn test_git_worktrees_and_submodules(cx: &mut gpui::TestAppContext) {|state| {state.head_contents- .insert(repo_path("c.txt"), "c".to_owned());+ .insert(repo_path("c.txt"), b"c".to_vec());state.index_contents- .insert(repo_path("c.txt"), "c".to_owned());+ .insert(repo_path("c.txt"), b"c".to_vec());},).unwrap();@@ -16695,14 +16768,16 @@ async fn test_staging_hunks_with_ambiguous_placement(cx: &mut gpui::TestAppConte}fn index_text(fs: &FakeFs) -> String {- fs.with_git_state(path!("/dir/.git").as_ref(), false, |state| {- state- .index_contents- .get(&repo_path("test.txt"))- .cloned()- .expect("file is present in the index")- })- .unwrap()+ let bytes = fs+ .with_git_state(path!("/dir/.git").as_ref(), false, |state| {+ state+ .index_contents+ .get(&repo_path("test.txt"))+ .cloned()+ .expect("file is present in the index")+ })+ .unwrap();+ String::from_utf8(bytes).expect("test index contents are valid UTF-8")}use DiffHunkSecondaryStatus::*;diff --git a/crates/worktree/Cargo.toml b/crates/worktree/Cargo.tomlindex 9a95eec9697..5a2b6204e99 100644--- a/crates/worktree/Cargo.toml+++ b/crates/worktree/Cargo.toml@@ -33,7 +33,6 @@ test-support = [anyhow.workspace = trueasync-channel.workspace = trueasync-lock.workspace = true-chardetng.workspace = trueclock.workspace = truecollections.workspace = trueencoding_rs.workspace = truediff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rsindex d09542a45c6..4a8a726a15d 100644--- a/crates/worktree/src/worktree.rs+++ b/crates/worktree/src/worktree.rs@@ -3,7 +3,6 @@ mod worktree_settings;use ::ignore::gitignore::{Gitignore, GitignoreBuilder};use anyhow::{Context as _, Result, anyhow};-use chardetng::EncodingDetector;use clock::ReplicaId;use collections::{BTreeMap, HashMap, HashSet, VecDeque};use encoding_rs::Encoding;@@ -32,7 +31,9 @@ use gpui::{Task,};pub use ignore::{IgnoreKind, IgnoreStack};-use language::{ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content};+use language::{+ ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,+};use async_channel::{self, Sender};use parking_lot::Mutex;@@ -1863,47 +1864,7 @@ impl LocalWorktree {LineEnding::Windows => text_string.replace('\n', "\r\n"),};- // Create the byte vector manually for UTF-16 encodings because encoding_rs encodes to UTF-8 by default (per WHATWG standards),- // which is not what we want for saving files.- let bytes = if encoding == encoding_rs::UTF_16BE {- let mut data = Vec::with_capacity(normalized_text.len() * 2 + 2);- if has_bom {- data.extend_from_slice(&[0xFE, 0xFF]); // BOM- }- let utf16be_bytes =- normalized_text.encode_utf16().flat_map(|u| u.to_be_bytes());- data.extend(utf16be_bytes);- data.into()- } else if encoding == encoding_rs::UTF_16LE {- let mut data = Vec::with_capacity(normalized_text.len() * 2 + 2);- if has_bom {- data.extend_from_slice(&[0xFF, 0xFE]); // BOM- }- let utf16le_bytes =- normalized_text.encode_utf16().flat_map(|u| u.to_le_bytes());- data.extend(utf16le_bytes);- data.into()- } else {- // For other encodings (Shift-JIS, UTF-8 with BOM, etc.), delegate to encoding_rs.- let bom_bytes = if has_bom {- if encoding == encoding_rs::UTF_8 {- vec![0xEF, 0xBB, 0xBF]- } else {- vec![]- }- } else {- vec![]- };- let (cow, _, _) = encoding.encode(&normalized_text);- if !bom_bytes.is_empty() {- let mut bytes = bom_bytes;- bytes.extend_from_slice(&cow);- bytes.into()- } else {- cow- }- };-+ let bytes = encode_text(normalized_text, encoding, has_bom);fs.write(&abs_path, &bytes).await}});@@ -7239,7 +7200,7 @@ pub async fn decode_file_text(}file_first_bytes.extend_from_slice(&buf[..n]);}- let (bom_encoding, byte_content) = decode_byte_header(&file_first_bytes);+ let (_, byte_content) = decode_byte_header(&file_first_bytes);anyhow::ensure!(byte_content != ByteContent::Binary,"Binary files are not supported"@@ -7259,7 +7220,8 @@ pub async fn decode_file_text(content.extend_from_slice(&buf[..n]);}}- decode_byte_full(content, bom_encoding, byte_content)+ let decoded = decode_text(content)?;+ Ok((decoded.text, decoded.encoding, decoded.has_bom))}pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {@@ -7269,63 +7231,6 @@ pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteCont(None, analyze_byte_content(prefix))}-fn decode_byte_full(- bytes: Vec<u8>,- bom_encoding: Option<&'static Encoding>,- byte_content: ByteContent,-) -> Result<(String, &'static Encoding, bool)> {- if let Some(encoding) = bom_encoding {- let (cow, _) = encoding.decode_with_bom_removal(&bytes);- return Ok((cow.into_owned(), encoding, true));- }-- match byte_content {- ByteContent::Utf16Le => {- let encoding = encoding_rs::UTF_16LE;- let (cow, _, _) = encoding.decode(&bytes);- return Ok((cow.into_owned(), encoding, false));- }- ByteContent::Utf16Be => {- let encoding = encoding_rs::UTF_16BE;- let (cow, _, _) = encoding.decode(&bytes);- return Ok((cow.into_owned(), encoding, false));- }- ByteContent::Binary => {- anyhow::bail!("Binary files are not supported");- }- ByteContent::Unknown => {}- }-- fn detect_encoding(bytes: Vec<u8>) -> (String, &'static Encoding) {- let mut detector = EncodingDetector::new();- detector.feed(&bytes, true);-- let encoding = detector.guess(None, true); // Use None for TLD hint to ensure neutral detection logic.-- let (cow, _, _) = encoding.decode(&bytes);- (cow.into_owned(), encoding)- }-- match String::from_utf8(bytes) {- Ok(text) => {- // ISO-2022-JP (and other ISO-2022 variants) consists entirely of 7-bit ASCII bytes,- // so it is valid UTF-8. However, it contains escape sequences starting with '\x1b'.- // If we find an escape character, we double-check the encoding to prevent- // displaying raw escape sequences instead of the correct characters.- if text.contains('\x1b') {- let (s, enc) = detect_encoding(text.into_bytes());- Ok((s, enc, false))- } else {- Ok((text, encoding_rs::UTF_8, false))- }- }- Err(e) => {- let (s, enc) = detect_encoding(e.into_bytes());- Ok((s, enc, false))- }- }-}-#[cfg(test)]mod tests {use super::*;