Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

git: Decode non-UTF-8 blobs for project diffs (#60821)

hvck8968bf78084f
diff --git a/Cargo.lock b/Cargo.lock
index 5eda1ece25f..795928677a4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9760,6 +9760,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
+ "chardetng",
"clock",
"collections",
"criterion",
@@ -22852,7 +22853,6 @@ dependencies = [
"anyhow",
"async-channel 2.5.0",
"async-lock",
- "chardetng",
"clock",
"collections",
"encoding_rs",
diff --git a/crates/collab/tests/integration/editor_tests.rs b/crates/collab/tests/integration/editor_tests.rs
index 3b16f57d04a..37fcd5150ff 100644
--- a/crates/collab/tests/integration/editor_tests.rs
+++ b/crates/collab/tests/integration/editor_tests.rs
@@ -4405,9 +4405,10 @@ async fn test_git_blame_is_forwarded(cx_a: &mut TestAppContext, cx_b: &mut TestA
.fs()
.with_git_state(Path::new(path!("/my-repo/.git")), true, |state| {
state.refs.insert("HEAD".into(), revision.to_string());
- state
- .head_contents
- .insert(repo_path("file.txt"), content_at_revision.to_string());
+ state.head_contents.insert(
+ repo_path("file.txt"),
+ content_at_revision.as_bytes().to_vec(),
+ );
state
.blames_at_revision
.insert((repo_path("file.txt"), revision), blame_at_revision.clone());
diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs
index ad84ff1dc52..6739552bf21 100644
--- a/crates/editor/src/editor_tests.rs
+++ b/crates/editor/src/editor_tests.rs
@@ -27071,12 +27071,9 @@ async fn test_merge_base_diff_hunks_are_read_only(cx: &mut TestAppContext) {
.unwrap();
assert_eq!(
index_contents,
- [(
- ::git::repository::repo_path("file.rs"),
- "head\n".to_string(),
- )]
- .into_iter()
- .collect::<HashMap<_, _>>()
+ [(::git::repository::repo_path("file.rs"), b"head\n".to_vec(),)]
+ .into_iter()
+ .collect::<HashMap<_, _>>()
);
}
diff --git a/crates/editor/src/test/editor_test_context.rs b/crates/editor/src/test/editor_test_context.rs
index 84b03d91ca1..517e3fa3b35 100644
--- a/crates/editor/src/test/editor_test_context.rs
+++ b/crates/editor/src/test/editor_test_context.rs
@@ -374,7 +374,7 @@ impl EditorTestContext {
.cloned();
})
.unwrap();
- assert_eq!(expected, found.as_deref());
+ assert_eq!(expected.map(str::as_bytes), found.as_deref());
}
/// Change the editor's text and selections using a string containing
diff --git a/crates/fs/src/fake_git_repo.rs b/crates/fs/src/fake_git_repo.rs
index 543a11a9c2f..95209c5a8f3 100644
--- a/crates/fs/src/fake_git_repo.rs
+++ b/crates/fs/src/fake_git_repo.rs
@@ -44,8 +44,8 @@ pub struct FakeGitRepository {
#[derive(Debug, Clone)]
pub struct FakeCommitSnapshot {
- pub head_contents: HashMap<RepoPath, String>,
- pub index_contents: HashMap<RepoPath, String>,
+ pub head_contents: HashMap<RepoPath, Vec<u8>>,
+ pub index_contents: HashMap<RepoPath, Vec<u8>>,
pub sha: String,
}
@@ -60,11 +60,11 @@ pub struct FakeGitRepositoryState {
pub commit_history: Vec<FakeCommitSnapshot>,
pub event_emitter: async_channel::Sender<PathBuf>,
pub unmerged_paths: HashMap<RepoPath, UnmergedStatus>,
- pub head_contents: HashMap<RepoPath, String>,
- pub index_contents: HashMap<RepoPath, String>,
+ pub head_contents: HashMap<RepoPath, Vec<u8>>,
+ pub index_contents: HashMap<RepoPath, Vec<u8>>,
// everything in commit contents is in oids
pub merge_base_contents: HashMap<RepoPath, Oid>,
- pub oids: HashMap<Oid, String>,
+ pub oids: HashMap<Oid, Vec<u8>>,
pub blames: HashMap<RepoPath, Blame>,
pub blames_at_revision: HashMap<(RepoPath, Oid), Blame>,
pub current_branch_name: Option<String>,
@@ -170,7 +170,7 @@ impl GitRepository for FakeGitRepository {
self.with_state_async(false, |state| Ok(state.commit_template.clone()))
}
- fn load_blob_content(&self, oid: git::Oid) -> BoxFuture<'_, Result<String>> {
+ fn load_blob_content(&self, oid: git::Oid) -> BoxFuture<'_, Result<Vec<u8>>> {
self.with_state_async(false, move |state| {
state.oids.get(&oid).cloned().context("oid does not exist")
})
@@ -188,7 +188,7 @@ impl GitRepository for FakeGitRepository {
fn set_index_text(
&self,
path: RepoPath,
- content: Option<String>,
+ content: Option<Vec<u8>>,
_env: Arc<HashMap<String, String>>,
_is_executable: bool,
) -> BoxFuture<'_, anyhow::Result<()>> {
@@ -219,7 +219,7 @@ impl GitRepository for FakeGitRepository {
.filter_map(|path| {
let path_in_repo = path.strip_prefix(workdir_path).ok()?;
let path_in_repo = RelPath::new(path_in_repo, PathStyle::local()).ok()?;
- let content = String::from_utf8(self.fs.read_file_sync(path).ok()?).ok()?;
+ let content = self.fs.read_file_sync(path).ok()?;
Some((RepoPath::from_rel_path(&path_in_repo), content))
})
.collect::<HashMap<_, _>>()
@@ -261,7 +261,10 @@ impl GitRepository for FakeGitRepository {
})
}
- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>> {
+ fn load_revisions(
+ &self,
+ revisions: Vec<String>,
+ ) -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>> {
let fut = self.with_state_async(false, move |state| {
Ok(revisions
.into_iter()
@@ -398,7 +401,7 @@ impl GitRepository for FakeGitRepository {
.collect::<Vec<_>>();
// Load working copy files.
- let git_files: HashMap<RepoPath, (String, bool)> = self
+ let git_files: HashMap<RepoPath, (Vec<u8>, bool)> = self
.fs
.files()
.iter()
@@ -413,11 +416,7 @@ impl GitRepository for FakeGitRepository {
ignore::Match::Whitelist(_) => break,
}
}
- let content = self
- .fs
- .read_file_sync(path)
- .ok()
- .map(|content| String::from_utf8(content).unwrap())?;
+ let content = self.fs.read_file_sync(path).ok()?;
let repo_path = RelPath::new(repo_path, PathStyle::local()).ok()?;
Some((RepoPath::from_rel_path(&repo_path), (content, is_ignored)))
})
@@ -1026,7 +1025,9 @@ impl GitRepository for FakeGitRepository {
.parent()
.unwrap()
.join(&path.as_std_path());
- Box::pin(async move { (path.clone(), self.fs.load(&abs_path).await.ok()) })
+ Box::pin(
+ async move { (path.clone(), self.fs.load_bytes(&abs_path).await.ok()) },
+ )
})
.collect::<Vec<_>>();
let contents = join_all(contents).await;
@@ -1200,11 +1201,11 @@ impl GitRepository for FakeGitRepository {
diff: git::repository::DiffStatType,
path_prefixes: &[RepoPath],
) -> BoxFuture<'static, Result<git::status::GitDiffStat>> {
- fn count_lines(s: &str) -> u32 {
- if s.is_empty() {
+ fn count_lines(bytes: &[u8]) -> u32 {
+ if bytes.is_empty() {
0
} else {
- s.lines().count() as u32
+ String::from_utf8_lossy(bytes).lines().count() as u32
}
}
@@ -1224,7 +1225,7 @@ impl GitRepository for FakeGitRepository {
let path_prefixes = path_prefixes.to_vec();
let workdir_path = self.dot_git_path.parent().unwrap().to_path_buf();
- let worktree_files: HashMap<RepoPath, String> = self
+ let worktree_files: HashMap<RepoPath, Vec<u8>> = self
.fs
.files()
.iter()
@@ -1233,11 +1234,7 @@ impl GitRepository for FakeGitRepository {
if repo_path.starts_with(".git") {
return None;
}
- let content = self
- .fs
- .read_file_sync(path)
- .ok()
- .and_then(|bytes| String::from_utf8(bytes).ok())?;
+ let content = self.fs.read_file_sync(path).ok()?;
let repo_path = RelPath::new(repo_path, PathStyle::local()).ok()?;
Some((RepoPath::from_rel_path(&repo_path), content))
})
diff --git a/crates/fs/src/fs.rs b/crates/fs/src/fs.rs
index 711aa53f279..de8ceaa4060 100644
--- a/crates/fs/src/fs.rs
+++ b/crates/fs/src/fs.rs
@@ -2294,7 +2294,7 @@ impl FakeFs {
state.index_contents.extend(
index_state
.iter()
- .map(|(path, content)| (repo_path(path), content.clone())),
+ .map(|(path, content)| (repo_path(path), content.as_bytes().to_vec())),
);
})
.unwrap();
@@ -2311,7 +2311,7 @@ impl FakeFs {
state.head_contents.extend(
head_state
.iter()
- .map(|(path, content)| (repo_path(path), content.clone())),
+ .map(|(path, content)| (repo_path(path), content.as_bytes().to_vec())),
);
state.refs.insert("HEAD".into(), sha.into());
})
@@ -2324,7 +2324,7 @@ impl FakeFs {
state.head_contents.extend(
contents_by_path
.iter()
- .map(|(path, contents)| (repo_path(path), contents.clone())),
+ .map(|(path, contents)| (repo_path(path), contents.as_bytes().to_vec())),
);
state.index_contents = state.head_contents.clone();
})
@@ -2345,7 +2345,7 @@ impl FakeFs {
.map(|n| Oid::from_bytes(n.repeat(20).as_bytes()).unwrap());
for ((path, content), oid) in contents_by_path.iter().zip(oids) {
state.merge_base_contents.insert(repo_path(path), oid);
- state.oids.insert(oid, content.clone());
+ state.oids.insert(oid, content.as_bytes().to_vec());
}
})
.unwrap();
@@ -2472,10 +2472,14 @@ impl FakeFs {
};
if let Some(content) = index_content {
- state.index_contents.insert(repo_path.clone(), content);
+ state
+ .index_contents
+ .insert(repo_path.clone(), content.into_bytes());
}
if let Some(content) = head_content {
- state.head_contents.insert(repo_path.clone(), content);
+ state
+ .head_contents
+ .insert(repo_path.clone(), content.into_bytes());
}
}
}).unwrap();
diff --git a/crates/git/src/repository.rs b/crates/git/src/repository.rs
index 7cddc70b1a5..4b52142d56e 100644
--- a/crates/git/src/repository.rs
+++ b/crates/git/src/repository.rs
@@ -547,14 +547,14 @@ pub enum CommitFileStatus {
#[derive(Debug)]
pub struct CommitFile {
pub path: RepoPath,
- pub old_text: Option<String>,
- pub new_text: Option<String>,
+ pub old_content: Option<Vec<u8>>,
+ pub new_content: Option<Vec<u8>>,
pub is_binary: bool,
}
impl CommitFile {
pub fn status(&self) -> CommitFileStatus {
- match (&self.old_text, &self.new_text) {
+ match (&self.old_content, &self.new_content) {
(None, Some(_)) => CommitFileStatus::Added,
(Some(_), None) => CommitFileStatus::Deleted,
_ => CommitFileStatus::Modified,
@@ -576,7 +576,7 @@ pub fn is_binary_content(content: &[u8]) -> bool {
}
struct LoadedCommitObject {
- text: String,
+ content: Vec<u8>,
is_binary: bool,
}
@@ -599,11 +599,7 @@ async fn read_commit_blob<R: smol::io::AsyncBufRead + Unpin>(
let is_binary = is_binary_content(&bytes);
Ok(LoadedCommitObject {
- text: if is_binary {
- String::new()
- } else {
- String::from_utf8_lossy(&bytes).to_string()
- },
+ content: bytes,
is_binary,
})
}
@@ -617,7 +613,7 @@ async fn load_commit_object<R: smol::io::AsyncBufRead + Unpin>(
match object {
Some(object) if object.kind == CommitDiffObjectKind::Gitlink => {
Ok(Some(LoadedCommitObject {
- text: format!("Subproject commit {}\n", object.oid),
+ content: format!("Subproject commit {}\n", object.oid).into_bytes(),
is_binary: false,
}))
}
@@ -754,7 +750,7 @@ pub trait GitRepository: Send + Sync {
/// Returns the contents of an entry in the repository's index, or None if there is no entry for the given path.
///
/// Also returns `None` for symlinks.
- fn load_index_text(&self, path: RepoPath) -> BoxFuture<'_, Option<String>> {
+ fn load_index_text(&self, path: RepoPath) -> BoxFuture<'_, Option<Vec<u8>>> {
let future = self.load_revisions(vec![format!(":{}", path.as_unix_str())]);
async move { future.await.ok()?.pop()? }.boxed()
}
@@ -762,16 +758,16 @@ pub trait GitRepository: Send + Sync {
/// Returns the contents of an entry in the repository's HEAD, or None if HEAD does not exist or has no entry for the given path.
///
/// Also returns `None` for symlinks.
- fn load_committed_text(&self, path: RepoPath) -> BoxFuture<'_, Option<String>> {
+ fn load_committed_text(&self, path: RepoPath) -> BoxFuture<'_, Option<Vec<u8>>> {
let future = self.load_revisions(vec![format!("HEAD:{}", path.as_unix_str())]);
async move { future.await.ok()?.pop()? }.boxed()
}
- fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<String>>;
+ fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<Vec<u8>>>;
fn set_index_text(
&self,
path: RepoPath,
- content: Option<String>,
+ content: Option<Vec<u8>>,
env: Arc<HashMap<String, String>>,
is_executable: bool,
) -> BoxFuture<'_, anyhow::Result<()>>;
@@ -789,7 +785,8 @@ pub trait GitRepository: Send + Sync {
/// Resolve a list of refs to SHAs.
fn revparse_batch(&self, revs: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>>;
- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>>;
+ fn load_revisions(&self, revisions: Vec<String>)
+ -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>>;
fn head_sha(&self) -> BoxFuture<'_, Option<String>> {
async move {
@@ -1461,25 +1458,13 @@ impl GitRepository for RealGitRepository {
.await?;
let is_binary = new_object.as_ref().is_some_and(|object| object.is_binary)
|| old_object.as_ref().is_some_and(|object| object.is_binary);
- let new_text = new_object.map(|object| {
- if is_binary {
- String::new()
- } else {
- object.text
- }
- });
- let old_text = old_object.map(|object| {
- if is_binary {
- String::new()
- } else {
- object.text
- }
- });
+ let new_content = new_object.map(|object| object.content);
+ let old_content = old_object.map(|object| object.content);
files.push(CommitFile {
path: RepoPath(Arc::from(rel_path)),
- old_text,
- new_text,
+ old_content,
+ new_content,
is_binary,
})
}
@@ -1547,11 +1532,23 @@ impl GitRepository for RealGitRepository {
.boxed()
}
- fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<String>> {
+ fn load_blob_content(&self, oid: Oid) -> BoxFuture<'_, Result<Vec<u8>>> {
let git_binary = self.git_binary();
let oid_str = oid.to_string();
self.executor
- .spawn(async move { git_binary.run_raw(&["cat-file", "blob", &oid_str]).await })
+ .spawn(async move {
+ let mut command = git_binary.build_command(&["cat-file", "blob", &oid_str]);
+ let output = command.output().await?;
+ anyhow::ensure!(
+ output.status.success(),
+ GitBinaryCommandError {
+ stdout: String::from_utf8_lossy(&output.stdout).to_string(),
+ stderr: String::from_utf8_lossy(&output.stderr).to_string(),
+ status: output.status,
+ }
+ );
+ Ok(output.stdout)
+ })
.boxed()
}
@@ -1600,7 +1597,7 @@ impl GitRepository for RealGitRepository {
fn set_index_text(
&self,
path: RepoPath,
- content: Option<String>,
+ content: Option<Vec<u8>>,
env: Arc<HashMap<String, String>>,
is_executable: bool,
) -> BoxFuture<'_, anyhow::Result<()>> {
@@ -1611,17 +1608,32 @@ impl GitRepository for RealGitRepository {
if let Some(content) = content {
let mut child = git
- .build_command(&["hash-object", "-w", "--stdin"])
+ .build_command(&[
+ "hash-object",
+ "-w",
+ "--stdin",
+ "--path",
+ path.as_unix_str(),
+ ])
.envs(env.iter())
.stdin(Stdio::piped())
.stdout(Stdio::piped())
+ .stderr(Stdio::piped())
.spawn()?;
- let mut stdin = child.stdin.take().unwrap();
- stdin.write_all(content.as_bytes()).await?;
+ let mut stdin = child.stdin.take().context("hash-object has no stdin")?;
+ stdin.write_all(&content).await?;
stdin.flush().await?;
drop(stdin);
- let output = child.output().await?.stdout;
- let sha = str::from_utf8(&output)?.trim();
+ let output = child.output().await?;
+ anyhow::ensure!(
+ output.status.success(),
+ GitBinaryCommandError {
+ stdout: String::from_utf8_lossy(&output.stdout).to_string(),
+ stderr: String::from_utf8_lossy(&output.stderr).to_string(),
+ status: output.status,
+ }
+ );
+ let sha = str::from_utf8(&output.stdout)?.trim();
log::debug!("indexing SHA: {sha}, path {path:?}");
@@ -1722,7 +1734,10 @@ impl GitRepository for RealGitRepository {
.boxed()
}
- fn load_revisions(&self, revisions: Vec<String>) -> BoxFuture<'_, Result<Vec<Option<String>>>> {
+ fn load_revisions(
+ &self,
+ revisions: Vec<String>,
+ ) -> BoxFuture<'_, Result<Vec<Option<Vec<u8>>>>> {
let git = self.git_binary();
self.executor
.spawn(async move {
@@ -1773,7 +1788,7 @@ impl GitRepository for RealGitRepository {
stdout.read_exact(&mut newline).await?;
if object_type == "blob" {
- results.push(String::from_utf8(content).ok());
+ results.push(Some(content));
} else {
results.push(None);
}
@@ -4510,8 +4525,11 @@ mod tests {
.first()
.expect("type-changed file should be present");
assert_eq!(file.path.as_unix_str(), "file.txt");
- assert_eq!(file.old_text.as_deref(), Some("regular contents\n"));
- assert_eq!(file.new_text.as_deref(), Some("target"));
+ assert_eq!(
+ file.old_content.as_deref(),
+ Some(b"regular contents\n".as_slice())
+ );
+ assert_eq!(file.new_content.as_deref(), Some(b"target".as_slice()));
assert_eq!(file.status(), CommitFileStatus::Modified);
}
@@ -4560,10 +4578,10 @@ mod tests {
.find(|file| file.path.as_unix_str() == "modules/example")
.expect("gitlink should be present alongside the regular file");
assert_eq!(gitlink.status(), CommitFileStatus::Added);
- assert_eq!(gitlink.old_text, None);
+ assert_eq!(gitlink.old_content, None);
assert_eq!(
- gitlink.new_text.as_deref(),
- Some("Subproject commit 1111111111111111111111111111111111111111\n")
+ gitlink.new_content.as_deref(),
+ Some(b"Subproject commit 1111111111111111111111111111111111111111\n".as_slice())
);
assert!(!gitlink.is_binary);
@@ -4588,12 +4606,12 @@ mod tests {
};
assert_eq!(gitlink.status(), CommitFileStatus::Modified);
assert_eq!(
- gitlink.old_text.as_deref(),
- Some("Subproject commit 1111111111111111111111111111111111111111\n")
+ gitlink.old_content.as_deref(),
+ Some(b"Subproject commit 1111111111111111111111111111111111111111\n".as_slice())
);
assert_eq!(
- gitlink.new_text.as_deref(),
- Some("Subproject commit 2222222222222222222222222222222222222222\n")
+ gitlink.new_content.as_deref(),
+ Some(b"Subproject commit 2222222222222222222222222222222222222222\n".as_slice())
);
assert!(!gitlink.is_binary);
@@ -4609,10 +4627,10 @@ mod tests {
};
assert_eq!(gitlink.status(), CommitFileStatus::Deleted);
assert_eq!(
- gitlink.old_text.as_deref(),
- Some("Subproject commit 2222222222222222222222222222222222222222\n")
+ gitlink.old_content.as_deref(),
+ Some(b"Subproject commit 2222222222222222222222222222222222222222\n".as_slice())
);
- assert_eq!(gitlink.new_text, None);
+ assert_eq!(gitlink.new_content, None);
assert!(!gitlink.is_binary);
}
@@ -5464,12 +5482,12 @@ mod tests {
assert_eq!(
results,
vec![
- Some("file1 committed contents".into()),
- Some("file1 index contents".into()),
- Some("file2 committed contents".into()),
- Some("file2 committed contents".into()), // untouched in index, should match HEAD
+ Some(b"file1 committed contents".to_vec()),
+ Some(b"file1 index contents".to_vec()),
+ Some(b"file2 committed contents".to_vec()),
+ Some(b"file2 committed contents".to_vec()), // untouched in index, should match HEAD
None,
- Some("space file committed contents".into()),
+ Some(b"space file committed contents".to_vec()),
None,
]
);
diff --git a/crates/git_ui/src/commit_view.rs b/crates/git_ui/src/commit_view.rs
index 009aa368add..7d720e2f7e2 100644
--- a/crates/git_ui/src/commit_view.rs
+++ b/crates/git_ui/src/commit_view.rs
@@ -6,7 +6,7 @@ use editor::{
SplittableEditor, hover_markdown_style, multibuffer_context_lines,
};
use futures_lite::future::yield_now;
-use git::repository::{CommitDetails, CommitDiff, RepoPath, is_binary_content};
+use git::repository::{CommitDetails, RepoPath};
use git::status::{FileStatus, StatusCode, TrackedStatus};
use git::{
BuildCommitPermalinkParams, GitHostingProviderRegistry, GitRemote, ParsedGitRemote,
@@ -24,7 +24,10 @@ use language::{
};
use markdown::{Markdown, MarkdownElement};
use multi_buffer::PathKey;
-use project::{Project, ProjectPath, WorktreeId, git_store::Repository};
+use project::{
+ Project, ProjectPath, WorktreeId,
+ git_store::{CommitDiff, Repository},
+};
use settings::{DiffViewStyle, Settings};
use std::{
any::{Any, TypeId},
@@ -322,11 +325,7 @@ impl CommitView {
let raw_new_text = file.new_text.unwrap_or_default();
let raw_old_text = file.old_text;
- let is_binary = file.is_binary
- || is_binary_content(raw_new_text.as_bytes())
- || raw_old_text
- .as_ref()
- .is_some_and(|text| is_binary_content(text.as_bytes()));
+ let is_binary = file.is_binary;
let new_text = if is_binary {
"(binary file not shown)".to_string()
diff --git a/crates/git_ui/src/git_graph.rs b/crates/git_ui/src/git_graph.rs
index 1003541824b..4e67356feca 100644
--- a/crates/git_ui/src/git_graph.rs
+++ b/crates/git_ui/src/git_graph.rs
@@ -11,10 +11,7 @@ use file_icons::FileIcons;
use git::{
BuildCommitPermalinkParams, GitHostingProviderRegistry, GitRemote, Oid, ParsedGitRemote,
parse_git_remote_url,
- repository::{
- CommitDiff, CommitFile, InitialGraphCommitData, LogOrder, LogSource, RepoPath,
- SearchCommitArgs,
- },
+ repository::{InitialGraphCommitData, LogOrder, LogSource, RepoPath, SearchCommitArgs},
status::{FileStatus, StatusCode, TrackedStatus},
};
use gpui::{
@@ -32,8 +29,8 @@ use picker::{Picker, PickerDelegate};
use project::{
ProjectPath,
git_store::{
- CommitDataState, GitGraphEvent, GitStore, GitStoreEvent, GraphDataResponse, Repository,
- RepositoryEvent, RepositoryId,
+ CommitDataState, CommitDiff, CommitFile, GitGraphEvent, GitStore, GitStoreEvent,
+ GraphDataResponse, Repository, RepositoryEvent, RepositoryId,
},
};
use smallvec::{SmallVec, smallvec};
diff --git a/crates/git_ui/src/staged_diff.rs b/crates/git_ui/src/staged_diff.rs
index c99ed2a58ce..648b1980eb5 100644
--- a/crates/git_ui/src/staged_diff.rs
+++ b/crates/git_ui/src/staged_diff.rs
@@ -976,7 +976,7 @@ mod tests {
repo.load_index_text(RepoPath::from_rel_path(rel_path("src/main.rs")))
.await
.unwrap(),
- committed_contents
+ committed_contents.as_bytes()
);
fs.unpause_events_and_flush();
diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml
index 69789da21ec..7fcef180cee 100644
--- a/crates/language/Cargo.toml
+++ b/crates/language/Cargo.toml
@@ -32,6 +32,7 @@ test-support = [
anyhow.workspace = true
async-trait.workspace = true
clock.workspace = true
+chardetng.workspace = true
collections.workspace = true
ec4rs.workspace = true
encoding_rs.workspace = true
diff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rs
index e02748257ba..4acfd5d04a8 100644
--- a/crates/language/src/file_content.rs
+++ b/crates/language/src/file_content.rs
@@ -1,5 +1,97 @@
+use anyhow::{Result, bail};
+use chardetng::EncodingDetector;
+use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};
+
pub const FILE_ANALYSIS_BYTES: usize = 1024;
+pub struct DecodedText {
+ pub text: String,
+ pub encoding: &'static Encoding,
+ pub has_bom: bool,
+}
+
+pub fn decode_text(bytes: Vec<u8>) -> Result<DecodedText> {
+ if let Some((encoding, _bom_len)) = Encoding::for_bom(&bytes) {
+ let (text, _) = encoding.decode_with_bom_removal(&bytes);
+ return Ok(DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: true,
+ });
+ }
+
+ let encoding = match analyze_byte_content(&bytes) {
+ ByteContent::Utf16Le => UTF_16LE,
+ ByteContent::Utf16Be => UTF_16BE,
+ ByteContent::Binary => bail!("Binary files are not supported"),
+ ByteContent::Unknown => {
+ return match String::from_utf8(bytes) {
+ Ok(text) if !text.contains('\x1b') => Ok(DecodedText {
+ text,
+ encoding: UTF_8,
+ has_bom: false,
+ }),
+ Ok(text) => Ok(decode_with_detected_encoding(text.into_bytes())),
+ Err(error) => Ok(decode_with_detected_encoding(error.into_bytes())),
+ };
+ }
+ };
+
+ let (text, _, _) = encoding.decode(&bytes);
+ Ok(DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: false,
+ })
+}
+
+pub fn encode_text(text: String, encoding: &'static Encoding, has_bom: bool) -> Vec<u8> {
+ if encoding == UTF_8 && !has_bom {
+ return text.into_bytes();
+ }
+
+ // encoding_rs follows the WHATWG standard and encodes UTF-16 labels as UTF-8.
+ if encoding == UTF_16BE {
+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
+ if has_bom {
+ bytes.extend_from_slice(&[0xFE, 0xFF]);
+ }
+ bytes.extend(text.encode_utf16().flat_map(u16::to_be_bytes));
+ return bytes;
+ }
+
+ if encoding == UTF_16LE {
+ let mut bytes = Vec::with_capacity(text.len() * 2 + 2);
+ if has_bom {
+ bytes.extend_from_slice(&[0xFF, 0xFE]);
+ }
+ bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes));
+ return bytes;
+ }
+
+ let (encoded, _, _) = encoding.encode(&text);
+ if has_bom && encoding == UTF_8 {
+ let mut bytes = Vec::with_capacity(encoded.len() + 3);
+ bytes.extend_from_slice(&[0xEF, 0xBB, 0xBF]);
+ bytes.extend_from_slice(&encoded);
+ bytes
+ } else {
+ encoded.into_owned()
+ }
+}
+
+fn decode_with_detected_encoding(bytes: Vec<u8>) -> DecodedText {
+ let mut detector = EncodingDetector::new();
+ detector.feed(&bytes, true);
+ let encoding = detector.guess(None, true);
+ let (text, _, _) = encoding.decode(&bytes);
+ DecodedText {
+ text: text.into_owned(),
+ encoding,
+ has_bom: false,
+ }
+}
+
#[derive(Debug, PartialEq)]
pub enum ByteContent {
Utf16Le,
@@ -176,3 +268,41 @@ fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option<u16> {
}
Some(u16::from_be_bytes(pair))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn decodes_and_encodes_windows_1251() {
+ let expected = "строка один\nстрока два\n";
+ let (bytes, _, _) = encoding_rs::WINDOWS_1251.encode(expected);
+
+ let decoded = decode_text(bytes.clone().into_owned()).unwrap();
+
+ assert_eq!(decoded.text, expected);
+ assert_eq!(decoded.encoding, encoding_rs::WINDOWS_1251);
+ assert!(!decoded.has_bom);
+ assert_eq!(
+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),
+ bytes.as_ref()
+ );
+ }
+
+ #[test]
+ fn preserves_unicode_boms() {
+ let expected = "Hello, мир\n";
+ for encoding in [UTF_8, UTF_16LE, UTF_16BE] {
+ let bytes = encode_text(expected.to_owned(), encoding, true);
+ let decoded = decode_text(bytes.clone()).unwrap();
+
+ assert_eq!(decoded.text, expected);
+ assert_eq!(decoded.encoding, encoding);
+ assert!(decoded.has_bom);
+ assert_eq!(
+ encode_text(decoded.text, decoded.encoding, decoded.has_bom),
+ bytes
+ );
+ }
+ }
+}
diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs
index 6ba6a834898..bf794425e2a 100644
--- a/crates/language/src/language.rs
+++ b/crates/language/src/language.rs
@@ -102,7 +102,9 @@ pub use buffer::Operation;
pub use buffer::*;
pub use diagnostic::{Diagnostic, DiagnosticSourceKind};
pub use diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup};
-pub use file_content::{ByteContent, FILE_ANALYSIS_BYTES, analyze_byte_content};
+pub use file_content::{
+ ByteContent, DecodedText, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,
+};
pub use language_registry::{
BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QUERY_FILENAME_PREFIXES,
};
diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml
index b3ee5a88a22..b42ddfcc950 100644
--- a/crates/project/Cargo.toml
+++ b/crates/project/Cargo.toml
@@ -48,6 +48,7 @@ collections.workspace = true
context_server.workspace = true
credentials_provider.workspace = true
dap.workspace = true
+encoding_rs.workspace = true
extension.workspace = true
fancy-regex.workspace = true
fs.workspace = true
@@ -109,7 +110,6 @@ ztracing.workspace = true
[dev-dependencies]
client = { workspace = true, features = ["test-support"] }
-encoding_rs.workspace = true
collections = { workspace = true, features = ["test-support"] }
context_server = { workspace = true, features = ["test-support"] }
buffer_diff = { workspace = true, features = ["test-support"] }
diff --git a/crates/project/src/git_store.rs b/crates/project/src/git_store.rs
index c4f77c78784..1df75d5192c 100644
--- a/crates/project/src/git_store.rs
+++ b/crates/project/src/git_store.rs
@@ -34,12 +34,12 @@ use git::{
blame::Blame,
parse_git_remote_url,
repository::{
- Branch, BranchesScanResult, CommitData, CommitDetails, CommitDiff, CommitFile,
- CommitOptions, CreateWorktreeTarget, DiffStatType, DiffType, FetchOptions,
- FileHistoryChangedFileSets, GitCommitTemplate, GitRepository, GitRepositoryCheckpoint,
- InitialGraphCommitData, LogOrder, LogSource, PushOptions, Remote, RemoteCommandOutput,
- RepoPath, ResetMode, SearchCommitArgs, UpstreamTrackingStatus, Worktree as GitWorktree,
- delete_branch_flag, is_binary_content,
+ Branch, BranchesScanResult, CommitData, CommitDetails, CommitFileStatus, CommitOptions,
+ CreateWorktreeTarget, DiffStatType, DiffType, FetchOptions, FileHistoryChangedFileSets,
+ GitCommitTemplate, GitRepository, GitRepositoryCheckpoint, InitialGraphCommitData,
+ LogOrder, LogSource, PushOptions, Remote, RemoteCommandOutput, RepoPath, ResetMode,
+ SearchCommitArgs, UpstreamTrackingStatus, Worktree as GitWorktree, delete_branch_flag,
+ is_binary_content,
},
stash::{GitStash, StashEntry},
status::{
@@ -52,7 +52,7 @@ use gpui::{
Subscription, Task, TaskExt, WeakEntity,
};
use language::{
- Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry,
+ Anchor, Buffer, BufferEvent, Capability, Language, LanguageRegistry, decode_text, encode_text,
proto::{deserialize_version, serialize_version},
};
use parking_lot::Mutex;
@@ -208,6 +208,85 @@ fn pending_hunks(
.collect()
}
+fn decode_git_text(bytes: Vec<u8>) -> Result<String> {
+ Ok(decode_text(bytes)?.text)
+}
+
+#[derive(Debug)]
+pub struct CommitDiff {
+ pub files: Vec<CommitFile>,
+}
+
+#[derive(Debug)]
+pub struct CommitFile {
+ pub path: RepoPath,
+ pub old_text: Option<String>,
+ pub new_text: Option<String>,
+ pub is_binary: bool,
+}
+
+impl CommitFile {
+ pub fn status(&self) -> CommitFileStatus {
+ match (&self.old_text, &self.new_text) {
+ (None, Some(_)) => CommitFileStatus::Added,
+ (Some(_), None) => CommitFileStatus::Deleted,
+ _ => CommitFileStatus::Modified,
+ }
+ }
+}
+
+fn decode_commit_diff(diff: git::repository::CommitDiff) -> CommitDiff {
+ let files = diff
+ .files
+ .into_iter()
+ .map(|file| {
+ let git::repository::CommitFile {
+ path,
+ old_content,
+ new_content,
+ mut is_binary,
+ } = file;
+
+ if is_binary {
+ return CommitFile {
+ path,
+ old_text: old_content.map(|_| String::new()),
+ new_text: new_content.map(|_| String::new()),
+ is_binary,
+ };
+ }
+
+ let mut decode_content = |content: Option<Vec<u8>>| {
+ content.map(|content| match decode_git_text(content) {
+ Ok(text) => text,
+ Err(error) => {
+ log::debug!(
+ "treating commit content for {} as binary after decoding failed: {error:#}",
+ path.as_unix_str()
+ );
+ is_binary = true;
+ String::new()
+ }
+ })
+ };
+ let mut old_text = decode_content(old_content);
+ let mut new_text = decode_content(new_content);
+ if is_binary {
+ old_text = old_text.map(|_| String::new());
+ new_text = new_text.map(|_| String::new());
+ }
+
+ CommitFile {
+ path,
+ old_text,
+ new_text,
+ is_binary,
+ }
+ })
+ .collect();
+ CommitDiff { files }
+}
+
#[derive(Clone, Debug)]
enum DiffBasesChange {
SetIndex(Option<String>),
@@ -2864,11 +2943,22 @@ impl GitStore {
let Some((repo, path)) = self.repository_and_path_for_buffer_id(buffer_id, cx) else {
return;
};
+ let (encoding, has_bom) = self
+ .buffer_store
+ .read(cx)
+ .get(buffer_id)
+ .map(|buffer| {
+ let buffer = buffer.read(cx);
+ (buffer.encoding(), buffer.has_bom())
+ })
+ .unwrap_or((encoding_rs::UTF_8, false));
let recv = repo.update(cx, |repo, cx| {
log::debug!("hunks changed for {}", path.as_unix_str());
repo.spawn_set_index_text_job(
path,
new_index_text,
+ encoding,
+ has_bom,
Some(hunk_staging_operation_count),
cx,
)
@@ -3435,11 +3525,30 @@ impl GitStore {
let repository_handle = Self::repository_for_request(&this, repository_id, &mut cx)?;
let repo_path = RepoPath::from_proto(&envelope.payload.path)?;
+ let (encoding, has_bom) = cx.update(|cx| {
+ repository_handle
+ .read(cx)
+ .repo_path_to_project_path(&repo_path, cx)
+ .and_then(|project_path| {
+ this.read(cx)
+ .buffer_store
+ .read(cx)
+ .get_by_path(&project_path)
+ })
+ .map(|buffer| {
+ let buffer = buffer.read(cx);
+ (buffer.encoding(), buffer.has_bom())
+ })
+ .unwrap_or((encoding_rs::UTF_8, false))
+ });
+
repository_handle
.update(&mut cx, |repository_handle, cx| {
repository_handle.spawn_set_index_text_job(
repo_path,
envelope.payload.text,
+ encoding,
+ has_bom,
None,
cx,
)
@@ -6346,11 +6455,13 @@ impl Repository {
{
let index_text = (current_index_text.is_some() && !*is_symlink)
.then(|| loaded_revisions.next().flatten())
- .flatten();
+ .flatten()
+ .and_then(|bytes| decode_git_text(bytes).log_err());
let head_text = (current_head_text.is_some() && !*is_symlink)
.then(|| loaded_revisions.next().flatten())
- .flatten();
+ .flatten()
+ .and_then(|bytes| decode_git_text(bytes).log_err());
let change =
match (current_index_text.as_ref(), current_head_text.as_ref()) {
@@ -6800,9 +6911,10 @@ impl Repository {
let id = self.id;
self.send_job("load_commit_diff", None, move |git_repo, cx| async move {
match git_repo {
- RepositoryState::Local(LocalRepositoryState { backend, .. }) => {
- backend.load_commit(commit, cx).await
- }
+ RepositoryState::Local(LocalRepositoryState { backend, .. }) => backend
+ .load_commit(commit, cx)
+ .await
+ .map(decode_commit_diff),
RepositoryState::Remote(RemoteRepositoryState {
client, project_id, ..
}) => {
@@ -8414,6 +8526,8 @@ impl Repository {
&mut self,
path: RepoPath,
content: Option<String>,
+ encoding: &'static encoding_rs::Encoding,
+ has_bom: bool,
hunk_staging_operation_count: Option<usize>,
cx: &mut Context<Self>,
) -> oneshot::Receiver<anyhow::Result<()>> {
@@ -8443,6 +8557,8 @@ impl Repository {
Ok(None) => false,
Err(_err) => false,
};
+ let content =
+ content.map(|content| encode_text(content, encoding, has_bom));
backend
.set_index_text(path.clone(), content, environment.clone(), executable)
.await?;
@@ -9748,9 +9864,11 @@ impl Repository {
) -> Task<Result<Option<String>>> {
let rx = self.send_job("load_staged_text", None, move |state, _| async move {
match state {
- RepositoryState::Local(LocalRepositoryState { backend, .. }) => {
- anyhow::Ok(backend.load_index_text(repo_path).await)
- }
+ RepositoryState::Local(LocalRepositoryState { backend, .. }) => backend
+ .load_index_text(repo_path)
+ .await
+ .map(decode_git_text)
+ .transpose(),
RepositoryState::Remote(RemoteRepositoryState { project_id, client }) => {
let response = client
.request(proto::OpenUnstagedDiff {
@@ -9779,8 +9897,16 @@ impl Repository {
format!(":{}", repo_path.as_unix_str()),
];
let mut loaded_revisions = backend.load_revisions(revisions).await?.into_iter();
- let committed_text = loaded_revisions.next().flatten();
- let staged_text = loaded_revisions.next().flatten();
+ let committed_text = loaded_revisions
+ .next()
+ .flatten()
+ .map(decode_git_text)
+ .transpose()?;
+ let staged_text = loaded_revisions
+ .next()
+ .flatten()
+ .map(decode_git_text)
+ .transpose()?;
let diff_bases_change = if committed_text == staged_text {
DiffBasesChange::SetBoth(committed_text)
} else {
@@ -9866,6 +9992,7 @@ impl Repository {
path.as_ref()
)
})?;
+ let content = decode_git_text(content)?;
anyhow::Ok((content, blame))
}
RepositoryState::Remote(RemoteRepositoryState { client, project_id }) => {
@@ -9903,7 +10030,7 @@ impl Repository {
let rx = self.send_job("load_blob_content", None, move |state, _| async move {
match state {
RepositoryState::Local(LocalRepositoryState { backend, .. }) => {
- backend.load_blob_content(oid).await
+ decode_git_text(backend.load_blob_content(oid).await?)
}
RepositoryState::Remote(RemoteRepositoryState { client, project_id }) => {
let response = client
@@ -10931,6 +11058,31 @@ mod tests {
assert_eq!(resolved, None);
}
+ #[gpui::test]
+ fn test_decode_git_text_windows_1251_one_line_change(cx: &mut TestAppContext) {
+ let old_text = "строка один\nстрока два\n";
+ let new_text = "строка один\nстрока три\n";
+ let (old_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(old_text);
+ let (new_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(new_text);
+
+ let decoded_old = decode_git_text(old_bytes.into_owned()).unwrap();
+ let decoded_new = decode_git_text(new_bytes.into_owned()).unwrap();
+ let buffer = cx.new(|cx| Buffer::local(decoded_new, cx));
+ let buffer_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ let diff = cx.new(|cx| BufferDiff::new_with_base_text(&decoded_old, &buffer_snapshot, cx));
+ let diff = diff.update(cx, |diff, cx| diff.snapshot(cx));
+ let hunks = diff.hunks(&buffer_snapshot).collect::<Vec<_>>();
+ let [hunk] = hunks.as_slice() else {
+ panic!("expected one modified hunk, got {hunks:?}");
+ };
+
+ assert_eq!(hunk.range, text::Point::new(1, 0)..text::Point::new(2, 0));
+ assert_eq!(
+ hunk.diff_base_byte_range,
+ old_text.find("строка два").unwrap()..old_text.len()
+ );
+ }
+
#[gpui::test]
async fn test_open_uncommitted_diff_skips_symlinks(cx: &mut TestAppContext) {
use util::rel_path::rel_path;
diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs
index 73acdc013d9..cba0f6cb8e4 100644
--- a/crates/project/tests/integration/project_tests.rs
+++ b/crates/project/tests/integration/project_tests.rs
@@ -11580,7 +11580,7 @@ async fn test_restaging_hunk_after_optimistic_unstage(cx: &mut gpui::TestAppCont
repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
.await
.unwrap(),
- file_contents,
+ file_contents.as_bytes(),
"the re-stage should have overridden the in-flight unstage"
);
@@ -11605,7 +11605,73 @@ async fn test_restaging_hunk_after_optimistic_unstage(cx: &mut gpui::TestAppCont
repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
.await
.unwrap(),
- file_contents
+ file_contents.as_bytes()
+ );
+}
+
+#[gpui::test]
+async fn test_staging_non_utf8_hunk_preserves_index_encoding(cx: &mut gpui::TestAppContext) {
+ init_test(cx);
+
+ let committed_text = "строка один\nстрока два\n";
+ let file_text = "строка один\nстрока три\n";
+ let (committed_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(committed_text);
+ let (file_bytes, _, _) = encoding_rs::WINDOWS_1251.encode(file_text);
+ let committed_bytes = committed_bytes.into_owned();
+ let file_bytes = file_bytes.into_owned();
+
+ let fs = FakeFs::new(cx.background_executor.clone());
+ fs.insert_tree(path!("/dir"), json!({ ".git": {} })).await;
+ fs.insert_file(path!("/dir/file.txt"), file_bytes.clone())
+ .await;
+ fs.with_git_state(path!("/dir/.git").as_ref(), true, |state| {
+ state
+ .head_contents
+ .insert(repo_path("file.txt"), committed_bytes.clone());
+ state
+ .index_contents
+ .insert(repo_path("file.txt"), committed_bytes.clone());
+ state.refs.insert("HEAD".into(), "deadbeef".into());
+ })
+ .unwrap();
+ let repo = fs
+ .open_repo(path!("/dir/.git").as_ref(), Some("git".as_ref()))
+ .unwrap();
+
+ let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await;
+ let buffer = project
+ .update(cx, |project, cx| {
+ project.open_local_buffer(path!("/dir/file.txt"), cx)
+ })
+ .await
+ .unwrap();
+ assert_eq!(
+ buffer.read_with(cx, |buffer, _| buffer.encoding()),
+ encoding_rs::WINDOWS_1251
+ );
+
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ let uncommitted_diff = project
+ .update(cx, |project, cx| {
+ project.open_uncommitted_diff(buffer.clone(), cx)
+ })
+ .await
+ .unwrap();
+ let range = snapshot.anchor_before(Point::new(1, 0))..snapshot.anchor_before(Point::new(2, 0));
+
+ project
+ .update(cx, |project, cx| {
+ let unstaged_diff = uncommitted_diff.read(cx).secondary_diff().unwrap();
+ project.stage_hunks(buffer.clone(), unstaged_diff, vec![range], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ assert_eq!(
+ repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
+ .await
+ .unwrap(),
+ file_bytes
);
}
@@ -11705,7 +11771,7 @@ async fn test_staging_random_hunks(
repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
.await
.unwrap(),
- index_text
+ index_text.as_bytes()
);
fs.unpause_events_and_flush();
cx.run_until_parked();
@@ -11762,9 +11828,12 @@ async fn test_staging_random_hunks(
log::info!(
"index text:\n{}",
- repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
- .await
- .unwrap()
+ String::from_utf8_lossy(
+ &repo
+ .load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
+ .await
+ .unwrap()
+ )
);
uncommitted_diff.update(cx, |diff, cx| {
@@ -11794,14 +11863,16 @@ async fn test_staging_random_hunks_with_edits(
init_test(cx);
fn disk_index_text(fs: &FakeFs) -> String {
- fs.with_git_state(path!("/dir/.git").as_ref(), false, |state| {
- state
- .index_contents
- .get(&repo_path("file.txt"))
- .cloned()
- .expect("file is always present in the index")
- })
- .unwrap()
+ let bytes = fs
+ .with_git_state(path!("/dir/.git").as_ref(), false, |state| {
+ state
+ .index_contents
+ .get(&repo_path("file.txt"))
+ .cloned()
+ .expect("file is always present in the index")
+ })
+ .unwrap();
+ String::from_utf8(bytes).expect("test index contents are valid UTF-8")
}
fn random_row_range(
@@ -12190,10 +12261,12 @@ async fn test_staging_random_hunks_with_edits(
let old_staged_state = capture_diff_state(&staged_diff, cx);
let old_uncommitted_state = capture_diff_state(&uncommitted_diff, cx);
- let disk_index_text = repo
- .load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
- .await
- .unwrap();
+ let disk_index_text = String::from_utf8(
+ repo.load_index_text(RepoPath::from_rel_path(rel_path("file.txt")))
+ .await
+ .unwrap(),
+ )
+ .unwrap();
assert_eq!(
old_unstaged_state.1.as_deref(),
Some(disk_index_text.as_str()),
@@ -14948,10 +15021,10 @@ async fn test_git_worktrees_and_submodules(cx: &mut gpui::TestAppContext) {
|state| {
state
.head_contents
- .insert(repo_path("src/b.txt"), "b".to_owned());
+ .insert(repo_path("src/b.txt"), b"b".to_vec());
state
.index_contents
- .insert(repo_path("src/b.txt"), "b".to_owned());
+ .insert(repo_path("src/b.txt"), b"b".to_vec());
},
)
.unwrap();
@@ -15001,10 +15074,10 @@ async fn test_git_worktrees_and_submodules(cx: &mut gpui::TestAppContext) {
|state| {
state
.head_contents
- .insert(repo_path("c.txt"), "c".to_owned());
+ .insert(repo_path("c.txt"), b"c".to_vec());
state
.index_contents
- .insert(repo_path("c.txt"), "c".to_owned());
+ .insert(repo_path("c.txt"), b"c".to_vec());
},
)
.unwrap();
@@ -16695,14 +16768,16 @@ async fn test_staging_hunks_with_ambiguous_placement(cx: &mut gpui::TestAppConte
}
fn index_text(fs: &FakeFs) -> String {
- fs.with_git_state(path!("/dir/.git").as_ref(), false, |state| {
- state
- .index_contents
- .get(&repo_path("test.txt"))
- .cloned()
- .expect("file is present in the index")
- })
- .unwrap()
+ let bytes = fs
+ .with_git_state(path!("/dir/.git").as_ref(), false, |state| {
+ state
+ .index_contents
+ .get(&repo_path("test.txt"))
+ .cloned()
+ .expect("file is present in the index")
+ })
+ .unwrap();
+ String::from_utf8(bytes).expect("test index contents are valid UTF-8")
}
use DiffHunkSecondaryStatus::*;
diff --git a/crates/worktree/Cargo.toml b/crates/worktree/Cargo.toml
index 9a95eec9697..5a2b6204e99 100644
--- a/crates/worktree/Cargo.toml
+++ b/crates/worktree/Cargo.toml
@@ -33,7 +33,6 @@ test-support = [
anyhow.workspace = true
async-channel.workspace = true
async-lock.workspace = true
-chardetng.workspace = true
clock.workspace = true
collections.workspace = true
encoding_rs.workspace = true
diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs
index d09542a45c6..4a8a726a15d 100644
--- a/crates/worktree/src/worktree.rs
+++ b/crates/worktree/src/worktree.rs
@@ -3,7 +3,6 @@ mod worktree_settings;
use ::ignore::gitignore::{Gitignore, GitignoreBuilder};
use anyhow::{Context as _, Result, anyhow};
-use chardetng::EncodingDetector;
use clock::ReplicaId;
use collections::{BTreeMap, HashMap, HashSet, VecDeque};
use encoding_rs::Encoding;
@@ -32,7 +31,9 @@ use gpui::{
Task,
};
pub use ignore::{IgnoreKind, IgnoreStack};
-use language::{ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content};
+use language::{
+ ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text,
+};
use async_channel::{self, Sender};
use parking_lot::Mutex;
@@ -1863,47 +1864,7 @@ impl LocalWorktree {
LineEnding::Windows => text_string.replace('\n', "\r\n"),
};
- // Create the byte vector manually for UTF-16 encodings because encoding_rs encodes to UTF-8 by default (per WHATWG standards),
- // which is not what we want for saving files.
- let bytes = if encoding == encoding_rs::UTF_16BE {
- let mut data = Vec::with_capacity(normalized_text.len() * 2 + 2);
- if has_bom {
- data.extend_from_slice(&[0xFE, 0xFF]); // BOM
- }
- let utf16be_bytes =
- normalized_text.encode_utf16().flat_map(|u| u.to_be_bytes());
- data.extend(utf16be_bytes);
- data.into()
- } else if encoding == encoding_rs::UTF_16LE {
- let mut data = Vec::with_capacity(normalized_text.len() * 2 + 2);
- if has_bom {
- data.extend_from_slice(&[0xFF, 0xFE]); // BOM
- }
- let utf16le_bytes =
- normalized_text.encode_utf16().flat_map(|u| u.to_le_bytes());
- data.extend(utf16le_bytes);
- data.into()
- } else {
- // For other encodings (Shift-JIS, UTF-8 with BOM, etc.), delegate to encoding_rs.
- let bom_bytes = if has_bom {
- if encoding == encoding_rs::UTF_8 {
- vec![0xEF, 0xBB, 0xBF]
- } else {
- vec![]
- }
- } else {
- vec![]
- };
- let (cow, _, _) = encoding.encode(&normalized_text);
- if !bom_bytes.is_empty() {
- let mut bytes = bom_bytes;
- bytes.extend_from_slice(&cow);
- bytes.into()
- } else {
- cow
- }
- };
-
+ let bytes = encode_text(normalized_text, encoding, has_bom);
fs.write(&abs_path, &bytes).await
}
});
@@ -7239,7 +7200,7 @@ pub async fn decode_file_text(
}
file_first_bytes.extend_from_slice(&buf[..n]);
}
- let (bom_encoding, byte_content) = decode_byte_header(&file_first_bytes);
+ let (_, byte_content) = decode_byte_header(&file_first_bytes);
anyhow::ensure!(
byte_content != ByteContent::Binary,
"Binary files are not supported"
@@ -7259,7 +7220,8 @@ pub async fn decode_file_text(
content.extend_from_slice(&buf[..n]);
}
}
- decode_byte_full(content, bom_encoding, byte_content)
+ let decoded = decode_text(content)?;
+ Ok((decoded.text, decoded.encoding, decoded.has_bom))
}
pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
@@ -7269,63 +7231,6 @@ pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteCont
(None, analyze_byte_content(prefix))
}
-fn decode_byte_full(
- bytes: Vec<u8>,
- bom_encoding: Option<&'static Encoding>,
- byte_content: ByteContent,
-) -> Result<(String, &'static Encoding, bool)> {
- if let Some(encoding) = bom_encoding {
- let (cow, _) = encoding.decode_with_bom_removal(&bytes);
- return Ok((cow.into_owned(), encoding, true));
- }
-
- match byte_content {
- ByteContent::Utf16Le => {
- let encoding = encoding_rs::UTF_16LE;
- let (cow, _, _) = encoding.decode(&bytes);
- return Ok((cow.into_owned(), encoding, false));
- }
- ByteContent::Utf16Be => {
- let encoding = encoding_rs::UTF_16BE;
- let (cow, _, _) = encoding.decode(&bytes);
- return Ok((cow.into_owned(), encoding, false));
- }
- ByteContent::Binary => {
- anyhow::bail!("Binary files are not supported");
- }
- ByteContent::Unknown => {}
- }
-
- fn detect_encoding(bytes: Vec<u8>) -> (String, &'static Encoding) {
- let mut detector = EncodingDetector::new();
- detector.feed(&bytes, true);
-
- let encoding = detector.guess(None, true); // Use None for TLD hint to ensure neutral detection logic.
-
- let (cow, _, _) = encoding.decode(&bytes);
- (cow.into_owned(), encoding)
- }
-
- match String::from_utf8(bytes) {
- Ok(text) => {
- // ISO-2022-JP (and other ISO-2022 variants) consists entirely of 7-bit ASCII bytes,
- // so it is valid UTF-8. However, it contains escape sequences starting with '\x1b'.
- // If we find an escape character, we double-check the encoding to prevent
- // displaying raw escape sequences instead of the correct characters.
- if text.contains('\x1b') {
- let (s, enc) = detect_encoding(text.into_bytes());
- Ok((s, enc, false))
- } else {
- Ok((text, encoding_rs::UTF_8, false))
- }
- }
- Err(e) => {
- let (s, enc) = detect_encoding(e.into_bytes());
- Ok((s, enc, false))
- }
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;