Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

fix(vcs): decode git blobs in the file's encoding (#301)

l0ng-aicdb6cc7c0d30
diff --git a/Cargo.lock b/Cargo.lock
index 6d49a14d5..8565c4eb6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4034,10 +4034,12 @@ name = "ide-agent"
version = "0.1.25"
dependencies = [
"dunce",
+ "encoding_rs",
"ide-core",
"ide-jsonrpc",
"ide-tools",
"ide-vcs",
+ "ide-vfs",
"parking_lot",
"serde",
"serde_json",
@@ -4202,6 +4204,7 @@ version = "0.1.25"
dependencies = [
"dirs 6.0.0",
"dunce",
+ "encoding_rs",
"ide-agent",
"ide-analysis",
"ide-build",
@@ -4603,6 +4606,7 @@ name = "ide-vcs"
version = "0.1.25"
dependencies = [
"dunce",
+ "encoding_rs",
"gix",
"ide-core",
"ignore",
diff --git a/crates/ide-agent/Cargo.toml b/crates/ide-agent/Cargo.toml
index 6e4a53fba..046d1b244 100644
--- a/crates/ide-agent/Cargo.toml
+++ b/crates/ide-agent/Cargo.toml
@@ -6,6 +6,8 @@ edition.workspace = true
license.workspace = true
[dependencies]
+encoding_rs.workspace = true
+ide-vfs.workspace = true
ide-core.workspace = true
ide-jsonrpc.workspace = true
ide-vcs.workspace = true
diff --git a/crates/ide-agent/src/worktree.rs b/crates/ide-agent/src/worktree.rs
index 7f385afca..b372604c3 100644
--- a/crates/ide-agent/src/worktree.rs
+++ b/crates/ide-agent/src/worktree.rs
@@ -96,7 +96,12 @@ pub(crate) fn without_untouched(changes: Vec<AgentChange>, before: &BTreeMap<Str
}
pub(crate) fn file_at(cwd: &Path, path: &Path, rev: &str) -> IdeResult<Option<String>> {
- repo_at(cwd)?.file_at(path, Some(rev))
+ let repo = repo_at(cwd)?;
+ let encoding = match ide_vfs::load_text(path) {
+ Ok(loaded) => encoding_rs::Encoding::for_label(loaded.encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8),
+ Err(_) => ide_vfs::detect_encoding(&repo.file_bytes_at(path, Some(rev))?.unwrap_or_default(), None),
+ };
+ repo.file_at(path, Some(rev), encoding)
}
/// A short ASCII branch-safe name from the task title.
diff --git a/crates/ide-host/Cargo.toml b/crates/ide-host/Cargo.toml
index adc1344ca..4518fa0ff 100644
--- a/crates/ide-host/Cargo.toml
+++ b/crates/ide-host/Cargo.toml
@@ -10,6 +10,7 @@ name = "openidea-server"
path = "src/bin/server.rs"
[dependencies]
+encoding_rs.workspace = true
ide-core.workspace = true
ide-text.workspace = true
ide-vfs.workspace = true
diff --git a/crates/ide-host/src/ops/vcs/commands.rs b/crates/ide-host/src/ops/vcs/commands.rs
index 6140487d1..0c639cd1d 100644
--- a/crates/ide-host/src/ops/vcs/commands.rs
+++ b/crates/ide-host/src/ops/vcs/commands.rs
@@ -187,18 +187,24 @@ pub async fn vcs_branch_info(host: &Host, window: &str, repo: Option<PathBuf>) -
pub fn vcs_line_hunks(host: &Host, window: &str, path: PathBuf) -> IdeResult<Vec<LineHunk>> {
let project = host.project(window)?;
let text = project.text_of(&path)?;
- project.with_repo_of(&path, |r| r.line_hunks(&path, &text, false))
+ project.with_repo_of(&path, |r| r.line_hunks(&path, &text, false, project.vcs_encoding(&path, None)?))
}
pub async fn vcs_file_at(host: &Host, window: &str, path: PathBuf, rev: Option<String>) -> IdeResult<Option<String>> {
let project = host.project(window)?;
- blocking(move || project.with_repo_of(&path, |r| r.file_at(&path, rev.as_deref()))).await
+ blocking(move || {
+ project.with_repo_of(&path, |r| r.file_at(&path, rev.as_deref(), project.vcs_encoding(&path, rev.as_deref())?))
+ })
+ .await
}
pub async fn vcs_blame(host: &Host, window: &str, path: PathBuf) -> IdeResult<Vec<BlameLine>> {
let project = host.project(window)?;
let text = project.documents.text_of(&path);
- blocking(move || project.with_repo_of(&path, |r| r.blame(&path, text.as_deref()))).await
+ blocking(move || {
+ project.with_repo_of(&path, |r| r.blame(&path, text.as_deref(), project.vcs_encoding(&path, None)?))
+ })
+ .await
}
pub async fn vcs_log(host: &Host, window: &str, repo: Option<PathBuf>, query: LogQuery) -> IdeResult<Vec<Commit>> {
@@ -276,7 +282,7 @@ pub async fn vcs_unstage(host: &Host, window: &str, paths: Vec<PathBuf>) -> IdeR
pub async fn vcs_stage_patch(host: &Host, window: &str, patch: String) -> IdeResult<()> {
let project = host.project(window)?;
- blocking(move || project.with_repo(|r| r.stage_patch(&patch))).await
+ blocking(move || project.with_repo(|r| r.stage_patch(&patch, &|path| project.vcs_encoding(path, Some(":0"))))).await
}
pub async fn vcs_discard(host: &Host, window: &str, paths: Vec<PathBuf>) -> IdeResult<()> {
@@ -293,7 +299,12 @@ pub async fn vcs_commit(
) -> IdeResult<String> {
// Commit hooks may run for a long time.
let project = host.project(window)?;
- blocking(move || project.with_repo_at(repo.as_deref(), |r| r.commit_with(&message, &options))).await
+ blocking(move || {
+ project.with_repo_at(repo.as_deref(), |r| {
+ r.commit_with(&message, &options, &|path| project.vcs_encoding(path, None))
+ })
+ })
+ .await
}
pub async fn vcs_checkout(
@@ -464,12 +475,14 @@ pub async fn vcs_remote(
/// The user's side is `ours` (sides swapped during a rebase, as in IDEA).
pub async fn vcs_conflict_versions(host: &Host, window: &str, path: PathBuf) -> IdeResult<ide_vcs::MergeVersions> {
let project = host.project(window)?;
- blocking(move || project.with_repo_of(&path, |r| r.merge_versions(&path))).await
+ blocking(move || project.with_repo_of(&path, |r| r.merge_versions(&path, project.vcs_encoding(&path, Some(":2"))?)))
+ .await
}
pub async fn vcs_diff_text(host: &Host, window: &str, path: PathBuf, staged: bool) -> IdeResult<String> {
let project = host.project(window)?;
- blocking(move || project.with_repo_of(&path, |r| r.diff_text(&path, staged))).await
+ blocking(move || project.with_repo_of(&path, |r| r.diff_text(&path, staged, project.vcs_encoding(&path, None)?)))
+ .await
}
/// Line hunks between two arbitrary texts (diff viewer, commit dialog).
@@ -481,13 +494,11 @@ pub fn vcs_diff_hunks(_host: &Host, _window: &str, old_text: String, new_text: S
/// Indices refer to `vcs_diff_hunks(index_text, worktree_text)`.
pub fn vcs_stage_hunks(host: &Host, window: &str, path: PathBuf, hunks: Vec<u32>) -> IdeResult<()> {
let project = host.project(window)?;
- let current = ide_vfs::load_text(&path)?.text;
+ let encoding = project.vcs_encoding(&path, Some(":0"))?;
+ let current = ide_vfs::load_text_as(&path, encoding.name())?.text;
project.with_repo_of(&path, |r| {
- let index_text = r.file_at(&path, Some(":0"))?.unwrap_or_default();
- let rel = path.strip_prefix(r.root()).unwrap_or(&path).to_string_lossy().replace('\\', "/");
let selected: Vec<usize> = hunks.iter().map(|&h| h as usize).collect();
- let patch = ide_vcs::build_partial_patch(&rel, &index_text, &current, &selected);
- if patch.is_empty() { Ok(()) } else { r.stage_patch(&patch) }
+ r.stage_hunks(&path, &current, &selected, encoding)
})
}
diff --git a/crates/ide-host/src/project.rs b/crates/ide-host/src/project.rs
index dcdeeb606..d959906aa 100644
--- a/crates/ide-host/src/project.rs
+++ b/crates/ide-host/src/project.rs
@@ -689,11 +689,29 @@ impl Project {
}
}
+ /// Use the open document's selected encoding, or the same detection/settings as opening it.
+ pub fn vcs_encoding(&self, path: &Path, rev: Option<&str>) -> IdeResult<&'static encoding_rs::Encoding> {
+ if let Some(id) = self.documents.by_path(path) {
+ let info = self.documents.snapshot(id)?;
+ return encoding_rs::Encoding::for_label(info.encoding.as_bytes())
+ .ok_or_else(|| ide_core::IdeError::invalid("unknown document encoding"));
+ }
+ let defaults = self.documents.file_defaults();
+ let fallback = defaults.fallback_for(path);
+ if path.exists() {
+ let loaded = ide_vfs::load_text_with_fallback(path, fallback)?;
+ return encoding_rs::Encoding::for_label(loaded.encoding.as_bytes())
+ .ok_or_else(|| ide_core::IdeError::invalid("unknown file encoding"));
+ }
+ let bytes = self.with_repo_of(path, |r| r.file_bytes_at(path, rev))?.unwrap_or_default();
+ Ok(ide_vfs::detect_encoding(&bytes, fallback))
+ }
+
/// Current text of `path`: the open buffer when available, else the file on disk.
pub fn text_of(&self, path: &Path) -> IdeResult<String> {
match self.documents.text_of(path) {
Some(text) => Ok(text),
- None => Ok(ide_vfs::load_text(path)?.text),
+ None => Ok(ide_vfs::load_text_with_fallback(path, self.documents.file_defaults().fallback_for(path))?.text),
}
}
}
diff --git a/crates/ide-host/tests/vcs_encoding.rs b/crates/ide-host/tests/vcs_encoding.rs
new file mode 100644
index 000000000..0b34d18b5
--- /dev/null
+++ b/crates/ide-host/tests/vcs_encoding.rs
@@ -0,0 +1,111 @@
+use ide_core::{ChangeSet, NullSink, TextChange};
+use ide_host::{Host, HostConfig, ops::vcs};
+use std::{path::Path, process::Command, sync::Arc, time::Duration};
+
+fn git(root: &Path, args: &[&str]) {
+ let out = Command::new("git").current_dir(root).args(args).output().unwrap();
+ assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn gbk_document_gutter_diff_stage_and_rollback_share_encoding() {
+ let temp = tempfile::tempdir().unwrap();
+ let base = dunce::canonicalize(temp.path()).unwrap();
+ let root = base.join("project");
+ std::fs::create_dir(&root).unwrap();
+ git(&root, &["init", "-b", "main"]);
+ git(&root, &["config", "user.name", "Test"]);
+ git(&root, &["config", "user.email", "[email protected]"]);
+ git(&root, &["config", "commit.gpgsign", "false"]);
+ git(&root, &["config", "core.autocrlf", "false"]);
+ let path = root.join("Example.java");
+ let old = "// 中文注释\nclass Example {\n // 保持不变\n int value = 1;\n}\n";
+ let original = ide_vfs::encode(old, "gbk", false).unwrap();
+ std::fs::write(&path, &original).unwrap();
+ std::fs::write(root.join("Utf8.java"), old).unwrap();
+ git(&root, &["add", "."]);
+ git(&root, &["commit", "-m", "GBK Java"]);
+ let host = Host::new(HostConfig {
+ config_dir: base.join("config"),
+ data_dir: base.join("data"),
+ languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()),
+ syntax: Arc::default(),
+ sink_for: Box::new(|_| Arc::new(NullSink)),
+ owner: |w| w,
+ runtime: tokio::runtime::Handle::current(),
+ project_environment: false,
+ });
+ let project = host.open_project("main", &root).unwrap();
+ for _ in 0..500 {
+ if !project.repos().is_empty() {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ let mut defaults = project.documents.file_defaults();
+ defaults.project_fallback = "gbk".into();
+ project.documents.set_file_defaults(defaults);
+ assert_eq!(project.text_of(&path).unwrap(), old);
+ assert_eq!(vcs::vcs_file_at(&host, "main", path.clone(), None).await.unwrap().as_deref(), Some(old));
+ let doc = project.documents.open(&path).unwrap();
+ assert_eq!(doc.encoding, "gbk");
+ // A live document's explicit selection wins over later project defaults.
+ let mut defaults = project.documents.file_defaults();
+ defaults.project_fallback = "windows-1252".into();
+ project.documents.set_file_defaults(defaults);
+ assert!(vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap().is_empty());
+ let from = old.find("value = 1").unwrap() + "value = ".len();
+ let offset = old[..from].encode_utf16().count() as u32;
+ project
+ .documents
+ .edit(
+ doc.id,
+ doc.version,
+ ChangeSet { changes: vec![TextChange { from: offset, to: offset + 1, insert: "2".into() }] },
+ )
+ .unwrap();
+ let hunks = vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap();
+ assert_eq!(hunks.len(), 1);
+ assert_eq!((hunks[0].start, hunks[0].lines), (3, 1));
+ assert_eq!(hunks[0].old_text, " int value = 1;\n");
+ project.documents.save(doc.id).unwrap();
+ vcs::vcs_stage_hunks(&host, "main", path.clone(), vec![0]).unwrap();
+ let repo = ide_vcs::Repo::discover(&root).unwrap().unwrap();
+ assert_eq!(
+ repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(),
+ ide_vfs::encode(&old.replace("value = 1", "value = 2"), "gbk", false).unwrap()
+ );
+ repo.unstage(std::slice::from_ref(&path)).unwrap();
+ let snapshot = project.documents.snapshot(doc.id).unwrap();
+ let line_start =
+ snapshot.text.split_inclusive('\n').take(3).map(|s| s.encode_utf16().count()).sum::<usize>() as u32;
+ project
+ .documents
+ .edit(
+ doc.id,
+ snapshot.version,
+ ChangeSet {
+ changes: vec![TextChange {
+ from: line_start,
+ to: line_start + " int value = 2;\n".encode_utf16().count() as u32,
+ insert: hunks[0].old_text.clone(),
+ }],
+ },
+ )
+ .unwrap();
+ project.documents.save(doc.id).unwrap();
+ assert_eq!(std::fs::read(&path).unwrap(), original);
+ assert!(vcs::vcs_line_hunks(&host, "main", path.clone()).unwrap().is_empty());
+ assert!(vcs::vcs_blame(&host, "main", path.clone()).await.unwrap().iter().all(|l| l.author == "Test"));
+ // Missing working files still use configured encoding for historical content.
+ project.documents.close(doc.id);
+ let mut defaults = project.documents.file_defaults();
+ defaults.project_fallback = "gbk".into();
+ project.documents.set_file_defaults(defaults);
+ std::fs::remove_file(&path).unwrap();
+ assert_eq!(vcs::vcs_file_at(&host, "main", path, None).await.unwrap().as_deref(), Some(old));
+ let utf8 = root.join("Utf8.java");
+ std::fs::remove_file(&utf8).unwrap();
+ assert_eq!(vcs::vcs_file_at(&host, "main", utf8, None).await.unwrap().as_deref(), Some(old));
+ host.close_project("main");
+}
diff --git a/crates/ide-vcs/Cargo.toml b/crates/ide-vcs/Cargo.toml
index 169063a40..9f74924e2 100644
--- a/crates/ide-vcs/Cargo.toml
+++ b/crates/ide-vcs/Cargo.toml
@@ -6,6 +6,7 @@ edition.workspace = true
license.workspace = true
[dependencies]
+encoding_rs.workspace = true
dunce.workspace = true
ide-core.workspace = true
ignore.workspace = true
diff --git a/crates/ide-vcs/src/diff.rs b/crates/ide-vcs/src/diff.rs
index f52f46406..9a2e935ff 100644
--- a/crates/ide-vcs/src/diff.rs
+++ b/crates/ide-vcs/src/diff.rs
@@ -174,6 +174,45 @@ fn quote_path(path: &str) -> String {
out
}
+/// Apply selected line hunks while retaining the base's line separators.
+pub(crate) fn selected_text(old: &str, new: &str, selected: &[usize]) -> String {
+ let old_norm = normalize_eol(old);
+ let new_norm = normalize_eol(new);
+ let old_lines = split_lines(&old_norm);
+ let new_lines = split_lines(&new_norm);
+ let old_raw = split_raw_lines(old);
+ let eol = if old.contains("\r\n") {
+ "\r\n"
+ } else if old.contains('\r') {
+ "\r"
+ } else {
+ "\n"
+ };
+ let mut result = String::new();
+ let mut cursor = 0;
+ for (index, (before, after)) in raw_hunks(&old_lines, &new_lines).iter().enumerate() {
+ if !selected.contains(&index) {
+ continue;
+ }
+ for line in &old_raw[cursor..before.start as usize] {
+ result.push_str(line);
+ }
+ for line in &new_lines[after.start as usize..after.end as usize] {
+ if let Some(body) = line.strip_suffix('\n') {
+ result.push_str(body);
+ result.push_str(eol);
+ } else {
+ result.push_str(line);
+ }
+ }
+ cursor = before.end as usize;
+ }
+ for line in &old_raw[cursor..] {
+ result.push_str(line);
+ }
+ result
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/ide-vcs/src/lib.rs b/crates/ide-vcs/src/lib.rs
index ceb0735c9..aa16e84f6 100644
--- a/crates/ide-vcs/src/lib.rs
+++ b/crates/ide-vcs/src/lib.rs
@@ -2,6 +2,7 @@
//! writes (commit, checkout, push/pull, stash, merge) shell out to `git` so
//! hooks, credentials and signing behave exactly as on the command line.
+use encoding_rs::Encoding;
use ide_core::ignored::IgnoredNames;
use ide_core::{CancellationToken, IdeResult};
use serde::{Deserialize, Serialize};
@@ -451,6 +452,7 @@ mod repo;
mod roots;
pub mod shelf;
mod status;
+mod text_encoding;
mod write;
pub use cli::{detected_git_executable, git_version, set_askpass_env, set_git_executable};
@@ -520,10 +522,24 @@ impl Repo {
}
/// Content of `path` at `rev` (default HEAD); `None` if absent there.
/// `rev` may be any revision git understands, or `":"` / `":0"` for the index.
- pub fn file_at(&self, path: &Path, rev: Option<&str>) -> IdeResult<Option<String>> {
+ pub fn file_at(&self, path: &Path, rev: Option<&str>, encoding: &'static Encoding) -> IdeResult<Option<String>> {
let rel = self.rel_path(path)?;
let repo = self.gix()?;
- Ok(repo::blob_at(&repo, &rel, rev.unwrap_or("HEAD"))?.map(|b| String::from_utf8_lossy(&b).into_owned()))
+ Ok(repo::blob_at(&repo, &rel, rev.unwrap_or("HEAD"))?.map(|b| text_encoding::decode(&b, encoding).into_owned()))
+ }
+ /// Raw revision bytes, for encoding detection when the working file is absent.
+ pub fn file_bytes_at(&self, path: &Path, rev: Option<&str>) -> IdeResult<Option<Vec<u8>>> {
+ repo::blob_at(&self.gix()?, &self.rel_path(path)?, rev.unwrap_or("HEAD"))
+ }
+ /// Stage selected decoded hunks, retaining the index encoding, BOM and line endings.
+ pub fn stage_hunks(
+ &self,
+ path: &Path,
+ current: &str,
+ selected: &[usize],
+ encoding: &'static Encoding,
+ ) -> IdeResult<()> {
+ write::stage_hunks(self, path, current, selected, encoding)
}
/// Hunks of `current_text` against HEAD (or the index when `against_index`).
///
@@ -531,7 +547,13 @@ impl Repo {
/// follows the removed block (equal to the line count when removed at the end).
/// Files absent from the base yield one `Added` hunk; ignored files, binary files and
/// paths outside the worktree yield no hunks.
- pub fn line_hunks(&self, path: &Path, current_text: &str, against_index: bool) -> IdeResult<Vec<LineHunk>> {
+ pub fn line_hunks(
+ &self,
+ path: &Path,
+ current_text: &str,
+ against_index: bool,
+ encoding: &'static Encoding,
+ ) -> IdeResult<Vec<LineHunk>> {
let Ok(rel) = self.rel_path(path) else { return Ok(Vec::new()) };
if rel.is_empty() || rel == ".git" || rel.starts_with(".git/") {
return Ok(Vec::new());
@@ -539,8 +561,8 @@ impl Repo {
let repo = self.gix()?;
let base = repo::blob_at(&repo, &rel, if against_index { ":0" } else { "HEAD" })?;
match base {
- Some(base) if base.contains(&0) => Ok(Vec::new()),
- Some(base) => Ok(diff::line_hunks(&String::from_utf8_lossy(&base), current_text)),
+ Some(base) if text_encoding::is_binary(&base, encoding) => Ok(Vec::new()),
+ Some(base) => Ok(diff::line_hunks(&text_encoding::decode(&base, encoding), current_text)),
None => {
if repo::is_ignored(&repo, &rel)? {
return Ok(Vec::new());
@@ -551,8 +573,13 @@ impl Repo {
}
/// One entry per line of the current text (`current_text`, or the file on disk).
/// Lines not yet committed carry an all-zero commit id and author "Not Committed Yet".
- pub fn blame(&self, path: &Path, current_text: Option<&str>) -> IdeResult<Vec<BlameLine>> {
- write::blame(self, path, current_text)
+ pub fn blame(
+ &self,
+ path: &Path,
+ current_text: Option<&str>,
+ encoding: &'static Encoding,
+ ) -> IdeResult<Vec<BlameLine>> {
+ write::blame(self, path, current_text, encoding)
}
/// Commits newest-first in topological order (children before parents).
///
@@ -583,8 +610,8 @@ impl Repo {
}
/// Unified diff of a file: index↔worktree (`staged == false`, untracked files diff
/// against /dev/null) or HEAD↔index (`staged == true`). Empty when unchanged.
- pub fn diff_text(&self, path: &Path, staged: bool) -> IdeResult<String> {
- write::diff_text(self, path, staged)
+ pub fn diff_text(&self, path: &Path, staged: bool, encoding: &'static Encoding) -> IdeResult<String> {
+ write::diff_text(self, path, staged, encoding)
}
/// `git config --get <key>` (all scopes); `None` when unset.
@@ -602,8 +629,8 @@ impl Repo {
}
/// Applies a unified-diff patch to the index (partial staging of hunks/lines).
/// Zero-context patches (see [`build_partial_patch`]) are accepted.
- pub fn stage_patch(&self, patch: &str) -> IdeResult<()> {
- write::stage_patch(self, patch)
+ pub fn stage_patch(&self, patch: &str, encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>) -> IdeResult<()> {
+ write::stage_patch(self, patch, encoding)
}
/// Discards unstaged changes: tracked paths are restored from the index (staged
/// changes are kept), untracked files and directories are deleted.
@@ -614,11 +641,16 @@ impl Repo {
/// `amend` keeps the previous message.
pub fn commit(&self, message: &str, amend: bool, sign_off: bool) -> IdeResult<String> {
let options = CommitOptions { amend, sign_off, author: None, run_hooks: true, partial: Vec::new() };
- write::commit(self, message, &options)
+ write::commit(self, message, &options, &|_| Ok(encoding_rs::UTF_8))
}
/// Commits with the Commit Options popup's choices.
- pub fn commit_with(&self, message: &str, options: &CommitOptions) -> IdeResult<String> {
- write::commit(self, message, options)
+ pub fn commit_with(
+ &self,
+ message: &str,
+ options: &CommitOptions,
+ encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>,
+ ) -> IdeResult<String> {
+ write::commit(self, message, options, encoding)
}
/// Switches to `branch`. With `create`, creates it at HEAD. Without, a remote branch
/// (`origin/x`) gets a local tracking branch and any other revision is checked out
@@ -867,8 +899,8 @@ impl Repo {
}
/// The versions of a conflicted file with the user's work as `ours`, as IDEA's merge
/// dialog shows them (sides swapped during a rebase).
- pub fn merge_versions(&self, path: &Path) -> IdeResult<MergeVersions> {
- let (base, ours, theirs) = self.conflict_versions(path)?;
+ pub fn merge_versions(&self, path: &Path, encoding: &'static Encoding) -> IdeResult<MergeVersions> {
+ let (base, ours, theirs) = self.conflict_versions(path, encoding)?;
let sides = self.merge_sides()?;
let (ours, theirs) = if sides.reversed { (theirs, ours) } else { (ours, theirs) };
let merged = match (&ours, &theirs) {
@@ -878,11 +910,15 @@ impl Repo {
Ok(MergeVersions { base, ours, theirs, sides, merged })
}
/// For conflict resolution: (base, ours, theirs) versions of a conflicted file.
- pub fn conflict_versions(&self, path: &Path) -> IdeResult<(Option<String>, Option<String>, Option<String>)> {
+ pub fn conflict_versions(
+ &self,
+ path: &Path,
+ encoding: &'static Encoding,
+ ) -> IdeResult<(Option<String>, Option<String>, Option<String>)> {
let rel = self.rel_path(path)?;
let repo = self.gix()?;
let stage = |n: u8| -> IdeResult<Option<String>> {
- Ok(repo::blob_at(&repo, &rel, &format!(":{n}"))?.map(|b| String::from_utf8_lossy(&b).into_owned()))
+ Ok(repo::blob_at(&repo, &rel, &format!(":{n}"))?.map(|b| text_encoding::decode(&b, encoding).into_owned()))
};
Ok((stage(1)?, stage(2)?, stage(3)?))
}
diff --git a/crates/ide-vcs/src/patch.rs b/crates/ide-vcs/src/patch.rs
index 9bb54feaf..8192641ca 100644
--- a/crates/ide-vcs/src/patch.rs
+++ b/crates/ide-vcs/src/patch.rs
@@ -50,11 +50,12 @@ pub(crate) fn working_patch(r: &Repo, rel_paths: &[String], reverse: bool) -> Id
let mut add: Vec<&str> = vec!["--literal-pathspecs", "add", "-A", "--"];
add.extend(rel_paths.iter().map(String::as_str));
run_env(r.root(), &add, None, &env)?;
- let mut diff = vec!["diff", "--cached", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff"];
+ let mut diff =
+ vec!["diff", "--cached", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff", "--no-textconv"];
if reverse {
diff.push("-R");
}
- let patch = run_env(r.root(), &diff, None, &env)?.stdout_str();
+ let patch = lossless_diff(r, &diff, &env)?;
let names = run_env(r.root(), &["diff", "--cached", "--name-status", "-z", "--no-renames"], None, &env)?;
Ok((patch, crate::ops::parse_name_status(&names.stdout_str())))
}
@@ -74,12 +75,12 @@ pub(crate) fn commits_patch(r: &Repo, commits: &[String], reverse: bool) -> IdeR
let parent = commit.parent_ids().next().map(|p| p.detach().to_string());
(parent.unwrap_or_else(|| EMPTY_TREE.to_string()), id.to_string())
};
- let mut args = vec!["diff", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff"];
+ let mut args = vec!["diff", "--binary", "--full-index", "-M", "--no-color", "--no-ext-diff", "--no-textconv"];
if reverse {
args.push("-R");
}
args.extend([parent.as_str(), id.as_str(), "--"]);
- out.push_str(&run(r.root(), &args, None)?.stdout_str());
+ out.push_str(&lossless_diff(r, &args, &[])?);
}
Ok(out)
}
@@ -266,6 +267,68 @@ pub(crate) fn parse_patch(patch: &str) -> Vec<PatchFile> {
out
}
+/// A patch crosses the string IPC boundary. Non-UTF-8 content must travel as a
+/// Git binary patch, not replacement characters or an encoding-dependent string.
+fn lossless_diff(r: &Repo, args: &[&str], env: &[(&str, &OsStr)]) -> IdeResult<String> {
+ let output = run_env(r.root(), args, None, env)?.stdout;
+ if let Ok(text) = String::from_utf8(output) {
+ return Ok(text);
+ }
+ // info/attributes has highest precedence. Isolate it from the real repository
+ // so concurrent Git operations and user attributes are never modified.
+ let temp = tempfile::tempdir()?;
+ let git_dir = temp.path().to_string_lossy().into_owned();
+ let format = run(r.root(), &["rev-parse", "--show-object-format"], None)?.stdout_str();
+ run(r.root(), &["init", "--bare", "--quiet", &format!("--object-format={}", format.trim()), &git_dir], None)?;
+ std::fs::write(temp.path().join("info/attributes"), "* -diff\n")?;
+ if let Some(head) = r.head()?.1 {
+ std::fs::write(temp.path().join("HEAD"), format!("{head}\n"))?;
+ }
+ let objects = run(r.root(), &["rev-parse", "--path-format=absolute", "--git-path", "objects"], None)?.stdout_str();
+ let mut isolated = env.to_vec();
+ isolated.extend([
+ ("GIT_DIR", temp.path().as_os_str()),
+ ("GIT_WORK_TREE", r.root().as_os_str()),
+ ("GIT_OBJECT_DIRECTORY", OsStr::new(objects.trim())),
+ ]);
+ String::from_utf8(run_env(r.root(), args, None, &isolated)?.stdout)
+ .map_err(|_| IdeError::invalid("Git produced a non-UTF-8 binary patch"))
+}
+
+/// Encode only hunk payloads; patch headers and Git paths always remain UTF-8.
+pub(crate) fn encode_text_patch(
+ r: &Repo,
+ patch: &str,
+ encoding: &dyn Fn(&std::path::Path) -> IdeResult<&'static encoding_rs::Encoding>,
+) -> IdeResult<Vec<u8>> {
+ let mut out = Vec::new();
+ let mut header = String::new();
+ let mut enc = None;
+ for line in patch.split_inclusive('\n') {
+ if line.starts_with("diff --git ") {
+ header.clear();
+ enc = None;
+ }
+ if line.starts_with("@@ ") {
+ let file = parse_patch(&header)
+ .into_iter()
+ .next()
+ .ok_or_else(|| IdeError::invalid("patch hunk has no file header"))?;
+ enc = Some(encoding(&r.abs_path(&file.path))?);
+ } else if let Some(enc) = enc {
+ if line.starts_with(['+', '-', ' ']) {
+ out.push(line.as_bytes()[0]);
+ out.extend(crate::text_encoding::encode(&line[1..], enc, false)?);
+ continue;
+ }
+ } else {
+ header.push_str(line);
+ }
+ out.extend_from_slice(line.as_bytes());
+ }
+ Ok(out)
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/ide-vcs/src/text_encoding.rs b/crates/ide-vcs/src/text_encoding.rs
new file mode 100644
index 000000000..9d23171a4
--- /dev/null
+++ b/crates/ide-vcs/src/text_encoding.rs
@@ -0,0 +1,46 @@
+//! File content uses the document's encoding; Git metadata remains UTF-8.
+use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};
+use ide_core::{IdeError, IdeResult};
+use std::borrow::Cow;
+
+pub(crate) fn has_bom(bytes: &[u8], encoding: &'static Encoding) -> bool {
+ Encoding::for_bom(bytes).is_some_and(|(enc, _)| enc == encoding)
+}
+
+pub(crate) fn decode<'a>(bytes: &'a [u8], encoding: &'static Encoding) -> Cow<'a, str> {
+ let body = match Encoding::for_bom(bytes) {
+ Some((enc, len)) if enc == encoding => &bytes[len..],
+ _ => bytes,
+ };
+ encoding.decode_without_bom_handling(body).0
+}
+
+pub(crate) fn is_binary(bytes: &[u8], encoding: &'static Encoding) -> bool {
+ encoding != UTF_16LE && encoding != UTF_16BE && bytes.contains(&0)
+}
+
+pub(crate) fn encode(text: &str, encoding: &'static Encoding, bom: bool) -> IdeResult<Vec<u8>> {
+ let mut out = Vec::new();
+ if encoding == UTF_16LE || encoding == UTF_16BE {
+ let le = encoding == UTF_16LE;
+ if bom {
+ out.extend_from_slice(if le { &[0xff, 0xfe] } else { &[0xfe, 0xff] });
+ }
+ for unit in text.encode_utf16() {
+ out.extend_from_slice(&if le { unit.to_le_bytes() } else { unit.to_be_bytes() });
+ }
+ } else {
+ let (bytes, _, errors) = encoding.encode(text);
+ if errors {
+ return Err(IdeError::invalid(format!(
+ "text contains characters not representable in {}",
+ encoding.name()
+ )));
+ }
+ if bom && encoding == UTF_8 {
+ out.extend_from_slice(&[0xef, 0xbb, 0xbf]);
+ }
+ out.extend_from_slice(&bytes);
+ }
+ Ok(out)
+}
diff --git a/crates/ide-vcs/src/write.rs b/crates/ide-vcs/src/write.rs
index d34e3eb5e..b636b1b3b 100644
--- a/crates/ide-vcs/src/write.rs
+++ b/crates/ide-vcs/src/write.rs
@@ -2,9 +2,11 @@
use crate::cli::{self, run};
use crate::repo::{gx, resolve};
+use crate::text_encoding::{decode, encode, has_bom, is_binary};
use crate::{
BlameLine, CheckoutMode, CheckoutOptions, CheckoutOutcome, CommitOptions, DeletedBranch, DivergedLocal, Repo,
};
+use encoding_rs::Encoding;
use gix::bstr::{BStr, ByteSlice};
use ide_core::{CancellationToken, IdeError, IdeResult};
use std::path::{Path, PathBuf};
@@ -40,11 +42,33 @@ fn run_paths(r: &Repo, args: &[&str], paths: &[String]) -> IdeResult<()> {
run(r.root(), &full, None).map(|_| ())
}
-pub(crate) fn blame(r: &Repo, path: &Path, current_text: Option<&str>) -> IdeResult<Vec<BlameLine>> {
+pub(crate) fn blame(
+ r: &Repo,
+ path: &Path,
+ current_text: Option<&str>,
+ encoding: &'static Encoding,
+) -> IdeResult<Vec<BlameLine>> {
let rel = r.rel_path(path)?;
- let text: String = match current_text {
- Some(t) => t.to_string(),
- None => String::from_utf8_lossy(&std::fs::read(r.abs_path(&rel))?).into_owned(),
+ let disk = match std::fs::read(r.abs_path(&rel)) {
+ Ok(bytes) => bytes,
+ Err(_) if current_text.is_some() => Vec::new(),
+ Err(error) => return Err(error.into()),
+ };
+ let text = current_text.map(str::to_owned).unwrap_or_else(|| decode(&disk, encoding).into_owned());
+ let contents = match current_text {
+ Some(_) => {
+ let disk_text = decode(&disk, encoding);
+ let eol = if disk_text.contains("\r\n") {
+ "\r\n"
+ } else if disk_text.contains('\r') {
+ "\r"
+ } else {
+ "\n"
+ };
+ let contents = crate::diff::normalize_eol(&text).replace('\n', eol);
+ encode(&contents, encoding, has_bom(&disk, encoding))?
+ }
+ None => disk,
};
let line_count = text.split_inclusive('\n').count();
let tracked_in_head = {
@@ -57,7 +81,7 @@ pub(crate) fn blame(r: &Repo, path: &Path, current_text: Option<&str>) -> IdeRes
let out = run(
r.root(),
&["--literal-pathspecs", "blame", "--porcelain", "--contents", "-", "--", &rel],
- Some(text.as_bytes()),
+ Some(&contents),
)?;
let mut lines = crate::blame::parse_porcelain(&out.stdout_str());
for line in &mut lines {
@@ -87,7 +111,7 @@ fn uncommitted_lines(n: usize) -> Vec<BlameLine> {
.collect()
}
-pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult<String> {
+pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool, encoding: &'static Encoding) -> IdeResult<String> {
let rel = r.rel_path(path)?;
let base =
["-c", "diff.noprefix=false", "-c", "diff.mnemonicPrefix=false", "-c", "diff.external=", "--literal-pathspecs"];
@@ -102,7 +126,7 @@ pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult<String
args.extend(["diff", "--no-color", "--no-ext-diff", "--no-index", "--", "/dev/null", &rel]);
let out = cli::run_raw(r.root(), &args, None)?;
return match out.code {
- Some(0) | Some(1) => Ok(out.stdout_str()),
+ Some(0) | Some(1) => Ok(decode_diff(&out.stdout, encoding)),
code => Err(cli::failure(&args, &out.stderr, &out.stdout, code)),
};
}
@@ -111,7 +135,7 @@ pub(crate) fn diff_text(r: &Repo, path: &Path, staged: bool) -> IdeResult<String
args.push("--cached");
}
args.extend(["--", &rel]);
- Ok(run(r.root(), &args, None)?.stdout_str())
+ Ok(decode_diff(&run(r.root(), &args, None)?.stdout, encoding))
}
pub(crate) fn stage(r: &Repo, paths: &[PathBuf]) -> IdeResult<()> {
@@ -128,7 +152,11 @@ pub(crate) fn unstage(r: &Repo, paths: &[PathBuf]) -> IdeResult<()> {
}
}
-pub(crate) fn stage_patch(r: &Repo, patch: &str) -> IdeResult<()> {
+pub(crate) fn stage_patch(
+ r: &Repo,
+ patch: &str,
+ encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>,
+) -> IdeResult<()> {
if patch.trim().is_empty() {
return Ok(());
}
@@ -136,12 +164,9 @@ pub(crate) fn stage_patch(r: &Repo, patch: &str) -> IdeResult<()> {
if !patch.ends_with('\n') {
patch.push('\n');
}
- run(
- r.root(),
- &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"],
- Some(patch.as_bytes()),
- )
- .map(|_| ())
+ let bytes = crate::patch::encode_text_patch(r, &patch, encoding)?;
+ run(r.root(), &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"], Some(&bytes))
+ .map(|_| ())
}
/// Paths with unresolved conflicts (index stages 1–3).
@@ -224,7 +249,12 @@ pub(crate) fn discard(r: &Repo, paths: &[PathBuf]) -> IdeResult<()> {
Ok(())
}
-pub(crate) fn commit(r: &Repo, message: &str, options: &CommitOptions) -> IdeResult<String> {
+pub(crate) fn commit(
+ r: &Repo,
+ message: &str,
+ options: &CommitOptions,
+ encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>,
+) -> IdeResult<String> {
let mut args = vec!["commit", "-q", "--cleanup=whitespace"];
if options.amend {
args.push("--amend");
@@ -252,7 +282,7 @@ pub(crate) fn commit(r: &Repo, message: &str, options: &CommitOptions) -> IdeRes
if options.partial.is_empty() {
run(r.root(), &args, stdin)?;
} else {
- commit_partial(r, &args, stdin, &options.partial)?;
+ commit_partial(r, &args, stdin, &options.partial, encoding)?;
}
let repo = r.gix()?;
Ok(repo.head_id().map_err(gx)?.detach().to_string())
@@ -277,6 +307,7 @@ fn commit_partial(
commit_args: &[&str],
stdin: Option<&[u8]>,
partial: &[crate::PartialCommitFile],
+ encoding: &dyn Fn(&Path) -> IdeResult<&'static Encoding>,
) -> IdeResult<()> {
let tmp = TempIndex(r.git_dir.join("openidea-partial-index"));
let index = r.git_dir.join("index");
@@ -295,10 +326,12 @@ fn commit_partial(
IdeError::invalid(format!("{rel}: only changes of a file in HEAD can be committed partially"))
})?;
let disk = std::fs::read(r.abs_path(&rel))?;
- if head.contains(&0) || disk.contains(&0) {
+ let encoding = encoding(&file.path.0)?;
+ let bom = has_bom(&head, encoding);
+ if is_binary(&head, encoding) || is_binary(&disk, encoding) {
return Err(IdeError::invalid(format!("{rel}: binary files cannot be committed partially")));
}
- let (head, disk) = (String::from_utf8_lossy(&head), String::from_utf8_lossy(&disk));
+ let (head, disk) = (decode(&head, encoding), decode(&disk, encoding));
let hunks = crate::diff::line_hunks(&head, &disk);
let mut selected = Vec::new();
for want in &file.hunks {
@@ -311,15 +344,9 @@ fn commit_partial(
}
}
cli::run_env(r.root(), &["--literal-pathspecs", "reset", "-q", "HEAD", "--", &rel], None, &env)?;
- let patch = crate::diff::build_partial_patch(&rel, &head, &disk, &selected);
- if !patch.is_empty() {
- cli::run_env(
- r.root(),
- &["apply", "--cached", "--unidiff-zero", "--whitespace=nowarn", "--recount", "-"],
- Some(patch.as_bytes()),
- &env,
- )?;
- }
+ let text = crate::diff::selected_text(&head, &disk, &selected);
+ let bytes = encode(&text, encoding, bom)?;
+ write_index_blob(r, &rel, &bytes, &env)?;
rels.push(rel);
}
cli::run_env(r.root(), commit_args, stdin, &env)?;
@@ -798,3 +825,57 @@ fn push_remote(r: &Repo) -> IdeResult<String> {
})
.ok_or_else(|| IdeError::invalid("no remote configured"))
}
+
+fn write_index_blob(r: &Repo, rel: &str, bytes: &[u8], env: &[(&str, &std::ffi::OsStr)]) -> IdeResult<()> {
+ let entry =
+ cli::run_env(r.root(), &["--literal-pathspecs", "ls-files", "--stage", "--", rel], None, env)?.stdout_str();
+ let mode = entry.split_whitespace().next().unwrap_or("100644");
+ let id = run(r.root(), &["hash-object", "-w", "--stdin"], Some(bytes))?.stdout_str();
+ cli::run_env(r.root(), &["update-index", "--add", "--cacheinfo", mode, id.trim(), rel], None, env)?;
+ Ok(())
+}
+
+pub(crate) fn stage_hunks(
+ r: &Repo,
+ path: &Path,
+ current: &str,
+ selected: &[usize],
+ encoding: &'static Encoding,
+) -> IdeResult<()> {
+ if selected.is_empty() {
+ return Ok(());
+ }
+ let rel = r.rel_path(path)?;
+ let base = r.file_bytes_at(path, Some(":0"))?.unwrap_or_default();
+ if is_binary(&base, encoding) {
+ return Err(IdeError::invalid("binary files cannot be staged partially"));
+ }
+ let old = decode(&base, encoding);
+ let hunks = crate::diff::line_hunks(&old, current);
+ if selected.iter().any(|&i| i >= hunks.len()) {
+ return Err(IdeError::invalid("hunk no longer exists"));
+ }
+ let text = crate::diff::selected_text(&old, current, selected);
+ let bytes = encode(&text, encoding, has_bom(&base, encoding))?;
+ write_index_blob(r, &rel, &bytes, &[])
+}
+
+// Git headers contain UTF-8 paths; only source lines use the file's encoding.
+fn decode_diff(bytes: &[u8], encoding: &'static Encoding) -> String {
+ let mut hunks = false;
+ let mut text = String::new();
+ for line in bytes.split_inclusive(|&b| b == b'\n') {
+ if line.starts_with(b"diff --git ") {
+ hunks = false;
+ }
+ if line.starts_with(b"@@ ") {
+ hunks = true;
+ }
+ if hunks {
+ text.push_str(&decode(line, encoding));
+ } else {
+ text.push_str(&String::from_utf8_lossy(line));
+ }
+ }
+ text
+}
diff --git a/crates/ide-vcs/tests/git_workflows.rs b/crates/ide-vcs/tests/git_workflows.rs
index 073161f19..e33706286 100644
--- a/crates/ide-vcs/tests/git_workflows.rs
+++ b/crates/ide-vcs/tests/git_workflows.rs
@@ -651,7 +651,7 @@ fn rollback_hunk_via_partial_patch_leaves_other_hunks() {
assert_eq!(hunks.len(), 2);
let patch = build_partial_patch("f.txt", &original, &edited, &[0]);
let repo = fx.repo();
- repo.stage_patch(&patch).unwrap();
+ repo.stage_patch(&patch, &|_| Ok(encoding_rs::UTF_8)).unwrap();
repo.discard(&[fx.path("f.txt")]).unwrap();
assert_eq!(fx.read("f.txt"), original.replace("line 2\n", "line two\n"));
}
@@ -777,7 +777,7 @@ fn add_to_vcs_annotate_and_history() {
repo.stage(&[fx.path("new/b.txt")]).unwrap();
assert_eq!(status_of(&repo.status().unwrap(), "new/b.txt").unwrap().staged, Some(ChangeKind::Added));
- let blame = repo.blame(&fx.path("a.txt"), Some("1\nchanged\nunsaved\n")).unwrap();
+ let blame = repo.blame(&fx.path("a.txt"), Some("1\nchanged\nunsaved\n"), encoding_rs::UTF_8).unwrap();
assert_eq!(blame.len(), 3);
assert_eq!(blame[0].commit, first);
assert_eq!(blame[1].commit, second);
@@ -785,8 +785,8 @@ fn add_to_vcs_annotate_and_history() {
let history = repo.log(&LogQuery { path: Some(fx.path("a.txt").into()), ..Default::default() }).unwrap();
assert_eq!(history.iter().map(|c| c.id.as_str()).collect::<Vec<_>>(), vec![second.as_str(), first.as_str()]);
- assert_eq!(repo.file_at(&fx.path("a.txt"), Some(&first)).unwrap().as_deref(), Some("1\n2\n"));
- let diff = repo.diff_text(&fx.path("new/b.txt"), true).unwrap();
+ assert_eq!(repo.file_at(&fx.path("a.txt"), Some(&first), encoding_rs::UTF_8).unwrap().as_deref(), Some("1\n2\n"));
+ let diff = repo.diff_text(&fx.path("new/b.txt"), true, encoding_rs::UTF_8).unwrap();
assert!(diff.contains("+b"), "{diff}");
}
diff --git a/crates/ide-vcs/tests/history_ops.rs b/crates/ide-vcs/tests/history_ops.rs
index 7116a98c4..9e2b756bf 100644
--- a/crates/ide-vcs/tests/history_ops.rs
+++ b/crates/ide-vcs/tests/history_ops.rs
@@ -113,7 +113,7 @@ fn merge_versions_put_the_users_work_on_the_left() {
let repo = fx.repo();
let topic = fx.git(&["rev-parse", "topic"]);
assert!(repo.rebase("main").is_err());
- let v = repo.merge_versions(&fx.path("f.txt")).unwrap();
+ let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap();
assert_eq!(v.ours.as_deref(), Some("topic\n"));
assert_eq!(v.theirs.as_deref(), Some("main\n"));
assert_eq!(v.base.as_deref(), Some("base\n"));
@@ -129,7 +129,7 @@ fn merge_versions_put_the_users_work_on_the_left() {
// Merge: not swapped; the merged branch is named.
assert!(repo.merge("main").is_err());
- let v = repo.merge_versions(&fx.path("f.txt")).unwrap();
+ let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap();
assert_eq!(v.ours.as_deref(), Some("topic\n"));
assert_eq!(v.theirs.as_deref(), Some("main\n"));
assert!(!v.sides.reversed);
@@ -140,7 +140,7 @@ fn merge_versions_put_the_users_work_on_the_left() {
// Cherry-pick: not swapped; the picked commit is named.
let main = fx.git(&["rev-parse", "main"]);
assert!(repo.cherry_pick(std::slice::from_ref(&main), &cancel, &out).is_err());
- let v = repo.merge_versions(&fx.path("f.txt")).unwrap();
+ let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap();
assert_eq!(v.ours.as_deref(), Some("topic\n"));
assert!(!v.sides.reversed);
assert_eq!(v.sides.operation.as_deref(), Some("cherry-pick"));
@@ -374,7 +374,7 @@ fn unshelve_with_conflict_leaves_merge_state() {
assert_eq!(outcome.conflicts, vec!["f.txt".to_string()]);
let text = fx.read("f.txt");
assert!(text.contains("<<<<<<<") && text.contains("SHELVED") && text.contains("COMMITTED"), "{text}");
- let (base, ours, theirs) = repo.conflict_versions(&fx.path("f.txt")).unwrap();
+ let (base, ours, theirs) = repo.conflict_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap();
assert!(base.is_some() && ours.unwrap().contains("COMMITTED") && theirs.unwrap().contains("SHELVED"));
// Kept on the shelf when not removing.
assert_eq!(shelf::list(&shelf).unwrap().len(), 1);
diff --git a/crates/ide-vcs/tests/repo.rs b/crates/ide-vcs/tests/repo.rs
index b202cd78f..6b97eeadf 100644
--- a/crates/ide-vcs/tests/repo.rs
+++ b/crates/ide-vcs/tests/repo.rs
@@ -121,11 +121,11 @@ fn repository_without_commits() {
assert!(repo.log(&LogQuery::default()).unwrap().is_empty());
assert!(repo.branches().unwrap().is_empty());
assert!(repo.stashes().unwrap().is_empty());
- assert_eq!(repo.file_at(Path::new("new.txt"), None).unwrap(), None);
+ assert_eq!(repo.file_at(Path::new("new.txt"), None, encoding_rs::UTF_8).unwrap(), None);
let st = repo.status().unwrap();
assert_eq!(status_of(&st, "new.txt").unstaged, Some(ChangeKind::Untracked));
- let hunks = repo.line_hunks(&fx.path("new.txt"), "hello\n", false).unwrap();
+ let hunks = repo.line_hunks(&fx.path("new.txt"), "hello\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!(hunks.len(), 1);
assert_eq!(hunks[0].kind, HunkKind::Added);
@@ -244,7 +244,7 @@ fn merge_conflict_and_versions() {
let st = repo.status().unwrap();
let s = status_of(&st, "c.txt");
assert_eq!((s.staged, s.unstaged), (None, Some(ChangeKind::Conflicted)));
- let (base, ours, theirs) = repo.conflict_versions(&fx.path("c.txt")).unwrap();
+ let (base, ours, theirs) = repo.conflict_versions(&fx.path("c.txt"), encoding_rs::UTF_8).unwrap();
assert_eq!(base.as_deref(), Some("base\n"));
assert_eq!(ours.as_deref(), Some("ours\n"));
assert_eq!(theirs.as_deref(), Some("theirs\n"));
@@ -378,36 +378,36 @@ fn line_hunks_against_head_and_index() {
let repo = fx.repo();
let p = fx.path("f.txt");
- assert!(repo.line_hunks(&p, "1\n2\n3\n4\n5\n", false).unwrap().is_empty());
- assert!(repo.line_hunks(&p, "1\r\n2\r\n3\r\n4\r\n5\r\n", false).unwrap().is_empty());
+ assert!(repo.line_hunks(&p, "1\n2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap().is_empty());
+ assert!(repo.line_hunks(&p, "1\r\n2\r\n3\r\n4\r\n5\r\n", false, encoding_rs::UTF_8).unwrap().is_empty());
- let h = repo.line_hunks(&p, "0\n1\n2\n3\n4\n5\n", false).unwrap();
+ let h = repo.line_hunks(&p, "0\n1\n2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h.len(), h[0].kind, h[0].start, h[0].lines), (1, HunkKind::Added, 0, 1));
- let h = repo.line_hunks(&p, "1\n2\n3\n4\n5\n6\n", false).unwrap();
+ let h = repo.line_hunks(&p, "1\n2\n3\n4\n5\n6\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h[0].kind, h[0].start, h[0].lines), (HunkKind::Added, 5, 1));
- let h = repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false).unwrap();
+ let h = repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h[0].kind, h[0].start, h[0].lines, h[0].old_text.as_str()), (HunkKind::Modified, 1, 1, "2\n"));
- let h = repo.line_hunks(&p, "1\n4\n5\n", false).unwrap();
+ let h = repo.line_hunks(&p, "1\n4\n5\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h[0].kind, h[0].start, h[0].lines, h[0].old_start, h[0].old_lines), (HunkKind::Deleted, 1, 0, 1, 2));
- let h = repo.line_hunks(&p, "2\n3\n4\n5\n", false).unwrap();
+ let h = repo.line_hunks(&p, "2\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h[0].kind, h[0].start), (HunkKind::Deleted, 0));
- let h = repo.line_hunks(&p, "1\n2\n3\n4\n", false).unwrap();
+ let h = repo.line_hunks(&p, "1\n2\n3\n4\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h[0].kind, h[0].start, h[0].old_start), (HunkKind::Deleted, 4, 4));
// Against the index: staged changes are not reported.
fx.write("f.txt", "1\nX\n3\n4\n5\n");
fx.git(&["add", "f.txt"]);
- assert!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", true).unwrap().is_empty());
- assert_eq!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false).unwrap().len(), 1);
+ assert!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", true, encoding_rs::UTF_8).unwrap().is_empty());
+ assert_eq!(repo.line_hunks(&p, "1\nX\n3\n4\n5\n", false, encoding_rs::UTF_8).unwrap().len(), 1);
// New and ignored files.
- let h = repo.line_hunks(&fx.path("new.txt"), "a\nb\n", false).unwrap();
+ let h = repo.line_hunks(&fx.path("new.txt"), "a\nb\n", false, encoding_rs::UTF_8).unwrap();
assert_eq!((h.len(), h[0].kind, h[0].start, h[0].lines), (1, HunkKind::Added, 0, 2));
- assert!(repo.line_hunks(&fx.path("ignored.txt"), "a\n", false).unwrap().is_empty());
- assert!(repo.line_hunks(Path::new("/definitely/outside"), "a\n", false).unwrap().is_empty());
+ assert!(repo.line_hunks(&fx.path("ignored.txt"), "a\n", false, encoding_rs::UTF_8).unwrap().is_empty());
+ assert!(repo.line_hunks(Path::new("/definitely/outside"), "a\n", false, encoding_rs::UTF_8).unwrap().is_empty());
}
#[test]
@@ -419,7 +419,7 @@ fn blame_with_unsaved_edits() {
let second = fx.commit_all("second");
let repo = fx.repo();
- let lines = repo.blame(&fx.path("b.txt"), None).unwrap();
+ let lines = repo.blame(&fx.path("b.txt"), None, encoding_rs::UTF_8).unwrap();
assert_eq!(lines.len(), 3);
assert_eq!(lines[0].commit, first);
assert_eq!(lines[1].commit, second);
@@ -427,7 +427,7 @@ fn blame_with_unsaved_edits() {
assert_eq!(lines[0].author, "Test User");
assert!(lines[0].time_ms > 1.0e12);
- let lines = repo.blame(&fx.path("b.txt"), Some("one\nedited\nthree\nfour")).unwrap();
+ let lines = repo.blame(&fx.path("b.txt"), Some("one\nedited\nthree\nfour"), encoding_rs::UTF_8).unwrap();
assert_eq!(lines.len(), 4);
assert_eq!(lines[0].commit, first);
assert!(lines[1].commit.bytes().all(|b| b == b'0'));
@@ -435,7 +435,7 @@ fn blame_with_unsaved_edits() {
assert_eq!(lines[2].commit, first);
assert_eq!(lines[3].author, "Not Committed Yet");
- let lines = repo.blame(&fx.path("untracked.txt"), Some("x\ny\n")).unwrap();
+ let lines = repo.blame(&fx.path("untracked.txt"), Some("x\ny\n"), encoding_rs::UTF_8).unwrap();
assert_eq!(lines.len(), 2);
assert!(lines.iter().all(|l| l.author == "Not Committed Yet"));
}
@@ -615,12 +615,12 @@ fn file_at_revisions_and_index() {
fx.git(&["add", "f.txt"]);
let repo = fx.repo();
let p = fx.path("f.txt");
- assert_eq!(repo.file_at(&p, None).unwrap().as_deref(), Some("v2\n"));
- assert_eq!(repo.file_at(&p, Some(&c1)).unwrap().as_deref(), Some("v1\n"));
- assert_eq!(repo.file_at(&p, Some("HEAD~1")).unwrap().as_deref(), Some("v1\n"));
- assert_eq!(repo.file_at(&p, Some(":0")).unwrap().as_deref(), Some("v3\n"));
- assert_eq!(repo.file_at(Path::new("missing.txt"), None).unwrap(), None);
- assert_eq!(repo.file_at(&p, Some("no-such-rev")).unwrap_err().kind, ErrorKind::NotFound);
+ assert_eq!(repo.file_at(&p, None, encoding_rs::UTF_8).unwrap().as_deref(), Some("v2\n"));
+ assert_eq!(repo.file_at(&p, Some(&c1), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n"));
+ assert_eq!(repo.file_at(&p, Some("HEAD~1"), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n"));
+ assert_eq!(repo.file_at(&p, Some(":0"), encoding_rs::UTF_8).unwrap().as_deref(), Some("v3\n"));
+ assert_eq!(repo.file_at(Path::new("missing.txt"), None, encoding_rs::UTF_8).unwrap(), None);
+ assert_eq!(repo.file_at(&p, Some("no-such-rev"), encoding_rs::UTF_8).unwrap_err().kind, ErrorKind::NotFound);
}
#[test]
@@ -636,7 +636,7 @@ fn file_at_in_a_blobless_clone() {
git_in(fx.dir.parent().unwrap(), &["clone", "-q", "--filter=blob:none", &url, "clone"]);
let repo = Repo::discover(&clone).unwrap().unwrap();
// v1's blob was never fetched; reading it fetches it on demand.
- assert_eq!(repo.file_at(&clone.join("f.txt"), Some(&c1)).unwrap().as_deref(), Some("v1\n"));
+ assert_eq!(repo.file_at(&clone.join("f.txt"), Some(&c1), encoding_rs::UTF_8).unwrap().as_deref(), Some("v1\n"));
}
#[test]
@@ -850,9 +850,12 @@ fn commit_with_author_and_without_hooks() {
fx.write("a", "2\n");
repo.stage(&[fx.path("a")]).unwrap();
let hooked = CommitOptions { run_hooks: true, ..Default::default() };
- assert!(repo.commit_with("blocked", &hooked).is_err(), "the failing pre-commit hook runs");
+ assert!(
+ repo.commit_with("blocked", &hooked, &|_| Ok(encoding_rs::UTF_8)).is_err(),
+ "the failing pre-commit hook runs"
+ );
let options = CommitOptions { author: Some(" Other <[email protected]> ".into()), ..Default::default() };
- repo.commit_with("by other", &options).unwrap();
+ repo.commit_with("by other", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap();
assert_eq!(fx.git(&["log", "-1", "--format=%an <%ae>"]), "Other <[email protected]>");
}
@@ -932,7 +935,7 @@ fn smart_and_force_checkout_with_local_changes_in_the_way() {
assert_eq!(repo.branch_info().unwrap().head.as_deref(), Some("other"));
assert_eq!(repo.stashes().unwrap().len(), 1, "git keeps the stash after a conflicting pop");
// The stashed edit is the user's side of the merge, the checked-out branch the other.
- let v = repo.merge_versions(&fx.path("f.txt")).unwrap();
+ let v = repo.merge_versions(&fx.path("f.txt"), encoding_rs::UTF_8).unwrap();
assert_eq!(v.sides.operation.as_deref(), Some("unstash"));
assert!(v.sides.reversed);
assert_eq!(v.ours.as_deref(), Some("1\nmine\n3\n4\n5\n"));
@@ -1049,7 +1052,7 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() {
partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }],
..Default::default()
};
- repo.commit_with("partial", &options).unwrap();
+ repo.commit_with("partial", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap();
assert_eq!(fx.git(&["show", "HEAD:f.txt"]), "A\nb\nc\nd\ne\nf\ng\nh\ni\nj");
assert_eq!(fx.git(&["show", "HEAD:other.txt"]), "o2");
@@ -1070,21 +1073,21 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() {
partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }],
..Default::default()
};
- let err = repo.commit_with("stale", &stale).unwrap_err();
+ let err = repo.commit_with("stale", &stale, &|_| Ok(encoding_rs::UTF_8)).unwrap_err();
assert!(err.message.contains("changed"), "{}", err.message);
assert_eq!(fx.git(&["rev-parse", "HEAD"]), before);
assert_eq!(fx.git(&["diff", "--cached", "--name-only"]), "");
assert!(!fx.dir.join(".git/openidea-partial-index").exists());
// The remaining hunk commits on its own; then nothing is left.
- let rest = diff_hunks(&repo.file_at(&fx.path("f.txt"), None).unwrap().unwrap(), edited);
+ let rest = diff_hunks(&repo.file_at(&fx.path("f.txt"), None, encoding_rs::UTF_8).unwrap().unwrap(), edited);
assert_eq!(rest.len(), 1);
let options = CommitOptions {
partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&rest[0])] }],
..Default::default()
};
- repo.commit_with("rest", &options).unwrap();
- assert_eq!(repo.file_at(&fx.path("f.txt"), None).unwrap().as_deref(), Some(edited));
+ repo.commit_with("rest", &options, &|_| Ok(encoding_rs::UTF_8)).unwrap();
+ assert_eq!(repo.file_at(&fx.path("f.txt"), None, encoding_rs::UTF_8).unwrap().as_deref(), Some(edited));
assert!(repo.status().unwrap().iter().all(|f| f.rel_path != "f.txt"));
assert_eq!(fx.read("staged.txt"), "s2\n");
@@ -1105,7 +1108,7 @@ fn partial_commit_of_chosen_hunks_leaves_worktree_and_index_alone() {
partial: vec![PartialCommitFile { path: fx.path("f.txt").into(), hunks: vec![range(&hunks[0])] }],
..Default::default()
};
- assert!(repo.commit_with("hooked", &options).is_err());
+ assert!(repo.commit_with("hooked", &options, &|_| Ok(encoding_rs::UTF_8)).is_err());
assert_eq!(fx.git(&["rev-parse", "HEAD"]), head);
assert_eq!(fx.git(&["diff", "--cached", "--name-only"]), "");
assert_eq!(fx.read("f.txt"), "Z\nb\nc\nd\ne\nf\ng\nh\ni\nJ\nk\nl\n");
@@ -1121,13 +1124,13 @@ fn partial_staging_with_built_patch() {
let new = "l0\nl1\nl2\nCHANGED\nl4\nl5\nl6\nl8\nl9\nl10\n";
fx.write("p.txt", new);
- let index_text = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap();
+ let index_text = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap();
let hunks = ide_vcs::diff_hunks(&index_text, new);
assert_eq!(hunks.len(), 4, "{hunks:#?}");
// Stage the modification and the trailing addition only.
let patch = build_partial_patch("p.txt", &index_text, new, &[1, 3]);
- repo.stage_patch(&patch).unwrap();
- let staged = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap();
+ repo.stage_patch(&patch, &|_| Ok(encoding_rs::UTF_8)).unwrap();
+ let staged = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap();
assert_eq!(staged, "l1\nl2\nCHANGED\nl4\nl5\nl6\nl7\nl8\nl9\nl10\n");
let st = repo.status().unwrap();
let s = status_of(&st, "p.txt");
@@ -1135,21 +1138,21 @@ fn partial_staging_with_built_patch() {
// Then the rest, one hunk at a time against the updated index.
loop {
- let index_text = repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap();
+ let index_text = repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap();
if ide_vcs::diff_hunks(&index_text, new).is_empty() {
break;
}
- repo.stage_patch(&build_partial_patch("p.txt", &index_text, new, &[0])).unwrap();
+ repo.stage_patch(&build_partial_patch("p.txt", &index_text, new, &[0]), &|_| Ok(encoding_rs::UTF_8)).unwrap();
}
- assert_eq!(repo.file_at(&fx.path("p.txt"), Some(":0")).unwrap().unwrap(), new);
+ assert_eq!(repo.file_at(&fx.path("p.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(), new);
// Patches also apply to files without a trailing newline and CRLF files.
fx.write("n.txt", "a\r\nb\r\nc");
fx.commit_all("crlf");
let new = "a\nB\nc\nd";
- let old = repo.file_at(&fx.path("n.txt"), Some(":0")).unwrap().unwrap();
- repo.stage_patch(&build_partial_patch("n.txt", &old, new, &[0, 1])).unwrap();
- assert_eq!(repo.file_at(&fx.path("n.txt"), Some(":0")).unwrap().unwrap(), "a\r\nB\r\nc\r\nd");
+ let old = repo.file_at(&fx.path("n.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap();
+ repo.stage_patch(&build_partial_patch("n.txt", &old, new, &[0, 1]), &|_| Ok(encoding_rs::UTF_8)).unwrap();
+ assert_eq!(repo.file_at(&fx.path("n.txt"), Some(":0"), encoding_rs::UTF_8).unwrap().unwrap(), "a\r\nB\r\nc\r\nd");
}
#[test]
@@ -1170,13 +1173,13 @@ fn discard_and_diff_text() {
fx.write("keep/new.txt", "n\n");
fx.write("keep/k.txt", "changed\n");
- let unstaged = repo.diff_text(&fx.path("both.txt"), false).unwrap();
+ let unstaged = repo.diff_text(&fx.path("both.txt"), false, encoding_rs::UTF_8).unwrap();
assert!(unstaged.contains("-staged\n+unstaged"), "{unstaged}");
- let staged = repo.diff_text(&fx.path("both.txt"), true).unwrap();
+ let staged = repo.diff_text(&fx.path("both.txt"), true, encoding_rs::UTF_8).unwrap();
assert!(staged.contains("-one\n+staged"), "{staged}");
- let untracked = repo.diff_text(&fx.path("u.txt"), false).unwrap();
+ let untracked = repo.diff_text(&fx.path("u.txt"), false, encoding_rs::UTF_8).unwrap();
assert!(untracked.contains("+untracked"), "{untracked}");
- assert!(repo.diff_text(&fx.path("t.txt"), true).unwrap().is_empty());
+ assert!(repo.diff_text(&fx.path("t.txt"), true, encoding_rs::UTF_8).unwrap().is_empty());
repo.discard(&[fx.path("t.txt"), fx.path("both.txt"), fx.path("u.txt"), fx.path("udir"), fx.path("keep")]).unwrap();
assert_eq!(fx.read("t.txt"), "one\n");
@@ -1322,3 +1325,161 @@ fn path_log_simplifies_merges() {
.collect();
assert_eq!(ids, vec![base]);
}
+
+#[test]
+fn gbk_java_unchanged_has_no_hunks() {
+ let fx = Fixture::new();
+ let text = "// 中文注释\nclass Example {\n // 保持不变\n}\n";
+ std::fs::write(fx.path("Example.java"), encoding_rs::GBK.encode(text).0).unwrap();
+ fx.commit_all("GBK Java");
+ for against_index in [false, true] {
+ assert!(
+ fx.repo().line_hunks(&fx.path("Example.java"), text, against_index, encoding_rs::GBK).unwrap().is_empty()
+ );
+ }
+}
+
+#[test]
+fn encoded_java_hunks_staging_blame_and_partial_commit_preserve_bytes() {
+ for encoding in [encoding_rs::GBK, encoding_rs::SHIFT_JIS, encoding_rs::UTF_8] {
+ for eol in ["\n", "\r\n"] {
+ let fx = Fixture::new();
+ let path = fx.path("中文.java");
+ let old = "// 中文注釈\nclass Example {\n // 保存\n int value = 1;\n}\n".replace('\n', eol);
+ let changed = old.replace("中文注釈", "中文変更").replace("value = 1", "value = 2");
+ let bytes = |text: &str| {
+ let (bytes, _, errors) = encoding.encode(text);
+ assert!(!errors);
+ bytes.into_owned()
+ };
+ std::fs::write(&path, bytes(&old)).unwrap();
+ fx.commit_all("encoded Java");
+ let repo = fx.repo();
+ assert_eq!(repo.file_at(&path, None, encoding).unwrap().as_deref(), Some(old.as_str()));
+ assert!(
+ repo.blame(&path, Some(&old.replace("\r\n", "\n")), encoding)
+ .unwrap()
+ .iter()
+ .all(|line| line.author == "Test User")
+ );
+ let hunks = repo.line_hunks(&path, &old.replace("value = 1", "value = 2"), false, encoding).unwrap();
+ assert_eq!(hunks.len(), 1);
+ assert_eq!((hunks[0].start, hunks[0].lines, hunks[0].kind), (3, 1, HunkKind::Modified));
+ std::fs::write(&path, bytes(&changed)).unwrap();
+ let hunks = repo.line_hunks(&path, &changed, false, encoding).unwrap();
+ assert_eq!(hunks.len(), 2);
+ repo.stage_hunks(&path, &changed, &[0], encoding).unwrap();
+ let selected = old.replace("中文注釈", "中文変更");
+ assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&selected));
+ assert_eq!(std::fs::read(&path).unwrap(), bytes(&changed));
+ assert!(repo.diff_text(&path, true, encoding).unwrap().contains("中文変更"));
+ repo.unstage(std::slice::from_ref(&path)).unwrap();
+ let options = CommitOptions {
+ partial: vec![PartialCommitFile {
+ path: path.clone().into(),
+ hunks: vec![HunkRange {
+ start: hunks[0].start,
+ lines: hunks[0].lines,
+ old_start: hunks[0].old_start,
+ old_lines: hunks[0].old_lines,
+ }],
+ }],
+ ..Default::default()
+ };
+ repo.commit_with("selected hunk", &options, &|_| Ok(encoding)).unwrap();
+ assert_eq!(repo.file_bytes_at(&path, None).unwrap().unwrap(), bytes(&selected));
+ assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&selected));
+ assert_eq!(std::fs::read(&path).unwrap(), bytes(&changed));
+ }
+ }
+}
+
+#[test]
+fn gbk_patches_round_trip_even_with_text_attributes() {
+ let fx = Fixture::new();
+ let path = fx.path("Example.java");
+ let old = encoding_rs::GBK.encode("// 中文注释\nclass Example {}\n").0.into_owned();
+ let new = encoding_rs::GBK.encode("// 修改注释\nclass Example {}\n").0.into_owned();
+ fx.write(".gitattributes", "*.java diff\n");
+ std::fs::write(&path, &old).unwrap();
+ fx.commit_all("base");
+ std::fs::write(&path, &new).unwrap();
+ let repo = fx.repo();
+ let patch = repo.create_patch(std::slice::from_ref(&path), false).unwrap();
+ assert!(patch.contains("GIT binary patch"));
+ repo.discard(std::slice::from_ref(&path)).unwrap();
+ repo.apply_patch(&patch).unwrap();
+ assert_eq!(std::fs::read(&path).unwrap(), new);
+ let commit = fx.commit_all("change");
+ let reverse = repo.create_commits_patch(&[commit], true).unwrap();
+ repo.apply_patch(&reverse).unwrap();
+ assert_eq!(std::fs::read(&path).unwrap(), old);
+}
+
+#[test]
+fn gbk_conflict_stages_are_readable() {
+ let fx = Fixture::new();
+ let path = fx.path("Example.java");
+ let write = |text: &str| std::fs::write(&path, encoding_rs::GBK.encode(text).0).unwrap();
+ write("// 原始注释\n");
+ fx.commit_all("base");
+ fx.git(&["checkout", "-b", "other"]);
+ write("// 分支注释\n");
+ fx.commit_all("other");
+ fx.git(&["checkout", "main"]);
+ write("// 当前注释\n");
+ fx.commit_all("ours");
+ let out = Command::new("git").current_dir(&fx.dir).args(["merge", "other"]).output().unwrap();
+ assert!(!out.status.success());
+ let versions = fx.repo().merge_versions(&path, encoding_rs::GBK).unwrap();
+ assert_eq!(versions.base.as_deref(), Some("// 原始注释\n"));
+ assert_eq!(versions.ours.as_deref(), Some("// 当前注释\n"));
+ assert_eq!(versions.theirs.as_deref(), Some("// 分支注释\n"));
+}
+
+#[test]
+fn utf16_and_utf8_boms_survive_partial_staging() {
+ for encoding in [encoding_rs::UTF_8, encoding_rs::UTF_16LE, encoding_rs::UTF_16BE] {
+ let fx = Fixture::new();
+ let path = fx.path("Example.java");
+ let old = "// 中文注释\r\nclass Example {}\r\n";
+ let changed = old.replace("中文", "修改");
+ let bytes = |text: &str| {
+ if encoding == encoding_rs::UTF_8 {
+ let mut bytes = vec![0xef, 0xbb, 0xbf];
+ bytes.extend_from_slice(text.as_bytes());
+ bytes
+ } else {
+ let le = encoding == encoding_rs::UTF_16LE;
+ let mut bytes = if le { vec![0xff, 0xfe] } else { vec![0xfe, 0xff] };
+ for unit in text.encode_utf16() {
+ bytes.extend_from_slice(&if le { unit.to_le_bytes() } else { unit.to_be_bytes() });
+ }
+ bytes
+ }
+ };
+ std::fs::write(&path, bytes(old)).unwrap();
+ fx.commit_all("BOM");
+ let repo = fx.repo();
+ assert!(repo.line_hunks(&path, old, false, encoding).unwrap().is_empty());
+ repo.stage_hunks(&path, &changed, &[0], encoding).unwrap();
+ assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), bytes(&changed));
+ }
+}
+
+#[test]
+fn gbk_text_patch_keeps_utf8_paths_and_encoded_payload() {
+ let fx = Fixture::new();
+ let path = fx.path("中文.java");
+ let old = "// 中文注释\n";
+ let new = "// 修改注释\n";
+ std::fs::write(&path, encoding_rs::GBK.encode(old).0).unwrap();
+ fx.commit_all("GBK");
+ let repo = fx.repo();
+ let patch = build_partial_patch("中文.java", old, new, &[0]);
+ repo.stage_patch(&patch, &|_| Ok(encoding_rs::GBK)).unwrap();
+ assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), encoding_rs::GBK.encode(new).0.as_ref());
+ let patch = build_partial_patch("中文.java", new, "// 😀\n", &[0]);
+ assert!(repo.stage_patch(&patch, &|_| Ok(encoding_rs::GBK)).is_err());
+ assert_eq!(repo.file_bytes_at(&path, Some(":0")).unwrap().unwrap(), encoding_rs::GBK.encode(new).0.as_ref());
+}
diff --git a/crates/ide-vfs/src/lib.rs b/crates/ide-vfs/src/lib.rs
index 24c882e70..fa56b02c2 100644
--- a/crates/ide-vfs/src/lib.rs
+++ b/crates/ide-vfs/src/lib.rs
@@ -313,6 +313,21 @@ impl FsBatch {
}
}
+/// Detect a revision's encoding with the same rules as files opened in the editor.
+pub fn detect_encoding(bytes: &[u8], fallback: Option<&str>) -> &'static encoding_rs::Encoding {
+ let (_, detected, has_bom, binary) = encoding::decode(bytes);
+ if !binary
+ && !has_bom
+ && detected != encoding_rs::UTF_8
+ && detected != encoding_rs::UTF_16LE
+ && detected != encoding_rs::UTF_16BE
+ {
+ fallback.and_then(|label| encoding_rs::Encoding::for_label(label.trim().as_bytes())).unwrap_or(detected)
+ } else {
+ detected
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;