Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

fix(build): decode tool output in the Windows code page (#299)

l0ng-aid44e2414caed
diff --git a/Cargo.lock b/Cargo.lock
index 54c5d2b75..6d49a14d5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4093,6 +4093,7 @@ name = "ide-core"
version = "0.1.25"
dependencies = [
"dunce",
+ "encoding_rs",
"serde",
"serde_json",
"specta",
@@ -4100,6 +4101,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.21",
"tokio-util",
+ "windows-sys 0.61.2",
]
[[package]]
diff --git a/crates/ide-agent/src/native/claude/mod.rs b/crates/ide-agent/src/native/claude/mod.rs
index a15c8860f..8fd87ad82 100644
--- a/crates/ide-agent/src/native/claude/mod.rs
+++ b/crates/ide-agent/src/native/claude/mod.rs
@@ -296,7 +296,7 @@ impl Shim {
let mut cmd = std::process::Command::new(&self.config.program);
ide_core::process::prepare(&mut cmd);
let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;
- let text = String::from_utf8_lossy(&out.stdout);
+ let text = ide_core::output_encoding::decode_tool_output(&out.stdout);
text.split_whitespace().next().filter(|v| out.status.success() && !v.is_empty()).map(str::to_owned)
}
diff --git a/crates/ide-agent/src/native/mod.rs b/crates/ide-agent/src/native/mod.rs
index 46de082e1..10f113092 100644
--- a/crates/ide-agent/src/native/mod.rs
+++ b/crates/ide-agent/src/native/mod.rs
@@ -64,7 +64,7 @@ fn version_of(program: &Path) -> Option<Vec<u64>> {
let mut cmd = std::process::Command::new(program);
ide_core::process::prepare(&mut cmd);
let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;
- parse_version(&String::from_utf8_lossy(&out.stdout))
+ parse_version(&ide_core::output_encoding::decode_tool_output(&out.stdout))
}
fn parse_version(text: &str) -> Option<Vec<u64>> {
diff --git a/crates/ide-build/src/process.rs b/crates/ide-build/src/process.rs
index 0ac375805..fcec4b454 100644
--- a/crates/ide-build/src/process.rs
+++ b/crates/ide-build/src/process.rs
@@ -68,7 +68,7 @@ pub(crate) fn run(
match reader.read_until(b'\n', &mut buf) {
Ok(0) | Err(_) => break,
Ok(_) => {
- let line = String::from_utf8_lossy(&buf);
+ let line = ide_core::output_encoding::decode_tool_output(&buf);
let line = line.trim_end_matches(['\n', '\r']).to_owned();
if tx.send((kind, line)).is_err() {
break;
diff --git a/crates/ide-core/Cargo.toml b/crates/ide-core/Cargo.toml
index 9daa648b4..af0d6b992 100644
--- a/crates/ide-core/Cargo.toml
+++ b/crates/ide-core/Cargo.toml
@@ -7,6 +7,7 @@ license.workspace = true
[dependencies]
dunce.workspace = true
+encoding_rs.workspace = true
serde.workspace = true
serde_json.workspace = true
specta.workspace = true
@@ -16,3 +17,6 @@ tokio-util.workspace = true
[dev-dependencies]
tempfile.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+windows-sys = { version = "0.61", features = ["Win32_Globalization"] }
diff --git a/crates/ide-core/src/output_encoding.rs b/crates/ide-core/src/output_encoding.rs
index 5d3a3688f..f5e85f3d1 100644
--- a/crates/ide-core/src/output_encoding.rs
+++ b/crates/ide-core/src/output_encoding.rs
@@ -18,6 +18,60 @@
use std::collections::BTreeMap;
+/// Decodes complete lines or collected output from IDE-managed tools, preferring UTF-8.
+/// Windows launcher/native diagnostics fall back to the system ANSI code page.
+/// Callers must buffer incomplete lines so multibyte characters are never split.
+/// Run consoles and protocol streams have their own encoding contracts.
+pub fn decode_tool_output(bytes: &[u8]) -> String {
+ #[cfg(windows)]
+ // SAFETY: GetACP is an argument-less query with no preconditions.
+ let code_page = unsafe { windows_sys::Win32::Globalization::GetACP() };
+ #[cfg(not(windows))]
+ let code_page = 65001;
+ decode_tool_output_with_code_page(bytes, code_page)
+}
+
+fn decode_tool_output_with_code_page(bytes: &[u8], code_page: u32) -> String {
+ if let Ok(text) = std::str::from_utf8(bytes) {
+ return text.to_owned();
+ }
+ let encoding = encoding_for_code_page(code_page).unwrap_or(encoding_rs::UTF_8);
+ // Collected output can mix UTF-8 JVM lines with native launcher diagnostics.
+ let mut text = String::new();
+ for line in bytes.split_inclusive(|b| *b == b'\n') {
+ match std::str::from_utf8(line) {
+ Ok(line) => text.push_str(line),
+ Err(_) => text.push_str(&encoding.decode_without_bom_handling(line).0),
+ }
+ }
+ text
+}
+
+fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> {
+ Some(match cp {
+ 65001 => encoding_rs::UTF_8,
+ 936 => encoding_rs::GBK,
+ 54936 => encoding_rs::GB18030,
+ 950 => encoding_rs::BIG5,
+ 932 => encoding_rs::SHIFT_JIS,
+ 949 => encoding_rs::EUC_KR,
+ 866 => encoding_rs::IBM866,
+ 874 => encoding_rs::WINDOWS_874,
+ 1250 => encoding_rs::WINDOWS_1250,
+ 1251 => encoding_rs::WINDOWS_1251,
+ 1252 => encoding_rs::WINDOWS_1252,
+ 1253 => encoding_rs::WINDOWS_1253,
+ 1254 => encoding_rs::WINDOWS_1254,
+ 1255 => encoding_rs::WINDOWS_1255,
+ 1256 => encoding_rs::WINDOWS_1256,
+ 1257 => encoding_rs::WINDOWS_1257,
+ 1258 => encoding_rs::WINDOWS_1258,
+ 20866 => encoding_rs::KOI8_R,
+ 21866 => encoding_rs::KOI8_U,
+ _ => return None,
+ })
+}
+
const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"];
const FILE_ENCODING: &str = "file.encoding";
const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS";
@@ -237,6 +291,29 @@ impl NoticeFilter {
mod tests {
use super::*;
+ #[test]
+ fn tool_output_prefers_utf8() {
+ for cp in [936, 1252, 65001, 0] {
+ for text in ["", "plain ASCII\r\n", "中文 café 🦀\n"] {
+ assert_eq!(decode_tool_output_with_code_page(text.as_bytes(), cp), text);
+ }
+ }
+ }
+
+ #[test]
+ fn tool_output_decodes_native_diagnostics_and_mixed_lines() {
+ const GBK: &[u8] = b"\xcf\xb5\xcd\xb3\xd5\xd2\xb2\xbb\xb5\xbd\xd6\xb8\xb6\xa8\xb5\xc4\xc2\xb7\xbe\xb6\xa1\xa3";
+ assert_eq!(decode_tool_output_with_code_page(GBK, 936), "系统找不到指定的路径。");
+ let mut mixed = "JVM: 中文\r\n".as_bytes().to_vec();
+ mixed.extend_from_slice(GBK);
+ mixed.extend_from_slice(b"\r\nlast");
+ assert_eq!(decode_tool_output_with_code_page(&mixed, 936), "JVM: 中文\r\n系统找不到指定的路径。\r\nlast");
+ assert_eq!(decode_tool_output_with_code_page(b"caf\xe9", 1252), "café");
+ for cp in [65001, 0] {
+ assert_eq!(decode_tool_output_with_code_page(b"a\xffb", cp), "a�b");
+ }
+ }
+
const STD_UTF8: [&str; 4] = [
"-Dstdout.encoding=UTF-8",
"-Dstderr.encoding=UTF-8",
diff --git a/crates/ide-coverage/src/lib.rs b/crates/ide-coverage/src/lib.rs
index 1fc48db27..757812036 100644
--- a/crates/ide-coverage/src/lib.rs
+++ b/crates/ide-coverage/src/lib.rs
@@ -213,8 +213,12 @@ pub fn collect(spec: &CoverageReportSpec) -> IdeResult<CoverageSuite> {
.output()
.map_err(|e| IdeError::unavailable(format!("cannot run {program}: {e}")))?;
if !out.status.success() {
- let text = String::from_utf8_lossy(&out.stderr).trim().to_owned();
- let text = if text.is_empty() { String::from_utf8_lossy(&out.stdout).trim().to_owned() } else { text };
+ let text = ide_core::output_encoding::decode_tool_output(&out.stderr).trim().to_owned();
+ let text = if text.is_empty() {
+ ide_core::output_encoding::decode_tool_output(&out.stdout).trim().to_owned()
+ } else {
+ text
+ };
return Err(IdeError::not_found(format!("exporting the coverage data failed: {text}")));
}
let text = read_report(lcov)?;
diff --git a/crates/ide-dap/src/cargo.rs b/crates/ide-dap/src/cargo.rs
index 6de0813ba..30fb88a76 100644
--- a/crates/ide-dap/src/cargo.rs
+++ b/crates/ide-dap/src/cargo.rs
@@ -70,8 +70,10 @@ pub async fn build(
let on_output = std::sync::Arc::new(on_output);
let err_out = on_output.clone();
let err_task = tokio::spawn(async move {
- let mut lines = BufReader::new(stderr).lines();
- while let Ok(Some(line)) = lines.next_line().await {
+ let mut lines = BufReader::new(stderr).split(b'\n');
+ while let Ok(Some(line)) = lines.next_segment().await {
+ let line = ide_core::output_encoding::decode_tool_output(&line);
+ let line = line.trim_end_matches('\r');
err_out(format!("{line}\n"));
}
});
diff --git a/crates/ide-docker/src/process.rs b/crates/ide-docker/src/process.rs
index e9dc25a3b..6c08c0050 100644
--- a/crates/ide-docker/src/process.rs
+++ b/crates/ide-docker/src/process.rs
@@ -54,7 +54,7 @@ fn read_all(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle<Str
std::thread::spawn(move || {
let mut buf = Vec::new();
let _ = pipe.read_to_end(&mut buf);
- String::from_utf8_lossy(&buf).into_owned()
+ ide_core::output_encoding::decode_tool_output(&buf)
})
}
@@ -115,7 +115,7 @@ fn spawn_reader(mut pipe: impl Read + Send + 'static, tx: mpsc::Sender<Event>, s
let mut pending: Vec<u8> = Vec::new();
let flush = |pending: &mut Vec<u8>| {
if !pending.is_empty() {
- let line = String::from_utf8_lossy(pending).trim_end().to_string();
+ let line = ide_core::output_encoding::decode_tool_output(pending).trim_end().to_string();
pending.clear();
let _ = tx.send(Event::Line(stream, line));
}
diff --git a/crates/ide-host/src/environment.rs b/crates/ide-host/src/environment.rs
index 4dabd3500..0ef84ec5f 100644
--- a/crates/ide-host/src/environment.rs
+++ b/crates/ide-host/src/environment.rs
@@ -281,7 +281,8 @@ fn run_one(argv: &[String], root: &Path, cancel: &CancellationToken, on_line: &d
for pipe in pipes.into_iter().flatten() {
let tx = tx.clone();
std::thread::spawn(move || {
- for line in BufReader::new(pipe).lines().map_while(Result::ok) {
+ for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) {
+ let line = ide_core::output_encoding::decode_tool_output(&line).trim_end_matches('\r').to_owned();
if tx.send(line).is_err() {
break;
}
diff --git a/crates/ide-notebook/src/kernel/process.rs b/crates/ide-notebook/src/kernel/process.rs
index 791939e0b..afcdda93e 100644
--- a/crates/ide-notebook/src/kernel/process.rs
+++ b/crates/ide-notebook/src/kernel/process.rs
@@ -1,13 +1,12 @@
use ide_core::{IdeError, IdeResult};
use jupyter_protocol::ConnectionInfo;
use parking_lot::Mutex;
-use std::collections::VecDeque;
use std::io::Write;
use std::path::Path;
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
-use tokio::io::AsyncReadExt;
+use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::task::JoinHandle;
use uuid::Uuid;
@@ -18,7 +17,7 @@ pub(super) struct Process {
child: Child,
#[cfg(unix)]
pid: Option<u32>,
- stderr: Arc<Mutex<VecDeque<u8>>>,
+ stderr: Arc<Mutex<String>>,
drains: Vec<JoinHandle<()>>,
// Kept until the process and sockets have stopped; contains the HMAC secret.
_directory: tempfile::TempDir,
@@ -103,26 +102,31 @@ impl Process {
child,
#[cfg(unix)]
pid,
- stderr: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_BYTES))),
+ stderr: Arc::new(Mutex::new(String::new())),
drains: Vec::new(),
_directory: directory,
#[cfg(windows)]
windows,
};
- if let Some(mut stderr) = process.child.stderr.take() {
+ if let Some(stderr) = process.child.stderr.take() {
let tail = process.stderr.clone();
process.drains.push(tokio::spawn(async move {
- let mut buffer = [0_u8; 4096];
- while let Ok(count) = stderr.read(&mut buffer).await {
- if count == 0 {
- break;
- }
- let mut tail = tail.lock();
- for byte in &buffer[..count] {
- if tail.len() == STDERR_BYTES {
- tail.pop_front();
+ let mut reader = BufReader::new(stderr);
+ let mut line = Vec::new();
+ loop {
+ line.clear();
+ match reader.read_until(b'\n', &mut line).await {
+ Ok(0) | Err(_) => break,
+ Ok(_) => {
+ let text = ide_core::output_encoding::decode_tool_output(&line);
+ let mut tail = tail.lock();
+ tail.push_str(&text);
+ let mut start = tail.len().saturating_sub(STDERR_BYTES);
+ while !tail.is_char_boundary(start) {
+ start += 1;
+ }
+ tail.drain(..start);
}
- tail.push_back(*byte);
}
}
}));
@@ -143,8 +147,7 @@ impl Process {
}
pub fn stderr(&self) -> String {
- let bytes = self.stderr.lock().iter().copied().collect::<Vec<_>>();
- String::from_utf8_lossy(&bytes).into_owned()
+ self.stderr.lock().clone()
}
pub fn death(&mut self) -> IdeResult<Option<String>> {
diff --git a/crates/ide-sdk/src/exec.rs b/crates/ide-sdk/src/exec.rs
index aa2296c67..03333c502 100644
--- a/crates/ide-sdk/src/exec.rs
+++ b/crates/ide-sdk/src/exec.rs
@@ -1,6 +1,7 @@
//! Running toolchain commands: short queries with a timeout and long-running
//! streaming commands that honour cancellation.
+use ide_core::output_encoding::decode_tool_output;
use ide_core::{CancellationToken, IdeError, IdeResult};
use std::ffi::OsString;
use std::io::{BufRead, BufReader, Read};
@@ -77,14 +78,14 @@ pub(crate) fn query_in(
let mut stdout = child.stdout.take()?;
let mut stderr = child.stderr.take()?;
let out = std::thread::spawn(move || {
- let mut s = String::new();
- let _ = stdout.read_to_string(&mut s);
- s
+ let mut s = Vec::new();
+ let _ = stdout.read_to_end(&mut s);
+ decode_tool_output(&s)
});
let err = std::thread::spawn(move || {
- let mut s = String::new();
- let _ = stderr.read_to_string(&mut s);
- s
+ let mut s = Vec::new();
+ let _ = stderr.read_to_end(&mut s);
+ decode_tool_output(&s)
});
let deadline = Instant::now() + timeout;
let status = loop {
@@ -131,16 +132,16 @@ pub(crate) fn stream(
if let Some(out) = child.stdout.take() {
let tx = tx.clone();
readers.push(std::thread::spawn(move || {
- for line in BufReader::new(out).lines().map_while(Result::ok) {
- let _ = tx.send(line);
+ for line in BufReader::new(out).split(b'\n').map_while(Result::ok) {
+ let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());
}
}));
}
if let Some(err) = child.stderr.take() {
let tx = tx.clone();
readers.push(std::thread::spawn(move || {
- for line in BufReader::new(err).lines().map_while(Result::ok) {
- let _ = tx.send(line);
+ for line in BufReader::new(err).split(b'\n').map_while(Result::ok) {
+ let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());
}
}));
}
diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs
index 159a7d606..81c44bf92 100644
--- a/crates/ide-tools/src/process.rs
+++ b/crates/ide-tools/src/process.rs
@@ -3,6 +3,7 @@
use crate::InstallEvent;
use ide_core::error::ErrorKind;
+use ide_core::output_encoding::decode_tool_output;
use ide_core::{CancellationToken, IdeError, IdeResult};
use std::io::{BufRead, BufReader, Read};
use std::path::Path;
@@ -86,8 +87,9 @@ pub fn run_collect(mut cmd: Command, cancel: &CancellationToken, emit: Emit) ->
for pipe in pipes.into_iter().flatten() {
let tx = tx.clone();
std::thread::spawn(move || {
- for line in BufReader::new(pipe).lines() {
+ for line in BufReader::new(pipe).split(b'\n') {
let Ok(line) = line else { break };
+ let line = decode_tool_output(&line).trim_end_matches('\r').to_owned();
if tx.send(line).is_err() {
break;
}
@@ -127,14 +129,14 @@ pub fn run_quiet(mut cmd: Command, timeout: Duration) -> Option<(bool, String)>
let mut stdout = child.stdout.take()?;
let mut stderr = child.stderr.take()?;
let out_t = std::thread::spawn(move || {
- let mut s = String::new();
- let _ = stdout.read_to_string(&mut s);
- s
+ let mut s = Vec::new();
+ let _ = stdout.read_to_end(&mut s);
+ decode_tool_output(&s)
});
let err_t = std::thread::spawn(move || {
- let mut s = String::new();
- let _ = stderr.read_to_string(&mut s);
- s
+ let mut s = Vec::new();
+ let _ = stderr.read_to_end(&mut s);
+ decode_tool_output(&s)
});
let deadline = Instant::now() + timeout;
let status = loop {
@@ -251,8 +253,9 @@ impl Http {
return Err(IdeError::new(ErrorKind::Io, format!("{url}: response is larger than {max} bytes")));
}
if !status.success() {
- let mut err = String::new();
- let _ = stderr.read_to_string(&mut err);
+ let mut err = Vec::new();
+ let _ = stderr.read_to_end(&mut err);
+ let err = decode_tool_output(&err);
return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim())));
}
String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8")))
@@ -272,9 +275,9 @@ impl Http {
let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?;
let mut stderr = child.stderr.take().expect("piped");
let err_t = std::thread::spawn(move || {
- let mut s = String::new();
- let _ = stderr.read_to_string(&mut s);
- s
+ let mut s = Vec::new();
+ let _ = stderr.read_to_end(&mut s);
+ decode_tool_output(&s)
});
let mut last = Instant::now();
let mut last_bytes = 0;