The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
fix(build): decode tool output in the Windows code page (#299)
l0ng-aid44e2414caed
diff --git a/Cargo.lock b/Cargo.lockindex 54c5d2b75..6d49a14d5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4093,6 +4093,7 @@ name = "ide-core"version = "0.1.25"dependencies = ["dunce",+ "encoding_rs","serde","serde_json","specta",@@ -4100,6 +4101,7 @@ dependencies = ["tempfile","thiserror 2.0.21","tokio-util",+ "windows-sys 0.61.2",][[package]]diff --git a/crates/ide-agent/src/native/claude/mod.rs b/crates/ide-agent/src/native/claude/mod.rsindex a15c8860f..8fd87ad82 100644--- a/crates/ide-agent/src/native/claude/mod.rs+++ b/crates/ide-agent/src/native/claude/mod.rs@@ -296,7 +296,7 @@ impl Shim {let mut cmd = std::process::Command::new(&self.config.program);ide_core::process::prepare(&mut cmd);let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;- let text = String::from_utf8_lossy(&out.stdout);+ let text = ide_core::output_encoding::decode_tool_output(&out.stdout);text.split_whitespace().next().filter(|v| out.status.success() && !v.is_empty()).map(str::to_owned)}diff --git a/crates/ide-agent/src/native/mod.rs b/crates/ide-agent/src/native/mod.rsindex 46de082e1..10f113092 100644--- a/crates/ide-agent/src/native/mod.rs+++ b/crates/ide-agent/src/native/mod.rs@@ -64,7 +64,7 @@ fn version_of(program: &Path) -> Option<Vec<u64>> {let mut cmd = std::process::Command::new(program);ide_core::process::prepare(&mut cmd);let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;- parse_version(&String::from_utf8_lossy(&out.stdout))+ parse_version(&ide_core::output_encoding::decode_tool_output(&out.stdout))}fn parse_version(text: &str) -> Option<Vec<u64>> {diff --git a/crates/ide-build/src/process.rs b/crates/ide-build/src/process.rsindex 0ac375805..fcec4b454 100644--- a/crates/ide-build/src/process.rs+++ b/crates/ide-build/src/process.rs@@ -68,7 +68,7 @@ pub(crate) fn run(match reader.read_until(b'\n', &mut buf) {Ok(0) | Err(_) => break,Ok(_) => {- let line = String::from_utf8_lossy(&buf);+ let line = ide_core::output_encoding::decode_tool_output(&buf);let line = line.trim_end_matches(['\n', '\r']).to_owned();if tx.send((kind, line)).is_err() {break;diff --git a/crates/ide-core/Cargo.toml b/crates/ide-core/Cargo.tomlindex 9daa648b4..af0d6b992 100644--- a/crates/ide-core/Cargo.toml+++ b/crates/ide-core/Cargo.toml@@ -7,6 +7,7 @@ license.workspace = true[dependencies]dunce.workspace = true+encoding_rs.workspace = trueserde.workspace = trueserde_json.workspace = truespecta.workspace = true@@ -16,3 +17,6 @@ tokio-util.workspace = true[dev-dependencies]tempfile.workspace = true++[target.'cfg(windows)'.dependencies]+windows-sys = { version = "0.61", features = ["Win32_Globalization"] }diff --git a/crates/ide-core/src/output_encoding.rs b/crates/ide-core/src/output_encoding.rsindex 5d3a3688f..f5e85f3d1 100644--- a/crates/ide-core/src/output_encoding.rs+++ b/crates/ide-core/src/output_encoding.rs@@ -18,6 +18,60 @@use std::collections::BTreeMap;+/// Decodes complete lines or collected output from IDE-managed tools, preferring UTF-8.+/// Windows launcher/native diagnostics fall back to the system ANSI code page.+/// Callers must buffer incomplete lines so multibyte characters are never split.+/// Run consoles and protocol streams have their own encoding contracts.+pub fn decode_tool_output(bytes: &[u8]) -> String {+ #[cfg(windows)]+ // SAFETY: GetACP is an argument-less query with no preconditions.+ let code_page = unsafe { windows_sys::Win32::Globalization::GetACP() };+ #[cfg(not(windows))]+ let code_page = 65001;+ decode_tool_output_with_code_page(bytes, code_page)+}++fn decode_tool_output_with_code_page(bytes: &[u8], code_page: u32) -> String {+ if let Ok(text) = std::str::from_utf8(bytes) {+ return text.to_owned();+ }+ let encoding = encoding_for_code_page(code_page).unwrap_or(encoding_rs::UTF_8);+ // Collected output can mix UTF-8 JVM lines with native launcher diagnostics.+ let mut text = String::new();+ for line in bytes.split_inclusive(|b| *b == b'\n') {+ match std::str::from_utf8(line) {+ Ok(line) => text.push_str(line),+ Err(_) => text.push_str(&encoding.decode_without_bom_handling(line).0),+ }+ }+ text+}++fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> {+ Some(match cp {+ 65001 => encoding_rs::UTF_8,+ 936 => encoding_rs::GBK,+ 54936 => encoding_rs::GB18030,+ 950 => encoding_rs::BIG5,+ 932 => encoding_rs::SHIFT_JIS,+ 949 => encoding_rs::EUC_KR,+ 866 => encoding_rs::IBM866,+ 874 => encoding_rs::WINDOWS_874,+ 1250 => encoding_rs::WINDOWS_1250,+ 1251 => encoding_rs::WINDOWS_1251,+ 1252 => encoding_rs::WINDOWS_1252,+ 1253 => encoding_rs::WINDOWS_1253,+ 1254 => encoding_rs::WINDOWS_1254,+ 1255 => encoding_rs::WINDOWS_1255,+ 1256 => encoding_rs::WINDOWS_1256,+ 1257 => encoding_rs::WINDOWS_1257,+ 1258 => encoding_rs::WINDOWS_1258,+ 20866 => encoding_rs::KOI8_R,+ 21866 => encoding_rs::KOI8_U,+ _ => return None,+ })+}+const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"];const FILE_ENCODING: &str = "file.encoding";const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS";@@ -237,6 +291,29 @@ impl NoticeFilter {mod tests {use super::*;+ #[test]+ fn tool_output_prefers_utf8() {+ for cp in [936, 1252, 65001, 0] {+ for text in ["", "plain ASCII\r\n", "中文 café 🦀\n"] {+ assert_eq!(decode_tool_output_with_code_page(text.as_bytes(), cp), text);+ }+ }+ }++ #[test]+ fn tool_output_decodes_native_diagnostics_and_mixed_lines() {+ const GBK: &[u8] = b"\xcf\xb5\xcd\xb3\xd5\xd2\xb2\xbb\xb5\xbd\xd6\xb8\xb6\xa8\xb5\xc4\xc2\xb7\xbe\xb6\xa1\xa3";+ assert_eq!(decode_tool_output_with_code_page(GBK, 936), "系统找不到指定的路径。");+ let mut mixed = "JVM: 中文\r\n".as_bytes().to_vec();+ mixed.extend_from_slice(GBK);+ mixed.extend_from_slice(b"\r\nlast");+ assert_eq!(decode_tool_output_with_code_page(&mixed, 936), "JVM: 中文\r\n系统找不到指定的路径。\r\nlast");+ assert_eq!(decode_tool_output_with_code_page(b"caf\xe9", 1252), "café");+ for cp in [65001, 0] {+ assert_eq!(decode_tool_output_with_code_page(b"a\xffb", cp), "a�b");+ }+ }+const STD_UTF8: [&str; 4] = ["-Dstdout.encoding=UTF-8","-Dstderr.encoding=UTF-8",diff --git a/crates/ide-coverage/src/lib.rs b/crates/ide-coverage/src/lib.rsindex 1fc48db27..757812036 100644--- a/crates/ide-coverage/src/lib.rs+++ b/crates/ide-coverage/src/lib.rs@@ -213,8 +213,12 @@ pub fn collect(spec: &CoverageReportSpec) -> IdeResult<CoverageSuite> {.output().map_err(|e| IdeError::unavailable(format!("cannot run {program}: {e}")))?;if !out.status.success() {- let text = String::from_utf8_lossy(&out.stderr).trim().to_owned();- let text = if text.is_empty() { String::from_utf8_lossy(&out.stdout).trim().to_owned() } else { text };+ let text = ide_core::output_encoding::decode_tool_output(&out.stderr).trim().to_owned();+ let text = if text.is_empty() {+ ide_core::output_encoding::decode_tool_output(&out.stdout).trim().to_owned()+ } else {+ text+ };return Err(IdeError::not_found(format!("exporting the coverage data failed: {text}")));}let text = read_report(lcov)?;diff --git a/crates/ide-dap/src/cargo.rs b/crates/ide-dap/src/cargo.rsindex 6de0813ba..30fb88a76 100644--- a/crates/ide-dap/src/cargo.rs+++ b/crates/ide-dap/src/cargo.rs@@ -70,8 +70,10 @@ pub async fn build(let on_output = std::sync::Arc::new(on_output);let err_out = on_output.clone();let err_task = tokio::spawn(async move {- let mut lines = BufReader::new(stderr).lines();- while let Ok(Some(line)) = lines.next_line().await {+ let mut lines = BufReader::new(stderr).split(b'\n');+ while let Ok(Some(line)) = lines.next_segment().await {+ let line = ide_core::output_encoding::decode_tool_output(&line);+ let line = line.trim_end_matches('\r');err_out(format!("{line}\n"));}});diff --git a/crates/ide-docker/src/process.rs b/crates/ide-docker/src/process.rsindex e9dc25a3b..6c08c0050 100644--- a/crates/ide-docker/src/process.rs+++ b/crates/ide-docker/src/process.rs@@ -54,7 +54,7 @@ fn read_all(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle<Strstd::thread::spawn(move || {let mut buf = Vec::new();let _ = pipe.read_to_end(&mut buf);- String::from_utf8_lossy(&buf).into_owned()+ ide_core::output_encoding::decode_tool_output(&buf)})}@@ -115,7 +115,7 @@ fn spawn_reader(mut pipe: impl Read + Send + 'static, tx: mpsc::Sender<Event>, slet mut pending: Vec<u8> = Vec::new();let flush = |pending: &mut Vec<u8>| {if !pending.is_empty() {- let line = String::from_utf8_lossy(pending).trim_end().to_string();+ let line = ide_core::output_encoding::decode_tool_output(pending).trim_end().to_string();pending.clear();let _ = tx.send(Event::Line(stream, line));}diff --git a/crates/ide-host/src/environment.rs b/crates/ide-host/src/environment.rsindex 4dabd3500..0ef84ec5f 100644--- a/crates/ide-host/src/environment.rs+++ b/crates/ide-host/src/environment.rs@@ -281,7 +281,8 @@ fn run_one(argv: &[String], root: &Path, cancel: &CancellationToken, on_line: &dfor pipe in pipes.into_iter().flatten() {let tx = tx.clone();std::thread::spawn(move || {- for line in BufReader::new(pipe).lines().map_while(Result::ok) {+ for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) {+ let line = ide_core::output_encoding::decode_tool_output(&line).trim_end_matches('\r').to_owned();if tx.send(line).is_err() {break;}diff --git a/crates/ide-notebook/src/kernel/process.rs b/crates/ide-notebook/src/kernel/process.rsindex 791939e0b..afcdda93e 100644--- a/crates/ide-notebook/src/kernel/process.rs+++ b/crates/ide-notebook/src/kernel/process.rs@@ -1,13 +1,12 @@use ide_core::{IdeError, IdeResult};use jupyter_protocol::ConnectionInfo;use parking_lot::Mutex;-use std::collections::VecDeque;use std::io::Write;use std::path::Path;use std::process::Stdio;use std::sync::Arc;use std::time::Duration;-use tokio::io::AsyncReadExt;+use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};use tokio::process::{Child, Command};use tokio::task::JoinHandle;use uuid::Uuid;@@ -18,7 +17,7 @@ pub(super) struct Process {child: Child,#[cfg(unix)]pid: Option<u32>,- stderr: Arc<Mutex<VecDeque<u8>>>,+ stderr: Arc<Mutex<String>>,drains: Vec<JoinHandle<()>>,// Kept until the process and sockets have stopped; contains the HMAC secret._directory: tempfile::TempDir,@@ -103,26 +102,31 @@ impl Process {child,#[cfg(unix)]pid,- stderr: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_BYTES))),+ stderr: Arc::new(Mutex::new(String::new())),drains: Vec::new(),_directory: directory,#[cfg(windows)]windows,};- if let Some(mut stderr) = process.child.stderr.take() {+ if let Some(stderr) = process.child.stderr.take() {let tail = process.stderr.clone();process.drains.push(tokio::spawn(async move {- let mut buffer = [0_u8; 4096];- while let Ok(count) = stderr.read(&mut buffer).await {- if count == 0 {- break;- }- let mut tail = tail.lock();- for byte in &buffer[..count] {- if tail.len() == STDERR_BYTES {- tail.pop_front();+ let mut reader = BufReader::new(stderr);+ let mut line = Vec::new();+ loop {+ line.clear();+ match reader.read_until(b'\n', &mut line).await {+ Ok(0) | Err(_) => break,+ Ok(_) => {+ let text = ide_core::output_encoding::decode_tool_output(&line);+ let mut tail = tail.lock();+ tail.push_str(&text);+ let mut start = tail.len().saturating_sub(STDERR_BYTES);+ while !tail.is_char_boundary(start) {+ start += 1;+ }+ tail.drain(..start);}- tail.push_back(*byte);}}}));@@ -143,8 +147,7 @@ impl Process {}pub fn stderr(&self) -> String {- let bytes = self.stderr.lock().iter().copied().collect::<Vec<_>>();- String::from_utf8_lossy(&bytes).into_owned()+ self.stderr.lock().clone()}pub fn death(&mut self) -> IdeResult<Option<String>> {diff --git a/crates/ide-sdk/src/exec.rs b/crates/ide-sdk/src/exec.rsindex aa2296c67..03333c502 100644--- a/crates/ide-sdk/src/exec.rs+++ b/crates/ide-sdk/src/exec.rs@@ -1,6 +1,7 @@//! Running toolchain commands: short queries with a timeout and long-running//! streaming commands that honour cancellation.+use ide_core::output_encoding::decode_tool_output;use ide_core::{CancellationToken, IdeError, IdeResult};use std::ffi::OsString;use std::io::{BufRead, BufReader, Read};@@ -77,14 +78,14 @@ pub(crate) fn query_in(let mut stdout = child.stdout.take()?;let mut stderr = child.stderr.take()?;let out = std::thread::spawn(move || {- let mut s = String::new();- let _ = stdout.read_to_string(&mut s);- s+ let mut s = Vec::new();+ let _ = stdout.read_to_end(&mut s);+ decode_tool_output(&s)});let err = std::thread::spawn(move || {- let mut s = String::new();- let _ = stderr.read_to_string(&mut s);- s+ let mut s = Vec::new();+ let _ = stderr.read_to_end(&mut s);+ decode_tool_output(&s)});let deadline = Instant::now() + timeout;let status = loop {@@ -131,16 +132,16 @@ pub(crate) fn stream(if let Some(out) = child.stdout.take() {let tx = tx.clone();readers.push(std::thread::spawn(move || {- for line in BufReader::new(out).lines().map_while(Result::ok) {- let _ = tx.send(line);+ for line in BufReader::new(out).split(b'\n').map_while(Result::ok) {+ let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());}}));}if let Some(err) = child.stderr.take() {let tx = tx.clone();readers.push(std::thread::spawn(move || {- for line in BufReader::new(err).lines().map_while(Result::ok) {- let _ = tx.send(line);+ for line in BufReader::new(err).split(b'\n').map_while(Result::ok) {+ let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());}}));}diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rsindex 159a7d606..81c44bf92 100644--- a/crates/ide-tools/src/process.rs+++ b/crates/ide-tools/src/process.rs@@ -3,6 +3,7 @@use crate::InstallEvent;use ide_core::error::ErrorKind;+use ide_core::output_encoding::decode_tool_output;use ide_core::{CancellationToken, IdeError, IdeResult};use std::io::{BufRead, BufReader, Read};use std::path::Path;@@ -86,8 +87,9 @@ pub fn run_collect(mut cmd: Command, cancel: &CancellationToken, emit: Emit) ->for pipe in pipes.into_iter().flatten() {let tx = tx.clone();std::thread::spawn(move || {- for line in BufReader::new(pipe).lines() {+ for line in BufReader::new(pipe).split(b'\n') {let Ok(line) = line else { break };+ let line = decode_tool_output(&line).trim_end_matches('\r').to_owned();if tx.send(line).is_err() {break;}@@ -127,14 +129,14 @@ pub fn run_quiet(mut cmd: Command, timeout: Duration) -> Option<(bool, String)>let mut stdout = child.stdout.take()?;let mut stderr = child.stderr.take()?;let out_t = std::thread::spawn(move || {- let mut s = String::new();- let _ = stdout.read_to_string(&mut s);- s+ let mut s = Vec::new();+ let _ = stdout.read_to_end(&mut s);+ decode_tool_output(&s)});let err_t = std::thread::spawn(move || {- let mut s = String::new();- let _ = stderr.read_to_string(&mut s);- s+ let mut s = Vec::new();+ let _ = stderr.read_to_end(&mut s);+ decode_tool_output(&s)});let deadline = Instant::now() + timeout;let status = loop {@@ -251,8 +253,9 @@ impl Http {return Err(IdeError::new(ErrorKind::Io, format!("{url}: response is larger than {max} bytes")));}if !status.success() {- let mut err = String::new();- let _ = stderr.read_to_string(&mut err);+ let mut err = Vec::new();+ let _ = stderr.read_to_end(&mut err);+ let err = decode_tool_output(&err);return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim())));}String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8")))@@ -272,9 +275,9 @@ impl Http {let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?;let mut stderr = child.stderr.take().expect("piped");let err_t = std::thread::spawn(move || {- let mut s = String::new();- let _ = stderr.read_to_string(&mut s);- s+ let mut s = Vec::new();+ let _ = stderr.read_to_end(&mut s);+ decode_tool_output(&s)});let mut last = Instant::now();let mut last_bytes = 0;