The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
fix(tools): resume partial downloads with curl -C - (#2029)
arookieofc7479b7ad9de0
diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rsindex 7d34423e4..5b42eda4d 100644--- a/crates/ide-tools/src/process.rs+++ b/crates/ide-tools/src/process.rs@@ -430,6 +430,9 @@ impl Http {}/// Downloads `url` to `dest`, reporting progress against `size` when known.+ /// A `.url` sidecar ties partial bytes to their source. Matching partials+ /// resume with `-C -`; untracked files or a changed URL start from zero.+ /// Rejected ranges (curl exit 33 or HTTP 416) also restart from zero.pub fn download(&self,url: &str,@@ -438,8 +441,67 @@ impl Http {cancel: &CancellationToken,emit: Emit,) -> IdeResult<()> {+ if cancel.is_cancelled() {+ return Err(IdeError::cancelled());+ }+ let mut url_path = dest.as_os_str().to_owned();+ url_path.push(".url");+ let url_path = std::path::PathBuf::from(url_path);+ if !std::fs::read_to_string(&url_path).is_ok_and(|previous| previous == url) {+ match std::fs::remove_file(dest) {+ Ok(()) => {}+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(e.into()),+ }+ }+ std::fs::write(&url_path, url)?;+ let had_partial = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0) > 0;+ let mut resume = true;+ loop {+ let attempt = self.fetch(url, dest, size, cancel, emit, resume);+ if cancel.is_cancelled() {+ let _ = std::fs::remove_file(&url_path);+ }+ let (status, http_code, stderr, token) = attempt?;+ // Some curl versions return success for 416 even for oversized partials.+ if resume && had_partial && (status.code() == Some(33) || http_code == 416) {+ resume = false;+ std::fs::remove_file(dest)?;+ continue;+ }+ if status.success() && http_code != 416 {+ break;+ }+ return Err(IdeError::new(+ ErrorKind::Io,+ format!("download of {url} failed: {}", redact_auth(stderr.trim(), token.as_deref())),+ ));+ }+ std::fs::remove_file(url_path)?;+ let bytes = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0);+ emit(progress_event(bytes, size.or(Some(bytes))));+ Ok(())+ }++ /// One curl download attempt: with `resume` it continues `dest`'s bytes,+ /// without it starts over. Returns curl's exit status, HTTP code, stderr,+ /// and auth token for redacting failures; spawn and cancellation failures+ /// are errors.+ fn fetch(+ &self,+ url: &str,+ dest: &Path,+ size: Option<u64>,+ cancel: &CancellationToken,+ emit: Emit,+ resume: bool,+ ) -> IdeResult<(std::process::ExitStatus, u16, String, Option<String>)> {let (mut cmd, token) = self.base(url, cancel)?;- cmd.arg("-o").arg(dest).arg(url).stdout(Stdio::null()).stderr(Stdio::piped());+ if resume {+ cmd.args(["-C", "-"]);+ }+ cmd.args(["--write-out", "%{http_code}"]).arg("-o").arg(dest).arg(url);+ cmd.stdout(Stdio::piped()).stderr(Stdio::piped());let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?;send_auth(&mut child, token.as_deref())?;let mut stderr = child.stderr.take().expect("piped");@@ -469,18 +531,9 @@ impl Http {}}};- if !status.success() {- return Err(IdeError::new(- ErrorKind::Io,- format!(- "download of {url} failed: {}",- redact_auth(err_t.join().unwrap_or_default().trim(), token.as_deref())- ),- ));- }- let bytes = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0);- emit(progress_event(bytes, size.or(Some(bytes))));- Ok(())+ let mut http_code = String::new();+ child.stdout.take().expect("piped").read_to_string(&mut http_code)?;+ Ok((status, http_code.trim().parse().unwrap_or(0), err_t.join().unwrap_or_default(), token))}}@@ -578,6 +631,9 @@ pub(crate) fn run_input(String::from_utf8(output).map_err(|e| IdeError::invalid(e.to_string()))}+#[cfg(test)]+mod download_tests;+#[cfg(test)]mod tests {use super::*;@@ -900,6 +956,46 @@ mod github_auth_tests {assert!(!error.contains("gho_fake_first"));}+ #[test]+ fn download_resumes_the_partial_file() {+ let f = Fixture::new();+ let dest = f.path("download");+ fs::write(&dest, "partial").unwrap();+ fs::write(f.path("download.url"), "https://example.test/file").unwrap();+ f.curl(&format!("printf done > {}", quote(&dest)));+ f.http.download("https://example.test/file", &dest, None, &CancellationToken::new(), &|_| {}).unwrap();+ assert!(+ fs::read_to_string(f.path("curl-args")).unwrap().contains("-C\n-\n"),+ "curl resumes the partial file with `-C -`"+ );+ }++ #[test]+ fn download_restarts_when_the_server_has_no_byte_ranges() {+ let f = Fixture::new();+ let dest = f.path("download");+ let count = f.path("attempts");+ fs::write(&dest, "partial").unwrap();+ fs::write(f.path("download.url"), "https://example.test/file").unwrap();+ // First attempt: curl reports it cannot resume (exit 33). Second:+ // a plain download from zero, which succeeds.+ f.curl(&format!(+ "n=$(cat {} 2>/dev/null || echo 0); echo $((n + 1)) > {}; \+ if [ \"$n\" = 0 ]; then echo 'curl: (33) no byte ranges' >&2; exit 33; fi; \+ printf done > {}",+ quote(&count),+ quote(&count),+ quote(&dest)+ ));+ f.http.download("https://example.test/file", &dest, None, &CancellationToken::new(), &|_| {}).unwrap();+ assert_eq!(fs::read_to_string(&dest).unwrap(), "done");+ assert_eq!(fs::read_to_string(&count).unwrap().trim(), "2");+ assert!(+ !fs::read_to_string(f.path("curl-args")).unwrap().contains("-C\n-\n"),+ "the restart downloads from zero, without `-C -`"+ );+ }+#[test]fn credential_lookup_is_bounded_and_cancellable_even_with_inherited_stdout() {let f = Fixture::new();diff --git a/crates/ide-tools/src/process/download_tests.rs b/crates/ide-tools/src/process/download_tests.rsnew file mode 100644index 000000000..042e1ad5c--- /dev/null+++ b/crates/ide-tools/src/process/download_tests.rs@@ -0,0 +1,177 @@+use super::*;+use std::fs;+use std::net::TcpListener;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex};++const CONTENT: &[u8] = b"a complete archive with enough bytes to interrupt and resume";+const PARTIAL: usize = 13;++struct Server {+ url: String,+ ranges: Arc<Mutex<Vec<Option<usize>>>>,+ stopped: Arc<AtomicBool>,+ thread: Option<std::thread::JoinHandle<()>>,+}++impl Server {+ fn new(supports_range: bool, interrupt_first: bool) -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ listener.set_nonblocking(true).unwrap();+ let url = format!("http://{}/archive", listener.local_addr().unwrap());+ let ranges = Arc::new(Mutex::new(Vec::new()));+ let stopped = Arc::new(AtomicBool::new(false));+ let requests = ranges.clone();+ let stop = stopped.clone();+ let thread = std::thread::spawn(move || {+ while !stop.load(Ordering::Relaxed) {+ let (mut stream, _) = match listener.accept() {+ Ok(connection) => connection,+ Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {+ std::thread::sleep(Duration::from_millis(5));+ continue;+ }+ Err(e) => panic!("accept: {e}"),+ };+ stream.set_nonblocking(false).unwrap();+ stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap();+ stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap();+ let mut request = String::new();+ let mut reader = BufReader::new(&stream);+ loop {+ let mut line = String::new();+ if reader.read_line(&mut line).unwrap() == 0 || line == "\r\n" {+ break;+ }+ request.push_str(&line);+ }+ let range = request.lines().find_map(|line| {+ line.to_ascii_lowercase().strip_prefix("range: bytes=")?.strip_suffix('-')?.parse::<usize>().ok()+ });+ let mut requests = requests.lock().unwrap();+ let interrupted = interrupt_first && requests.is_empty();+ requests.push(range);+ drop(requests);+ let (status, content_range, body) = match range.filter(|_| supports_range) {+ Some(start) if start >= CONTENT.len() => {+ ("416 Range Not Satisfiable", format!("Content-Range: bytes */{}\r\n", CONTENT.len()), &b""[..])+ }+ Some(start) => (+ "206 Partial Content",+ format!("Content-Range: bytes {start}-{}/{}\r\n", CONTENT.len() - 1, CONTENT.len()),+ &CONTENT[start..],+ ),+ None => ("200 OK", String::new(), CONTENT),+ };+ let header = format!(+ "HTTP/1.1 {status}\r\n{content_range}Content-Length: {}\r\nConnection: close\r\n\r\n",+ body.len()+ );+ let _ = stream.write_all(header.as_bytes());+ let _ = stream.write_all(if interrupted { &body[..PARTIAL] } else { body });+ }+ });+ Self { url, ranges, stopped, thread: Some(thread) }+ }++ fn ranges(&self) -> Vec<Option<usize>> {+ self.ranges.lock().unwrap().clone()+ }+}++impl Drop for Server {+ fn drop(&mut self) {+ self.stopped.store(true, Ordering::Relaxed);+ let result = self.thread.take().unwrap().join();+ if !std::thread::panicking() {+ result.unwrap();+ }+ }+}++fn http() -> Option<Http> {+ let Ok(curl) = which::which("curl") else {+ eprintln!("skipping download test: curl is not installed");+ return None;+ };+ Some(Http { curl, gh: None, explicit_github_token: None, https_only: false, revoke_best_effort: false })+}++#[test]+fn interrupted_download_resumes_with_range() {+ let Some(http) = http() else { return };+ let server = Server::new(true, true);+ let dir = tempfile::tempdir().unwrap();+ let dest = dir.path().join("archive.tar.gz");+ let cancel = CancellationToken::new();+ assert!(http.download(&server.url, &dest, None, &cancel, &|_| {}).is_err());+ assert_eq!(fs::read(&dest).unwrap(), &CONTENT[..PARTIAL]);+ assert_eq!(fs::read_to_string(dir.path().join("archive.tar.gz.url")).unwrap(), server.url);+ http.download(&server.url, &dest, None, &cancel, &|_| {}).unwrap();+ assert_eq!(fs::read(&dest).unwrap(), CONTENT);+ assert_eq!(server.ranges(), vec![None, Some(PARTIAL)]);+ assert!(!dir.path().join("archive.tar.gz.url").exists());+}++#[test]+fn interrupted_download_restarts_without_range_support() {+ let Some(http) = http() else { return };+ let server = Server::new(false, true);+ let dir = tempfile::tempdir().unwrap();+ let dest = dir.path().join("archive");+ let cancel = CancellationToken::new();+ assert!(http.download(&server.url, &dest, None, &cancel, &|_| {}).is_err());+ http.download(&server.url, &dest, None, &cancel, &|_| {}).unwrap();+ assert_eq!(fs::read(&dest).unwrap(), CONTENT);+ assert_eq!(server.ranges(), vec![None, Some(PARTIAL), None]);+ assert!(!dir.path().join("archive.url").exists());+}++#[test]+fn different_or_unknown_url_discards_stale_bytes_including_same_size_files() {+ let Some(http) = http() else { return };+ for previous_url in [Some("https://example.test/older-version"), None] {+ for length in [PARTIAL, CONTENT.len()] {+ let server = Server::new(true, false);+ let dir = tempfile::tempdir().unwrap();+ let dest = dir.path().join("archive");+ fs::write(&dest, vec![b'x'; length]).unwrap();+ if let Some(url) = previous_url {+ fs::write(dir.path().join("archive.url"), url).unwrap();+ }+ http.download(&server.url, &dest, None, &CancellationToken::new(), &|_| {}).unwrap();+ assert_eq!(fs::read(&dest).unwrap(), CONTENT);+ assert_eq!(server.ranges(), vec![None]);+ assert!(!dir.path().join("archive.url").exists());+ }+ }+}++#[test]+fn oversized_partial_restarts_after_http_416() {+ let Some(http) = http() else { return };+ let server = Server::new(true, false);+ let dir = tempfile::tempdir().unwrap();+ let dest = dir.path().join("archive");+ let length = CONTENT.len() + 10;+ fs::write(&dest, vec![b'x'; length]).unwrap();+ fs::write(dir.path().join("archive.url"), &server.url).unwrap();+ http.download(&server.url, &dest, None, &CancellationToken::new(), &|_| {}).unwrap();+ assert_eq!(fs::read(&dest).unwrap(), CONTENT);+ assert_eq!(server.ranges(), vec![Some(length), None]);+ assert!(!dir.path().join("archive.url").exists());+}++#[test]+fn complete_file_for_same_url_succeeds() {+ let Some(http) = http() else { return };+ let server = Server::new(true, false);+ let dir = tempfile::tempdir().unwrap();+ let dest = dir.path().join("archive");+ fs::write(&dest, CONTENT).unwrap();+ fs::write(dir.path().join("archive.url"), &server.url).unwrap();+ http.download(&server.url, &dest, None, &CancellationToken::new(), &|_| {}).unwrap();+ assert_eq!(fs::read(&dest).unwrap(), CONTENT);+ assert_eq!(server.ranges()[0], Some(CONTENT.len()));+ assert!(!dir.path().join("archive.url").exists());+}