Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
reqwest_client: Fix streamed request bodies being truncated (#60314)
Ben Kunkle552fc9f3c3c7
diff --git a/crates/reqwest_client/src/reqwest_client.rs b/crates/reqwest_client/src/reqwest_client.rsindex 60908d8a8d4..066e95fc655 100644--- a/crates/reqwest_client/src/reqwest_client.rs+++ b/crates/reqwest_client/src/reqwest_client.rs@@ -156,7 +156,11 @@ impl futures::Stream for StreamReader {}match poll_read_buf(&mut reader, cx, &mut this.buf) {- Poll::Pending => Poll::Pending,+ Poll::Pending => {+ self.reader = Some(reader);++ Poll::Pending+ }Poll::Ready(Err(err)) => {self.reader = None;@@ -177,7 +181,7 @@ impl futures::Stream for StreamReader {/// Implementation from <https://docs.rs/tokio-util/0.7.12/src/tokio_util/util/poll_buf.rs.html>/// Specialized for this use case-pub fn poll_read_buf(+fn poll_read_buf(io: &mut Pin<Box<dyn futures::AsyncRead + Send + Sync>>,cx: &mut std::task::Context<'_>,buf: &mut BytesMut,@@ -190,22 +194,22 @@ pub fn poll_read_buf(let dst = buf.chunk_mut();// Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a- // transparent wrapper around `[MaybeUninit<u8>]`.+ // transparent wrapper around `[std::mem::MaybeUninit<u8>]`.let dst = unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>]) };- let mut buf = tokio::io::ReadBuf::uninit(dst);- let ptr = buf.filled().as_ptr();- let unfilled_portion = buf.initialize_unfilled();+ let mut read_buf = tokio::io::ReadBuf::uninit(dst);+ let unfilled_portion = read_buf.initialize_unfilled();// SAFETY: Pin projectionlet io_pin = unsafe { Pin::new_unchecked(io) };- std::task::ready!(io_pin.poll_read(cx, unfilled_portion)?);-- // Ensure the pointer does not change from under us- assert_eq!(ptr, buf.filled().as_ptr());- buf.filled().len()+ // `futures::AsyncRead` reports the byte count as the poll's return+ // value; `read_buf.filled()` stays empty because the reader writes+ // through the initialized slice without advancing the `ReadBuf`.+ std::task::ready!(io_pin.poll_read(cx, unfilled_portion)?)};- // Safety: This is guaranteed to be the number of initialized (and read)- // bytes due to the invariants provided by `ReadBuf::filled`.+ // Safety: `initialize_unfilled()` zero-initialized the entire spare+ // capacity, so the first `n` bytes are initialized no matter how many the+ // reader actually wrote, and `advance_mut` panics rather than exceeding+ // the capacity if `n` overstates the slice length.unsafe {buf.advance_mut(n);}@@ -286,10 +290,90 @@ impl http_client::HttpClient for ReqwestClient {#[cfg(test)]mod tests {- use http_client::{HttpClient, Url};+ use std::io::{Read as _, Write as _};+ use std::net::TcpListener;++ use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest, Url};use crate::ReqwestClient;+ /// Regression test: `StreamReader::poll_next` used to drop the reader it+ /// `take()`s whenever the reader returned `Poll::Pending`, so the next+ /// poll reported end-of-stream and streamed request bodies were silently+ /// truncated. Readers backed by real I/O (e.g. `async_fs::File`) return+ /// `Pending` on their very first read, so their uploads sent zero bytes.+ #[test]+ fn test_streamed_body_survives_pending_reader() {+ let payload: Vec<u8> = (0..30_000usize).map(|byte| (byte % 251) as u8).collect();++ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let expected_payload = payload.clone();+ let server = std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ let mut request = Vec::new();+ let mut buffer = [0u8; 8192];+ loop {+ let read = stream.read(&mut buffer).unwrap();+ assert_ne!(read, 0, "client closed the connection mid-request");+ request.extend_from_slice(&buffer[..read]);+ if let Some(position) = request.windows(4).position(|w| w == b"\r\n\r\n") {+ let body_start = position + 4;+ while request.len() - body_start < expected_payload.len() {+ let read = stream.read(&mut buffer).unwrap();+ assert_ne!(read, 0, "client closed the connection mid-body");+ request.extend_from_slice(&buffer[..read]);+ }+ assert_eq!(&request[body_start..], &expected_payload);+ break;+ }+ }+ stream+ .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")+ .unwrap();+ });++ // A reader that returns `Pending` before every chunk, like a reader+ // backed by real I/O would.+ struct PendingFirstReader {+ data: std::io::Cursor<Vec<u8>>,+ ready: bool,+ }++ impl futures::AsyncRead for PendingFirstReader {+ fn poll_read(+ mut self: std::pin::Pin<&mut Self>,+ cx: &mut std::task::Context<'_>,+ buf: &mut [u8],+ ) -> std::task::Poll<std::io::Result<usize>> {+ if self.ready {+ self.ready = false;+ std::task::Poll::Ready(self.data.read(buf))+ } else {+ self.ready = true;+ cx.waker().wake_by_ref();+ std::task::Poll::Pending+ }+ }+ }++ let reader = PendingFirstReader {+ data: std::io::Cursor::new(payload.clone()),+ ready: false,+ };++ let client = ReqwestClient::new();+ let request = HttpRequest::builder()+ .method(Method::PUT)+ .uri(format!("http://{address}/upload"))+ .header("Content-Length", payload.len().to_string())+ .body(AsyncBody::from_reader(reader))+ .unwrap();+ let response = futures::executor::block_on(client.send(request)).unwrap();+ assert!(response.status().is_success());+ server.join().unwrap();+ }+#[test]fn test_proxy_uri() {let client = ReqwestClient::new();