Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

reqwest_client: Fix streamed request bodies being truncated (#60314)

Ben Kunkle552fc9f3c3c7
diff --git a/crates/reqwest_client/src/reqwest_client.rs b/crates/reqwest_client/src/reqwest_client.rs
index 60908d8a8d4..066e95fc655 100644
--- a/crates/reqwest_client/src/reqwest_client.rs
+++ b/crates/reqwest_client/src/reqwest_client.rs
@@ -156,7 +156,11 @@ impl futures::Stream for StreamReader {
}
match poll_read_buf(&mut reader, cx, &mut this.buf) {
- Poll::Pending => Poll::Pending,
+ Poll::Pending => {
+ self.reader = Some(reader);
+
+ Poll::Pending
+ }
Poll::Ready(Err(err)) => {
self.reader = None;
@@ -177,7 +181,7 @@ impl futures::Stream for StreamReader {
/// Implementation from <https://docs.rs/tokio-util/0.7.12/src/tokio_util/util/poll_buf.rs.html>
/// Specialized for this use case
-pub fn poll_read_buf(
+fn poll_read_buf(
io: &mut Pin<Box<dyn futures::AsyncRead + Send + Sync>>,
cx: &mut std::task::Context<'_>,
buf: &mut BytesMut,
@@ -190,22 +194,22 @@ pub fn poll_read_buf(
let dst = buf.chunk_mut();
// Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
- // transparent wrapper around `[MaybeUninit<u8>]`.
+ // transparent wrapper around `[std::mem::MaybeUninit<u8>]`.
let dst = unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>]) };
- let mut buf = tokio::io::ReadBuf::uninit(dst);
- let ptr = buf.filled().as_ptr();
- let unfilled_portion = buf.initialize_unfilled();
+ let mut read_buf = tokio::io::ReadBuf::uninit(dst);
+ let unfilled_portion = read_buf.initialize_unfilled();
// SAFETY: Pin projection
let io_pin = unsafe { Pin::new_unchecked(io) };
- std::task::ready!(io_pin.poll_read(cx, unfilled_portion)?);
-
- // Ensure the pointer does not change from under us
- assert_eq!(ptr, buf.filled().as_ptr());
- buf.filled().len()
+ // `futures::AsyncRead` reports the byte count as the poll's return
+ // value; `read_buf.filled()` stays empty because the reader writes
+ // through the initialized slice without advancing the `ReadBuf`.
+ std::task::ready!(io_pin.poll_read(cx, unfilled_portion)?)
};
- // Safety: This is guaranteed to be the number of initialized (and read)
- // bytes due to the invariants provided by `ReadBuf::filled`.
+ // Safety: `initialize_unfilled()` zero-initialized the entire spare
+ // capacity, so the first `n` bytes are initialized no matter how many the
+ // reader actually wrote, and `advance_mut` panics rather than exceeding
+ // the capacity if `n` overstates the slice length.
unsafe {
buf.advance_mut(n);
}
@@ -286,10 +290,90 @@ impl http_client::HttpClient for ReqwestClient {
#[cfg(test)]
mod tests {
- use http_client::{HttpClient, Url};
+ use std::io::{Read as _, Write as _};
+ use std::net::TcpListener;
+
+ use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest, Url};
use crate::ReqwestClient;
+ /// Regression test: `StreamReader::poll_next` used to drop the reader it
+ /// `take()`s whenever the reader returned `Poll::Pending`, so the next
+ /// poll reported end-of-stream and streamed request bodies were silently
+ /// truncated. Readers backed by real I/O (e.g. `async_fs::File`) return
+ /// `Pending` on their very first read, so their uploads sent zero bytes.
+ #[test]
+ fn test_streamed_body_survives_pending_reader() {
+ let payload: Vec<u8> = (0..30_000usize).map(|byte| (byte % 251) as u8).collect();
+
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let expected_payload = payload.clone();
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut request = Vec::new();
+ let mut buffer = [0u8; 8192];
+ loop {
+ let read = stream.read(&mut buffer).unwrap();
+ assert_ne!(read, 0, "client closed the connection mid-request");
+ request.extend_from_slice(&buffer[..read]);
+ if let Some(position) = request.windows(4).position(|w| w == b"\r\n\r\n") {
+ let body_start = position + 4;
+ while request.len() - body_start < expected_payload.len() {
+ let read = stream.read(&mut buffer).unwrap();
+ assert_ne!(read, 0, "client closed the connection mid-body");
+ request.extend_from_slice(&buffer[..read]);
+ }
+ assert_eq!(&request[body_start..], &expected_payload);
+ break;
+ }
+ }
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+ .unwrap();
+ });
+
+ // A reader that returns `Pending` before every chunk, like a reader
+ // backed by real I/O would.
+ struct PendingFirstReader {
+ data: std::io::Cursor<Vec<u8>>,
+ ready: bool,
+ }
+
+ impl futures::AsyncRead for PendingFirstReader {
+ fn poll_read(
+ mut self: std::pin::Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ buf: &mut [u8],
+ ) -> std::task::Poll<std::io::Result<usize>> {
+ if self.ready {
+ self.ready = false;
+ std::task::Poll::Ready(self.data.read(buf))
+ } else {
+ self.ready = true;
+ cx.waker().wake_by_ref();
+ std::task::Poll::Pending
+ }
+ }
+ }
+
+ let reader = PendingFirstReader {
+ data: std::io::Cursor::new(payload.clone()),
+ ready: false,
+ };
+
+ let client = ReqwestClient::new();
+ let request = HttpRequest::builder()
+ .method(Method::PUT)
+ .uri(format!("http://{address}/upload"))
+ .header("Content-Length", payload.len().to_string())
+ .body(AsyncBody::from_reader(reader))
+ .unwrap();
+ let response = futures::executor::block_on(client.send(request)).unwrap();
+ assert!(response.status().is_success());
+ server.join().unwrap();
+ }
+
#[test]
fn test_proxy_uri() {
let client = ReqwestClient::new();