Minecraft: Java Edition 26.3 rewritten from scratch in Rust, checked against vanilla with 281 differential harnesses. Get it with the Horizon launcher.
Rust differential-testinggame-enginegamedevlinuxminecraftminecraft-javareimplementationrustvoxelvulkanwgpu
Receive path: one buffer read into in place, one inflater, block pipe reads
Ludovic Mariec53bea9730cc
diff --git a/crates/horizon-protocol/src/connection.rs b/crates/horizon-protocol/src/connection.rsindex 7136a60a9..a7e1d3060 100644--- a/crates/horizon-protocol/src/connection.rs+++ b/crates/horizon-protocol/src/connection.rs@@ -83,11 +83,7 @@ impl Read for Pipe {}};}- let n = out.len().min(state.0.len());- for (o, b) in out.iter_mut().zip(state.0.drain(..n)) {- *o = b;- }- Ok(n)+ state.0.read(out)}}impl Write for Pipe {@@ -169,7 +165,12 @@ pub struct Counters {pub struct ConnReader {stream: Stream,+ /// Bytes read: `buffer[start..end]` are not consumed yet, and reads go+ /// straight into `buffer[end..]`.buffer: Vec<u8>,+ start: usize,+ end: usize,+ inflater: frame::Inflater,decrypt: Option<Cfb8>,pub threshold: Option<usize>,pub counters: Arc<Counters>,@@ -291,6 +292,9 @@ impl Connection {reader: ConnReader {stream,buffer: Vec::new(),+ start: 0,+ end: 0,+ inflater: frame::Inflater::default(),decrypt: None,threshold: None,counters: counters.clone(),@@ -317,7 +321,8 @@ impl Connection {pub fn enable_encryption(&mut self, secret: &[u8; 16]) {let mut decrypt = Cfb8::new(secret, secret);// Bytes already read past the last packet were sent encrypted.- decrypt.decrypt(&mut self.reader.buffer);+ let r = &mut self.reader;+ decrypt.decrypt(&mut r.buffer[r.start..r.end]);self.reader.decrypt = Some(decrypt);self.writer.encrypt = Some(Cfb8::new(secret, secret));}@@ -341,46 +346,84 @@ impl ConnReader {/// Block until a whole packet arrives: (id, payload).pub fn recv(&mut self) -> Result<(i32, Vec<u8>)> {loop {- if let Some(frame) = frame::split(&mut self.buffer)? {- let body = frame::decompress(frame, self.threshold)?;- if body.is_empty() {- // A frame with no packet in it. Vanilla's PacketDecoder- // only decodes a packet from a readable buffer, so an- // empty compressed frame is skipped and the connection- // stays up.+ let pending = &self.buffer[self.start..self.end];+ let mut wanted = 0;+ if let Some((prefix, len)) = frame::header(pending)? {+ if pending.len() >= prefix + len {+ let frame = self.start + prefix..self.start + prefix + len;+ self.start = frame.end;+ if let Some(packet) = self.packet(frame)? {+ self.counters.received.fetch_add(1, Ordering::Relaxed);+ return Ok(packet);+ }continue;}+ wanted = prefix + len;+ }+ self.fill(wanted)?;+ }+ }+ /// The packet in `buffer[frame]`, or `None` for a frame with no packet in it:+ /// vanilla's PacketDecoder only decodes a packet from a readable buffer, so an+ /// empty compressed frame is skipped and the connection stays up.+ fn packet(&mut self, frame: std::ops::Range<usize>) -> Result<Option<(i32, Vec<u8>)>> {+ let frame = &self.buffer[frame];+ match frame::body(frame, self.threshold, &mut self.inflater)? {+ frame::Body::Plain(at) => {+ let body = &frame[at..];+ if body.is_empty() {+ return Ok(None);+ }+ let mut r = Reader::new(body);+ let id = r.varint()?;+ Ok(Some((id, body[r.position()..].to_vec())))+ }+ frame::Body::Inflated(mut body) => {let mut r = Reader::new(&body);let id = r.varint()?;- let start = r.position();- self.counters.received.fetch_add(1, Ordering::Relaxed);- return Ok((id, body[start..].to_vec()));+ let at = r.position();+ body.drain(..at);+ Ok(Some((id, body)))}- let mut chunk = [0u8; 16384];- let n = match self.stream.read(&mut chunk) {- Ok(0) => return Err(Error::end_of_stream()),- Ok(n) => n,- // The read timeout passed: Windows reports TimedOut, Unix- // WouldBlock.- Err(e)- if matches!(- e.kind(),- std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock- ) =>- {- return Err(Error::timed_out());- }- Err(e) => return Err(e.into()),- };- self.counters- .bytes_in- .fetch_add(n as i64, Ordering::Relaxed);- let data = &mut chunk[..n];- if let Some(c) = &mut self.decrypt {- c.decrypt(data);+ }+ }+ /// Read once from the stream into the buffer, with room for at least+ /// `frame` bytes of unconsumed data (a whole frame, once its length is known).+ fn fill(&mut self, frame: usize) -> Result<()> {+ const READ: usize = 1 << 16;+ let pending = self.end - self.start;+ let room = READ.max(frame.saturating_sub(pending));+ if self.buffer.len() - self.end < room {+ self.buffer.copy_within(self.start..self.end, 0);+ (self.start, self.end) = (0, pending);+ if self.buffer.len() < pending + room {+ self.buffer.resize(pending + room, 0);}- self.buffer.extend_from_slice(data);}+ let n = match self.stream.read(&mut self.buffer[self.end..]) {+ Ok(0) => return Err(Error::end_of_stream()),+ Ok(n) => n,+ // The read timeout passed: Windows reports TimedOut, Unix+ // WouldBlock.+ Err(e)+ if matches!(+ e.kind(),+ std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock+ ) =>+ {+ return Err(Error::timed_out());+ }+ Err(e) => return Err(e.into()),+ };+ self.counters+ .bytes_in+ .fetch_add(n as i64, Ordering::Relaxed);+ let data = &mut self.buffer[self.end..self.end + n];+ if let Some(c) = &mut self.decrypt {+ c.decrypt(data);+ }+ self.end += n;+ Ok(())}/// A [`Closer`] for this connection.pub fn closer(&self) -> Result<Closer> {@@ -599,6 +642,38 @@ mod tests {assert!(c.recv().is_err());}+ /// Packets from one byte to past the read size, compressed and+ /// encrypted, arrive whole and in order however the reads cut them.+ #[test]+ fn packets_of_every_size_arrive_whole() {+ let (a, b) = Pipe::pair();+ let bodies: Vec<Vec<u8>> = (0..400u32)+ .map(|i| {+ let len = [1, 2, 63, 64, 65, 300, 4000, 70_000, 200_000][i as usize % 9];+ (0..len).map(|j| (j as u32 * 31 + i) as u8 & 0x7f).collect()+ })+ .collect();+ let sent = bodies.clone();+ let server = std::thread::spawn(move || {+ let mut c = Connection::local(b);+ c.set_compression(64);+ c.enable_encryption(&[3; 16]);+ for body in &sent {+ c.send(body).unwrap();+ }+ c+ });+ let mut c = Connection::local(a);+ c.set_compression(64);+ c.enable_encryption(&[3; 16]);+ for body in &bodies {+ let (id, payload) = c.recv().unwrap();+ assert_eq!(id, i32::from(body[0]));+ assert_eq!(payload, body[1..]);+ }+ drop(server.join().unwrap());+ }+#[test]fn dropping_a_pipe_end_closes_it() {let (a, b) = Pipe::pair();diff --git a/crates/horizon-protocol/src/frame.rs b/crates/horizon-protocol/src/frame.rsindex 25d113f47..243a07c87 100644--- a/crates/horizon-protocol/src/frame.rs+++ b/crates/horizon-protocol/src/frame.rs@@ -4,8 +4,8 @@ use crate::{Error, Result,buf::{Reader, write_varint},};-use flate2::{Compression, read::ZlibDecoder, write::ZlibEncoder};-use std::io::{Read, Write};+use flate2::{Compression, Decompress, FlushDecompress, Status, write::ZlibEncoder};+use std::io::Write;/// Frames are at most three VarInt bytes long (21 bits).pub const MAX_FRAME: usize = (1 << 21) - 1;@@ -55,12 +55,12 @@ pub fn prefixed(contents: &[u8]) -> Vec<u8> {out}-/// Split one complete frame off the front of `buffer`, if available.-/// Returns the frame contents (without the length prefix).-pub fn split(buffer: &mut Vec<u8>) -> Result<Option<Vec<u8>>> {+/// The length prefix at the front of `bytes`, once it is complete: its own+/// size and the frame's length.+pub fn header(bytes: &[u8]) -> Result<Option<(usize, usize)>> {let mut len = 0usize;for i in 0..3 {- let Some(&b) = buffer.get(i) else {+ let Some(&b) = bytes.get(i) else {return Ok(None);};len |= usize::from(b & 0x7F) << (7 * i);@@ -70,27 +70,55 @@ pub fn split(buffer: &mut Vec<u8>) -> Result<Option<Vec<u8>>> {// corrupted frame, not an empty packet.return Err(Error::new("Frame length cannot be zero"));}- let start = i + 1;- if buffer.len() < start + len {- return Ok(None);- }- let frame = buffer[start..start + len].to_vec();- buffer.drain(..start + len);- return Ok(Some(frame));+ return Ok(Some((i + 1, len)));}}Err(Error::new("length wider than 21 bits"))}-/// Undo compression on a frame, giving the packet body (id + payload).-pub fn decompress(frame: Vec<u8>, threshold: Option<usize>) -> Result<Vec<u8>> {+/// Inflates compressed frames, keeping its state from one to the next+/// (`CompressionDecoder`'s one `Inflater`).+pub struct Inflater(Decompress);+impl Default for Inflater {+ fn default() -> Self {+ Self(Decompress::new(true))+ }+}+impl Inflater {+ /// The zlib stream at the front of `input`, which must hold exactly `size` bytes.+ fn inflate(&mut self, input: &[u8], size: usize) -> Result<Vec<u8>> {+ self.0.reset(true);+ // One byte more than declared: a longer stream fills it.+ let mut out = Vec::with_capacity(size + 1);+ let status = self+ .0+ .decompress_vec(input, &mut out, FlushDecompress::Finish)+ .map_err(|e| Error::new(format!("Badly compressed packet - {e}")))?;+ if out.len() != size || !matches!(status, Status::StreamEnd) {+ return Err(Error::new(format!(+ "Badly compressed packet - actual length {} does not match declared size {size}",+ out.len()+ )));+ }+ Ok(out)+ }+}++/// A frame's packet body: where it starts in the frame, or inflated.+pub enum Body {+ Plain(usize),+ Inflated(Vec<u8>),+}++/// `CompressionDecoder`: the body of a frame, inflated when its data length says so.+pub fn body(frame: &[u8], threshold: Option<usize>, inflater: &mut Inflater) -> Result<Body> {let Some(threshold) = threshold else {- return Ok(frame);+ return Ok(Body::Plain(0));};- let mut r = Reader::new(&frame);+ let mut r = Reader::new(frame);let size = r.varint()?;if size == 0 {- return Ok(frame[r.position()..].to_vec());+ return Ok(Body::Plain(r.position()));}let size = usize::try_from(size).map_err(|_| Error::new("negative uncompressed length"))?;if size < threshold {@@ -103,24 +131,36 @@ pub fn decompress(frame: Vec<u8>, threshold: Option<usize>) -> Result<Vec<u8>> {"Badly compressed packet - size of {size} is larger than protocol maximum")));}- let start = r.position();- let mut out = Vec::with_capacity(size);- ZlibDecoder::new(&frame[start..])- .take(size as u64 + 1)- .read_to_end(&mut out)?;- if out.len() != size {- return Err(Error::new(format!(- "Badly compressed packet - actual length {} does not match declared size {size}",- out.len()- )));- }- Ok(out)+ inflater+ .inflate(&frame[r.position()..], size)+ .map(Body::Inflated)}#[cfg(test)]mod tests {use super::*;+ /// Split one complete frame off the front of `buffer`, if available.+ fn split(buffer: &mut Vec<u8>) -> Result<Option<Vec<u8>>> {+ let Some((start, len)) = header(buffer)? else {+ return Ok(None);+ };+ if buffer.len() < start + len {+ return Ok(None);+ }+ let frame = buffer[start..start + len].to_vec();+ buffer.drain(..start + len);+ Ok(Some(frame))+ }++ /// The packet body (id + payload) of a frame.+ fn decompress(frame: Vec<u8>, threshold: Option<usize>) -> Result<Vec<u8>> {+ Ok(match body(&frame, threshold, &mut Inflater::default())? {+ Body::Plain(at) => frame[at..].to_vec(),+ Body::Inflated(body) => body,+ })+ }+#[test]fn uncompressed_frames_round_trip() {let body = vec![0x00, 1, 2, 3];@@ -169,6 +209,33 @@ mod tests {assert!(decompress(frame, Some(256)).is_err());}+ /// One inflater serves every frame, also after a bad one.+ #[test]+ fn the_inflater_is_reused() {+ let mut inflater = Inflater::default();+ let frame = |body: &[u8], declared: usize| {+ let mut frame = Vec::new();+ write_varint(&mut frame, declared as i32);+ let mut z = ZlibEncoder::new(frame, Compression::default());+ z.write_all(body).unwrap();+ z.finish().unwrap()+ };+ for n in [300usize, 70_000, 256, 5000] {+ let body: Vec<u8> = (0..n).map(|i| (i * 7 % 13) as u8).collect();+ match body_of(&frame(&body, n), &mut inflater) {+ Ok(Body::Inflated(out)) => assert_eq!(out, body),+ _ => panic!("{n} bytes did not inflate"),+ }+ // A stream longer than declared, then one shorter.+ assert!(body_of(&frame(&body, n - 1), &mut inflater).is_err());+ assert!(body_of(&frame(&body, n + 1), &mut inflater).is_err());+ }+ }++ fn body_of(frame: &[u8], inflater: &mut Inflater) -> Result<Body> {+ body(frame, Some(256), inflater)+ }+#[test]fn a_noncanonical_length_prefix_still_reads() {// The data-length VarInt is read, not measured: zero as `0x80 0x00`.