Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(websocket): rely on native frame validation
ldm031bb71c43053
diff --git a/moli-curl/src/websocket.rs b/moli-curl/src/websocket.rsindex d36c85d54..5b7c876d8 100644--- a/moli-curl/src/websocket.rs+++ b/moli-curl/src/websocket.rs@@ -68,10 +68,14 @@ pub enum CurlWebSocketEvent {response: Vec<u8>,result: std::result::Result<(), String>,},- Chunk {- data: Vec<u8>,- frame: WsFrame,- },+ /// Decoded payload from libcurl with raw mode and automatic Pong disabled.+ /// Invalid opcodes/RSV, masking, fragmentation and control sizes fail at the+ /// native decoder. Each chunk belongs to one frame: len equals data.len(),+ /// offsets advance from zero, and bytes_left counts the remaining payload.+ /// Empty frames yield an empty chunk. Continuations retain TEXT/BINARY;+ /// CONT marks every non-final fragment, including empty ones.+ /// UTF-8, Close contents and application size limits belong to the caller.+ Chunk { data: Vec<u8>, frame: WsFrame },/// Emitted once, after all previously admitted events. Ok means TCP EOF;/// it does not assert that a WebSocket close handshake was completed.Closed {diff --git a/moli-curl/src/websocket/tests.rs b/moli-curl/src/websocket/tests.rsindex e278cfecb..9e622bfcc 100644--- a/moli-curl/src/websocket/tests.rs+++ b/moli-curl/src/websocket/tests.rs@@ -109,6 +109,134 @@ async fn native_upgrade_retains_socket_and_same_packet_empty_frame() {task.join().unwrap();}+#[tokio::test]+async fn native_decoder_rejects_invalid_framing_before_delivering_payload() {+ let mut cases = Vec::new();+ // Reserved opcodes/RSV, orphan continuations and fragmented control frames.+ for first in [+ 0x83, 0x84, 0x85, 0x86, 0x87, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0xc1, 0xa1, 0x91, 0x00, 0x80,+ 0x08, 0x09, 0x0a,+ ] {+ cases.push((vec![first, 0], None));+ }+ cases.push((vec![0x81, 0x80, 0, 0, 0, 0], None)); // masked server frame+ cases.push((vec![0x82, 127, 128, 0, 0, 0, 0, 0, 0, 0], None)); // >63-bit size+ for first in [0x88, 0x89, 0x8a] {+ cases.push((vec![first, 126, 0, 126], None)); // control payload >125+ }+ // A new message cannot interrupt a fragmented one, even with the same type.+ for (first, kind) in [(0x01, WsFlags::TEXT), (0x02, WsFlags::BINARY)] {+ for next in [0x01, 0x02, 0x81, 0x82] {+ cases.push((vec![first, 1, b'x', next, 0], Some(kind | WsFlags::CONT)));+ }+ }+ let runtime = CurlWebSocketRuntime::new().unwrap();+ for (wire, prefix) in cases {+ let description = format!("{wire:02x?}");+ let (finish_tx, finish_rx) = std::sync::mpsc::channel();+ let (url, task) = server(move |mut stream| {+ upgrade(&mut stream, &wire);+ // Keep TCP open so failure must come from decoding, not peer EOF.+ finish_rx.recv_timeout(DEADLINE).unwrap();+ });+ let mut connection = runtime.connect(CurlWebSocketRequest::new(url)).unwrap();+ opened(&mut connection).await;+ connection.sender().set_reading(true);+ if let Some(flags) = prefix {+ assert!(+ matches!(event(&mut connection).await,+ CurlWebSocketEvent::Chunk { data, frame }+ if data == b"x" && frame.flags() == flags),+ "{description}"+ );+ }+ match event(&mut connection).await {+ CurlWebSocketEvent::Closed { result } => {+ let error = result.expect_err(&description);+ assert!(+ error.starts_with("WebSocket receive failed:"),+ "{description}: {error}"+ );+ }+ unexpected => {+ panic!("{description}: invalid frame escaped the decoder: {unexpected:?}")+ }+ }+ assert!(connection.recv().await.is_none());+ finish_tx.send(()).unwrap();+ task.join().unwrap();+ }+}++#[tokio::test]+async fn native_chunks_preserve_frame_boundaries_and_continuation_types() {+ let frames = [+ (+ 0x01,+ WsFlags::TEXT | WsFlags::CONT,+ vec![b'a'; 40 * 1024 + 1],+ ),+ (0x89, WsFlags::PING, vec![b'p'; 125]),+ (0x00, WsFlags::TEXT | WsFlags::CONT, Vec::new()),+ (0x8a, WsFlags::PONG, vec![b'q'; 125]),+ (0x80, WsFlags::TEXT, vec![b'z'; 40 * 1024 + 3]),+ (0x81, WsFlags::TEXT, Vec::new()),+ (+ 0x02,+ WsFlags::BINARY | WsFlags::CONT,+ vec![0; 20 * 1024 + 1],+ ),+ (0x80, WsFlags::BINARY, vec![255; 20 * 1024 + 3]),+ (0x82, WsFlags::BINARY, Vec::new()),+ (0x88, WsFlags::CLOSE, vec![0x03, 0xe8]),+ ];+ let mut wire = Vec::new();+ for (first, _, payload) in &frames {+ wire.push(*first);+ if payload.len() < 126 {+ wire.push(payload.len() as u8);+ } else {+ wire.push(126);+ wire.extend_from_slice(&(payload.len() as u16).to_be_bytes());+ }+ wire.extend_from_slice(payload);+ }+ let (url, task) = server(move |mut stream| {+ upgrade(&mut stream, &wire);+ assert_eq!(stream.read(&mut [0]).unwrap(), 0);+ });+ let runtime = CurlWebSocketRuntime::new().unwrap();+ let mut connection = runtime.connect(CurlWebSocketRequest::new(url)).unwrap();+ opened(&mut connection).await;+ connection.sender().set_reading(true);+ for (_, flags, expected) in frames {+ let mut offset = 0;+ let mut chunks = 0;+ loop {+ let CurlWebSocketEvent::Chunk { data, frame } = event(&mut connection).await else {+ panic!("expected frame {flags:?} at offset {offset}");+ };+ assert_eq!(frame.flags(), flags);+ assert_eq!(frame.offset(), offset as u64);+ assert_eq!(frame.len(), data.len());+ assert!(!data.is_empty() || expected.is_empty());+ assert_eq!(data, expected[offset..offset + data.len()]);+ offset += data.len();+ chunks += 1;+ assert_eq!(frame.bytes_left(), (expected.len() - offset) as u64);+ if frame.bytes_left() == 0 {+ break;+ }+ }+ assert_eq!(offset, expected.len());+ if expected.len() > 16 * 1024 {+ assert!(chunks > 1, "exercise metadata across native reads");+ }+ }+ drop(connection);+ task.join().unwrap();+}+#[tokio::test]async fn native_fragmented_send_completes_without_echo() {let payload: Vec<_> = (0..MAX_SEND_FRAME_BYTES * 2)diff --git a/moli-websocket/src/frames.rs b/moli-websocket/src/frames.rsindex 7f546efbd..8383eebef 100644--- a/moli-websocket/src/frames.rs+++ b/moli-websocket/src/frames.rs@@ -1,23 +1,15 @@-//! Reassemble native chunks and validate the browser's message boundary.+//! Reassemble validated native chunks and enforce browser message policy.use moli_curl::websocket::{WsFlags, WsFrame};const MAX_FRAME_BYTES: u64 = 16 * 1024 * 1024;-const MAX_MESSAGE_BYTES: usize = 64 * 1024 * 1024;+const MAX_MESSAGE_BYTES: u64 = 64 * 1024 * 1024;#[derive(Default)]pub(super) struct Assembler {- frame: Option<FrameProgress>,- message_kind: Option<WsFlags>,message: Vec<u8>,control: Vec<u8>,}-struct FrameProgress {- flags: WsFlags,- size: u64,- offset: u64,-}-pub(super) enum Received {Text(String),Binary(Vec<u8>),@@ -35,51 +27,18 @@ impl Assembler {let flags = meta.flags();let kind = WsFlags::from_bits(flags.bits() & !WsFlags::CONT.bits());let control = matches!(kind, WsFlags::CLOSE | WsFlags::PING | WsFlags::PONG);- if !control && kind != WsFlags::TEXT && kind != WsFlags::BINARY {- return Err("WebSocket received invalid frame flags".to_owned());- }- let end = meta- .offset()- .checked_add(data.len() as u64)- .ok_or("WebSocket frame length overflow")?;- let size = end- .checked_add(meta.bytes_left())- .ok_or("WebSocket frame length overflow")?;- if size > MAX_FRAME_BYTES || data.len() != meta.len() {- return Err("WebSocket frame exceeds size limit or has invalid metadata".to_owned());- }- if control && (size > 125 || flags.contains(WsFlags::CONT)) {- return Err("WebSocket received invalid control frame".to_owned());- }- if self.frame.is_none() {- if meta.offset() != 0 {- return Err("WebSocket frame starts at a nonzero offset".to_owned());+ // Native Chunk metadata starts each frame at zero and retains the data+ // type on continuations. Check our memory limits before buffering it;+ // libcurl already validates framing and the nonnegative 63-bit length.+ if meta.offset() == 0 {+ let size = meta.len() as u64 + meta.bytes_left();+ if size > MAX_FRAME_BYTES {+ return Err("WebSocket frame exceeds size limit".to_owned());}- self.frame = Some(FrameProgress {- flags,- size,- offset: 0,- });- if !control {- if self.message_kind.is_some_and(|previous| previous != kind) {- return Err("WebSocket fragmented message changed type".to_owned());- }- self.message_kind = Some(kind);- if self- .message- .len()- .checked_add(size as usize)- .is_none_or(|len| len > MAX_MESSAGE_BYTES)- {- return Err("WebSocket message exceeds size limit".to_owned());- }+ if !control && self.message.len() as u64 + size > MAX_MESSAGE_BYTES {+ return Err("WebSocket message exceeds size limit".to_owned());}}- let frame = self.frame.as_mut().expect("frame started");- if frame.offset != meta.offset() || frame.size != size || frame.flags != flags {- return Err("WebSocket received inconsistent frame chunks".to_owned());- }- frame.offset = end;if control {self.control.extend_from_slice(&data);} else {@@ -88,7 +47,6 @@ impl Assembler {if meta.bytes_left() != 0 {return Ok(None);}- self.frame = None;if control {let payload = std::mem::take(&mut self.control);return Ok(Some(match kind {@@ -108,7 +66,6 @@ impl Assembler {if flags.contains(WsFlags::CONT) {return Ok(None);}- self.message_kind = None;let message = std::mem::take(&mut self.message);Ok(Some(if kind == WsFlags::TEXT {Received::Text(diff --git a/moli-websocket/src/native_tests.rs b/moli-websocket/src/native_tests.rsindex 29e84a88b..06e145077 100644--- a/moli-websocket/src/native_tests.rs+++ b/moli-websocket/src/native_tests.rs@@ -90,9 +90,10 @@ where#[tokio::test]async fn native_fragmented_utf8_and_ping_are_assembled_as_one_message() {- // UTF-8 U+1F642 spans continuation frames, with a Ping in between.+ // UTF-8 U+1F642 spans fragments, with empty continuation and control frames.let frames = vec![- 0x01, 2, 0xf0, 0x9f, 0x89, 2, b'h', b'i', 0x80, 2, 0x99, 0x82, 0x82, 0,+ 0x01, 2, 0xf0, 0x9f, 0x00, 0, 0x89, 2, b'h', b'i', 0x8a, 0, 0x80, 2, 0x99, 0x82, 0x81, 0,+ 0x82, 0,];let (url, server) = frames_server(frames, async |mut socket| {assert_eq!(@@ -107,6 +108,7 @@ async fn native_fragmented_utf8_and_ping_are_assembled_as_one_message() {let handle = spawn_connection(72, url, Vec::new(), test_websocket_context(), tx);recv_open_event(&mut rx).await;assert_text_message(&mut rx, 72, "π").await;+ assert_text_message(&mut rx, 72, "").await;assert_binary_message(&mut rx, 72, &[]).await;handle.close(Some(1000), String::new()).unwrap();assert_closing(&mut rx, 72).await;@@ -115,15 +117,38 @@ async fn native_fragmented_utf8_and_ping_are_assembled_as_one_message() {}#[tokio::test]-async fn native_invalid_utf8_close_code_and_frame_size_fail_without_message_delivery() {+async fn native_invalid_frames_and_messages_fail_before_delivery() {let cases = [- vec![0x81, 2, 0xc0, 0xaf], // overlong UTF-8- vec![0x88, 2, 0x03, 0xee], // reserved close code 1006- vec![0x88, 3, 0x03, 0xe8, 0xff], // invalid close reason+ (vec![0x83, 0], "WebSocket receive failed:"),+ // A native framing failure must also discard a partially assembled message.+ (+ vec![0x01, 1, b'a', 0x82, 1, b'b'],+ "WebSocket receive failed:",+ ),+ (vec![0x81, 2, 0xc0, 0xaf], "text is not valid UTF-8"), // overlong+ (vec![0x81, 1, 0xf0], "text is not valid UTF-8"), // incomplete+ (+ vec![0x01, 1, 0xf0, 0x80, 1, b'a'],+ "text is not valid UTF-8",+ ),+ (vec![0x88, 1, 0x03], "close payload has invalid length"),+ (vec![0x88, 2, 0x03, 0xee], "invalid close code 1006"),+ (+ vec![0x88, 3, 0x03, 0xe8, 0xff],+ "close reason is not valid UTF-8",+ ),// A 16 MiB + 1 frame header followed by one byte is enough to reject it.- vec![0x82, 127, 0, 0, 0, 0, 1, 0, 0, 1, 7],+ (+ vec![0x82, 127, 0, 0, 0, 0, 1, 0, 0, 1, 7],+ "frame exceeds size limit",+ ),+ // The largest legal wire length must also fail before buffering payload.+ (+ vec![0x82, 127, 127, 255, 255, 255, 255, 255, 255, 255, 7],+ "frame exceeds size limit",+ ),];- for frames in cases {+ for (frames, expected_error) in cases {let (url, server) = frames_server(frames, async |mut socket| {assert!(matches!(socket.next().await, None | Some(Err(_))));})@@ -131,15 +156,84 @@ async fn native_invalid_utf8_close_code_and_frame_size_fail_without_message_delilet (tx, mut rx) = mpsc::channel(8);let _handle = spawn_connection(73, url, Vec::new(), test_websocket_context(), tx);recv_open_event(&mut rx).await;- assert!(matches!(- timeout(Duration::from_secs(3), rx.recv()).await.unwrap(),- Some(Event::Error { .. })- ));+ match timeout(Duration::from_secs(3), rx.recv()).await.unwrap() {+ Some(Event::Error { message, .. }) => {+ assert!(+ message.contains(expected_error),+ "{message}: expected {expected_error}"+ );+ }+ unexpected => panic!("expected {expected_error} before delivery, got {unexpected:?}"),+ }assert_close(&mut rx, 73, 1006, "", false).await;server.await.unwrap();}}+#[tokio::test]+async fn native_message_size_limit_counts_fragments_and_resets_after_delivery() {+ const FRAME_BYTES: usize = 16 * 1024 * 1024;+ for overflow in [false, true] {+ let (url, server) = frames_server(Vec::new(), async move |mut socket| {+ // Accept exactly 64 MiB; leave 1 MiB available in the rejection case.+ let payload = vec![0xab; FRAME_BYTES];+ for (index, first) in [0x02, 0x00, 0x00, 0x00].into_iter().enumerate() {+ let size = if overflow && index == 3 {+ FRAME_BYTES - 1024 * 1024+ } else {+ FRAME_BYTES+ };+ let mut header = vec![first, 127];+ header.extend_from_slice(&(size as u64).to_be_bytes());+ socket.get_mut().write_all(&header).await.unwrap();+ socket.get_mut().write_all(&payload[..size]).await.unwrap();+ }+ if overflow {+ // Only one byte arrives, but the declared 2 MiB final frame would+ // exceed the message limit. Do not wait for its remaining payload.+ socket+ .get_mut()+ .write_all(&[0x80, 127, 0, 0, 0, 0, 0, 0x20, 0, 0, 7])+ .await+ .unwrap();+ assert!(matches!(socket.next().await, None | Some(Err(_))));+ } else {+ socket.get_mut().write_all(&[0x80, 0]).await.unwrap();+ socket+ .send(Message::Text("next message".into()))+ .await+ .unwrap();+ assert!(matches!(socket.next().await, Some(Ok(Message::Close(_)))));+ let _ = socket.flush().await;+ }+ })+ .await;+ let (tx, mut rx) = mpsc::channel(8);+ let handle = spawn_connection(76, url, Vec::new(), test_websocket_context(), tx);+ recv_open_event(&mut rx).await;+ let event = timeout(Duration::from_secs(10), rx.recv()).await.unwrap();+ if overflow {+ assert!(matches!(event, Some(Event::Error { message, .. })+ if message == "WebSocket message exceeds size limit"));+ assert_close(&mut rx, 76, 1006, "", false).await;+ } else {+ let Some(Event::BinaryMessage { data, .. }) = event else {+ panic!("expected complete binary message, got {event:?}");+ };+ assert_eq!(data.len(), 4 * FRAME_BYTES);+ assert!(data.iter().all(|byte| *byte == 0xab));+ assert_text_message(&mut rx, 76, "next message").await;+ handle.close(Some(1000), String::new()).unwrap();+ assert_closing(&mut rx, 76).await;+ assert_close(&mut rx, 76, 1000, "", true).await;+ }+ timeout(Duration::from_secs(3), server)+ .await+ .unwrap()+ .unwrap();+ }+}+#[tokio::test]async fn native_unsolicited_extensions_are_rejected_before_open() {let message = websocket_computed_accept_handshake_failure_message(@@ -157,7 +251,7 @@ async fn native_unsolicited_extensions_are_rejected_before_open() {#[tokio::test]async fn native_large_incoming_frame_keeps_chunk_offsets_and_utf8() {- let message = "π".repeat(20_000);+ let message = format!("a{}", "π".repeat(20_000));let expected = message.clone();let (url, server) = frames_server(Vec::new(), async move |mut socket| {socket.send(Message::Text(message.into())).await.unwrap();