Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(websocket): encapsulate native session lifecycle
ldm0cd3a2b167930
diff --git a/moli-curl/src/websocket.rs b/moli-curl/src/websocket.rsindex 5b7c876d8..39bad96ab 100644--- a/moli-curl/src/websocket.rs+++ b/moli-curl/src/websocket.rs@@ -6,6 +6,7 @@mod owner;mod request;+mod session;#[cfg(test)]mod tests;diff --git a/moli-curl/src/websocket/owner.rs b/moli-curl/src/websocket/owner.rsindex b41f2bb64..97c541c23 100644--- a/moli-curl/src/websocket/owner.rs+++ b/moli-curl/src/websocket/owner.rs@@ -1,3 +1,6 @@+//! Coordinates admission, DNS, handshakes and scheduling. Native frame I/O and+//! the lifetime of an attached easy handle belong to Session.+use std::{collections::HashMap,sync::{@@ -9,17 +12,16 @@ use std::{use curl::{easy::Easy2,- multi::{Easy2Handle, Multi, MultiWaker, WaitFd},+ multi::{Multi, MultiWaker},};use super::{- CurlWebSocketEvent, SessionIo, Submission,+ SessionIo, Submission,request::{self, Handshake},+ session::{Session, Step},};use crate::{CurlDnsResolution, CurlTransferId, dns_adapter::CurlDnsOwnerResidence};-const CHUNK_BYTES: usize = 16 * 1024;-const IO_BUDGET: usize = 8;const IDLE_WAIT: Duration = Duration::from_secs(1);struct Pending {@@ -30,12 +32,10 @@ struct Pending {io: SessionIo,}-struct Session {- handle: Easy2Handle<Handshake>,- deadline: Instant,- open: bool,- received_close: bool,- io: SessionIo,+struct Owner {+ multi: Multi,+ sessions: HashMap<CurlTransferId, Session>,+ dns: CurlDnsOwnerResidence<CurlTransferId, Pending>,}pub(super) fn run(@@ -43,11 +43,30 @@ pub(super) fn run(waker_tx: crossbeam_channel::Sender<MultiWaker>,shutdown: Arc<AtomicBool>,) {- let mut multi = Multi::new();- let _ = waker_tx.send(multi.waker());- let mut sessions = HashMap::<CurlTransferId, Session>::new();- let mut dns = CurlDnsOwnerResidence::<CurlTransferId, Pending>::default();+ let mut owner = Owner {+ multi: Multi::new(),+ sessions: HashMap::new(),+ dns: CurlDnsOwnerResidence::default(),+ };+ let _ = waker_tx.send(owner.multi.waker());while !shutdown.load(Ordering::Acquire) {+ owner.accept_submissions(&submissions);+ owner.resolve_dns();+ owner.advance_handshakes();+ if !owner.advance_sessions() {+ owner.wait_for_work();+ }+ }+ owner.fail_sessions("curl WebSocket runtime shut down");+ for pending in owner.dns.drain() {+ pending+ .io+ .finish(Err("curl WebSocket runtime shut down".to_owned()));+ }+}++impl Owner {+ fn accept_submissions(&mut self, submissions: &crossbeam_channel::Receiver<Submission>) {for submission in submissions.try_iter().take(super::SESSION_CAPACITY) {if submission.io.cancelled() {continue;@@ -74,18 +93,24 @@ pub(super) fn run(io: submission.io,};match pending.dns.target().cloned() {- Some(target) => dns.start(pending.id, pending, target, multi.waker()),- None => start(&mut multi, &mut sessions, pending),+ Some(target) => self+ .dns+ .start(pending.id, pending, target, self.multi.waker()),+ None => self.start(pending),}}- for pending in dns+ }++ fn resolve_dns(&mut self) {+ for pending in self+ .dns.take_matching(|pending| pending.io.cancelled() || pending.deadline <= Instant::now()){pending.io.finish(Err("WebSocket DNS cancelled or timed out".to_owned()));}- while let Some(ready) = dns.try_claim_next() {+ while let Some(ready) = self.dns.try_claim_next() {let mut pending = ready.pending;if pending.io.cancelled() {continue;@@ -100,17 +125,30 @@ pub(super) fn run(.map_err(|error| error.to_string())});match result {- Ok(()) => start(&mut multi, &mut sessions, pending),+ Ok(()) => self.start(pending),Err(error) => pending.io.finish(Err(error)),}}- if let Err(error) = multi.perform() {- for (_, session) in sessions.drain() {- finish(&mut multi, session, Err(error.to_string()));- }+ }++ fn start(&mut self, pending: Pending) {+ if let Some(session) = Session::attach(+ &mut self.multi,+ pending.id,+ pending.easy,+ pending.io,+ pending.deadline,+ ) {+ self.sessions.insert(pending.id, session);+ }+ }++ fn advance_handshakes(&mut self) {+ if let Err(error) = self.multi.perform() {+ self.fail_sessions(&error.to_string());}let mut completed = Vec::new();- multi.messages(|message| {+ self.multi.messages(|message| {if let (Ok(token), Some(result)) = (message.token(), message.result())&& let Some(id) = CurlTransferId::from_token(token){@@ -118,60 +156,51 @@ pub(super) fn run(}});for (id, result) in completed {- let Some(session) = sessions.get_mut(&id) else {- continue;- };- if session.open {- continue;- }- let handshake = session.handle.get_mut();- let native_result = match handshake.error.take() {- Some(error) => Err(error),- None => result.map_err(|error| error.to_string()),- };- let event = CurlWebSocketEvent::Handshake {- request: std::mem::take(&mut handshake.request),- response: std::mem::take(&mut handshake.response),- result: native_result.clone(),- };- // No data events precede the handshake, so this queue has a free slot.- if session.io.events.try_send(event).is_err() || native_result.is_err() {- let session = sessions.remove(&id).expect("completed session is resident");- finish(&mut multi, session, native_result);- } else {- session.open = true;+ if let Some(session) = self.sessions.get_mut(&id) {+ let step = session.complete_handshake(result);+ self.apply_step(id, step);}}+ }+ fn advance_sessions(&mut self) -> bool {let mut retired = Vec::new();let mut progressed = false;- for (id, session) in &mut sessions {- if session.io.cancelled() {- retired.push((*id, Ok(())));- } else if !session.open && session.deadline <= Instant::now() {- retired.push((*id, Err("WebSocket handshake timed out".to_owned())));- } else if session.open {- match session.drive() {- Ok(progress) => progressed |= progress,- Err(result) => retired.push((*id, result)),- }+ for (id, session) in &mut self.sessions {+ match session.advance() {+ Ok(Step::Progress) => progressed = true,+ Ok(Step::Idle) => {}+ terminal => retired.push((*id, terminal)),}}- for (id, result) in retired {- if let Some(session) = sessions.remove(&id) {- finish(&mut multi, session, result);- }+ for (id, step) in retired {+ self.apply_step(id, step);}- if progressed {- continue;+ progressed+ }++ fn apply_step(&mut self, id: CurlTransferId, step: Result<Step, String>) {+ let result = match step {+ Ok(Step::Progress | Step::Idle) => return,+ Ok(Step::Closed) => Ok(()),+ Err(error) => Err(error),+ };+ if let Some(session) = self.sessions.remove(&id) {+ session.finish(&mut self.multi, result);}+ }- let mut fds: Vec<_> = sessions.values().filter_map(Session::wait_fd).collect();- let deadline = sessions+ fn wait_for_work(&mut self) {+ let mut fds: Vec<_> = self+ .sessions.values()- .filter(|session| !session.open)- .map(|session| session.deadline)- .chain(dns.next_deadline(|pending| Some(pending.deadline)))+ .filter_map(Session::wait_fd)+ .collect();+ let deadline = self+ .sessions+ .values()+ .filter_map(Session::handshake_deadline)+ .chain(self.dns.next_deadline(|pending| Some(pending.deadline))).min();let timeout = deadline.map(|deadline| {@@ -180,149 +209,14 @@ pub(super) fn run(.min(IDLE_WAIT)}).unwrap_or(IDLE_WAIT);- // poll includes libcurl's handshake sockets plus our upgraded sockets.- // Completed CONNECT_ONLY handles deliberately remain attached to this Multi.- if let Err(error) = multi.poll(&mut fds, timeout) {- for (_, session) in sessions.drain() {- finish(&mut multi, session, Err(error.to_string()));- }+ if let Err(error) = self.multi.poll(&mut fds, timeout) {+ self.fail_sessions(&error.to_string());}}- for (_, session) in sessions.drain() {- finish(- &mut multi,- session,- Err("curl WebSocket runtime shut down".to_owned()),- );- }- for pending in dns.drain() {- pending- .io- .finish(Err("curl WebSocket runtime shut down".to_owned()));- }-}--fn start(multi: &mut Multi, sessions: &mut HashMap<CurlTransferId, Session>, mut pending: Pending) {- if pending.io.cancelled() {- return;- }- let remaining = pending.deadline.saturating_duration_since(Instant::now());- if remaining.is_zero() {- pending- .io- .finish(Err("WebSocket handshake timed out".to_owned()));- return;- }- // timeout belongs to the transfer (opening handshake), not the upgraded session.- if let Err(error) = pending.easy.timeout(remaining) {- pending.io.finish(Err(error.to_string()));- return;- }- match multi.add2(pending.easy) {- Ok(mut handle) => {- if let Err(error) = handle.set_token(pending.id.token()) {- let _ = multi.remove2(handle);- pending.io.finish(Err(error.to_string()));- return;- }- sessions.insert(- pending.id,- Session {- handle,- deadline: pending.deadline,- open: false,- received_close: false,- io: pending.io,- },- );- }- Err(error) => pending.io.finish(Err(error.to_string())),- }-}-fn finish(multi: &mut Multi, session: Session, result: std::result::Result<(), String>) {- // Physically release the socket before publishing terminal channel closure.- let _ = multi.remove2(session.handle);- session.io.finish(result);-}--impl Session {- /// Err carries the terminal transport result; Ok reports useful work only.- fn drive(&mut self) -> std::result::Result<bool, std::result::Result<(), String>> {- let mut progressed = false;- for _ in 0..IO_BUDGET {- if self.io.cancelled() {- return Err(Ok(()));- }- {- // Keep the single frame resident across partial nonblocking writes.- let mut pending = self.io.control.send.lock();- if let Some(send) = &mut *pending {- match self- .handle- .ws_send(&send.frame.data[send.offset..], 0, send.frame.flags)- {- Ok(count) => {- send.offset += count;- progressed = true;- if send.offset == send.frame.data.len() {- let send = pending.take().expect("completed frame");- let _ = send.completed.send(send.offset);- }- }- Err(error) if error.is_again() => {- #[cfg(test)]- self.io.control.write_blocked.notify_one();- }- Err(error) => return Err(Err(format!("WebSocket send failed: {error}"))),- }- }- }- if self.received_close || !self.io.control.reading.load(Ordering::Acquire) {- break;- }- let permit = match self.io.events.try_reserve() {- Ok(permit) => permit,- Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {- #[cfg(test)]- self.io.control.read_blocked.notify_one();- break;- }- Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => return Err(Ok(())),- };- let mut data = vec![0; CHUNK_BYTES];- match self.handle.ws_recv(&mut data) {- Ok((count, frame)) => {- data.truncate(count);- // Let the caller answer Close before probing EOF. A peer can- // half-close TCP in the same packet as its Close frame.- self.received_close =- frame.flags().contains(super::WsFlags::CLOSE) && frame.bytes_left() == 0;- permit.send(CurlWebSocketEvent::Chunk { data, frame });- progressed = true;- }- Err(error) if error.is_again() => break,- Err(error) if error.is_got_nothing() => return Err(Ok(())),- Err(error) => return Err(Err(format!("WebSocket receive failed: {error}"))),- }- }- Ok(progressed)- }-- fn wait_fd(&self) -> Option<WaitFd> {- if !self.open {- return None;- }- let reading = self.io.events.capacity() > 0- && !self.received_close- && self.io.control.reading.load(Ordering::Acquire);- let writing = self.io.control.send.lock().is_some();- if !reading && !writing {- return None;+ fn fail_sessions(&mut self, error: &str) {+ for (_, session) in self.sessions.drain() {+ session.finish(&mut self.multi, Err(error.to_owned()));}- let mut fd = WaitFd::new();- fd.set_fd(self.handle.active_socket().ok()??);- fd.poll_on_read(reading).poll_on_write(writing);- Some(fd)}}diff --git a/moli-curl/src/websocket/session.rs b/moli-curl/src/websocket/session.rsnew file mode 100644index 000000000..812ebc0d9--- /dev/null+++ b/moli-curl/src/websocket/session.rs@@ -0,0 +1,225 @@+//! One attached libcurl handle. A completed CONNECT_ONLY transfer becomes an+//! open session; it stays in Multi until finish releases the socket.++use std::{sync::atomic::Ordering, time::Instant};++use curl::{+ easy::Easy2,+ multi::{Easy2Handle, Multi, WaitFd},+};++use super::{CurlWebSocketEvent, SessionIo, WsFlags, request::Handshake};+use crate::CurlTransferId;++const CHUNK_BYTES: usize = 16 * 1024;+const IO_BUDGET: usize = 8;++/// Result of advancing one native operation. Failures use Result::Err.+pub(super) enum Step {+ Progress,+ Idle,+ /// EOF, cancellation or a dropped receiver. Browser close policy is external.+ Closed,+}++enum Phase {+ Opening { deadline: Instant },+ Open,+ ReceivedClose,+}++pub(super) struct Session {+ handle: Easy2Handle<Handshake>,+ io: SessionIo,+ phase: Phase,+}++impl Session {+ pub(super) fn attach(+ multi: &mut Multi,+ id: CurlTransferId,+ mut easy: Easy2<Handshake>,+ io: SessionIo,+ deadline: Instant,+ ) -> Option<Self> {+ if io.cancelled() {+ return None;+ }+ let remaining = deadline.saturating_duration_since(Instant::now());+ if remaining.is_zero() {+ io.finish(Err("WebSocket handshake timed out".to_owned()));+ return None;+ }+ // timeout belongs to the opening handshake, not the upgraded session.+ if let Err(error) = easy.timeout(remaining) {+ io.finish(Err(error.to_string()));+ return None;+ }+ match multi.add2(easy) {+ Ok(mut handle) => {+ if let Err(error) = handle.set_token(id.token()) {+ let _ = multi.remove2(handle);+ io.finish(Err(error.to_string()));+ return None;+ }+ Some(Self {+ handle,+ io,+ phase: Phase::Opening { deadline },+ })+ }+ Err(error) => {+ io.finish(Err(error.to_string()));+ None+ }+ }+ }++ pub(super) fn handshake_deadline(&self) -> Option<Instant> {+ match self.phase {+ Phase::Opening { deadline } => Some(deadline),+ Phase::Open | Phase::ReceivedClose => None,+ }+ }++ pub(super) fn complete_handshake(+ &mut self,+ result: Result<(), curl::Error>,+ ) -> Result<Step, String> {+ if self.handshake_deadline().is_none() {+ return Ok(Step::Idle);+ }+ let handshake = self.handle.get_mut();+ let result = match handshake.error.take() {+ Some(error) => Err(error),+ None => result.map_err(|error| error.to_string()),+ };+ let event = CurlWebSocketEvent::Handshake {+ request: std::mem::take(&mut handshake.request),+ response: std::mem::take(&mut handshake.response),+ result: result.clone(),+ };+ // No data events precede the handshake, so this queue has a free slot.+ let delivered = self.io.events.try_send(event).is_ok();+ result?;+ if !delivered {+ return Ok(Step::Closed);+ }+ self.phase = Phase::Open;+ Ok(Step::Idle)+ }++ pub(super) fn advance(&mut self) -> Result<Step, String> {+ if self.io.cancelled() {+ return Ok(Step::Closed);+ }+ if let Some(deadline) = self.handshake_deadline() {+ return if deadline <= Instant::now() {+ Err("WebSocket handshake timed out".to_owned())+ } else {+ Ok(Step::Idle)+ };+ }+ let mut progressed = false;+ for _ in 0..IO_BUDGET {+ if self.io.cancelled() {+ return Ok(Step::Closed);+ }+ // Receive delivery must never hold up a pending write.+ progressed |= self.write_pending()?;+ match self.read_chunk()? {+ Step::Progress => progressed = true,+ Step::Idle => break,+ Step::Closed => return Ok(Step::Closed),+ }+ }+ Ok(if progressed {+ Step::Progress+ } else {+ Step::Idle+ })+ }++ fn write_pending(&mut self) -> Result<bool, String> {+ // Keep the single frame resident across partial nonblocking writes.+ let mut pending = self.io.control.send.lock();+ let Some(send) = &mut *pending else {+ return Ok(false);+ };+ match self+ .handle+ .ws_send(&send.frame.data[send.offset..], 0, send.frame.flags)+ {+ Ok(count) => {+ send.offset += count;+ if send.offset == send.frame.data.len() {+ let send = pending.take().expect("completed frame");+ let _ = send.completed.send(send.offset);+ }+ Ok(true)+ }+ Err(error) if error.is_again() => {+ #[cfg(test)]+ self.io.control.write_blocked.notify_one();+ Ok(false)+ }+ Err(error) => Err(format!("WebSocket send failed: {error}")),+ }+ }++ fn reading_enabled(&self) -> bool {+ matches!(self.phase, Phase::Open) && self.io.control.reading.load(Ordering::Acquire)+ }++ fn read_chunk(&mut self) -> Result<Step, String> {+ if !self.reading_enabled() {+ return Ok(Step::Idle);+ }+ let permit = match self.io.events.try_reserve() {+ Ok(permit) => permit,+ Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {+ #[cfg(test)]+ self.io.control.read_blocked.notify_one();+ return Ok(Step::Idle);+ }+ Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => return Ok(Step::Closed),+ };+ let mut data = vec![0; CHUNK_BYTES];+ match self.handle.ws_recv(&mut data) {+ Ok((count, frame)) => {+ data.truncate(count);+ // Let the caller answer Close before probing EOF. A peer can+ // half-close TCP in the same packet as its Close frame.+ if frame.flags().contains(WsFlags::CLOSE) && frame.bytes_left() == 0 {+ self.phase = Phase::ReceivedClose;+ }+ permit.send(CurlWebSocketEvent::Chunk { data, frame });+ Ok(Step::Progress)+ }+ Err(error) if error.is_again() => Ok(Step::Idle),+ Err(error) if error.is_got_nothing() => Ok(Step::Closed),+ Err(error) => Err(format!("WebSocket receive failed: {error}")),+ }+ }++ pub(super) fn wait_fd(&self) -> Option<WaitFd> {+ if self.handshake_deadline().is_some() {+ return None;+ }+ let reading = self.reading_enabled() && self.io.events.capacity() > 0;+ let writing = self.io.control.send.lock().is_some();+ if !reading && !writing {+ return None;+ }+ let mut fd = WaitFd::new();+ fd.set_fd(self.handle.active_socket().ok()??);+ fd.poll_on_read(reading).poll_on_write(writing);+ Some(fd)+ }++ pub(super) fn finish(self, multi: &mut Multi, result: Result<(), String>) {+ // Physically release the socket before publishing terminal channel closure.+ let _ = multi.remove2(self.handle);+ self.io.finish(result);+ }+}