Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(encoding): add a native streaming XHR response decoder
ldm098a342af737d
diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rsindex d8ff46f62..ae170f03d 100644--- a/moli-encoding/src/lib.rs+++ b/moli-encoding/src/lib.rs@@ -7,6 +7,7 @@ mod legacy_text;mod percent;mod query;mod script;+mod xhr;pub use document::{DocumentStreamingDecoder, LegacyEncodingDetector, decode_html_document,@@ -23,6 +24,7 @@ pub use labels::{pub use legacy_text::decode_text_for_legacy_web;pub use query::encode_url_query_for_legacy_web;pub use script::{decode_classic_script_source, decode_utf8};+pub use xhr::{XhrResponseDecoder, XhrResponseTextKind};#[cfg(test)]mod tests;diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 4e9f50398..5ddf29831 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -1415,3 +1415,135 @@ fn header_charset_keeps_an_escaped_quote_as_data() {assert_eq!(label, "utf-8\"");assert!(encoding_for_label(&label).is_none());}++#[test]+fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() {+ use XhrResponseTextKind::{Html, Text, Xml};+ let xml = b"<?xml version='1.0' encoding='windows-1252'?><x>\xff</x>";+ let xml_utf16 = "<?xml version='1.0' encoding='UTF-16'?><x>é</x>";+ let cases = [+ (+ Text,+ None,+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),+ "機",+ "UTF-8",+ ),+ (+ Text,+ Some(encoding_rs::WINDOWS_1252),+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),+ "機",+ "UTF-8",+ ),+ (+ Text,+ None,+ b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(),+ "\u{feff}",+ "UTF-8",+ ),+ (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"),+ (+ Text,+ Some(encoding_rs::UTF_8),+ b"\xfe\xff\0A\0\xe9".to_vec(),+ "Aé",+ "UTF-16BE",+ ),+ (+ Text,+ Some(encoding_rs::UTF_16LE),+ b"\xe9\0".to_vec(),+ "é",+ "UTF-16LE",+ ),+ (+ Text,+ Some(encoding_rs::SHIFT_JIS),+ b"\x83e\x83X\x83g".to_vec(),+ "テスト",+ "Shift_JIS",+ ),+ (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"),+ (+ Xml,+ None,+ xml.to_vec(),+ "<?xml version='1.0' encoding='windows-1252'?><x>ÿ</x>",+ "windows-1252",+ ),+ (+ Xml,+ Some(encoding_rs::UTF_8),+ xml.to_vec(),+ "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",+ "UTF-8",+ ),+ (+ Text,+ None,+ xml.to_vec(),+ "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",+ "UTF-8",+ ),+ (+ Xml,+ None,+ xml_utf16+ .encode_utf16()+ .flat_map(u16::to_le_bytes)+ .collect(),+ xml_utf16,+ "UTF-16LE",+ ),+ (+ Html,+ None,+ b"<meta charset=windows-1252><x>\xff</x>".to_vec(),+ "<meta charset=windows-1252><x>ÿ</x>",+ "windows-1252",+ ),+ (+ Html,+ Some(encoding_rs::UTF_8),+ b"<meta charset=windows-1252><x>\xff</x>".to_vec(),+ "<meta charset=windows-1252><x>\u{fffd}</x>",+ "UTF-8",+ ),+ (+ Html,+ None,+ b"<x>\xe6\xa9\x9f</x>".to_vec(),+ "<x>機</x>",+ "UTF-8",+ ),+ ];+ for (kind, transport, bytes, expected, encoding) in cases {+ for boundary in 0..=bytes.len() {+ let mut decoder = XhrResponseDecoder::new(kind, transport);+ let mut text = decoder.push(&bytes[..boundary]);+ text.push_str(&decoder.push(&bytes[boundary..]));+ text.push_str(&decoder.finish());+ assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}");+ assert_eq!(decoder.encoding_name(), encoding);+ }+ }+}++#[test]+fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() {+ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);+ assert_eq!(decoder.push(b"\xef"), "");+ assert_eq!(decoder.push(b"\xbb"), "");+ assert_eq!(decoder.push(b"\xbf"), "");+ assert_eq!(decoder.push(b"a\xc2"), "a");+ assert_eq!(decoder.finish(), "\u{fffd}");++ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);+ assert_eq!(decoder.push(b"\xff"), "");+ assert_eq!(decoder.push(b"\xfeA"), "");+ assert_eq!(decoder.push(b"\0\xe9"), "A");+ assert_eq!(decoder.push(b"\0"), "é");+ assert_eq!(decoder.finish(), "");+}diff --git a/moli-encoding/src/xhr.rs b/moli-encoding/src/xhr.rsnew file mode 100644index 000000000..9b3acd696--- /dev/null+++ b/moli-encoding/src/xhr.rs@@ -0,0 +1,171 @@+use crate::document::DocumentStreamingDecoder;+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE};++/// In-band encoding declarations are only meaningful for legacy XML text+/// responses and document responses. Explicit `responseType = "text"` does+/// not inspect markup.+#[derive(Clone, Copy)]+pub enum XhrResponseTextKind {+ Text,+ Xml,+ Html,+}++pub struct XhrResponseDecoder {+ kind: XhrResponseTextKind,+ transport_encoding: Option<&'static Encoding>,+ prefix: Vec<u8>,+ html: Option<DocumentStreamingDecoder>,+ decoder: Option<Decoder>,+ selected_encoding: Option<&'static Encoding>,+}++impl XhrResponseDecoder {+ pub fn new(kind: XhrResponseTextKind, transport_encoding: Option<&'static Encoding>) -> Self {+ let html = matches!(kind, XhrResponseTextKind::Html).then(|| {+ let headers = transport_encoding+ .map(|encoding| {+ vec![(+ "content-type".to_owned(),+ format!("text/html;charset={}", encoding.name()).into_bytes(),+ )]+ })+ .unwrap_or_default();+ DocumentStreamingDecoder::new_with_fallback(&headers, Some("UTF-8"))+ });+ Self {+ kind,+ transport_encoding,+ prefix: Vec::new(),+ html,+ decoder: None,+ selected_encoding: None,+ }+ }++ pub fn encoding_name(&self) -> &'static str {+ self.html+ .as_ref()+ .map(DocumentStreamingDecoder::document_encoding_name)+ .unwrap_or_else(|| self.selected_encoding.unwrap_or(UTF_8).name())+ }++ pub fn push(&mut self, bytes: &[u8]) -> String {+ if let Some(html) = self.html.as_mut() {+ return html.push(bytes).concat();+ }+ self.decode(bytes, false)+ }++ pub fn finish(&mut self) -> String {+ if let Some(html) = self.html.as_mut() {+ return html.finish().unwrap_or_default();+ }+ self.decode(&[], true)+ }++ fn decode(&mut self, bytes: &[u8], last: bool) -> String {+ if self.decoder.is_some() {+ return self.decode_selected(bytes, last);+ }+ self.prefix.extend_from_slice(bytes);+ let Some(encoding) = self.select_encoding(last) else {+ return String::new();+ };+ self.selected_encoding = Some(encoding);+ self.decoder = Some(encoding.new_decoder_with_bom_removal());+ let prefix = std::mem::take(&mut self.prefix);+ self.decode_selected(&prefix, last)+ }++ fn select_encoding(&mut self, last: bool) -> Option<&'static Encoding> {+ if let Some((encoding, _)) = Encoding::for_bom(&self.prefix) {+ return Some(encoding);+ }+ if !last+ && matches!(+ self.prefix.as_slice(),+ [] | [0xef] | [0xef, 0xbb] | [0xff] | [0xfe]+ )+ {+ return None;+ }+ if let Some(encoding) = self.transport_encoding {+ return Some(encoding);+ }+ match self.kind {+ XhrResponseTextKind::Text | XhrResponseTextKind::Html => Some(UTF_8),+ XhrResponseTextKind::Xml => {+ for (signature, encoding) in+ [(b"<\0?\0".as_slice(), UTF_16LE), (b"\0<\0?", UTF_16BE)]+ {+ if self.prefix.starts_with(signature) {+ return Some(encoding);+ }+ if !last && signature.starts_with(&self.prefix) {+ return None;+ }+ }+ if !last && b"<?xml".starts_with(&self.prefix) {+ return None;+ }+ if self.prefix.starts_with(b"<?xml") {+ if let Some(end) = self.prefix.iter().position(|byte| *byte == b'>') {+ if let Some(label) = xml_declaration_encoding(&self.prefix[..end]) {+ let encoding = Encoding::for_label(label);+ return Some(encoding.unwrap_or(UTF_8));+ }+ } else if !last {+ return None;+ }+ }+ Some(UTF_8)+ }+ }+ }++ fn decode_selected(&mut self, bytes: &[u8], last: bool) -> String {+ let decoder = self.decoder.as_mut().expect("XHR encoding was selected");+ let mut output = String::new();+ let mut read = 0;+ loop {+ let input = &bytes[read..];+ output.reserve(+ decoder+ .max_utf8_buffer_length(input.len())+ .unwrap_or(input.len().saturating_mul(3).saturating_add(16)),+ );+ let (result, consumed, _) = decoder.decode_to_string(input, &mut output, last);+ read += consumed;+ if result == CoderResult::InputEmpty {+ return output;+ }+ }+ }+}++fn xml_declaration_encoding(declaration: &[u8]) -> Option<&[u8]> {+ let mut rest = declaration.strip_prefix(b"<?xml")?;+ while rest.first().is_some_and(u8::is_ascii_whitespace) {+ rest = rest.trim_ascii_start();+ let end = rest+ .iter()+ .position(|byte| byte.is_ascii_whitespace() || *byte == b'=')?;+ let name = &rest[..end];+ rest = rest[end..]+ .trim_ascii_start()+ .strip_prefix(b"=")?+ .trim_ascii_start();+ let quote = *rest.first()?;+ if !matches!(quote, b'\'' | b'"') {+ return None;+ }+ rest = &rest[1..];+ let end = rest.iter().position(|byte| *byte == quote)?;+ if name == b"encoding" {+ return Some(&rest[..end]);+ }+ rest = &rest[end + 1..];+ }+ None+}