Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(encoding): add a native streaming XHR response decoder

ldm098a342af737d
diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rs
index d8ff46f62..ae170f03d 100644
--- a/moli-encoding/src/lib.rs
+++ b/moli-encoding/src/lib.rs
@@ -7,6 +7,7 @@ mod legacy_text;
mod percent;
mod query;
mod script;
+mod xhr;
pub use document::{
DocumentStreamingDecoder, LegacyEncodingDetector, decode_html_document,
@@ -23,6 +24,7 @@ pub use labels::{
pub use legacy_text::decode_text_for_legacy_web;
pub use query::encode_url_query_for_legacy_web;
pub use script::{decode_classic_script_source, decode_utf8};
+pub use xhr::{XhrResponseDecoder, XhrResponseTextKind};
#[cfg(test)]
mod tests;
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 4e9f50398..5ddf29831 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -1415,3 +1415,135 @@ fn header_charset_keeps_an_escaped_quote_as_data() {
assert_eq!(label, "utf-8\"");
assert!(encoding_for_label(&label).is_none());
}
+
+#[test]
+fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() {
+ use XhrResponseTextKind::{Html, Text, Xml};
+ let xml = b"<?xml version='1.0' encoding='windows-1252'?><x>\xff</x>";
+ let xml_utf16 = "<?xml version='1.0' encoding='UTF-16'?><x>é</x>";
+ let cases = [
+ (
+ Text,
+ None,
+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
+ "機",
+ "UTF-8",
+ ),
+ (
+ Text,
+ Some(encoding_rs::WINDOWS_1252),
+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
+ "機",
+ "UTF-8",
+ ),
+ (
+ Text,
+ None,
+ b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(),
+ "\u{feff}",
+ "UTF-8",
+ ),
+ (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"),
+ (
+ Text,
+ Some(encoding_rs::UTF_8),
+ b"\xfe\xff\0A\0\xe9".to_vec(),
+ "Aé",
+ "UTF-16BE",
+ ),
+ (
+ Text,
+ Some(encoding_rs::UTF_16LE),
+ b"\xe9\0".to_vec(),
+ "é",
+ "UTF-16LE",
+ ),
+ (
+ Text,
+ Some(encoding_rs::SHIFT_JIS),
+ b"\x83e\x83X\x83g".to_vec(),
+ "テスト",
+ "Shift_JIS",
+ ),
+ (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"),
+ (
+ Xml,
+ None,
+ xml.to_vec(),
+ "<?xml version='1.0' encoding='windows-1252'?><x>ÿ</x>",
+ "windows-1252",
+ ),
+ (
+ Xml,
+ Some(encoding_rs::UTF_8),
+ xml.to_vec(),
+ "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
+ "UTF-8",
+ ),
+ (
+ Text,
+ None,
+ xml.to_vec(),
+ "<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
+ "UTF-8",
+ ),
+ (
+ Xml,
+ None,
+ xml_utf16
+ .encode_utf16()
+ .flat_map(u16::to_le_bytes)
+ .collect(),
+ xml_utf16,
+ "UTF-16LE",
+ ),
+ (
+ Html,
+ None,
+ b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
+ "<meta charset=windows-1252><x>ÿ</x>",
+ "windows-1252",
+ ),
+ (
+ Html,
+ Some(encoding_rs::UTF_8),
+ b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
+ "<meta charset=windows-1252><x>\u{fffd}</x>",
+ "UTF-8",
+ ),
+ (
+ Html,
+ None,
+ b"<x>\xe6\xa9\x9f</x>".to_vec(),
+ "<x>機</x>",
+ "UTF-8",
+ ),
+ ];
+ for (kind, transport, bytes, expected, encoding) in cases {
+ for boundary in 0..=bytes.len() {
+ let mut decoder = XhrResponseDecoder::new(kind, transport);
+ let mut text = decoder.push(&bytes[..boundary]);
+ text.push_str(&decoder.push(&bytes[boundary..]));
+ text.push_str(&decoder.finish());
+ assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}");
+ assert_eq!(decoder.encoding_name(), encoding);
+ }
+ }
+}
+
+#[test]
+fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() {
+ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
+ assert_eq!(decoder.push(b"\xef"), "");
+ assert_eq!(decoder.push(b"\xbb"), "");
+ assert_eq!(decoder.push(b"\xbf"), "");
+ assert_eq!(decoder.push(b"a\xc2"), "a");
+ assert_eq!(decoder.finish(), "\u{fffd}");
+
+ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
+ assert_eq!(decoder.push(b"\xff"), "");
+ assert_eq!(decoder.push(b"\xfeA"), "");
+ assert_eq!(decoder.push(b"\0\xe9"), "A");
+ assert_eq!(decoder.push(b"\0"), "é");
+ assert_eq!(decoder.finish(), "");
+}
diff --git a/moli-encoding/src/xhr.rs b/moli-encoding/src/xhr.rs
new file mode 100644
index 000000000..9b3acd696
--- /dev/null
+++ b/moli-encoding/src/xhr.rs
@@ -0,0 +1,171 @@
+use crate::document::DocumentStreamingDecoder;
+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8, UTF_16BE, UTF_16LE};
+
+/// In-band encoding declarations are only meaningful for legacy XML text
+/// responses and document responses. Explicit `responseType = "text"` does
+/// not inspect markup.
+#[derive(Clone, Copy)]
+pub enum XhrResponseTextKind {
+ Text,
+ Xml,
+ Html,
+}
+
+pub struct XhrResponseDecoder {
+ kind: XhrResponseTextKind,
+ transport_encoding: Option<&'static Encoding>,
+ prefix: Vec<u8>,
+ html: Option<DocumentStreamingDecoder>,
+ decoder: Option<Decoder>,
+ selected_encoding: Option<&'static Encoding>,
+}
+
+impl XhrResponseDecoder {
+ pub fn new(kind: XhrResponseTextKind, transport_encoding: Option<&'static Encoding>) -> Self {
+ let html = matches!(kind, XhrResponseTextKind::Html).then(|| {
+ let headers = transport_encoding
+ .map(|encoding| {
+ vec![(
+ "content-type".to_owned(),
+ format!("text/html;charset={}", encoding.name()).into_bytes(),
+ )]
+ })
+ .unwrap_or_default();
+ DocumentStreamingDecoder::new_with_fallback(&headers, Some("UTF-8"))
+ });
+ Self {
+ kind,
+ transport_encoding,
+ prefix: Vec::new(),
+ html,
+ decoder: None,
+ selected_encoding: None,
+ }
+ }
+
+ pub fn encoding_name(&self) -> &'static str {
+ self.html
+ .as_ref()
+ .map(DocumentStreamingDecoder::document_encoding_name)
+ .unwrap_or_else(|| self.selected_encoding.unwrap_or(UTF_8).name())
+ }
+
+ pub fn push(&mut self, bytes: &[u8]) -> String {
+ if let Some(html) = self.html.as_mut() {
+ return html.push(bytes).concat();
+ }
+ self.decode(bytes, false)
+ }
+
+ pub fn finish(&mut self) -> String {
+ if let Some(html) = self.html.as_mut() {
+ return html.finish().unwrap_or_default();
+ }
+ self.decode(&[], true)
+ }
+
+ fn decode(&mut self, bytes: &[u8], last: bool) -> String {
+ if self.decoder.is_some() {
+ return self.decode_selected(bytes, last);
+ }
+ self.prefix.extend_from_slice(bytes);
+ let Some(encoding) = self.select_encoding(last) else {
+ return String::new();
+ };
+ self.selected_encoding = Some(encoding);
+ self.decoder = Some(encoding.new_decoder_with_bom_removal());
+ let prefix = std::mem::take(&mut self.prefix);
+ self.decode_selected(&prefix, last)
+ }
+
+ fn select_encoding(&mut self, last: bool) -> Option<&'static Encoding> {
+ if let Some((encoding, _)) = Encoding::for_bom(&self.prefix) {
+ return Some(encoding);
+ }
+ if !last
+ && matches!(
+ self.prefix.as_slice(),
+ [] | [0xef] | [0xef, 0xbb] | [0xff] | [0xfe]
+ )
+ {
+ return None;
+ }
+ if let Some(encoding) = self.transport_encoding {
+ return Some(encoding);
+ }
+ match self.kind {
+ XhrResponseTextKind::Text | XhrResponseTextKind::Html => Some(UTF_8),
+ XhrResponseTextKind::Xml => {
+ for (signature, encoding) in
+ [(b"<\0?\0".as_slice(), UTF_16LE), (b"\0<\0?", UTF_16BE)]
+ {
+ if self.prefix.starts_with(signature) {
+ return Some(encoding);
+ }
+ if !last && signature.starts_with(&self.prefix) {
+ return None;
+ }
+ }
+ if !last && b"<?xml".starts_with(&self.prefix) {
+ return None;
+ }
+ if self.prefix.starts_with(b"<?xml") {
+ if let Some(end) = self.prefix.iter().position(|byte| *byte == b'>') {
+ if let Some(label) = xml_declaration_encoding(&self.prefix[..end]) {
+ let encoding = Encoding::for_label(label);
+ return Some(encoding.unwrap_or(UTF_8));
+ }
+ } else if !last {
+ return None;
+ }
+ }
+ Some(UTF_8)
+ }
+ }
+ }
+
+ fn decode_selected(&mut self, bytes: &[u8], last: bool) -> String {
+ let decoder = self.decoder.as_mut().expect("XHR encoding was selected");
+ let mut output = String::new();
+ let mut read = 0;
+ loop {
+ let input = &bytes[read..];
+ output.reserve(
+ decoder
+ .max_utf8_buffer_length(input.len())
+ .unwrap_or(input.len().saturating_mul(3).saturating_add(16)),
+ );
+ let (result, consumed, _) = decoder.decode_to_string(input, &mut output, last);
+ read += consumed;
+ if result == CoderResult::InputEmpty {
+ return output;
+ }
+ }
+ }
+}
+
+fn xml_declaration_encoding(declaration: &[u8]) -> Option<&[u8]> {
+ let mut rest = declaration.strip_prefix(b"<?xml")?;
+ while rest.first().is_some_and(u8::is_ascii_whitespace) {
+ rest = rest.trim_ascii_start();
+ let end = rest
+ .iter()
+ .position(|byte| byte.is_ascii_whitespace() || *byte == b'=')?;
+ let name = &rest[..end];
+ rest = rest[end..]
+ .trim_ascii_start()
+ .strip_prefix(b"=")?
+ .trim_ascii_start();
+ let quote = *rest.first()?;
+ if !matches!(quote, b'\'' | b'"') {
+ return None;
+ }
+ rest = &rest[1..];
+ let end = rest.iter().position(|byte| *byte == quote)?;
+ if name == b"encoding" {
+ return Some(&rest[..end]);
+ }
+ rest = &rest[end + 1..];
+ }
+ None
+}