Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): preserve split UTF-16 XML signatures

ldm06fa68cadbed0
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index f6ea1e237..b7e3da79b 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -4,6 +4,8 @@ use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};
use crate::{charset_from_headers, encoding_for_label};
const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";
+const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";
+const UTF16BE_XML_PREFIX: &[u8; 6] = b"\0<\0?\0x";
pub fn decode_html_document(bytes: &[u8], headers: &[(String, String)]) -> (String, &'static str) {
decode_html_document_with_fallback(bytes, headers, None)
@@ -189,6 +191,7 @@ impl HtmlDocumentStreamingDecoder {
fn take_safe_ascii_sniff_prefix(&mut self) -> String {
if self.emitted_sniff_len >= self.sniff_buffer.len()
|| bytes_could_still_be_bom_prefix(&self.sniff_buffer)
+ || bytes_could_still_be_utf16_xml_prefix(&self.sniff_buffer)
{
return String::new();
}
@@ -221,6 +224,11 @@ fn bytes_could_still_be_bom_prefix(bytes: &[u8]) -> bool {
matches!(bytes, [] | [0xEF] | [0xEF, 0xBB] | [0xFF] | [0xFE])
}
+fn bytes_could_still_be_utf16_xml_prefix(bytes: &[u8]) -> bool {
+ bytes.len() < UTF16LE_XML_PREFIX.len()
+ && (UTF16LE_XML_PREFIX.starts_with(bytes) || UTF16BE_XML_PREFIX.starts_with(bytes))
+}
+
fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {
if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
return Some(encoding_rs::UTF_8);
@@ -242,24 +250,10 @@ fn encoding_for_document_bom(bytes: &[u8]) -> Option<&'static Encoding> {
///
/// These match the start of `<?xml` without requiring a BOM or transport charset.
fn encoding_for_document_xml_declaration(bytes: &[u8]) -> Option<&'static Encoding> {
- if bytes.len() >= 6
- && bytes[0] == 0x3C
- && bytes[1] == 0x00
- && bytes[2] == 0x3F
- && bytes[3] == 0x00
- && bytes[4] == 0x78
- && bytes[5] == 0x00
- {
+ if bytes.starts_with(UTF16LE_XML_PREFIX) {
return Some(encoding_rs::UTF_16LE);
}
- if bytes.len() >= 6
- && bytes[0] == 0x00
- && bytes[1] == 0x3C
- && bytes[2] == 0x00
- && bytes[3] == 0x3F
- && bytes[4] == 0x00
- && bytes[5] == 0x78
- {
+ if bytes.starts_with(UTF16BE_XML_PREFIX) {
return Some(encoding_rs::UTF_16BE);
}
None
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 6efc38a7f..91c4b7bf1 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -527,6 +527,77 @@ fn utf16be_bytes(input: &str) -> Vec<u8> {
.collect()
}
+fn assert_bom_less_utf16_xml_decodes_across_signature_splits(
+ input: &[u8],
+ expected_text: &str,
+ expected_encoding: &str,
+) {
+ for split in 1..6 {
+ let mut decoder = HtmlDocumentStreamingDecoder::new(&[]);
+ assert_eq!(
+ decoder.push(&input[..split]),
+ Vec::<String>::new(),
+ "split after signature byte {split} must remain buffered"
+ );
+
+ let mut decoded = decoder.push(&input[split..]).concat();
+ if let Some(tail) = decoder.finish() {
+ decoded.push_str(&tail);
+ }
+
+ assert_eq!(
+ decoder.selected_encoding_name(),
+ Some(expected_encoding),
+ "wrong encoding after signature byte {split} split"
+ );
+ assert_eq!(
+ decoded, expected_text,
+ "misdecoded document after signature byte {split} split"
+ );
+ }
+}
+
+#[test]
+fn bom_less_utf16le_xml_signature_survives_every_streaming_split() {
+ let source = "<?xml version=\"1.0\"?><html><head></head><body>little endian</body></html>";
+ let input = utf16le_bytes(source);
+
+ assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16LE");
+}
+
+#[test]
+fn bom_less_utf16be_xml_signature_survives_every_streaming_split() {
+ let source = "<?xml version=\"1.0\"?><html><head></head><body>big endian</body></html>";
+ let input = utf16be_bytes(source);
+
+ assert_bom_less_utf16_xml_decodes_across_signature_splits(&input, source, "UTF-16BE");
+}
+
+#[test]
+fn diverged_utf16_xml_signature_prefix_resumes_ascii_streaming() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new(&[]);
+
+ assert!(decoder.push(b"<").is_empty());
+ assert_eq!(
+ decoder.push(b"!doctype html><p>ordinary html"),
+ vec!["<!doctype html><p>ordinary html"]
+ );
+ assert_eq!(decoder.finish(), None);
+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
+}
+
+#[test]
+fn transport_charset_does_not_wait_for_a_utf16_xml_signature_prefix() {
+ let headers = vec![(
+ "Content-Type".to_owned(),
+ "text/html; charset=windows-1252".to_owned(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new(&headers);
+
+ assert_eq!(decoder.push(b"<"), vec!["<"]);
+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
+}
+
#[test]
fn bom_less_utf16be_xml_declaration_is_detected() {
let input = utf16be_bytes(
@@ -561,9 +632,10 @@ fn bom_less_utf16le_uppercase_x_not_detected() {
fn bom_less_utf16le_truncated_not_detected() {
let input: Vec<u8> = vec![0x3C, 0x00, 0x3F, 0x00];
- let (_, encoding) = decode_html_document(&input, &[]);
+ let (text, encoding) = decode_html_document(&input, &[]);
assert_eq!(encoding, "windows-1252");
+ assert_eq!(text, "<\0?\0");
}
#[test]
@@ -588,7 +660,8 @@ fn bom_less_utf16be_uppercase_x_not_detected() {
fn bom_less_utf16be_truncated_not_detected() {
let input: Vec<u8> = vec![0x00, 0x3C, 0x00, 0x3F];
- let (_, encoding) = decode_html_document(&input, &[]);
+ let (text, encoding) = decode_html_document(&input, &[]);
assert_eq!(encoding, "windows-1252");
+ assert_eq!(text, "\0<\0?");
}