Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(encoding): extract legacy charset detector
ldm05e173f3a5a02
diff --git a/Cargo.lock b/Cargo.lockindex f5b54c1c5..68ffecd43 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2447,12 +2447,19 @@ dependencies = [name = "moli-encoding"version = "0.1.0"dependencies = [- "compact-enc-det","encoding_rs","moli-charset-parser","moli-content-type",]+[[package]]+name = "moli-encoding-detector"+version = "0.1.0"+dependencies = [+ "compact-enc-det",+ "encoding_rs",+]+[[package]]name = "moli-fetch"version = "0.1.0"@@ -2828,6 +2835,7 @@ dependencies = ["moli-css-parse","moli-dom","moli-encoding",+ "moli-encoding-detector","moli-fetch","moli-file-api","moli-geometry",diff --git a/Cargo.toml b/Cargo.tomlindex bd8c29e00..5e8bff65e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -19,6 +19,7 @@ members = ["moli-dns-resolver","moli-protocol","moli-encoding",+ "moli-encoding-detector","moli-core","moli-import-map","moli-image",diff --git a/moli-encoding-detector/Cargo.toml b/moli-encoding-detector/Cargo.tomlnew file mode 100644index 000000000..a33635d9f--- /dev/null+++ b/moli-encoding-detector/Cargo.toml@@ -0,0 +1,12 @@+[package]+license.workspace = true+name = "moli-encoding-detector"+version = "0.1.0"+edition = "2024"++[dependencies]+compact-enc-det = "0.1"+encoding_rs = "0.8"++[lints]+workspace = truediff --git a/moli-encoding/src/detector.rs b/moli-encoding-detector/src/lib.rssimilarity index 88%rename from moli-encoding/src/detector.rsrename to moli-encoding-detector/src/lib.rsindex af1add771..58d76988b 100644--- a/moli-encoding/src/detector.rs+++ b/moli-encoding-detector/src/lib.rs@@ -1,7 +1,13 @@+//! Heuristic detection of legacy encodings used by web documents.+use compact_enc_det::{DetectHints, Encoding as CedEncoding, TextCorpusType, detect_encoding};use encoding_rs::Encoding;-pub(crate) fn detect_legacy_html_encoding(+/// Detects a legacy HTML encoding using Chromium's CED configuration.+///+/// UTF encodings are deliberately excluded: browser-side heuristics exist for+/// legacy content, not as a substitute for labelling modern UTF-8 documents.+pub fn detect_legacy_html_encoding(bytes: &[u8],url_hint: Option<&str>,) -> Option<&'static Encoding> {diff --git a/moli-encoding/Cargo.toml b/moli-encoding/Cargo.tomlindex 5dd413cff..908d89776 100644--- a/moli-encoding/Cargo.toml+++ b/moli-encoding/Cargo.toml@@ -5,7 +5,6 @@ version = "0.1.0"edition = "2024"[dependencies]-compact-enc-det = "0.1"encoding_rs = "0.8"moli-charset-parser = { path = "../moli-charset-parser" }moli-content-type = { path = "../moli-content-type" }diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex 02df2ce05..d1a95a50e 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -1,14 +1,18 @@use encoding_rs::{CoderResult, Decoder, Encoding};use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};-use crate::{- detector::detect_legacy_html_encoding, encoding_for_label, encoding_from_response_headers,-};+use crate::{encoding_for_label, encoding_from_response_headers};const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";const UTF16BE_XML_PREFIX: &[u8; 6] = b"\0<\0?\0x";+/// Stateless hook for optional, heuristic detection of legacy document+/// encodings. Deterministic BOM, transport and in-document declarations are+/// resolved before this hook is called.+pub type LegacyEncodingDetector =+ fn(bytes: &[u8], url_hint: Option<&str>) -> Option<&'static Encoding>;+pub fn decode_html_document(bytes: &[u8], headers: &[(String, String)]) -> (String, &'static str) {decode_html_document_with_fallback(bytes, headers, None)}@@ -39,7 +43,7 @@ pub struct HtmlDocumentStreamingDecoder {emitted_sniff_len: usize,meta_prescan_fed_len: usize,meta_charset_parser: HtmlMetaCharsetParser,- allow_legacy_content_detection: bool,+ legacy_encoding_detector: Option<LegacyEncodingDetector>,url_hint: Option<String>,decoder: Option<Decoder>,selected_encoding: Option<&'static Encoding>,@@ -47,24 +51,29 @@ pub struct HtmlDocumentStreamingDecoder {impl HtmlDocumentStreamingDecoder {pub fn new(headers: &[(String, String)]) -> Self {- Self::new_with_fallback_and_url(headers, None, None)+ Self::new_with_options(headers, None, None, None)}- pub fn new_for_url(headers: &[(String, String)], url_hint: &str) -> Self {- Self::new_with_fallback_and_url(headers, None, Some(url_hint))+ pub fn new_with_legacy_encoding_detector(+ headers: &[(String, String)],+ url_hint: &str,+ detector: LegacyEncodingDetector,+ ) -> Self {+ Self::new_with_options(headers, None, Some(url_hint), Some(detector))}pub fn new_with_fallback(headers: &[(String, String)],fallback_encoding: Option<&str>,) -> Self {- Self::new_with_fallback_and_url(headers, fallback_encoding, None)+ Self::new_with_options(headers, fallback_encoding, None, None)}- fn new_with_fallback_and_url(+ fn new_with_options(headers: &[(String, String)],fallback_encoding: Option<&str>,url_hint: Option<&str>,+ legacy_encoding_detector: Option<LegacyEncodingDetector>,) -> Self {Self {transport_encoding: encoding_from_response_headers(headers),@@ -75,10 +84,7 @@ impl HtmlDocumentStreamingDecoder {emitted_sniff_len: 0,meta_prescan_fed_len: 0,meta_charset_parser: HtmlMetaCharsetParser::new(),- // A supplied fallback represents an inherited or otherwise- // authoritative default. Blink only lets content detection replace- // its own default, not an ordinary parent-frame encoding.- allow_legacy_content_detection: fallback_encoding.is_none(),+ legacy_encoding_detector,url_hint: url_hint.map(str::to_owned),decoder: None,selected_encoding: None,@@ -162,9 +168,8 @@ impl HtmlDocumentStreamingDecoder {}fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> {- self.allow_legacy_content_detection- .then(|| detect_legacy_html_encoding(&self.sniff_buffer, self.url_hint.as_deref()))- .flatten()+ self.legacy_encoding_detector+ .and_then(|detector| detector(&self.sniff_buffer, self.url_hint.as_deref()))}fn feed_meta_charset_prescan(&mut self, finishing: bool) -> HtmlMetaCharsetScanResult {diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rsindex 09f1f001e..f706fc46a 100644--- a/moli-encoding/src/lib.rs+++ b/moli-encoding/src/lib.rs@@ -1,6 +1,5 @@//! Browser-compatible text, document, form, script, and URL encoding helpers.-mod detector;mod document;mod form;mod labels;@@ -10,7 +9,8 @@ mod query;mod script;pub use document::{- HtmlDocumentStreamingDecoder, decode_html_document, decode_html_document_with_fallback,+ HtmlDocumentStreamingDecoder, LegacyEncodingDetector, decode_html_document,+ decode_html_document_with_fallback,};pub use form::{encode_text_for_legacy_web, form_output_encoding, form_output_encoding_for_label,diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 33e16f298..dbec8f5fe 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -9,6 +9,21 @@ fn gbk_bytes(input: &str) -> Vec<u8> {encoding_rs::GBK.encode(input).0.into_owned()}+fn test_legacy_encoding_detector(+ bytes: &[u8],+ url_hint: Option<&str>,+) -> Option<&'static Encoding> {+ (!bytes.is_empty() && url_hint == Some("https://legacy.example/"))+ .then_some(encoding_rs::IBM866)+}++fn unexpected_legacy_encoding_detector(+ _bytes: &[u8],+ _url_hint: Option<&str>,+) -> Option<&'static Encoding> {+ panic!("declared encoding must take precedence over heuristic detection")+}+#[test]fn content_type_charset_is_selected() {let headers = vec![(@@ -114,10 +129,7 @@ fn meta_charset_after_1024_bytes_after_head_is_ignored() {input.extend(vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - input.len()]);input.extend_from_slice(b"<meta charset=\"gbk\"><p>");input.extend_from_slice(&gbk_bytes("家居"));- // Suppress heuristic detection so this test isolates the meta prescan's- // head boundary rather than CED independently recognizing the GBK body.- let mut decoder =- HtmlDocumentStreamingDecoder::new_with_fallback(&headers, Some("windows-1252"));+ let mut decoder = HtmlDocumentStreamingDecoder::new(&headers);let decoded = decoder.push(&input).join("");@@ -222,15 +234,64 @@ fn unknown_charset_falls_back_to_html_default_on_finish() {}#[test]-fn unlabelled_html_document_uses_legacy_content_detection() {+fn unlabelled_html_document_without_detector_uses_html_default() {let (text, encoding) = decode_html_document(b"\x80\x80 Hello", &[]);+ assert_eq!(encoding, "windows-1252");+ assert_eq!(text, "\u{20ac}\u{20ac} Hello");+}++#[test]+fn injected_legacy_content_detector_receives_url_hint() {+ let headers = vec![(+ "Content-Type".to_owned(),+ "text/html; charset=x-unknown".to_owned(),+ )];+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ &headers,+ "https://legacy.example/",+ test_legacy_encoding_detector,+ );+ let mut text = decoder.push(b"\x80\x80 Hello").join("");+ if let Some(tail) = decoder.finish() {+ text.push_str(&tail);+ }++ let encoding = decoder+ .selected_encoding_name()+ .expect("finishing must select an encoding");assert_eq!(encoding, "IBM866");assert_eq!(text, "\u{410}\u{410} Hello");}#[test]-fn inherited_fallback_suppresses_legacy_content_detection() {+fn declared_encodings_precede_injected_legacy_detector() {+ let headers = vec![(+ "Content-Type".to_owned(),+ "text/html; charset=gbk".to_owned(),+ )];+ let mut header_decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ &headers,+ "https://legacy.example/",+ unexpected_legacy_encoding_detector,+ );+ assert_eq!(header_decoder.push(&gbk_bytes("太平洋")), vec!["太平洋"]);++ let mut meta_decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ &[],+ "https://legacy.example/",+ unexpected_legacy_encoding_detector,+ );+ let mut input = b"<meta charset=\"gbk\">".to_vec();+ input.extend_from_slice(&gbk_bytes("太平洋"));+ assert_eq!(+ meta_decoder.push(&input),+ vec!["<meta charset=\"gbk\">太平洋"]+ );+}++#[test]+fn explicit_fallback_keeps_core_decoder_deterministic() {let (text, encoding) =decode_html_document_with_fallback(b"\x80\x80 Hello", &[], Some("windows-1252"));@@ -804,7 +865,7 @@ fn header_charset_exact_lookup_rejects_non_http_whitespace() {assert_eq!(encoding_from_response_headers(&headers), None);assert_eq!(decode_html_document(&gbk_bytes("太平洋"), &headers).1,- "GBK"+ "windows-1252");}}diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.tomlindex 66f207291..f4b24bdfc 100644--- a/moli-renderer-v8/Cargo.toml+++ b/moli-renderer-v8/Cargo.toml@@ -45,6 +45,7 @@ moli-storage-service = { path = "../moli-storage-service" }moli-cookie-jar = { path = "../moli-cookie-jar" }moli-canvas = { path = "../moli-canvas" }moli-encoding = { path = "../moli-encoding" }+moli-encoding-detector = { path = "../moli-encoding-detector" }moli-fetch = { path = "../moli-fetch" }moli-file-api = { path = "../moli-file-api" }moli-geometry = { path = "../moli-geometry" }diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rsindex e565ba05d..c7c7d3c21 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs@@ -4,6 +4,7 @@ use super::parser_blocking_pending::main_parser_blocking_classic_script_item;use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;use super::*;use moli_encoding::HtmlDocumentStreamingDecoder;+use moli_encoding_detector::detect_legacy_html_encoding;use moli_web_mime::response_headers_indicate_attachment_download;use tokio::sync::{mpsc, oneshot};@@ -123,8 +124,11 @@ impl ConcurrentParseTimeRuntime {)});state.service_worker_preload_context = service_worker_preload_context.clone();- let mut decoder =- HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ &response_headers,+ state.final_url.as_str(),+ detect_legacy_html_encoding,+ );// Raw navigation bodies are decoded during prebootstrap scan. The decoder// is carried forward so split multibyte sequences are not decoded twice or// lost between the scan and parser handoff.@@ -358,8 +362,11 @@ impl ConcurrentParseTimeRuntime {)});state.service_worker_preload_context = service_worker_preload_context.clone();- let mut decoder =- HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ &response_headers,+ state.final_url.as_str(),+ detect_legacy_html_encoding,+ );// External raw bodies replay captured chunks. Pre-scan only what is already// buffered before bootstrap so the producer's backpressure boundary still// controls how far ahead the parser can get.