Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor(encoding): extract legacy charset detector

ldm05e173f3a5a02
diff --git a/Cargo.lock b/Cargo.lock
index f5b54c1c5..68ffecd43 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2447,12 +2447,19 @@ dependencies = [
name = "moli-encoding"
version = "0.1.0"
dependencies = [
- "compact-enc-det",
"encoding_rs",
"moli-charset-parser",
"moli-content-type",
]
+[[package]]
+name = "moli-encoding-detector"
+version = "0.1.0"
+dependencies = [
+ "compact-enc-det",
+ "encoding_rs",
+]
+
[[package]]
name = "moli-fetch"
version = "0.1.0"
@@ -2828,6 +2835,7 @@ dependencies = [
"moli-css-parse",
"moli-dom",
"moli-encoding",
+ "moli-encoding-detector",
"moli-fetch",
"moli-file-api",
"moli-geometry",
diff --git a/Cargo.toml b/Cargo.toml
index bd8c29e00..5e8bff65e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,7 @@ members = [
"moli-dns-resolver",
"moli-protocol",
"moli-encoding",
+ "moli-encoding-detector",
"moli-core",
"moli-import-map",
"moli-image",
diff --git a/moli-encoding-detector/Cargo.toml b/moli-encoding-detector/Cargo.toml
new file mode 100644
index 000000000..a33635d9f
--- /dev/null
+++ b/moli-encoding-detector/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+license.workspace = true
+name = "moli-encoding-detector"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+compact-enc-det = "0.1"
+encoding_rs = "0.8"
+
+[lints]
+workspace = true
diff --git a/moli-encoding/src/detector.rs b/moli-encoding-detector/src/lib.rs
similarity index 88%
rename from moli-encoding/src/detector.rs
rename to moli-encoding-detector/src/lib.rs
index af1add771..58d76988b 100644
--- a/moli-encoding/src/detector.rs
+++ b/moli-encoding-detector/src/lib.rs
@@ -1,7 +1,13 @@
+//! Heuristic detection of legacy encodings used by web documents.
+
use compact_enc_det::{DetectHints, Encoding as CedEncoding, TextCorpusType, detect_encoding};
use encoding_rs::Encoding;
-pub(crate) fn detect_legacy_html_encoding(
+/// Detects a legacy HTML encoding using Chromium's CED configuration.
+///
+/// UTF encodings are deliberately excluded: browser-side heuristics exist for
+/// legacy content, not as a substitute for labelling modern UTF-8 documents.
+pub fn detect_legacy_html_encoding(
bytes: &[u8],
url_hint: Option<&str>,
) -> Option<&'static Encoding> {
diff --git a/moli-encoding/Cargo.toml b/moli-encoding/Cargo.toml
index 5dd413cff..908d89776 100644
--- a/moli-encoding/Cargo.toml
+++ b/moli-encoding/Cargo.toml
@@ -5,7 +5,6 @@ version = "0.1.0"
edition = "2024"
[dependencies]
-compact-enc-det = "0.1"
encoding_rs = "0.8"
moli-charset-parser = { path = "../moli-charset-parser" }
moli-content-type = { path = "../moli-content-type" }
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index 02df2ce05..d1a95a50e 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -1,14 +1,18 @@
use encoding_rs::{CoderResult, Decoder, Encoding};
use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};
-use crate::{
- detector::detect_legacy_html_encoding, encoding_for_label, encoding_from_response_headers,
-};
+use crate::{encoding_for_label, encoding_from_response_headers};
const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";
const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";
const UTF16BE_XML_PREFIX: &[u8; 6] = b"\0<\0?\0x";
+/// Stateless hook for optional, heuristic detection of legacy document
+/// encodings. Deterministic BOM, transport and in-document declarations are
+/// resolved before this hook is called.
+pub type LegacyEncodingDetector =
+ fn(bytes: &[u8], url_hint: Option<&str>) -> Option<&'static Encoding>;
+
pub fn decode_html_document(bytes: &[u8], headers: &[(String, String)]) -> (String, &'static str) {
decode_html_document_with_fallback(bytes, headers, None)
}
@@ -39,7 +43,7 @@ pub struct HtmlDocumentStreamingDecoder {
emitted_sniff_len: usize,
meta_prescan_fed_len: usize,
meta_charset_parser: HtmlMetaCharsetParser,
- allow_legacy_content_detection: bool,
+ legacy_encoding_detector: Option<LegacyEncodingDetector>,
url_hint: Option<String>,
decoder: Option<Decoder>,
selected_encoding: Option<&'static Encoding>,
@@ -47,24 +51,29 @@ pub struct HtmlDocumentStreamingDecoder {
impl HtmlDocumentStreamingDecoder {
pub fn new(headers: &[(String, String)]) -> Self {
- Self::new_with_fallback_and_url(headers, None, None)
+ Self::new_with_options(headers, None, None, None)
}
- pub fn new_for_url(headers: &[(String, String)], url_hint: &str) -> Self {
- Self::new_with_fallback_and_url(headers, None, Some(url_hint))
+ pub fn new_with_legacy_encoding_detector(
+ headers: &[(String, String)],
+ url_hint: &str,
+ detector: LegacyEncodingDetector,
+ ) -> Self {
+ Self::new_with_options(headers, None, Some(url_hint), Some(detector))
}
pub fn new_with_fallback(
headers: &[(String, String)],
fallback_encoding: Option<&str>,
) -> Self {
- Self::new_with_fallback_and_url(headers, fallback_encoding, None)
+ Self::new_with_options(headers, fallback_encoding, None, None)
}
- fn new_with_fallback_and_url(
+ fn new_with_options(
headers: &[(String, String)],
fallback_encoding: Option<&str>,
url_hint: Option<&str>,
+ legacy_encoding_detector: Option<LegacyEncodingDetector>,
) -> Self {
Self {
transport_encoding: encoding_from_response_headers(headers),
@@ -75,10 +84,7 @@ impl HtmlDocumentStreamingDecoder {
emitted_sniff_len: 0,
meta_prescan_fed_len: 0,
meta_charset_parser: HtmlMetaCharsetParser::new(),
- // A supplied fallback represents an inherited or otherwise
- // authoritative default. Blink only lets content detection replace
- // its own default, not an ordinary parent-frame encoding.
- allow_legacy_content_detection: fallback_encoding.is_none(),
+ legacy_encoding_detector,
url_hint: url_hint.map(str::to_owned),
decoder: None,
selected_encoding: None,
@@ -162,9 +168,8 @@ impl HtmlDocumentStreamingDecoder {
}
fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> {
- self.allow_legacy_content_detection
- .then(|| detect_legacy_html_encoding(&self.sniff_buffer, self.url_hint.as_deref()))
- .flatten()
+ self.legacy_encoding_detector
+ .and_then(|detector| detector(&self.sniff_buffer, self.url_hint.as_deref()))
}
fn feed_meta_charset_prescan(&mut self, finishing: bool) -> HtmlMetaCharsetScanResult {
diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rs
index 09f1f001e..f706fc46a 100644
--- a/moli-encoding/src/lib.rs
+++ b/moli-encoding/src/lib.rs
@@ -1,6 +1,5 @@
//! Browser-compatible text, document, form, script, and URL encoding helpers.
-mod detector;
mod document;
mod form;
mod labels;
@@ -10,7 +9,8 @@ mod query;
mod script;
pub use document::{
- HtmlDocumentStreamingDecoder, decode_html_document, decode_html_document_with_fallback,
+ HtmlDocumentStreamingDecoder, LegacyEncodingDetector, decode_html_document,
+ decode_html_document_with_fallback,
};
pub use form::{
encode_text_for_legacy_web, form_output_encoding, form_output_encoding_for_label,
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 33e16f298..dbec8f5fe 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -9,6 +9,21 @@ fn gbk_bytes(input: &str) -> Vec<u8> {
encoding_rs::GBK.encode(input).0.into_owned()
}
+fn test_legacy_encoding_detector(
+ bytes: &[u8],
+ url_hint: Option<&str>,
+) -> Option<&'static Encoding> {
+ (!bytes.is_empty() && url_hint == Some("https://legacy.example/"))
+ .then_some(encoding_rs::IBM866)
+}
+
+fn unexpected_legacy_encoding_detector(
+ _bytes: &[u8],
+ _url_hint: Option<&str>,
+) -> Option<&'static Encoding> {
+ panic!("declared encoding must take precedence over heuristic detection")
+}
+
#[test]
fn content_type_charset_is_selected() {
let headers = vec![(
@@ -114,10 +129,7 @@ fn meta_charset_after_1024_bytes_after_head_is_ignored() {
input.extend(vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - input.len()]);
input.extend_from_slice(b"<meta charset=\"gbk\"><p>");
input.extend_from_slice(&gbk_bytes("家居"));
- // Suppress heuristic detection so this test isolates the meta prescan's
- // head boundary rather than CED independently recognizing the GBK body.
- let mut decoder =
- HtmlDocumentStreamingDecoder::new_with_fallback(&headers, Some("windows-1252"));
+ let mut decoder = HtmlDocumentStreamingDecoder::new(&headers);
let decoded = decoder.push(&input).join("");
@@ -222,15 +234,64 @@ fn unknown_charset_falls_back_to_html_default_on_finish() {
}
#[test]
-fn unlabelled_html_document_uses_legacy_content_detection() {
+fn unlabelled_html_document_without_detector_uses_html_default() {
let (text, encoding) = decode_html_document(b"\x80\x80 Hello", &[]);
+ assert_eq!(encoding, "windows-1252");
+ assert_eq!(text, "\u{20ac}\u{20ac} Hello");
+}
+
+#[test]
+fn injected_legacy_content_detector_receives_url_hint() {
+ let headers = vec![(
+ "Content-Type".to_owned(),
+ "text/html; charset=x-unknown".to_owned(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ &headers,
+ "https://legacy.example/",
+ test_legacy_encoding_detector,
+ );
+ let mut text = decoder.push(b"\x80\x80 Hello").join("");
+ if let Some(tail) = decoder.finish() {
+ text.push_str(&tail);
+ }
+
+ let encoding = decoder
+ .selected_encoding_name()
+ .expect("finishing must select an encoding");
assert_eq!(encoding, "IBM866");
assert_eq!(text, "\u{410}\u{410} Hello");
}
#[test]
-fn inherited_fallback_suppresses_legacy_content_detection() {
+fn declared_encodings_precede_injected_legacy_detector() {
+ let headers = vec![(
+ "Content-Type".to_owned(),
+ "text/html; charset=gbk".to_owned(),
+ )];
+ let mut header_decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ &headers,
+ "https://legacy.example/",
+ unexpected_legacy_encoding_detector,
+ );
+ assert_eq!(header_decoder.push(&gbk_bytes("太平洋")), vec!["太平洋"]);
+
+ let mut meta_decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ &[],
+ "https://legacy.example/",
+ unexpected_legacy_encoding_detector,
+ );
+ let mut input = b"<meta charset=\"gbk\">".to_vec();
+ input.extend_from_slice(&gbk_bytes("太平洋"));
+ assert_eq!(
+ meta_decoder.push(&input),
+ vec!["<meta charset=\"gbk\">太平洋"]
+ );
+}
+
+#[test]
+fn explicit_fallback_keeps_core_decoder_deterministic() {
let (text, encoding) =
decode_html_document_with_fallback(b"\x80\x80 Hello", &[], Some("windows-1252"));
@@ -804,7 +865,7 @@ fn header_charset_exact_lookup_rejects_non_http_whitespace() {
assert_eq!(encoding_from_response_headers(&headers), None);
assert_eq!(
decode_html_document(&gbk_bytes("太平洋"), &headers).1,
- "GBK"
+ "windows-1252"
);
}
}
diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.toml
index 66f207291..f4b24bdfc 100644
--- a/moli-renderer-v8/Cargo.toml
+++ b/moli-renderer-v8/Cargo.toml
@@ -45,6 +45,7 @@ moli-storage-service = { path = "../moli-storage-service" }
moli-cookie-jar = { path = "../moli-cookie-jar" }
moli-canvas = { path = "../moli-canvas" }
moli-encoding = { path = "../moli-encoding" }
+moli-encoding-detector = { path = "../moli-encoding-detector" }
moli-fetch = { path = "../moli-fetch" }
moli-file-api = { path = "../moli-file-api" }
moli-geometry = { path = "../moli-geometry" }
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index e565ba05d..c7c7d3c21 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -4,6 +4,7 @@ use super::parser_blocking_pending::main_parser_blocking_classic_script_item;
use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;
use super::*;
use moli_encoding::HtmlDocumentStreamingDecoder;
+use moli_encoding_detector::detect_legacy_html_encoding;
use moli_web_mime::response_headers_indicate_attachment_download;
use tokio::sync::{mpsc, oneshot};
@@ -123,8 +124,11 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder =
- HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());
+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ &response_headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ );
// Raw navigation bodies are decoded during prebootstrap scan. The decoder
// is carried forward so split multibyte sequences are not decoded twice or
// lost between the scan and parser handoff.
@@ -358,8 +362,11 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder =
- HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());
+ let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ &response_headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ );
// External raw bodies replay captured chunks. Pre-scan only what is already
// buffered before bootstrap so the producer's backpressure boundary still
// controls how far ahead the parser can get.