Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(dom): retain encodings in native Document state
ldm029393a6cfb0a
diff --git a/moli-dom/src/native/document.rs b/moli-dom/src/native/document.rsindex d0ea9f2f8..89c2c9cc0 100644--- a/moli-dom/src/native/document.rs+++ b/moli-dom/src/native/document.rs@@ -48,6 +48,7 @@ impl std::fmt::Display for DocumentReadyState {pub struct Document {url: Url,content_type: Box<str>,+ character_set: Box<str>,ready_state: DocumentReadyState,quirks_mode: QuirksMode,kind: DocumentKind,@@ -68,6 +69,7 @@ impl Document {base_url_state: DocumentBaseUrlState::new(&url),url,content_type: "text/html".into(),+ character_set: "UTF-8".into(),ready_state: DocumentReadyState::Complete,quirks_mode: QuirksMode::NoQuirks,kind: DocumentKind::Html,@@ -83,6 +85,7 @@ impl Document {base_url_state: DocumentBaseUrlState::new(&url),url,content_type: "application/xml".into(),+ character_set: "UTF-8".into(),ready_state: DocumentReadyState::Complete,quirks_mode: QuirksMode::NoQuirks,kind: DocumentKind::Xml,@@ -101,6 +104,14 @@ impl Document {&self.content_type}+ pub fn character_set(&self) -> &str {+ &self.character_set+ }++ pub fn set_character_set(&mut self, character_set: impl Into<String>) {+ self.character_set = character_set.into().into_boxed_str();+ }+pub fn css_target(&self) -> Option<NativeNodeId> {self.css_target}diff --git a/moli-dom/src/native/host/document.rs b/moli-dom/src/native/host/document.rsindex 517b78630..c93b8714e 100644--- a/moli-dom/src/native/host/document.rs+++ b/moli-dom/src/native/host/document.rs@@ -1207,6 +1207,29 @@ impl DomHost {true}+ pub fn document_character_set_for_handle(&self, document_handle: DomHandle) -> Option<&str> {+ Some(self.node(document_handle)?.as_document()?.character_set())+ }++ pub fn set_document_character_set_for_handle(+ &mut self,+ document_handle: DomHandle,+ character_set: &str,+ ) -> bool {+ let Some(document) = self+ .node_mut(document_handle)+ .and_then(|node| node.data_mut().as_document_mut())+ else {+ return false;+ };+ if document.character_set() == character_set {+ return false;+ }+ document.set_character_set(character_set);+ self.record_mutation(MutationScope::LocalState);+ true+ }+pub fn set_document_content_type_for_handle(&mut self,document_handle: DomHandle,diff --git a/moli-renderer-v8/src/document_runtime.rs b/moli-renderer-v8/src/document_runtime.rsindex 93b3f7127..ab67a0c6d 100644--- a/moli-renderer-v8/src/document_runtime.rs+++ b/moli-renderer-v8/src/document_runtime.rs@@ -733,7 +733,6 @@ pub(super) struct DocumentRuntime {policy_container: DocumentPolicyContainer,delivered_meta_content_security_policies: RefCell<HashMap<DomHandle, Vec<String>>>,processed_meta_content_security_policy_handles: RefCell<HashSet<(DomHandle, DomHandle)>>,- document_character_set: String,resource_loader_binding: Option<DocumentResourceLoaderBinding>,script_context_stack: Vec<CurrentScriptContext>,root_document_parser: Option<DocumentParserSession>,diff --git a/moli-renderer-v8/src/document_runtime/runtime_core.rs b/moli-renderer-v8/src/document_runtime/runtime_core.rsindex 396309c8c..dc5698c00 100644--- a/moli-renderer-v8/src/document_runtime/runtime_core.rs+++ b/moli-renderer-v8/src/document_runtime/runtime_core.rs@@ -79,7 +79,6 @@ impl DocumentRuntime {policy_container: DocumentPolicyContainer::default(),delivered_meta_content_security_policies: RefCell::new(HashMap::new()),processed_meta_content_security_policy_handles: RefCell::new(HashSet::new()),- document_character_set: "UTF-8".to_owned(),resource_loader_binding: None,script_context_stack: Vec::new(),root_document_parser: None,@@ -207,7 +206,6 @@ impl DocumentRuntime {policy_container: _,delivered_meta_content_security_policies: _,processed_meta_content_security_policy_handles: _,- document_character_set: _,resource_loader_binding: _,script_context_stack: _,root_document_parser: _,@@ -257,7 +255,12 @@ impl DocumentRuntime {}pub(crate) fn document_character_set(&self) -> &str {- &self.document_character_set+ self.document_character_set_for_handle(self.dom_host.document_handle())+ .unwrap_or("UTF-8")+ }++ pub(crate) fn document_character_set_for_handle(&self, handle: DomHandle) -> Option<&str> {+ self.dom_host.document_character_set_for_handle(handle)}pub(crate) fn set_script_execution_disabled(&mut self, disabled: bool) {@@ -294,7 +297,10 @@ impl DocumentRuntime {}pub(crate) fn set_document_character_set(&mut self, character_set: impl Into<String>) {- self.document_character_set = character_set.into();+ let character_set = character_set.into();+ let handle = self.dom_host.document_handle();+ self.dom_host+ .set_document_character_set_for_handle(handle, &character_set);}pub(crate) fn set_document_default_language(&mut self, language: Option<String>) {diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rsindex 743cc557e..a1b9b2f4c 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs@@ -192,6 +192,8 @@ impl JsContextHost {);document_handle};+ self.dom_host_mut()+ .set_document_character_set_for_handle(document_handle, &snapshot.character_set);let document_url = self.document_url_for_handle(document_handle);let document_base_url = self.document_base_url_for_handle(document_handle);let parser_base_url = document_base_url.clone();diff --git a/moli-renderer-v8/src/native_bridge/document.rs b/moli-renderer-v8/src/native_bridge/document.rsindex 871f6a8c4..6dc341c02 100644--- a/moli-renderer-v8/src/native_bridge/document.rs+++ b/moli-renderer-v8/src/native_bridge/document.rs@@ -209,7 +209,6 @@ pub(in crate::native_bridge) use detached_objects::{detached_shadow_root_active_element_value, detached_shadow_root_selection_value,};pub(in crate::native_bridge::document) use detached_surface::create_detached_element_for_document;-pub(in crate::native_bridge::document) use detached_surface::set_detached_document_content_type;pub(super) use detached_surface::{bridge_adopt_node_into_document_callback, bridge_clone_node_into_document_callback,bridge_create_cdata_section_not_supported_callback, bridge_create_detached_comment_callback,@@ -263,6 +262,9 @@ pub(super) use detached_surface::{pub(in crate::native_bridge) use detached_surface::{detached_element_local_name, detached_element_namespace_uri, detached_element_prefix,};+pub(in crate::native_bridge::document) use detached_surface::{+ set_detached_document_character_set, set_detached_document_content_type,+};pub(crate) use hit_test::install_caret_position_template_bindings;pub(super) use hit_test::{node_document_caret_position_from_point_callback, node_document_element_from_point_callback,@@ -900,20 +902,14 @@ fn document_character_set_getter_function<'s>(mut rv: v8::ReturnValue<'s, v8::Value>,) {let receiver = args.this();- if detached_state_object(scope, receiver).is_some() {- let character_set = detached_state_string(scope, receiver, "characterSet")- .unwrap_or_else(|| "UTF-8".into());- set_document_string_return_value(scope, &mut rv, &character_set);- return;- }let Some((runtime_ptr, handle)) = document_receiver_runtime_and_handle(scope, receiver) else {rv.set_undefined();return;};let runtime = unsafe { &*runtime_ptr };let character_set = runtime- .child_browsing_context_character_set_for_document_handle(handle)- .unwrap_or_else(|| runtime.document_character_set())+ .document_character_set_for_handle(handle)+ .unwrap_or("UTF-8").to_owned();set_document_string_return_value(scope, &mut rv, &character_set);}diff --git a/moli-renderer-v8/src/native_bridge/document/detached_objects/builders/documents.rs b/moli-renderer-v8/src/native_bridge/document/detached_objects/builders/documents.rsindex 1930be84e..a2a1eff45 100644--- a/moli-renderer-v8/src/native_bridge/document/detached_objects/builders/documents.rs+++ b/moli-renderer-v8/src/native_bridge/document/detached_objects/builders/documents.rs@@ -207,6 +207,7 @@ pub(crate) fn build_detached_document_object_from_dom_host_with_content_type<'s>"CSS1Compat"};let content_type = content_type.unwrap_or(parsed.dom().document()?.content_type());+ let character_set = character_set.unwrap_or(parsed.dom().document()?.character_set());let document = new_detached_document_shell(scope, kind, content_type, url)?;if let Some(state) = detached_state_object(scope, document) {let _ = state.set(@@ -214,13 +215,6 @@ pub(crate) fn build_detached_document_object_from_dom_host_with_content_type<'s>v8str(scope, "compatMode").into(),v8_string(scope, compat_mode)?.into(),);- if let Some(character_set) = character_set {- let _ = state.set(- scope,- v8str(scope, "characterSet").into(),- v8_string(scope, character_set)?.into(),- );- }}if let Some(document_handle) = detached_native_handle(scope, document)&& let Some(runtime_ptr) = context_host_ptr_from_global_bridge(scope)@@ -229,6 +223,7 @@ pub(crate) fn build_detached_document_object_from_dom_host_with_content_type<'s>.dom_host_mut().set_document_quirks_mode_for_handle(document_handle, quirks_mode);}+ set_detached_document_character_set(scope, document, character_set)?;import_detached_document_children_from_host(scope, document, &parsed)?;Some(document)}diff --git a/moli-renderer-v8/src/native_bridge/document/detached_surface/creation/clone_adopt.rs b/moli-renderer-v8/src/native_bridge/document/detached_surface/creation/clone_adopt.rsindex 063b8b1ad..6caf59dba 100644--- a/moli-renderer-v8/src/native_bridge/document/detached_surface/creation/clone_adopt.rs+++ b/moli-renderer-v8/src/native_bridge/document/detached_surface/creation/clone_adopt.rs@@ -66,6 +66,8 @@ fn clone_detached_document_shell<'s>(.and_then(|value| v8::Local::<v8::Object>::try_from(value).ok())?;let content_type = detached_document_content_type_value(scope, document)?;set_detached_document_content_type(scope, cloned, &content_type)?;+ let character_set = detached_document_character_set_value(scope, document)?;+ set_detached_document_character_set(scope, cloned, &character_set)?;if html_shell {for child in detached_child_node_objects(scope, cloned) {detached_detach_from_parent(scope, child);diff --git a/moli-renderer-v8/src/native_bridge/document/detached_surface/document_state.rs b/moli-renderer-v8/src/native_bridge/document/detached_surface/document_state.rsindex b1df185e3..73b626180 100644--- a/moli-renderer-v8/src/native_bridge/document/detached_surface/document_state.rs+++ b/moli-renderer-v8/src/native_bridge/document/detached_surface/document_state.rs@@ -252,6 +252,28 @@ pub(in crate::native_bridge::document) fn set_detached_document_content_type<'a>Some(())}+pub(in crate::native_bridge::document) fn detached_document_character_set_value<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ document: v8::Local<'s, v8::Object>,+) -> Option<String> {+ let (runtime_ptr, handle) = document_receiver_runtime_and_handle(scope, document)?;+ unsafe { &*runtime_ptr }+ .document_character_set_for_handle(handle)+ .map(str::to_owned)+}++pub(in crate::native_bridge::document) fn set_detached_document_character_set<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ document: v8::Local<'s, v8::Object>,+ character_set: &str,+) -> Option<()> {+ let (runtime_ptr, handle) = document_receiver_runtime_and_handle(scope, document)?;+ unsafe { &mut *runtime_ptr }+ .dom_host_mut()+ .set_document_character_set_for_handle(handle, character_set);+ Some(())+}+pub(in crate::native_bridge) fn bridge_detached_document_character_set_callback<'a>(scope: &mut v8::PinScope<'a, '_>,args: v8::FunctionCallbackArguments<'a>,@@ -261,7 +283,10 @@ pub(in crate::native_bridge) fn bridge_detached_document_character_set_callback<rv.set_empty_string();return;};- let character_set = detached_document_state_string(scope, document, "characterSet", "UTF-8");+ let Some(character_set) = detached_document_character_set_value(scope, document) else {+ rv.set_empty_string();+ return;+ };set_string_return_value(scope, &mut rv, &character_set);}diff --git a/moli-renderer-v8/src/script_vm/tests/document_encodings.rs b/moli-renderer-v8/src/script_vm/tests/document_encodings.rsnew file mode 100644index 000000000..30fb9ab93--- /dev/null+++ b/moli-renderer-v8/src/script_vm/tests/document_encodings.rs@@ -0,0 +1,88 @@+use super::*;++#[test]+fn native_document_encodings_survive_live_and_detached_clones() {+ let mut vm = new_storage_test_vm("https://document-encodings.test/");+ vm.document_runtime.set_document_character_set("GBK");+ assert_eq!(+ vm.eval(+ r#"+ globalThis.shallow = document.cloneNode(false);+ globalThis.deep = document.cloneNode(true);+ JSON.stringify([document.characterSet, document.charset, document.inputEncoding,+ shallow.characterSet, deep.characterSet,+ document.implementation.createHTMLDocument('').characterSet,+ new DOMParser().parseFromString('<root/>', 'application/xml').characterSet])+ "#+ )+ .unwrap(),+ r#"["GBK","GBK","GBK","GBK","GBK","UTF-8","UTF-8"]"#+ );+ vm.document_runtime+ .set_document_character_set("windows-1252");+ assert_eq!(+ vm.eval(+ r#"+ JSON.stringify([document.characterSet, shallow.characterSet, deep.characterSet,+ shallow.cloneNode(true).characterSet, deep.cloneNode(false).characterSet])+ "#+ )+ .unwrap(),+ r#"["windows-1252","GBK","GBK","GBK","GBK"]"#+ );+}++#[tokio::test]+async fn native_document_encodings_follow_child_commit_and_stay_document_local() {+ let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).unwrap();+ let mut vm = new_storage_page_task_executor_test_vm_with_loader(+ "https://document-encodings.test/",+ &loader,+ );+ vm.document_runtime+ .set_document_character_set("windows-1252");+ vm.exec(+ r#"+ const frame = document.body.appendChild(document.createElement('iframe'));+ frame.id = 'encoding-frame'; frame.srcdoc = '<!doctype html><p>child</p>';+ "#,+ None,+ )+ .unwrap();+ drain_pending_page_child_frame_work_for_test(&mut vm).await;+ let frame = vm+ .document_runtime+ .get_element_by_id("encoding-frame")+ .unwrap();+ let child_document = vm+ ._context_host+ .borrow()+ .child_browsing_context_document_handle(frame)+ .unwrap();+ assert_eq!(+ vm.document_runtime+ .document_character_set_for_handle(child_document),+ Some("windows-1252")+ );+ vm.document_runtime+ .dom_host_mut()+ .set_document_character_set_for_handle(child_document, "GBK");+ assert_eq!(vm.eval(r#"+ const child = document.getElementById('encoding-frame').contentDocument;+ globalThis.childClone = child.cloneNode(true);+ JSON.stringify([document.characterSet, child.characterSet, child.charset, child.inputEncoding,+ childClone.characterSet, childClone.cloneNode(false).characterSet])+ "#).unwrap(), r#"["windows-1252","GBK","GBK","GBK","GBK","GBK"]"#);+ vm.document_runtime.set_document_character_set("UTF-8");+ assert_eq!(+ vm.eval(+ r#"+ JSON.stringify([document.characterSet,+ document.getElementById('encoding-frame').contentDocument.characterSet,+ childClone.characterSet])+ "#+ )+ .unwrap(),+ r#"["UTF-8","GBK","GBK"]"#+ );+}diff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rsindex 4b39e95e9..21f4ffae1 100644--- a/moli-renderer-v8/src/script_vm/tests/mod.rs+++ b/moli-renderer-v8/src/script_vm/tests/mod.rs@@ -4,6 +4,7 @@ mod credential_container;mod credential_json_options;mod cross_origin_navigation;mod current_pass_css_fonts;+mod document_encodings;mod message_port_transfer;mod picture_in_picture_interfaces;mod visual_viewport_receiver;