Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(fetch): centralize header list projections
ldm037ffa1ec5647
diff --git a/moli-renderer-v8/src/network_host/fetch_surface.rs b/moli-renderer-v8/src/network_host/fetch_surface.rsindex 570a8ce75..cfb67e1ba 100644--- a/moli-renderer-v8/src/network_host/fetch_surface.rs+++ b/moli-renderer-v8/src/network_host/fetch_surface.rs@@ -1,6 +1,4 @@-use super::headers::{- build_headers_object, headers_entries, mark_headers_immutable, normalized_headers_entries,-};+use super::headers::{build_headers_object, headers_entries, headers_list, mark_headers_immutable};use super::response::{ParsedResponseInit, install_response_body_methods, parse_response_init};use super::*;use crate::context_bootstrap::readable_stream_disturbed;@@ -188,7 +186,7 @@ pub(crate) fn request_headers_entries<'s>(request: v8::Local<'s, v8::Object>,) -> Vec<(String, String)> {request_slot_object(scope, request, REQUEST_HEADERS_SLOT)- .map(|headers| headers_entries(scope, headers))+ .map(|headers| headers_list(scope, headers)).unwrap_or_default()}@@ -462,10 +460,8 @@ fn response_content_type<'s>() -> Option<String> {let headers = response_slot_object(scope, response, RESPONSE_HEADERS_SLOT)?;moli_web_mime::response_header_value(- &moli_fetch::headers_from_byte_strings(&normalized_headers_entries(&headers_entries(- scope, headers,- )))- .expect("Headers contain ByteStrings"),+ &moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))+ .expect("Headers contain ByteStrings"),"content-type",)}diff --git a/moli-renderer-v8/src/network_host/headers.rs b/moli-renderer-v8/src/network_host/headers.rsindex 8b789082b..4fba186a2 100644--- a/moli-renderer-v8/src/network_host/headers.rs+++ b/moli-renderer-v8/src/network_host/headers.rs@@ -9,6 +9,6 @@ pub(crate) use self::methods::install_headers_template_bindings;pub(crate) use self::store::headers_entries;pub(crate) use self::store::{HeadersGuard, filter_headers_for_guard};pub(super) use self::store::{- build_headers_object, build_headers_object_with_state, headers_entries_from_init,+ build_headers_object, build_headers_object_with_state, headers_entries_from_init, headers_list,mark_headers_immutable, normalized_headers_entries,};diff --git a/moli-renderer-v8/src/network_host/headers/methods/access.rs b/moli-renderer-v8/src/network_host/headers/methods/access.rsindex 2515d1a1b..82ecb890c 100644--- a/moli-renderer-v8/src/network_host/headers/methods/access.rs+++ b/moli-renderer-v8/src/network_host/headers/methods/access.rs@@ -1,4 +1,4 @@-use super::super::store::{get_header_prop, headers_entries, normalized_header_name_or_throw};+use super::super::store::{get_header_prop, headers_list, normalized_header_name_or_throw};use super::*;use crate::webidl;@@ -58,7 +58,7 @@ pub(in crate::network_host::headers) fn headers_get_set_cookie_callback<'s>(let Some(this) = require_headers_receiver(scope, args.this()) else {return;};- let values = headers_entries(scope, this)+ let values = headers_list(scope, this).into_iter().filter_map(|(name, value)| (name == "set-cookie").then_some(value)).filter_map(|value| v8_string(scope, &value).map(Into::into))diff --git a/moli-renderer-v8/src/network_host/headers/methods/iteration.rs b/moli-renderer-v8/src/network_host/headers/methods/iteration.rsindex 282c0b025..80708ecbe 100644--- a/moli-renderer-v8/src/network_host/headers/methods/iteration.rs+++ b/moli-renderer-v8/src/network_host/headers/methods/iteration.rs@@ -1,4 +1,4 @@-use super::super::store::{headers_entries, normalized_headers_entries};+use super::super::store::headers_entries;use super::*;use crate::web_api_interfaces;use crate::{@@ -129,7 +129,7 @@ pub(in crate::network_host::headers) fn headers_for_each_callback<'s>(let mut index = 0;loop {// Script can change both the values and the sorted pair order.- let entries = normalized_headers_entries(&headers_entries(scope, this));+ let entries = headers_entries(scope, this);let Some((name, value)) = entries.get(index) else {break;};@@ -212,7 +212,7 @@ fn headers_iterator_next_callback<'s>(return;};let index = index_value.integer_value(scope).unwrap_or(0).max(0) as usize;- let entries = normalized_headers_entries(&headers_entries(scope, target));+ let entries = headers_entries(scope, target);if index >= entries.len() {let result = HeadersIteratorResultDeclaration::new(true, v8::undefined(scope).into()).bind(scope)diff --git a/moli-renderer-v8/src/network_host/headers/methods/mutation.rs b/moli-renderer-v8/src/network_host/headers/methods/mutation.rsindex bd8e9a1d7..61fe73481 100644--- a/moli-renderer-v8/src/network_host/headers/methods/mutation.rs+++ b/moli-renderer-v8/src/network_host/headers/methods/mutation.rs@@ -1,7 +1,7 @@use super::super::store::{- header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable,- headers_entries, headers_guard, normalized_header_name_or_throw,- normalized_header_value_or_throw, set_headers_entries,+ header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable, headers_guard,+ headers_list, normalized_header_name_or_throw, normalized_header_value_or_throw,+ set_headers_entries,};use super::*;use crate::webidl;@@ -59,7 +59,7 @@ pub(in crate::network_host::headers) fn headers_set_callback<'s>(if !header_allowed_by_guard(guard, &lower, &value) {return;}- let mut entries = headers_entries(scope, this);+ let mut entries = headers_list(scope, this);let insert_at = entries.iter().position(|(entry_name, _)| *entry_name == lower)@@ -87,7 +87,7 @@ pub(in crate::network_host::headers) fn headers_delete_callback<'s>(let Some(lower) = normalized_header_name_or_throw(scope, &name) else {return;};- let mut entries = headers_entries(scope, this);+ let mut entries = headers_list(scope, this);entries.retain(|(entry_name, _)| *entry_name != lower);set_headers_entries(scope, this, &entries);}@@ -115,7 +115,7 @@ pub(in crate::network_host::headers) fn headers_append_callback<'s>(return;};let guard = headers_guard(scope, this);- let mut entries = headers_entries(scope, this);+ let mut entries = headers_list(scope, this);if !header_append_allowed_by_guard(guard, &name, &value, &entries) {return;}diff --git a/moli-renderer-v8/src/network_host/headers/store.rs b/moli-renderer-v8/src/network_host/headers/store.rsindex f6b2004b2..2709b9f52 100644--- a/moli-renderer-v8/src/network_host/headers/store.rs+++ b/moli-renderer-v8/src/network_host/headers/store.rs@@ -7,7 +7,7 @@ pub(in crate::network_host::headers) use self::entries::{header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable, headers_guard,};pub(in crate::network_host) use self::entries::{- mark_headers_immutable, normalized_headers_entries, set_headers_entries,+ headers_list, mark_headers_immutable, normalized_headers_entries, set_headers_entries,};pub(in crate::network_host::headers) use self::entries::{normalized_header_name_or_throw, normalized_header_value_or_throw,diff --git a/moli-renderer-v8/src/network_host/headers/store/entries.rs b/moli-renderer-v8/src/network_host/headers/store/entries.rsindex 2ff3dcb5f..601485470 100644--- a/moli-renderer-v8/src/network_host/headers/store/entries.rs+++ b/moli-renderer-v8/src/network_host/headers/store/entries.rs@@ -246,19 +246,22 @@ fn is_http_whitespace(ch: char) -> bool {matches!(ch, '\t' | '\n' | '\r' | ' ')}+/// The sorted, combined view used by public iteration and header consumers.pub(crate) fn headers_entries<'s>(scope: &mut v8::PinScope<'s, '_>,obj: v8::Local<'s, v8::Object>,) -> Vec<(String, String)> {- headers_entries_if_present(scope, obj).unwrap_or_default()+ normalized_headers_entries(&headers_list(scope, obj))}-pub(in crate::network_host) fn headers_entries_if_present<'s>(+/// The internal header list, before the public sort-and-combine projection.+pub(in crate::network_host) fn headers_list<'s>(scope: &mut v8::PinScope<'s, '_>,obj: v8::Local<'s, v8::Object>,-) -> Option<Vec<(String, String)>> {+) -> Vec<(String, String)> {private_string_value(scope, obj, HEADERS_ENTRIES_SLOT).and_then(|json| serde_json::from_str::<Vec<(String, String)>>(&json).ok())+ .unwrap_or_default()}fn private_string_value<'s>(diff --git a/moli-renderer-v8/src/network_host/headers/store/projection.rs b/moli-renderer-v8/src/network_host/headers/store/projection.rsindex b855f5490..1a7487bea 100644--- a/moli-renderer-v8/src/network_host/headers/store/projection.rs+++ b/moli-renderer-v8/src/network_host/headers/store/projection.rs@@ -1,7 +1,7 @@use super::super::*;use super::entries::{HEADERS_ENTRIES_SLOT, HEADERS_GUARD_SLOT, HEADERS_IMMUTABLE_SLOT, HeadersGuard,- headers_entries, headers_entries_json, normalized_header_name_or_throw,+ headers_entries_json, headers_list, normalized_header_name_or_throw,};use crate::web_api_interfaces;use moli_webapi_declare::WebApiObject;@@ -23,7 +23,7 @@ pub(in crate::network_host) fn get_header_prop<'s>(name: &str,) -> Option<v8::Local<'s, v8::Value>> {let lower = normalized_header_name_or_throw(scope, name)?;- let values = headers_entries(scope, obj)+ let values = headers_list(scope, obj).into_iter().filter_map(|(entry_name, value)| (entry_name == lower).then_some(value)).collect::<Vec<_>>();diff --git a/moli-renderer-v8/src/network_host/response/body_methods.rs b/moli-renderer-v8/src/network_host/response/body_methods.rsindex 870fc37a1..7e592c5d5 100644--- a/moli-renderer-v8/src/network_host/response/body_methods.rs+++ b/moli-renderer-v8/src/network_host/response/body_methods.rs@@ -4,7 +4,6 @@ mod text_json;use super::super::fetch_surface::{REQUEST_BODY_USED_SLOT, REQUEST_HEADERS_SLOT, RESPONSE_BODY_USED_SLOT, RESPONSE_HEADERS_SLOT,};-use super::super::headers::normalized_headers_entries;use super::*;use moli_web_mime::{is_form_urlencoded_mime, multipart_form_data_boundary, response_blob_mime_type,@@ -185,8 +184,7 @@ fn body_headers<'s>(.map(|headers| headers_entries(scope, headers)),}.map(|headers| {- moli_fetch::headers_from_byte_strings(&normalized_headers_entries(&headers))- .expect("Headers contain ByteStrings")+ moli_fetch::headers_from_byte_strings(&headers).expect("Headers contain ByteStrings")})}diff --git a/moli-renderer-v8/src/network_host/response/materialize.rs b/moli-renderer-v8/src/network_host/response/materialize.rsindex c67185acb..1f8856cdb 100644--- a/moli-renderer-v8/src/network_host/response/materialize.rs+++ b/moli-renderer-v8/src/network_host/response/materialize.rs@@ -4,6 +4,7 @@ use super::super::fetch_surface::{response_slot_number,};use super::*;+use crate::network_host::headers::headers_list;use crate::types::NetworkBodySourceId;use crate::web_api_interfaces;use moli_fetch::RequestMode;@@ -585,7 +586,7 @@ pub(crate) fn materialize_response_object_head<'s>(response_slot_string(scope, response, RESPONSE_STATUS_TEXT_SLOT).unwrap_or_default();let headers = response_slot_object(scope, response, RESPONSE_HEADERS_SLOT).map(|headers| {- moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))+ moli_fetch::headers_from_byte_strings(&headers_list(scope, headers)).expect("Headers contain ByteStrings")}).unwrap_or_default();@@ -619,7 +620,7 @@ pub(crate) fn materialize_response_object_head_for_service_worker_respond_with<'.unwrap_or_default();head.headers = response_slot_object(scope, response, RESPONSE_INTERNAL_HEADERS_SLOT).map(|headers| {- moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))+ moli_fetch::headers_from_byte_strings(&headers_list(scope, headers)).expect("Headers contain ByteStrings")}).unwrap_or_default();diff --git a/moli-renderer-v8/src/script_vm/tests/headers_list.rs b/moli-renderer-v8/src/script_vm/tests/headers_list.rsnew file mode 100644index 000000000..feaedc68d--- /dev/null+++ b/moli-renderer-v8/src/script_vm/tests/headers_list.rs@@ -0,0 +1,55 @@+use super::*;++#[tokio::test(flavor = "current_thread")]+async fn headers_preserve_internal_lists_in_window_and_worker() {+ let fixture = include_str!("../../../tests/fixtures/headers-list.js");+ for worker in [false, true] {+ let loader = static_http_loader([]);+ let mut vm =+ new_page_task_executor_test_vm_with_loader("https://headers-list.test/", &loader);+ vm.eval("globalThis.headerListResult = null;").unwrap();+ let script = if worker {+ let source = format!(+ "{fixture}\nheadersListProbe('https://headers-list.test').then(postMessage, error => postMessage({{error: String(error.stack || error)}}));"+ );+ format!(+ r#"+ const workerUrl = URL.createObjectURL(new Blob([{}], {{type: 'text/javascript'}}));+ const worker = new Worker(workerUrl);+ const finish = value => {{+ headerListResult = value;+ worker.terminate();+ URL.revokeObjectURL(workerUrl);+ }};+ worker.onmessage = event => finish(event.data);+ worker.onerror = event => {{ finish({{error: event.message}}); event.preventDefault(); }};+ "#,+ serde_json::to_string(&source).unwrap()+ )+ } else {+ format!(+ "{fixture}\nheadersListProbe('https://headers-list.test').then(value => {{ headerListResult = value; }}, error => {{ headerListResult = {{error: String(error.stack || error)}}; }});"+ )+ };+ vm.eval(&script).unwrap();+ advance_page_task_executor_until_eval_equals(+ &mut vm,+ &loader,+ "String(headerListResult !== null)",+ "true",+ "Headers list checks should finish",+ )+ .await;+ let result: serde_json::Value =+ serde_json::from_str(&vm.eval("JSON.stringify(headerListResult)").unwrap()).unwrap();+ let checks = result["checks"]+ .as_array()+ .unwrap_or_else(|| panic!("worker={worker}: {result}"));+ let failures: Vec<_> = checks+ .iter()+ .filter(|check| check["pass"] != true)+ .collect();+ assert_eq!(result["state"], "pass", "worker={worker}: {failures:?}");+ assert_eq!(checks.len(), 174, "worker={worker}");+ }+}diff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rsindex 02ca19a5a..263ed499b 100644--- a/moli-renderer-v8/src/script_vm/tests/mod.rs+++ b/moli-renderer-v8/src/script_vm/tests/mod.rs@@ -15150,6 +15150,7 @@ mod child_dynamic_inline_scripts;mod dom_elements;mod dom_xhr;mod element_click;+mod headers_list;mod http_fixture;mod import_meta;mod indexed_db;diff --git a/moli-renderer-v8/tests/fixtures/headers-list.js b/moli-renderer-v8/tests/fixtures/headers-list.jsnew file mode 100644index 000000000..195a016c9--- /dev/null+++ b/moli-renderer-v8/tests/fixtures/headers-list.js@@ -0,0 +1,98 @@+async function headersListProbe(base) {+ const checks = [];+ const check = (label, actual, wanted) => {+ if (typeof actual === 'function') {+ try { actual = actual(); } catch (error) { actual = 'throw:' + error.message; }+ }+ checks.push({label, actual, wanted, pass: JSON.stringify(actual) === JSON.stringify(wanted)});+ };+ const cases = [];+ for (const name of ['Accept', 'Accept-Language', 'Content-Language']) {+ for (const [label, values, wanted, mergedAllowed] of [+ ['overflow', ['a'.repeat(126), 'b'], 'a'.repeat(126), false],+ ['at-limit', ['a'.repeat(125), 'b'], 'a'.repeat(125) + ', b', true],+ ['after-limit', ['a'.repeat(125), 'b', 'c'], 'a'.repeat(125) + ', b', false],+ ['empty-first', ['', 'b'.repeat(127)], '', false],+ ]) cases.push([name + '/' + label, name, values, wanted, mergedAllowed]);+ }+ const longType = 'text/plain;x=' + 'a'.repeat(114);+ cases.push(+ ['type/duplicate', 'Content-Type', ['text/plain', 'text/plain'], 'text/plain', false],+ ['type/parameter', 'Content-Type', ['text/plain;charset=utf8', 'extra'], 'text/plain;charset=utf8, extra', true],+ ['type/parameter-second-type', 'Content-Type', ['text/plain;charset=utf8', 'application/json'], 'text/plain;charset=utf8, application/json', true],+ ['type/unsafe', 'Content-Type', ['text/plain;charset=utf8', '"'], 'text/plain;charset=utf8', false],+ ['type/invalid-first', 'Content-Type', ['application/json', 'text/plain'], 'text/plain', false],+ ['type/overflow', 'Content-Type', [longType, 'ok'], longType, false],+ );+ for (const [label, name, values, wanted, mergedAllowed] of cases) {+ const names = [name, name.toLowerCase(), name.toUpperCase()];+ const pairs = values.map((value, index) => [names[index], value]);+ const url = base + '/echo?case=' + encodeURIComponent(label);+ const source = new Request(url, {headers: pairs});+ const joined = values.join(', ');+ const refill = request => new Request(request, {mode: 'no-cors'});+ check(label + '/mode-change', refill(source).headers.get(name), wanted);+ check(label + '/clone-refill', refill(source.clone()).headers.get(name), wanted);+ check(label + '/construct-copy', refill(new Request(source)).headers.get(name), wanted);+ check(label + '/refill-cors', refill(new Request(source, {cache: 'no-store'})).headers.get(name), wanted);+ check(label + '/input-unmodified', source.headers.get(name), joined);+ check(label + '/guarded-clone', refill(source).clone().headers.get(name), wanted);+ check(label + '/public-headers-init', new Request(url, {mode: 'no-cors', headers: source.headers}).headers.get(name), mergedAllowed ? joined : null);+ source.headers.append('X-Unrelated', 'value');+ check(label + '/after-unrelated-mutation', refill(source).headers.get(name), wanted);++ }++ // Mutating another name must not trim separators generated by empty fields.+ const pairs = [['X-B', 'b'], ['X-A', ' a '], ['Set-Cookie', 'a=1'], ['x-a', ''], ['X-A', ''], ['set-cookie', 'b=2']];+ const headers = new Headers(pairs);+ headers.append('X-C', 'c');+ headers.set('X-B', 'new');+ headers.delete('X-C');+ const expectedEntries = [['set-cookie', 'a=1'], ['set-cookie', 'b=2'], ['x-a', 'a, , '], ['x-b', 'new']];+ check('projection/get', headers.get('x-a'), 'a, , ');+ check('projection/entries', [...headers], expectedEntries);+ check('projection/keys', [...headers.keys()], expectedEntries.map(entry => entry[0]));+ check('projection/values', [...headers.values()], expectedEntries.map(entry => entry[1]));+ check('projection/cookies', headers.getSetCookie(), ['a=1', 'b=2']);+ const visited = [];+ headers.forEach((value, key, owner) => visited.push([key, value, owner === headers]));+ check('projection/forEach', visited, expectedEntries.map(entry => [...entry, true]));+ check('projection/public-copy-normalizes', new Headers(headers).get('x-a'), 'a, ,');+ const iterator = headers.entries();+ check('projection/live-first', iterator.next().value, expectedEntries[0]);+ headers.append('X-B', 'tail');+ check('projection/live-rest', [...iterator], [expectedEntries[1], expectedEntries[2], ['x-b', 'new, tail']]);+ headers.set('x-a', 'reset');+ headers.append('X-A', 'end');+ check('projection/set-replaces-all', headers.get('x-a'), 'reset, end');+ headers.delete('x-a');+ check('projection/delete-all', headers.has('x-a'), false);++ const long = 'a'.repeat(126);+ for (const poisoned of ['request-getter', 'headers-iterator', 'both']) {+ const request = new Request(base + '/echo?case=' + poisoned, {headers: [['Accept', long], ['accept', 'b']]});+ const associated = request.headers;+ let getterCalls = 0;+ let iteratorCalls = 0;+ if (poisoned !== 'headers-iterator') {+ Object.defineProperty(request, 'headers', {get() { getterCalls++; throw new Error('headers getter'); }});+ }+ if (poisoned !== 'request-getter') {+ Object.defineProperty(associated, Symbol.iterator, {get() { iteratorCalls++; throw new Error('headers iterator'); }});+ }+ check(poisoned + '/inherit', () => new Request(request).headers.get('Accept'), long + ', b');+ check(poisoned + '/refill', () => new Request(request, {mode: 'no-cors'}).headers.get('Accept'), long);+ check(poisoned + '/clone', () => request.clone().headers.get('Accept'), long + ', b');++ check(poisoned + '/getter-not-read', getterCalls, 0);+ check(poisoned + '/iterator-not-read', iteratorCalls, 0);+ if (poisoned !== 'request-getter') {+ check(poisoned + '/explicit-init-observes-iterator', () => new Request(base, {headers: associated}), 'throw:headers iterator');+ check(poisoned + '/explicit-init-reads-once', iteratorCalls, 1);+ }+ }+++ return {state: checks.every(check => check.pass) ? 'pass' : 'fail', checks};+}