Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor(fetch): centralize header list projections

ldm037ffa1ec5647
diff --git a/moli-renderer-v8/src/network_host/fetch_surface.rs b/moli-renderer-v8/src/network_host/fetch_surface.rs
index 570a8ce75..cfb67e1ba 100644
--- a/moli-renderer-v8/src/network_host/fetch_surface.rs
+++ b/moli-renderer-v8/src/network_host/fetch_surface.rs
@@ -1,6 +1,4 @@
-use super::headers::{
- build_headers_object, headers_entries, mark_headers_immutable, normalized_headers_entries,
-};
+use super::headers::{build_headers_object, headers_entries, headers_list, mark_headers_immutable};
use super::response::{ParsedResponseInit, install_response_body_methods, parse_response_init};
use super::*;
use crate::context_bootstrap::readable_stream_disturbed;
@@ -188,7 +186,7 @@ pub(crate) fn request_headers_entries<'s>(
request: v8::Local<'s, v8::Object>,
) -> Vec<(String, String)> {
request_slot_object(scope, request, REQUEST_HEADERS_SLOT)
- .map(|headers| headers_entries(scope, headers))
+ .map(|headers| headers_list(scope, headers))
.unwrap_or_default()
}
@@ -462,10 +460,8 @@ fn response_content_type<'s>(
) -> Option<String> {
let headers = response_slot_object(scope, response, RESPONSE_HEADERS_SLOT)?;
moli_web_mime::response_header_value(
- &moli_fetch::headers_from_byte_strings(&normalized_headers_entries(&headers_entries(
- scope, headers,
- )))
- .expect("Headers contain ByteStrings"),
+ &moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))
+ .expect("Headers contain ByteStrings"),
"content-type",
)
}
diff --git a/moli-renderer-v8/src/network_host/headers.rs b/moli-renderer-v8/src/network_host/headers.rs
index 8b789082b..4fba186a2 100644
--- a/moli-renderer-v8/src/network_host/headers.rs
+++ b/moli-renderer-v8/src/network_host/headers.rs
@@ -9,6 +9,6 @@ pub(crate) use self::methods::install_headers_template_bindings;
pub(crate) use self::store::headers_entries;
pub(crate) use self::store::{HeadersGuard, filter_headers_for_guard};
pub(super) use self::store::{
- build_headers_object, build_headers_object_with_state, headers_entries_from_init,
+ build_headers_object, build_headers_object_with_state, headers_entries_from_init, headers_list,
mark_headers_immutable, normalized_headers_entries,
};
diff --git a/moli-renderer-v8/src/network_host/headers/methods/access.rs b/moli-renderer-v8/src/network_host/headers/methods/access.rs
index 2515d1a1b..82ecb890c 100644
--- a/moli-renderer-v8/src/network_host/headers/methods/access.rs
+++ b/moli-renderer-v8/src/network_host/headers/methods/access.rs
@@ -1,4 +1,4 @@
-use super::super::store::{get_header_prop, headers_entries, normalized_header_name_or_throw};
+use super::super::store::{get_header_prop, headers_list, normalized_header_name_or_throw};
use super::*;
use crate::webidl;
@@ -58,7 +58,7 @@ pub(in crate::network_host::headers) fn headers_get_set_cookie_callback<'s>(
let Some(this) = require_headers_receiver(scope, args.this()) else {
return;
};
- let values = headers_entries(scope, this)
+ let values = headers_list(scope, this)
.into_iter()
.filter_map(|(name, value)| (name == "set-cookie").then_some(value))
.filter_map(|value| v8_string(scope, &value).map(Into::into))
diff --git a/moli-renderer-v8/src/network_host/headers/methods/iteration.rs b/moli-renderer-v8/src/network_host/headers/methods/iteration.rs
index 282c0b025..80708ecbe 100644
--- a/moli-renderer-v8/src/network_host/headers/methods/iteration.rs
+++ b/moli-renderer-v8/src/network_host/headers/methods/iteration.rs
@@ -1,4 +1,4 @@
-use super::super::store::{headers_entries, normalized_headers_entries};
+use super::super::store::headers_entries;
use super::*;
use crate::web_api_interfaces;
use crate::{
@@ -129,7 +129,7 @@ pub(in crate::network_host::headers) fn headers_for_each_callback<'s>(
let mut index = 0;
loop {
// Script can change both the values and the sorted pair order.
- let entries = normalized_headers_entries(&headers_entries(scope, this));
+ let entries = headers_entries(scope, this);
let Some((name, value)) = entries.get(index) else {
break;
};
@@ -212,7 +212,7 @@ fn headers_iterator_next_callback<'s>(
return;
};
let index = index_value.integer_value(scope).unwrap_or(0).max(0) as usize;
- let entries = normalized_headers_entries(&headers_entries(scope, target));
+ let entries = headers_entries(scope, target);
if index >= entries.len() {
let result = HeadersIteratorResultDeclaration::new(true, v8::undefined(scope).into())
.bind(scope)
diff --git a/moli-renderer-v8/src/network_host/headers/methods/mutation.rs b/moli-renderer-v8/src/network_host/headers/methods/mutation.rs
index bd8e9a1d7..61fe73481 100644
--- a/moli-renderer-v8/src/network_host/headers/methods/mutation.rs
+++ b/moli-renderer-v8/src/network_host/headers/methods/mutation.rs
@@ -1,7 +1,7 @@
use super::super::store::{
- header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable,
- headers_entries, headers_guard, normalized_header_name_or_throw,
- normalized_header_value_or_throw, set_headers_entries,
+ header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable, headers_guard,
+ headers_list, normalized_header_name_or_throw, normalized_header_value_or_throw,
+ set_headers_entries,
};
use super::*;
use crate::webidl;
@@ -59,7 +59,7 @@ pub(in crate::network_host::headers) fn headers_set_callback<'s>(
if !header_allowed_by_guard(guard, &lower, &value) {
return;
}
- let mut entries = headers_entries(scope, this);
+ let mut entries = headers_list(scope, this);
let insert_at = entries
.iter()
.position(|(entry_name, _)| *entry_name == lower)
@@ -87,7 +87,7 @@ pub(in crate::network_host::headers) fn headers_delete_callback<'s>(
let Some(lower) = normalized_header_name_or_throw(scope, &name) else {
return;
};
- let mut entries = headers_entries(scope, this);
+ let mut entries = headers_list(scope, this);
entries.retain(|(entry_name, _)| *entry_name != lower);
set_headers_entries(scope, this, &entries);
}
@@ -115,7 +115,7 @@ pub(in crate::network_host::headers) fn headers_append_callback<'s>(
return;
};
let guard = headers_guard(scope, this);
- let mut entries = headers_entries(scope, this);
+ let mut entries = headers_list(scope, this);
if !header_append_allowed_by_guard(guard, &name, &value, &entries) {
return;
}
diff --git a/moli-renderer-v8/src/network_host/headers/store.rs b/moli-renderer-v8/src/network_host/headers/store.rs
index f6b2004b2..2709b9f52 100644
--- a/moli-renderer-v8/src/network_host/headers/store.rs
+++ b/moli-renderer-v8/src/network_host/headers/store.rs
@@ -7,7 +7,7 @@ pub(in crate::network_host::headers) use self::entries::{
header_allowed_by_guard, header_append_allowed_by_guard, headers_are_immutable, headers_guard,
};
pub(in crate::network_host) use self::entries::{
- mark_headers_immutable, normalized_headers_entries, set_headers_entries,
+ headers_list, mark_headers_immutable, normalized_headers_entries, set_headers_entries,
};
pub(in crate::network_host::headers) use self::entries::{
normalized_header_name_or_throw, normalized_header_value_or_throw,
diff --git a/moli-renderer-v8/src/network_host/headers/store/entries.rs b/moli-renderer-v8/src/network_host/headers/store/entries.rs
index 2ff3dcb5f..601485470 100644
--- a/moli-renderer-v8/src/network_host/headers/store/entries.rs
+++ b/moli-renderer-v8/src/network_host/headers/store/entries.rs
@@ -246,19 +246,22 @@ fn is_http_whitespace(ch: char) -> bool {
matches!(ch, '\t' | '\n' | '\r' | ' ')
}
+/// The sorted, combined view used by public iteration and header consumers.
pub(crate) fn headers_entries<'s>(
scope: &mut v8::PinScope<'s, '_>,
obj: v8::Local<'s, v8::Object>,
) -> Vec<(String, String)> {
- headers_entries_if_present(scope, obj).unwrap_or_default()
+ normalized_headers_entries(&headers_list(scope, obj))
}
-pub(in crate::network_host) fn headers_entries_if_present<'s>(
+/// The internal header list, before the public sort-and-combine projection.
+pub(in crate::network_host) fn headers_list<'s>(
scope: &mut v8::PinScope<'s, '_>,
obj: v8::Local<'s, v8::Object>,
-) -> Option<Vec<(String, String)>> {
+) -> Vec<(String, String)> {
private_string_value(scope, obj, HEADERS_ENTRIES_SLOT)
.and_then(|json| serde_json::from_str::<Vec<(String, String)>>(&json).ok())
+ .unwrap_or_default()
}
fn private_string_value<'s>(
diff --git a/moli-renderer-v8/src/network_host/headers/store/projection.rs b/moli-renderer-v8/src/network_host/headers/store/projection.rs
index b855f5490..1a7487bea 100644
--- a/moli-renderer-v8/src/network_host/headers/store/projection.rs
+++ b/moli-renderer-v8/src/network_host/headers/store/projection.rs
@@ -1,7 +1,7 @@
use super::super::*;
use super::entries::{
HEADERS_ENTRIES_SLOT, HEADERS_GUARD_SLOT, HEADERS_IMMUTABLE_SLOT, HeadersGuard,
- headers_entries, headers_entries_json, normalized_header_name_or_throw,
+ headers_entries_json, headers_list, normalized_header_name_or_throw,
};
use crate::web_api_interfaces;
use moli_webapi_declare::WebApiObject;
@@ -23,7 +23,7 @@ pub(in crate::network_host) fn get_header_prop<'s>(
name: &str,
) -> Option<v8::Local<'s, v8::Value>> {
let lower = normalized_header_name_or_throw(scope, name)?;
- let values = headers_entries(scope, obj)
+ let values = headers_list(scope, obj)
.into_iter()
.filter_map(|(entry_name, value)| (entry_name == lower).then_some(value))
.collect::<Vec<_>>();
diff --git a/moli-renderer-v8/src/network_host/response/body_methods.rs b/moli-renderer-v8/src/network_host/response/body_methods.rs
index 870fc37a1..7e592c5d5 100644
--- a/moli-renderer-v8/src/network_host/response/body_methods.rs
+++ b/moli-renderer-v8/src/network_host/response/body_methods.rs
@@ -4,7 +4,6 @@ mod text_json;
use super::super::fetch_surface::{
REQUEST_BODY_USED_SLOT, REQUEST_HEADERS_SLOT, RESPONSE_BODY_USED_SLOT, RESPONSE_HEADERS_SLOT,
};
-use super::super::headers::normalized_headers_entries;
use super::*;
use moli_web_mime::{
is_form_urlencoded_mime, multipart_form_data_boundary, response_blob_mime_type,
@@ -185,8 +184,7 @@ fn body_headers<'s>(
.map(|headers| headers_entries(scope, headers)),
}
.map(|headers| {
- moli_fetch::headers_from_byte_strings(&normalized_headers_entries(&headers))
- .expect("Headers contain ByteStrings")
+ moli_fetch::headers_from_byte_strings(&headers).expect("Headers contain ByteStrings")
})
}
diff --git a/moli-renderer-v8/src/network_host/response/materialize.rs b/moli-renderer-v8/src/network_host/response/materialize.rs
index c67185acb..1f8856cdb 100644
--- a/moli-renderer-v8/src/network_host/response/materialize.rs
+++ b/moli-renderer-v8/src/network_host/response/materialize.rs
@@ -4,6 +4,7 @@ use super::super::fetch_surface::{
response_slot_number,
};
use super::*;
+use crate::network_host::headers::headers_list;
use crate::types::NetworkBodySourceId;
use crate::web_api_interfaces;
use moli_fetch::RequestMode;
@@ -585,7 +586,7 @@ pub(crate) fn materialize_response_object_head<'s>(
response_slot_string(scope, response, RESPONSE_STATUS_TEXT_SLOT).unwrap_or_default();
let headers = response_slot_object(scope, response, RESPONSE_HEADERS_SLOT)
.map(|headers| {
- moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))
+ moli_fetch::headers_from_byte_strings(&headers_list(scope, headers))
.expect("Headers contain ByteStrings")
})
.unwrap_or_default();
@@ -619,7 +620,7 @@ pub(crate) fn materialize_response_object_head_for_service_worker_respond_with<'
.unwrap_or_default();
head.headers = response_slot_object(scope, response, RESPONSE_INTERNAL_HEADERS_SLOT)
.map(|headers| {
- moli_fetch::headers_from_byte_strings(&headers_entries(scope, headers))
+ moli_fetch::headers_from_byte_strings(&headers_list(scope, headers))
.expect("Headers contain ByteStrings")
})
.unwrap_or_default();
diff --git a/moli-renderer-v8/src/script_vm/tests/headers_list.rs b/moli-renderer-v8/src/script_vm/tests/headers_list.rs
new file mode 100644
index 000000000..feaedc68d
--- /dev/null
+++ b/moli-renderer-v8/src/script_vm/tests/headers_list.rs
@@ -0,0 +1,55 @@
+use super::*;
+
+#[tokio::test(flavor = "current_thread")]
+async fn headers_preserve_internal_lists_in_window_and_worker() {
+ let fixture = include_str!("../../../tests/fixtures/headers-list.js");
+ for worker in [false, true] {
+ let loader = static_http_loader([]);
+ let mut vm =
+ new_page_task_executor_test_vm_with_loader("https://headers-list.test/", &loader);
+ vm.eval("globalThis.headerListResult = null;").unwrap();
+ let script = if worker {
+ let source = format!(
+ "{fixture}\nheadersListProbe('https://headers-list.test').then(postMessage, error => postMessage({{error: String(error.stack || error)}}));"
+ );
+ format!(
+ r#"
+ const workerUrl = URL.createObjectURL(new Blob([{}], {{type: 'text/javascript'}}));
+ const worker = new Worker(workerUrl);
+ const finish = value => {{
+ headerListResult = value;
+ worker.terminate();
+ URL.revokeObjectURL(workerUrl);
+ }};
+ worker.onmessage = event => finish(event.data);
+ worker.onerror = event => {{ finish({{error: event.message}}); event.preventDefault(); }};
+ "#,
+ serde_json::to_string(&source).unwrap()
+ )
+ } else {
+ format!(
+ "{fixture}\nheadersListProbe('https://headers-list.test').then(value => {{ headerListResult = value; }}, error => {{ headerListResult = {{error: String(error.stack || error)}}; }});"
+ )
+ };
+ vm.eval(&script).unwrap();
+ advance_page_task_executor_until_eval_equals(
+ &mut vm,
+ &loader,
+ "String(headerListResult !== null)",
+ "true",
+ "Headers list checks should finish",
+ )
+ .await;
+ let result: serde_json::Value =
+ serde_json::from_str(&vm.eval("JSON.stringify(headerListResult)").unwrap()).unwrap();
+ let checks = result["checks"]
+ .as_array()
+ .unwrap_or_else(|| panic!("worker={worker}: {result}"));
+ let failures: Vec<_> = checks
+ .iter()
+ .filter(|check| check["pass"] != true)
+ .collect();
+ assert_eq!(result["state"], "pass", "worker={worker}: {failures:?}");
+ assert_eq!(checks.len(), 174, "worker={worker}");
+ }
+}
diff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rs
index 02ca19a5a..263ed499b 100644
--- a/moli-renderer-v8/src/script_vm/tests/mod.rs
+++ b/moli-renderer-v8/src/script_vm/tests/mod.rs
@@ -15150,6 +15150,7 @@ mod child_dynamic_inline_scripts;
mod dom_elements;
mod dom_xhr;
mod element_click;
+mod headers_list;
mod http_fixture;
mod import_meta;
mod indexed_db;
diff --git a/moli-renderer-v8/tests/fixtures/headers-list.js b/moli-renderer-v8/tests/fixtures/headers-list.js
new file mode 100644
index 000000000..195a016c9
--- /dev/null
+++ b/moli-renderer-v8/tests/fixtures/headers-list.js
@@ -0,0 +1,98 @@
+async function headersListProbe(base) {
+ const checks = [];
+ const check = (label, actual, wanted) => {
+ if (typeof actual === 'function') {
+ try { actual = actual(); } catch (error) { actual = 'throw:' + error.message; }
+ }
+ checks.push({label, actual, wanted, pass: JSON.stringify(actual) === JSON.stringify(wanted)});
+ };
+ const cases = [];
+ for (const name of ['Accept', 'Accept-Language', 'Content-Language']) {
+ for (const [label, values, wanted, mergedAllowed] of [
+ ['overflow', ['a'.repeat(126), 'b'], 'a'.repeat(126), false],
+ ['at-limit', ['a'.repeat(125), 'b'], 'a'.repeat(125) + ', b', true],
+ ['after-limit', ['a'.repeat(125), 'b', 'c'], 'a'.repeat(125) + ', b', false],
+ ['empty-first', ['', 'b'.repeat(127)], '', false],
+ ]) cases.push([name + '/' + label, name, values, wanted, mergedAllowed]);
+ }
+ const longType = 'text/plain;x=' + 'a'.repeat(114);
+ cases.push(
+ ['type/duplicate', 'Content-Type', ['text/plain', 'text/plain'], 'text/plain', false],
+ ['type/parameter', 'Content-Type', ['text/plain;charset=utf8', 'extra'], 'text/plain;charset=utf8, extra', true],
+ ['type/parameter-second-type', 'Content-Type', ['text/plain;charset=utf8', 'application/json'], 'text/plain;charset=utf8, application/json', true],
+ ['type/unsafe', 'Content-Type', ['text/plain;charset=utf8', '"'], 'text/plain;charset=utf8', false],
+ ['type/invalid-first', 'Content-Type', ['application/json', 'text/plain'], 'text/plain', false],
+ ['type/overflow', 'Content-Type', [longType, 'ok'], longType, false],
+ );
+ for (const [label, name, values, wanted, mergedAllowed] of cases) {
+ const names = [name, name.toLowerCase(), name.toUpperCase()];
+ const pairs = values.map((value, index) => [names[index], value]);
+ const url = base + '/echo?case=' + encodeURIComponent(label);
+ const source = new Request(url, {headers: pairs});
+ const joined = values.join(', ');
+ const refill = request => new Request(request, {mode: 'no-cors'});
+ check(label + '/mode-change', refill(source).headers.get(name), wanted);
+ check(label + '/clone-refill', refill(source.clone()).headers.get(name), wanted);
+ check(label + '/construct-copy', refill(new Request(source)).headers.get(name), wanted);
+ check(label + '/refill-cors', refill(new Request(source, {cache: 'no-store'})).headers.get(name), wanted);
+ check(label + '/input-unmodified', source.headers.get(name), joined);
+ check(label + '/guarded-clone', refill(source).clone().headers.get(name), wanted);
+ check(label + '/public-headers-init', new Request(url, {mode: 'no-cors', headers: source.headers}).headers.get(name), mergedAllowed ? joined : null);
+ source.headers.append('X-Unrelated', 'value');
+ check(label + '/after-unrelated-mutation', refill(source).headers.get(name), wanted);
+
+ }
+
+ // Mutating another name must not trim separators generated by empty fields.
+ const pairs = [['X-B', 'b'], ['X-A', ' a '], ['Set-Cookie', 'a=1'], ['x-a', ''], ['X-A', ''], ['set-cookie', 'b=2']];
+ const headers = new Headers(pairs);
+ headers.append('X-C', 'c');
+ headers.set('X-B', 'new');
+ headers.delete('X-C');
+ const expectedEntries = [['set-cookie', 'a=1'], ['set-cookie', 'b=2'], ['x-a', 'a, , '], ['x-b', 'new']];
+ check('projection/get', headers.get('x-a'), 'a, , ');
+ check('projection/entries', [...headers], expectedEntries);
+ check('projection/keys', [...headers.keys()], expectedEntries.map(entry => entry[0]));
+ check('projection/values', [...headers.values()], expectedEntries.map(entry => entry[1]));
+ check('projection/cookies', headers.getSetCookie(), ['a=1', 'b=2']);
+ const visited = [];
+ headers.forEach((value, key, owner) => visited.push([key, value, owner === headers]));
+ check('projection/forEach', visited, expectedEntries.map(entry => [...entry, true]));
+ check('projection/public-copy-normalizes', new Headers(headers).get('x-a'), 'a, ,');
+ const iterator = headers.entries();
+ check('projection/live-first', iterator.next().value, expectedEntries[0]);
+ headers.append('X-B', 'tail');
+ check('projection/live-rest', [...iterator], [expectedEntries[1], expectedEntries[2], ['x-b', 'new, tail']]);
+ headers.set('x-a', 'reset');
+ headers.append('X-A', 'end');
+ check('projection/set-replaces-all', headers.get('x-a'), 'reset, end');
+ headers.delete('x-a');
+ check('projection/delete-all', headers.has('x-a'), false);
+
+ const long = 'a'.repeat(126);
+ for (const poisoned of ['request-getter', 'headers-iterator', 'both']) {
+ const request = new Request(base + '/echo?case=' + poisoned, {headers: [['Accept', long], ['accept', 'b']]});
+ const associated = request.headers;
+ let getterCalls = 0;
+ let iteratorCalls = 0;
+ if (poisoned !== 'headers-iterator') {
+ Object.defineProperty(request, 'headers', {get() { getterCalls++; throw new Error('headers getter'); }});
+ }
+ if (poisoned !== 'request-getter') {
+ Object.defineProperty(associated, Symbol.iterator, {get() { iteratorCalls++; throw new Error('headers iterator'); }});
+ }
+ check(poisoned + '/inherit', () => new Request(request).headers.get('Accept'), long + ', b');
+ check(poisoned + '/refill', () => new Request(request, {mode: 'no-cors'}).headers.get('Accept'), long);
+ check(poisoned + '/clone', () => request.clone().headers.get('Accept'), long + ', b');
+
+ check(poisoned + '/getter-not-read', getterCalls, 0);
+ check(poisoned + '/iterator-not-read', iteratorCalls, 0);
+ if (poisoned !== 'request-getter') {
+ check(poisoned + '/explicit-init-observes-iterator', () => new Request(base, {headers: associated}), 'throw:headers iterator');
+ check(poisoned + '/explicit-init-reads-once', iteratorCalls, 1);
+ }
+ }
+
+
+ return {state: checks.every(check => check.pass) ? 'pass' : 'fail', checks};
+}