Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): remove one initial UTF-8 BOM from Body text and JSON

ldm08f94ad95286f
diff --git a/moli-renderer-v8/src/network_host/body_source.rs b/moli-renderer-v8/src/network_host/body_source.rs
index d088f19e4..dd52105e8 100644
--- a/moli-renderer-v8/src/network_host/body_source.rs
+++ b/moli-renderer-v8/src/network_host/body_source.rs
@@ -1846,11 +1846,16 @@ fn body_materialization_value<'s>(
kind: PendingBodyMaterializationKind,
) -> Result<v8::Local<'s, v8::Value>, v8::Local<'s, v8::Value>> {
match kind {
- PendingBodyMaterializationKind::Text => v8_string(scope, &String::from_utf8_lossy(bytes))
- .map(Into::into)
- .ok_or_else(|| v8::undefined(scope).into()),
+ PendingBodyMaterializationKind::Text => {
+ // Fetch decodes as UTF-8 while removing exactly one initial BOM,
+ // independent of the body's MIME type or any UTF-16 byte marker.
+ let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;
+ v8_string(scope, &text)
+ .map(Into::into)
+ .ok_or_else(|| v8::undefined(scope).into())
+ }
PendingBodyMaterializationKind::Json => {
- let text = String::from_utf8_lossy(bytes).into_owned();
+ let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;
v8_json_parse(scope, &text).ok_or_else(|| {
v8_string(scope, "SyntaxError: JSON parse error")
.map(|message| v8::Exception::syntax_error(scope, message))
diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs
index 1ac51cc62..56aa76811 100644
--- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs
+++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs
@@ -9498,3 +9498,123 @@ fn materialize_response_object_rejects_locked_response_body() {
})
.expect("locked response should reject materialization");
}
+
+fn assert_body_utf8_bom_probe(scenario: &str) {
+ let mut vm = new_storage_test_vm("https://body-utf8.test/");
+ let script = r#"
+globalThis.__bodyUtf8Result = 'pending';
+async function probeBodyUtf8(scenario) {
+ const encode = value => new TextEncoder().encode(value);
+ const make = (source, bytes, type = 'text/plain;charset=UTF-16') => {
+ const headers = {'Content-Type': type};
+ if (source === 'request') {
+ return new Request('https://body-utf8.test/', {method: 'POST', body: bytes, headers});
+ }
+ if (source === 'response') return new Response(bytes, {headers});
+ let offset = 0;
+ const stream = new ReadableStream({async pull(controller) {
+ await Promise.resolve();
+ if (offset === bytes.length) controller.close();
+ else controller.enqueue(bytes.slice(offset, ++offset));
+ }});
+ return new Response(stream, {headers});
+ };
+ const check = (value, label) => {if (!value) throw new Error(label);};
+ for (const source of ['request', 'response', 'response-stream']) {
+ if (scenario === 'text') {
+ const cases = [
+ [encode('\uFEFFA中'), 'A中'],
+ [encode('\uFEFF\uFEFFx'), '\uFEFFx'],
+ [encode('x\uFEFF'), 'x\uFEFF'],
+ [encode('\uFEFF'), ''],
+ [new Uint8Array([0xEF]), '\uFFFD'],
+ [new Uint8Array([0xEF, 0xBB]), '\uFFFD'],
+ [new Uint8Array([0xEF, 0xBB, 0x41]), '\uFFFDA'],
+ [new Uint8Array([0xFF, 0xFE, 0x41, 0]), '\uFFFD\uFFFDA\0'],
+ [new Uint8Array([0xEF, 0xBB, 0xBF, 0xF0, 0x9F, 0x41]), '\uFFFDA'],
+ [encode('A中'), 'A中'],
+ [new Uint8Array(), ''],
+ ];
+ for (const [index, [bytes, expected]] of cases.entries()) {
+ const body = make(source, bytes);
+ check(await body.text() === expected, source + '/text/' + index);
+ check(body.bodyUsed, source + '/text/' + index + ': bodyUsed');
+ }
+ } else if (scenario === 'json') {
+ const cases = [
+ [encode('\uFEFF{"value":1}'), {value: 1}],
+ [encode('\uFEFF"\uFEFFvalue"'), '\uFEFFvalue'],
+ [encode('\uFEFF\uFEFF{}'), undefined],
+ [encode(' \uFEFF{}'), undefined],
+ [encode('\uFEFF'), undefined],
+ [new Uint8Array([0xFF, 0xFE, 0x7B, 0, 0x7D, 0]), undefined],
+ [new Uint8Array([0xEF, 0xBB, 0xBF, 0x22, 0xFF, 0x22]), '\uFFFD'],
+ [encode('0'), 0],
+ ];
+ for (const [index, [bytes, expected]] of cases.entries()) {
+ const body = make(source, bytes);
+ let value, error;
+ try {value = await body.json();} catch (caught) {error = caught;}
+ if (expected === undefined) {
+ check(error instanceof SyntaxError, source + '/json/' + index + ': SyntaxError');
+ } else {
+ check(!error && JSON.stringify(value) === JSON.stringify(expected), source + '/json/' + index);
+ }
+ check(body.bodyUsed, source + '/json/' + index + ': bodyUsed');
+ }
+ } else {
+ const bytes = encode('\uFEFF\uFEFFvalue\uFEFF');
+ for (const method of ['bytes', 'arrayBuffer', 'blob', 'stream']) {
+ const body = make(source, bytes);
+ let actual;
+ if (method === 'stream') {
+ const reader = body.body.getReader();
+ const values = [];
+ for (;;) {
+ const {done, value} = await reader.read();
+ if (done) break;
+ values.push(...value);
+ }
+ actual = new Uint8Array(values);
+ } else {
+ const value = await body[method]();
+ actual = method === 'bytes' ? value : new Uint8Array(method === 'blob' ? await value.arrayBuffer() : value);
+ }
+ check(actual.length === bytes.length && actual.every((value, index) => value === bytes[index]),
+ source + '/' + method + ': raw BOM bytes');
+ }
+ const fields = await make(source, encode('\uFEFFname=\uFEFFvalue'),
+ 'application/x-www-form-urlencoded').formData();
+ check(fields.get('\uFEFFname') === '\uFEFFvalue', source + ': form values');
+ }
+ }
+ return 'ok';
+}
+"#;
+ vm.eval(&format!(
+ "{script}\nprobeBodyUtf8({scenario:?}).then(value => __bodyUtf8Result = value, error => __bodyUtf8Result = String(error));"
+ ))
+ .expect("Body UTF-8 probe should schedule");
+ vm.eval("0").expect("Body UTF-8 promises should drain");
+ assert_eq!(
+ vm.eval("__bodyUtf8Result")
+ .expect("Body UTF-8 probe should finish"),
+ "ok",
+ "{scenario}"
+ );
+}
+
+#[test]
+fn fetch_body_text_removes_one_initial_utf8_bom() {
+ assert_body_utf8_bom_probe("text");
+}
+
+#[test]
+fn fetch_body_json_removes_initial_utf8_bom_and_preserves_syntax_errors() {
+ assert_body_utf8_bom_probe("json");
+}
+
+#[test]
+fn fetch_body_binary_and_form_methods_preserve_bom_bytes() {
+ assert_body_utf8_bom_probe("bytes");
+}