Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): remove one initial UTF-8 BOM from Body text and JSON
ldm08f94ad95286f
diff --git a/moli-renderer-v8/src/network_host/body_source.rs b/moli-renderer-v8/src/network_host/body_source.rsindex d088f19e4..dd52105e8 100644--- a/moli-renderer-v8/src/network_host/body_source.rs+++ b/moli-renderer-v8/src/network_host/body_source.rs@@ -1846,11 +1846,16 @@ fn body_materialization_value<'s>(kind: PendingBodyMaterializationKind,) -> Result<v8::Local<'s, v8::Value>, v8::Local<'s, v8::Value>> {match kind {- PendingBodyMaterializationKind::Text => v8_string(scope, &String::from_utf8_lossy(bytes))- .map(Into::into)- .ok_or_else(|| v8::undefined(scope).into()),+ PendingBodyMaterializationKind::Text => {+ // Fetch decodes as UTF-8 while removing exactly one initial BOM,+ // independent of the body's MIME type or any UTF-16 byte marker.+ let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;+ v8_string(scope, &text)+ .map(Into::into)+ .ok_or_else(|| v8::undefined(scope).into())+ }PendingBodyMaterializationKind::Json => {- let text = String::from_utf8_lossy(bytes).into_owned();+ let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;v8_json_parse(scope, &text).ok_or_else(|| {v8_string(scope, "SyntaxError: JSON parse error").map(|message| v8::Exception::syntax_error(scope, message))diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rsindex 1ac51cc62..56aa76811 100644--- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs+++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch.rs@@ -9498,3 +9498,123 @@ fn materialize_response_object_rejects_locked_response_body() {}).expect("locked response should reject materialization");}++fn assert_body_utf8_bom_probe(scenario: &str) {+ let mut vm = new_storage_test_vm("https://body-utf8.test/");+ let script = r#"+globalThis.__bodyUtf8Result = 'pending';+async function probeBodyUtf8(scenario) {+ const encode = value => new TextEncoder().encode(value);+ const make = (source, bytes, type = 'text/plain;charset=UTF-16') => {+ const headers = {'Content-Type': type};+ if (source === 'request') {+ return new Request('https://body-utf8.test/', {method: 'POST', body: bytes, headers});+ }+ if (source === 'response') return new Response(bytes, {headers});+ let offset = 0;+ const stream = new ReadableStream({async pull(controller) {+ await Promise.resolve();+ if (offset === bytes.length) controller.close();+ else controller.enqueue(bytes.slice(offset, ++offset));+ }});+ return new Response(stream, {headers});+ };+ const check = (value, label) => {if (!value) throw new Error(label);};+ for (const source of ['request', 'response', 'response-stream']) {+ if (scenario === 'text') {+ const cases = [+ [encode('\uFEFFA中'), 'A中'],+ [encode('\uFEFF\uFEFFx'), '\uFEFFx'],+ [encode('x\uFEFF'), 'x\uFEFF'],+ [encode('\uFEFF'), ''],+ [new Uint8Array([0xEF]), '\uFFFD'],+ [new Uint8Array([0xEF, 0xBB]), '\uFFFD'],+ [new Uint8Array([0xEF, 0xBB, 0x41]), '\uFFFDA'],+ [new Uint8Array([0xFF, 0xFE, 0x41, 0]), '\uFFFD\uFFFDA\0'],+ [new Uint8Array([0xEF, 0xBB, 0xBF, 0xF0, 0x9F, 0x41]), '\uFFFDA'],+ [encode('A中'), 'A中'],+ [new Uint8Array(), ''],+ ];+ for (const [index, [bytes, expected]] of cases.entries()) {+ const body = make(source, bytes);+ check(await body.text() === expected, source + '/text/' + index);+ check(body.bodyUsed, source + '/text/' + index + ': bodyUsed');+ }+ } else if (scenario === 'json') {+ const cases = [+ [encode('\uFEFF{"value":1}'), {value: 1}],+ [encode('\uFEFF"\uFEFFvalue"'), '\uFEFFvalue'],+ [encode('\uFEFF\uFEFF{}'), undefined],+ [encode(' \uFEFF{}'), undefined],+ [encode('\uFEFF'), undefined],+ [new Uint8Array([0xFF, 0xFE, 0x7B, 0, 0x7D, 0]), undefined],+ [new Uint8Array([0xEF, 0xBB, 0xBF, 0x22, 0xFF, 0x22]), '\uFFFD'],+ [encode('0'), 0],+ ];+ for (const [index, [bytes, expected]] of cases.entries()) {+ const body = make(source, bytes);+ let value, error;+ try {value = await body.json();} catch (caught) {error = caught;}+ if (expected === undefined) {+ check(error instanceof SyntaxError, source + '/json/' + index + ': SyntaxError');+ } else {+ check(!error && JSON.stringify(value) === JSON.stringify(expected), source + '/json/' + index);+ }+ check(body.bodyUsed, source + '/json/' + index + ': bodyUsed');+ }+ } else {+ const bytes = encode('\uFEFF\uFEFFvalue\uFEFF');+ for (const method of ['bytes', 'arrayBuffer', 'blob', 'stream']) {+ const body = make(source, bytes);+ let actual;+ if (method === 'stream') {+ const reader = body.body.getReader();+ const values = [];+ for (;;) {+ const {done, value} = await reader.read();+ if (done) break;+ values.push(...value);+ }+ actual = new Uint8Array(values);+ } else {+ const value = await body[method]();+ actual = method === 'bytes' ? value : new Uint8Array(method === 'blob' ? await value.arrayBuffer() : value);+ }+ check(actual.length === bytes.length && actual.every((value, index) => value === bytes[index]),+ source + '/' + method + ': raw BOM bytes');+ }+ const fields = await make(source, encode('\uFEFFname=\uFEFFvalue'),+ 'application/x-www-form-urlencoded').formData();+ check(fields.get('\uFEFFname') === '\uFEFFvalue', source + ': form values');+ }+ }+ return 'ok';+}+"#;+ vm.eval(&format!(+ "{script}\nprobeBodyUtf8({scenario:?}).then(value => __bodyUtf8Result = value, error => __bodyUtf8Result = String(error));"+ ))+ .expect("Body UTF-8 probe should schedule");+ vm.eval("0").expect("Body UTF-8 promises should drain");+ assert_eq!(+ vm.eval("__bodyUtf8Result")+ .expect("Body UTF-8 probe should finish"),+ "ok",+ "{scenario}"+ );+}++#[test]+fn fetch_body_text_removes_one_initial_utf8_bom() {+ assert_body_utf8_bom_probe("text");+}++#[test]+fn fetch_body_json_removes_initial_utf8_bom_and_preserves_syntax_errors() {+ assert_body_utf8_bom_probe("json");+}++#[test]+fn fetch_body_binary_and_form_methods_preserve_bom_bytes() {+ assert_body_utf8_bom_probe("bytes");+}