Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(encoding): preserve UTF-16 boundaries in encoder streams
ldm07458b20a02b1
diff --git a/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs b/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rsindex 088deebe5..cb20eba4e 100644--- a/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs+++ b/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs@@ -23,9 +23,9 @@ use super::utils::{};use super::writable::{attach_transform_writable_close_settlement, error_writable_stream_with_value,- flush_text_decoder_stream, invoke_writable_stream_algorithm, normalize_stream_algorithm_result,- transform_stream_snapshot, with_transform_stream_relevant_realm, writable_stream_snapshot,- writable_stream_stored_error,+ flush_text_decoder_stream, flush_text_encoder_stream, invoke_writable_stream_algorithm,+ normalize_stream_algorithm_result, transform_stream_snapshot,+ with_transform_stream_relevant_realm, writable_stream_snapshot, writable_stream_stored_error,};use super::{TRANSFORM_STREAM_CONTROLLER_FINISH_PROMISE_SLOT,@@ -458,6 +458,9 @@ fn perform_transform_stream_close_in_relevant_realm<'s>(unreachable!("writable close must claim the transform flush algorithm")};let flush_result = match algorithm {+ TransformFlushAlgorithm::TextEncoder => {+ Ok(flush_text_encoder_stream(scope, stream, readable))+ }TransformFlushAlgorithm::TextDecoder => {Ok(flush_text_decoder_stream(scope, stream, readable))}diff --git a/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs b/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rsindex 374e5c224..2cf5e759e 100644--- a/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs+++ b/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs@@ -2,8 +2,8 @@ use super::transform_finish::{perform_transform_stream_close, transform_stream_sink_abort_algorithm,};use super::*;-use crate::text_codec::{TextCodecStore, TextDecodeError};-use crate::util::get_private_value;+use crate::text_codec::{TextCodecStore, TextDecodeError, TextEncoderState};+use crate::util::{get_private_value, v8_string_to_u16_string};use crate::webidl;use moli_streams::queue::{QueueBounds, QueueRemainderPlan};use moli_streams::strategy::StrategySnapshot;@@ -43,6 +43,7 @@ const WRITABLE_PENDING_ABORT_PROMISE_INDEX: u32 = 0;const WRITABLE_PENDING_ABORT_RESIDENCE_INDEX: u32 = 1;const WRITABLE_PENDING_ABORT_REASON_INDEX: u32 = 2;const WRITABLE_PENDING_ABORT_ALREADY_ERRORING_INDEX: u32 = 3;+const TEXT_ENCODER_LEADING_SURROGATE_SLOT: &str = "__moliTextEncoderLeadingSurrogate";fn writable_stream_kind<'s>(scope: &mut v8::PinScope<'s, '_>,@@ -1102,20 +1103,16 @@ fn perform_transform_stream_write_in_relevant_realm<'s>(transform_stream_snapshot(scope, stream, readable).plan_queued_write_algorithm();match algorithm {TransformWriteAlgorithm::TextEncoder => {- let input = match text_encoder_stream_chunk_string(scope, chunk) {+ let input = match text_encoder_stream_chunk_utf16(scope, chunk) {Ok(input) => input,Err(error) => {return reject_transform_stream_with_value(scope, stream, readable, error);}};- let bytes = new_uint8_array_from_bytes(scope, input.into_bytes())- .expect("TextEncoderStream output allocation must succeed");- if let Err(error) =- enqueue_transform_readable_chunk(scope, stream, readable, bytes.into())- {- return rejected_promise_value(scope, error);- }- return None;+ let mut state = text_encoder_stream_state(scope, stream);+ let bytes = state.encode(&input);+ store_text_encoder_stream_state(scope, stream, &state);+ return enqueue_text_encoder_stream_bytes(scope, stream, readable, bytes);}TransformWriteAlgorithm::TextDecoder => {let input = match webidl::convert::<webidl::AllowSharedBufferSource<'s>>(@@ -2273,25 +2270,82 @@ pub(super) fn flush_text_decoder_stream<'s>(}}-fn text_encoder_stream_chunk_string<'s>(+fn text_encoder_stream_chunk_utf16<'s>(scope: &mut v8::PinScope<'s, '_>,chunk: v8::Local<'s, v8::Value>,-) -> Result<String, v8::Local<'s, v8::Value>> {+) -> Result<Vec<u16>, v8::Local<'s, v8::Value>> {let try_catch = std::pin::pin!(v8::TryCatch::new(scope));- let scope = try_catch.init();- match chunk.to_string(&scope) {- Some(value) => Ok(value.to_rust_string_lossy(&scope)),+ let scope = &mut try_catch.init();+ match chunk.to_string(scope) {+ Some(value) => Ok(v8_string_to_u16_string(scope, value).into_vec()),None if scope.has_caught() => Err(scope.exception().unwrap_or_else(|| {- let message = v8str(&scope, "Failed to convert chunk");- v8::Exception::type_error(&scope, message)+ let message = v8str(scope, "Failed to convert chunk");+ v8::Exception::type_error(scope, message)})),None => {- let message = v8str(&scope, "Failed to convert chunk");- Err(v8::Exception::type_error(&scope, message))+ let message = v8str(scope, "Failed to convert chunk");+ Err(v8::Exception::type_error(scope, message))}}}+fn text_encoder_stream_state<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ stream: v8::Local<'s, v8::Object>,+) -> TextEncoderState {+ let leading_surrogate = stream_slot_value(scope, stream, TEXT_ENCODER_LEADING_SURROGATE_SLOT)+ .filter(|value| value.is_uint32())+ .and_then(|value| value.uint32_value(scope))+ .and_then(|value| u16::try_from(value).ok());+ TextEncoderState::new(leading_surrogate)+}++fn store_text_encoder_stream_state<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ stream: v8::Local<'s, v8::Object>,+ state: &TextEncoderState,+) {+ let value = match state.leading_surrogate() {+ Some(unit) => v8::Integer::new_from_unsigned(scope, u32::from(unit)).into(),+ None => v8::null(scope).into(),+ };+ set_required_stream_slot_value(+ scope,+ stream,+ TEXT_ENCODER_LEADING_SURROGATE_SLOT,+ value,+ "TextEncoderStream leading surrogate",+ );+}++fn enqueue_text_encoder_stream_bytes<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ stream: v8::Local<'s, v8::Object>,+ readable: v8::Local<'s, v8::Object>,+ bytes: Vec<u8>,+) -> Option<v8::Local<'s, v8::Value>> {+ if bytes.is_empty() {+ return None;+ }+ let bytes = new_uint8_array_from_bytes(scope, bytes)+ .expect("TextEncoderStream output allocation must succeed");+ match enqueue_transform_readable_chunk(scope, stream, readable, bytes.into()) {+ Ok(()) => None,+ Err(error) => rejected_promise_value(scope, error),+ }+}++pub(super) fn flush_text_encoder_stream<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ stream: v8::Local<'s, v8::Object>,+ readable: v8::Local<'s, v8::Object>,+) -> Option<v8::Local<'s, v8::Value>> {+ let mut state = text_encoder_stream_state(scope, stream);+ let bytes = state.flush();+ store_text_encoder_stream_state(scope, stream, &state);+ enqueue_text_encoder_stream_bytes(scope, stream, readable, bytes)+}+fn decode_text_decoder_stream<'s>(scope: &mut v8::PinScope<'s, '_>,stream: v8::Local<'s, v8::Object>,diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rsindex 59a1783be..ac10aec70 100644--- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs+++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs@@ -1724,3 +1724,26 @@ fn text_decoder_constructors_follow_encoding_label_whitespace_rules() {assert!(failures.is_empty(), "{failures:#?}");assert_eq!(checks.len(), 512);}++#[test]+fn text_encoder_stream_preserves_utf16_chunk_boundaries() {+ let mut vm = new_storage_test_vm("https://text-encoder-boundaries.test/");+ vm.eval(include_str!(+ "../../../../tests/fixtures/text-encoder-boundaries.js"+ ))+ .expect("text encoder boundary contract should load");+ vm.eval("runTextEncoderBoundaryContract().then(result => globalThis.__textEncoderBoundaryResults = result)")+ .expect("text encoder boundary contract should execute");+ let facts: serde_json::Value = serde_json::from_str(+ &vm.eval("JSON.stringify(__textEncoderBoundaryResults)")+ .expect("text encoder boundary results should serialize"),+ )+ .expect("text encoder boundary results should be valid JSON");+ let checks = facts["checks"].as_array().expect("boundary checks");+ let failures = checks+ .iter()+ .filter(|check| check["passed"] != true)+ .collect::<Vec<_>>();+ assert!(failures.is_empty(), "{failures:#?}");+ assert_eq!(checks.len(), 49);+}diff --git a/moli-renderer-v8/src/text_codec.rs b/moli-renderer-v8/src/text_codec.rsindex 5fe8ccd92..11c6befe6 100644--- a/moli-renderer-v8/src/text_codec.rs+++ b/moli-renderer-v8/src/text_codec.rs@@ -10,6 +10,48 @@ use super::util::{get_private_value, set_private_value};const TEXT_DECODER_ID_SLOT: &str = "__lmTextDecoderId";+#[derive(Default)]+pub(super) struct TextEncoderState {+ leading_surrogate: Option<u16>,+}++impl TextEncoderState {+ pub(super) fn new(leading_surrogate: Option<u16>) -> Self {+ debug_assert!(leading_surrogate.is_none_or(|unit| (0xd800..=0xdbff).contains(&unit)));+ Self { leading_surrogate }+ }++ pub(super) fn leading_surrogate(&self) -> Option<u16> {+ self.leading_surrogate+ }++ pub(super) fn encode(&mut self, input: &[u16]) -> Vec<u8> {+ if input.is_empty() {+ return Vec::new();+ }+ let previous = self.leading_surrogate.take();+ let body = match input.split_last() {+ Some((last, rest)) if (0xd800..=0xdbff).contains(last) => {+ self.leading_surrogate = Some(*last);+ rest+ }+ _ => input,+ };+ char::decode_utf16(previous.into_iter().chain(body.iter().copied()))+ .map(|result| result.unwrap_or(char::REPLACEMENT_CHARACTER))+ .collect::<String>()+ .into_bytes()+ }++ pub(super) fn flush(&mut self) -> Vec<u8> {+ if self.leading_surrogate.take().is_some() {+ vec![0xef, 0xbf, 0xbd]+ } else {+ Vec::new()+ }+ }+}+#[derive(Default)]pub(super) struct TextCodecStore {state: Arc<Mutex<TextCodecStoreState>>,@@ -325,4 +367,34 @@ mod tests {assert_eq!(state.decode(&[0x80], false).unwrap(), "\u{20AC}");}+ #[test]+ fn text_encoder_state_preserves_utf16_across_all_three_chunk_partitions() {+ let cases: &[&[u16]] = &[+ &[],+ &[0, 0x41, 0x20ac],+ &[0xd800],+ &[0xdfff],+ &[0xd800, 0xdc00],+ &[0xdbff, 0xdfff],+ &[0x41, 0xd83d, 0xdc99, 0x42],+ &[0xd800, 0xd800, 0xdc00, 0xdfff],+ &[0xd800, 0x41, 0xdc00, 0xdbff],+ ];+ for input in cases {+ let expected = String::from_utf16_lossy(input).into_bytes();+ for first in 0..=input.len() {+ for second in first..=input.len() {+ let mut state = TextEncoderState::default();+ let mut actual = Vec::new();+ for chunk in [&input[..first], &input[first..second], &input[second..]] {+ actual.extend(state.encode(chunk));+ assert!(state.encode(&[]).is_empty());+ }+ actual.extend(state.flush());+ assert_eq!(actual, expected, "input={input:?}, cuts={first}/{second}");+ assert!(state.flush().is_empty());+ }+ }+ }+ }}diff --git a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rsindex ba50efe84..8664e71d2 100644--- a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs+++ b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs@@ -1881,3 +1881,36 @@ runDecoderStreamSourceContract().then(assert!(failures.is_empty(), "{failures:#?}");assert_eq!(checks.len(), 64);}++#[tokio::test]+async fn worker_text_encoder_stream_preserves_utf16_chunk_boundaries() {+ ensure_v8();+ let source = [+ include_str!("../../../../../tests/fixtures/text-encoder-boundaries.js"),+ r#"+runTextEncoderBoundaryContract().then(+ result => { postMessage(result); close(); },+ error => { postMessage({ failure: String(error) }); close(); }+);+"#,+ ]+ .join("\n");+ let mut handle = spawn_worker(+ source,+ "https://text-encoder-boundaries.test/worker.js".into(),+ );+ let message = timeout(TIMEOUT, handle.recv())+ .await+ .expect("text encoder boundary contract should complete")+ .expect("worker result channel should stay open");+ handle.terminate_and_join();+ let facts: serde_json::Value = serde_json::from_str(&expect_post_json(message))+ .expect("text encoder boundary results should be valid JSON");+ let checks = facts["checks"].as_array().expect("boundary checks");+ let failures = checks+ .iter()+ .filter(|check| check["passed"] != true)+ .collect::<Vec<_>>();+ assert!(failures.is_empty(), "{failures:#?}");+ assert_eq!(checks.len(), 49);+}diff --git a/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.js b/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.jsnew file mode 100644index 000000000..12292ce1a--- /dev/null+++ b/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.js@@ -0,0 +1,111 @@+globalThis.runTextEncoderBoundaryContract = async function () {+ const checks = [];+ async function check(name, operation) {+ try { checks.push({ name, passed: await operation() === true }); }+ catch (error) { checks.push({ name, passed: false, error: String(error) }); }+ }+ function collect(readable) {+ const reader = readable.getReader();+ const result = { chunks: [], error: undefined };+ result.done = (async () => {+ try {+ while (true) {+ const item = await reader.read();+ if (item.done) return;+ result.chunks.push(Array.from(item.value));+ }+ } catch (error) { result.error = error; }+ })();+ return result;+ }+ const inputs = [+ "", "abc", "\u0000\u20ac\u4e2d", "A\ud83d\udc99B",+ "\ud800", "\udfff", "\ud800\udc00", "\udbff\udfff",+ "\ud800A\udc00", "\ud800\udbff\udc00", "\udbff\udfff\ud800",+ "\ud800\ud800\udc00\udfff",+ ];+ for (const input of ["abc", "\ud83d\udc99"]) {+ await check(`whole input ${input}`, async () => {+ const encoder = new TextEncoderStream();+ const result = collect(encoder.readable);+ const writer = encoder.writable.getWriter();+ await writer.write(input);+ await writer.close();+ await result.done;+ return result.error === undefined && JSON.stringify(result.chunks) ===+ JSON.stringify([Array.from(new TextEncoder().encode(input))]);+ });+ }+ for (const [index, input] of inputs.entries()) {+ const expected = Array.from(new TextEncoder().encode(input));+ for (let split = 0; split <= input.length; split++) {+ await check(`partition ${index} at ${split}`, async () => {+ const parts = [input.slice(0, split), "", input.slice(split), ""];+ const source = new ReadableStream({ start(controller) {+ for (const part of parts) controller.enqueue(part);+ controller.close();+ } });+ const result = collect(source.pipeThrough(new TextEncoderStream()));+ await result.done;+ return result.error === undefined && result.chunks.every(chunk => chunk.length > 0) &&+ JSON.stringify(result.chunks.flat()) === JSON.stringify(expected);+ });+ }+ }+ for (const end of ["close", "pair", "abort", "throw"]) {+ await check(`pending leading surrogate: ${end}`, async () => {+ const encoder = new TextEncoderStream();+ const result = collect(encoder.readable);+ const writer = encoder.writable.getWriter();+ writer.closed.catch(() => {});+ await writer.write("\ud83d");+ await writer.write("");+ const held = result.chunks.length === 0;+ const sentinel = {};+ if (end === "pair") {+ await writer.write("\udc99");+ await writer.close();+ } else if (end === "close") {+ await writer.close();+ } else if (end === "abort") {+ await writer.abort(sentinel);+ } else {+ try {+ await writer.write({ toString() { throw sentinel; } });+ return false;+ } catch (error) { if (error !== sentinel) return false; }+ }+ await result.done;+ const expected = end === "close" ? [[0xef, 0xbf, 0xbd]] :+ end === "pair" ? [[0xf0, 0x9f, 0x92, 0x99]] : [];+ return held && JSON.stringify(result.chunks) === JSON.stringify(expected) &&+ result.error === (end === "abort" || end === "throw" ? sentinel : undefined);+ });+ }+ await check("DOMString conversion preserves a surrogate pair split across objects", async () => {+ const encoder = new TextEncoderStream();+ const result = collect(encoder.readable);+ const writer = encoder.writable.getWriter();+ let calls = 0;+ for (const value of ["\ud83d", "", "\udc99"]) {+ await writer.write({ toString() { calls++; return value; } });+ }+ await writer.close();+ await result.done;+ return calls === 3 && result.error === undefined &&+ JSON.stringify(result.chunks) === JSON.stringify([[0xf0, 0x9f, 0x92, 0x99]]);+ });+ await check("interleaved encoders have separate pending surrogates", async () => {+ const left = new TextEncoderStream(), right = new TextEncoderStream();+ const a = collect(left.readable), b = collect(right.readable);+ const aw = left.writable.getWriter(), bw = right.writable.getWriter();+ await aw.write("\ud83d");+ await bw.write("A");+ await aw.write("\udc99");+ await Promise.all([aw.close(), bw.close(), a.done, b.done]);+ return a.error === undefined && b.error === undefined &&+ JSON.stringify(a.chunks) === JSON.stringify([[0xf0, 0x9f, 0x92, 0x99]]) &&+ JSON.stringify(b.chunks) === JSON.stringify([[65]]);+ });+ return { checks };+};diff --git a/moli-streams/src/transform.rs b/moli-streams/src/transform.rsindex 7a5a5b086..0db767246 100644--- a/moli-streams/src/transform.rs+++ b/moli-streams/src/transform.rs@@ -42,8 +42,9 @@ impl TransformMode {pub const fn flush_algorithm(self) -> TransformFlushAlgorithm {match self {Self::Callback => TransformFlushAlgorithm::Callback,+ Self::TextEncoder => TransformFlushAlgorithm::TextEncoder,Self::TextDecoder => TransformFlushAlgorithm::TextDecoder,- Self::Identity | Self::TextEncoder => TransformFlushAlgorithm::None,+ Self::Identity => TransformFlushAlgorithm::None,}}@@ -70,6 +71,7 @@ pub enum TransformWriteAlgorithm {pub enum TransformFlushAlgorithm {None,Callback,+ TextEncoder,TextDecoder,}@@ -561,7 +563,7 @@ mod tests {true,TransformMode::TextEncoder,TransformWriteAlgorithm::TextEncoder,- TransformFlushAlgorithm::None,+ TransformFlushAlgorithm::TextEncoder,TransformCancelAlgorithm::None,),(