Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(encoding): preserve UTF-16 boundaries in encoder streams

ldm07458b20a02b1
diff --git a/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs b/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs
index 088deebe5..cb20eba4e 100644
--- a/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs
+++ b/moli-renderer-v8/src/context_bootstrap/stream_adapter/transform_finish.rs
@@ -23,9 +23,9 @@ use super::utils::{
};
use super::writable::{
attach_transform_writable_close_settlement, error_writable_stream_with_value,
- flush_text_decoder_stream, invoke_writable_stream_algorithm, normalize_stream_algorithm_result,
- transform_stream_snapshot, with_transform_stream_relevant_realm, writable_stream_snapshot,
- writable_stream_stored_error,
+ flush_text_decoder_stream, flush_text_encoder_stream, invoke_writable_stream_algorithm,
+ normalize_stream_algorithm_result, transform_stream_snapshot,
+ with_transform_stream_relevant_realm, writable_stream_snapshot, writable_stream_stored_error,
};
use super::{
TRANSFORM_STREAM_CONTROLLER_FINISH_PROMISE_SLOT,
@@ -458,6 +458,9 @@ fn perform_transform_stream_close_in_relevant_realm<'s>(
unreachable!("writable close must claim the transform flush algorithm")
};
let flush_result = match algorithm {
+ TransformFlushAlgorithm::TextEncoder => {
+ Ok(flush_text_encoder_stream(scope, stream, readable))
+ }
TransformFlushAlgorithm::TextDecoder => {
Ok(flush_text_decoder_stream(scope, stream, readable))
}
diff --git a/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs b/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs
index 374e5c224..2cf5e759e 100644
--- a/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs
+++ b/moli-renderer-v8/src/context_bootstrap/stream_adapter/writable.rs
@@ -2,8 +2,8 @@ use super::transform_finish::{
perform_transform_stream_close, transform_stream_sink_abort_algorithm,
};
use super::*;
-use crate::text_codec::{TextCodecStore, TextDecodeError};
-use crate::util::get_private_value;
+use crate::text_codec::{TextCodecStore, TextDecodeError, TextEncoderState};
+use crate::util::{get_private_value, v8_string_to_u16_string};
use crate::webidl;
use moli_streams::queue::{QueueBounds, QueueRemainderPlan};
use moli_streams::strategy::StrategySnapshot;
@@ -43,6 +43,7 @@ const WRITABLE_PENDING_ABORT_PROMISE_INDEX: u32 = 0;
const WRITABLE_PENDING_ABORT_RESIDENCE_INDEX: u32 = 1;
const WRITABLE_PENDING_ABORT_REASON_INDEX: u32 = 2;
const WRITABLE_PENDING_ABORT_ALREADY_ERRORING_INDEX: u32 = 3;
+const TEXT_ENCODER_LEADING_SURROGATE_SLOT: &str = "__moliTextEncoderLeadingSurrogate";
fn writable_stream_kind<'s>(
scope: &mut v8::PinScope<'s, '_>,
@@ -1102,20 +1103,16 @@ fn perform_transform_stream_write_in_relevant_realm<'s>(
transform_stream_snapshot(scope, stream, readable).plan_queued_write_algorithm();
match algorithm {
TransformWriteAlgorithm::TextEncoder => {
- let input = match text_encoder_stream_chunk_string(scope, chunk) {
+ let input = match text_encoder_stream_chunk_utf16(scope, chunk) {
Ok(input) => input,
Err(error) => {
return reject_transform_stream_with_value(scope, stream, readable, error);
}
};
- let bytes = new_uint8_array_from_bytes(scope, input.into_bytes())
- .expect("TextEncoderStream output allocation must succeed");
- if let Err(error) =
- enqueue_transform_readable_chunk(scope, stream, readable, bytes.into())
- {
- return rejected_promise_value(scope, error);
- }
- return None;
+ let mut state = text_encoder_stream_state(scope, stream);
+ let bytes = state.encode(&input);
+ store_text_encoder_stream_state(scope, stream, &state);
+ return enqueue_text_encoder_stream_bytes(scope, stream, readable, bytes);
}
TransformWriteAlgorithm::TextDecoder => {
let input = match webidl::convert::<webidl::AllowSharedBufferSource<'s>>(
@@ -2273,25 +2270,82 @@ pub(super) fn flush_text_decoder_stream<'s>(
}
}
-fn text_encoder_stream_chunk_string<'s>(
+fn text_encoder_stream_chunk_utf16<'s>(
scope: &mut v8::PinScope<'s, '_>,
chunk: v8::Local<'s, v8::Value>,
-) -> Result<String, v8::Local<'s, v8::Value>> {
+) -> Result<Vec<u16>, v8::Local<'s, v8::Value>> {
let try_catch = std::pin::pin!(v8::TryCatch::new(scope));
- let scope = try_catch.init();
- match chunk.to_string(&scope) {
- Some(value) => Ok(value.to_rust_string_lossy(&scope)),
+ let scope = &mut try_catch.init();
+ match chunk.to_string(scope) {
+ Some(value) => Ok(v8_string_to_u16_string(scope, value).into_vec()),
None if scope.has_caught() => Err(scope.exception().unwrap_or_else(|| {
- let message = v8str(&scope, "Failed to convert chunk");
- v8::Exception::type_error(&scope, message)
+ let message = v8str(scope, "Failed to convert chunk");
+ v8::Exception::type_error(scope, message)
})),
None => {
- let message = v8str(&scope, "Failed to convert chunk");
- Err(v8::Exception::type_error(&scope, message))
+ let message = v8str(scope, "Failed to convert chunk");
+ Err(v8::Exception::type_error(scope, message))
}
}
}
+fn text_encoder_stream_state<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ stream: v8::Local<'s, v8::Object>,
+) -> TextEncoderState {
+ let leading_surrogate = stream_slot_value(scope, stream, TEXT_ENCODER_LEADING_SURROGATE_SLOT)
+ .filter(|value| value.is_uint32())
+ .and_then(|value| value.uint32_value(scope))
+ .and_then(|value| u16::try_from(value).ok());
+ TextEncoderState::new(leading_surrogate)
+}
+
+fn store_text_encoder_stream_state<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ stream: v8::Local<'s, v8::Object>,
+ state: &TextEncoderState,
+) {
+ let value = match state.leading_surrogate() {
+ Some(unit) => v8::Integer::new_from_unsigned(scope, u32::from(unit)).into(),
+ None => v8::null(scope).into(),
+ };
+ set_required_stream_slot_value(
+ scope,
+ stream,
+ TEXT_ENCODER_LEADING_SURROGATE_SLOT,
+ value,
+ "TextEncoderStream leading surrogate",
+ );
+}
+
+fn enqueue_text_encoder_stream_bytes<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ stream: v8::Local<'s, v8::Object>,
+ readable: v8::Local<'s, v8::Object>,
+ bytes: Vec<u8>,
+) -> Option<v8::Local<'s, v8::Value>> {
+ if bytes.is_empty() {
+ return None;
+ }
+ let bytes = new_uint8_array_from_bytes(scope, bytes)
+ .expect("TextEncoderStream output allocation must succeed");
+ match enqueue_transform_readable_chunk(scope, stream, readable, bytes.into()) {
+ Ok(()) => None,
+ Err(error) => rejected_promise_value(scope, error),
+ }
+}
+
+pub(super) fn flush_text_encoder_stream<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ stream: v8::Local<'s, v8::Object>,
+ readable: v8::Local<'s, v8::Object>,
+) -> Option<v8::Local<'s, v8::Value>> {
+ let mut state = text_encoder_stream_state(scope, stream);
+ let bytes = state.flush();
+ store_text_encoder_stream_state(scope, stream, &state);
+ enqueue_text_encoder_stream_bytes(scope, stream, readable, bytes)
+}
+
fn decode_text_decoder_stream<'s>(
scope: &mut v8::PinScope<'s, '_>,
stream: v8::Local<'s, v8::Object>,
diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs
index 59a1783be..ac10aec70 100644
--- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs
+++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs
@@ -1724,3 +1724,26 @@ fn text_decoder_constructors_follow_encoding_label_whitespace_rules() {
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 512);
}
+
+#[test]
+fn text_encoder_stream_preserves_utf16_chunk_boundaries() {
+ let mut vm = new_storage_test_vm("https://text-encoder-boundaries.test/");
+ vm.eval(include_str!(
+ "../../../../tests/fixtures/text-encoder-boundaries.js"
+ ))
+ .expect("text encoder boundary contract should load");
+ vm.eval("runTextEncoderBoundaryContract().then(result => globalThis.__textEncoderBoundaryResults = result)")
+ .expect("text encoder boundary contract should execute");
+ let facts: serde_json::Value = serde_json::from_str(
+ &vm.eval("JSON.stringify(__textEncoderBoundaryResults)")
+ .expect("text encoder boundary results should serialize"),
+ )
+ .expect("text encoder boundary results should be valid JSON");
+ let checks = facts["checks"].as_array().expect("boundary checks");
+ let failures = checks
+ .iter()
+ .filter(|check| check["passed"] != true)
+ .collect::<Vec<_>>();
+ assert!(failures.is_empty(), "{failures:#?}");
+ assert_eq!(checks.len(), 49);
+}
diff --git a/moli-renderer-v8/src/text_codec.rs b/moli-renderer-v8/src/text_codec.rs
index 5fe8ccd92..11c6befe6 100644
--- a/moli-renderer-v8/src/text_codec.rs
+++ b/moli-renderer-v8/src/text_codec.rs
@@ -10,6 +10,48 @@ use super::util::{get_private_value, set_private_value};
const TEXT_DECODER_ID_SLOT: &str = "__lmTextDecoderId";
+#[derive(Default)]
+pub(super) struct TextEncoderState {
+ leading_surrogate: Option<u16>,
+}
+
+impl TextEncoderState {
+ pub(super) fn new(leading_surrogate: Option<u16>) -> Self {
+ debug_assert!(leading_surrogate.is_none_or(|unit| (0xd800..=0xdbff).contains(&unit)));
+ Self { leading_surrogate }
+ }
+
+ pub(super) fn leading_surrogate(&self) -> Option<u16> {
+ self.leading_surrogate
+ }
+
+ pub(super) fn encode(&mut self, input: &[u16]) -> Vec<u8> {
+ if input.is_empty() {
+ return Vec::new();
+ }
+ let previous = self.leading_surrogate.take();
+ let body = match input.split_last() {
+ Some((last, rest)) if (0xd800..=0xdbff).contains(last) => {
+ self.leading_surrogate = Some(*last);
+ rest
+ }
+ _ => input,
+ };
+ char::decode_utf16(previous.into_iter().chain(body.iter().copied()))
+ .map(|result| result.unwrap_or(char::REPLACEMENT_CHARACTER))
+ .collect::<String>()
+ .into_bytes()
+ }
+
+ pub(super) fn flush(&mut self) -> Vec<u8> {
+ if self.leading_surrogate.take().is_some() {
+ vec![0xef, 0xbf, 0xbd]
+ } else {
+ Vec::new()
+ }
+ }
+}
+
#[derive(Default)]
pub(super) struct TextCodecStore {
state: Arc<Mutex<TextCodecStoreState>>,
@@ -325,4 +367,34 @@ mod tests {
assert_eq!(state.decode(&[0x80], false).unwrap(), "\u{20AC}");
}
+ #[test]
+ fn text_encoder_state_preserves_utf16_across_all_three_chunk_partitions() {
+ let cases: &[&[u16]] = &[
+ &[],
+ &[0, 0x41, 0x20ac],
+ &[0xd800],
+ &[0xdfff],
+ &[0xd800, 0xdc00],
+ &[0xdbff, 0xdfff],
+ &[0x41, 0xd83d, 0xdc99, 0x42],
+ &[0xd800, 0xd800, 0xdc00, 0xdfff],
+ &[0xd800, 0x41, 0xdc00, 0xdbff],
+ ];
+ for input in cases {
+ let expected = String::from_utf16_lossy(input).into_bytes();
+ for first in 0..=input.len() {
+ for second in first..=input.len() {
+ let mut state = TextEncoderState::default();
+ let mut actual = Vec::new();
+ for chunk in [&input[..first], &input[first..second], &input[second..]] {
+ actual.extend(state.encode(chunk));
+ assert!(state.encode(&[]).is_empty());
+ }
+ actual.extend(state.flush());
+ assert_eq!(actual, expected, "input={input:?}, cuts={first}/{second}");
+ assert!(state.flush().is_empty());
+ }
+ }
+ }
+ }
}
diff --git a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs
index ba50efe84..8664e71d2 100644
--- a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs
+++ b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs
@@ -1881,3 +1881,36 @@ runDecoderStreamSourceContract().then(
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 64);
}
+
+#[tokio::test]
+async fn worker_text_encoder_stream_preserves_utf16_chunk_boundaries() {
+ ensure_v8();
+ let source = [
+ include_str!("../../../../../tests/fixtures/text-encoder-boundaries.js"),
+ r#"
+runTextEncoderBoundaryContract().then(
+ result => { postMessage(result); close(); },
+ error => { postMessage({ failure: String(error) }); close(); }
+);
+"#,
+ ]
+ .join("\n");
+ let mut handle = spawn_worker(
+ source,
+ "https://text-encoder-boundaries.test/worker.js".into(),
+ );
+ let message = timeout(TIMEOUT, handle.recv())
+ .await
+ .expect("text encoder boundary contract should complete")
+ .expect("worker result channel should stay open");
+ handle.terminate_and_join();
+ let facts: serde_json::Value = serde_json::from_str(&expect_post_json(message))
+ .expect("text encoder boundary results should be valid JSON");
+ let checks = facts["checks"].as_array().expect("boundary checks");
+ let failures = checks
+ .iter()
+ .filter(|check| check["passed"] != true)
+ .collect::<Vec<_>>();
+ assert!(failures.is_empty(), "{failures:#?}");
+ assert_eq!(checks.len(), 49);
+}
diff --git a/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.js b/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.js
new file mode 100644
index 000000000..12292ce1a
--- /dev/null
+++ b/moli-renderer-v8/tests/fixtures/text-encoder-boundaries.js
@@ -0,0 +1,111 @@
+globalThis.runTextEncoderBoundaryContract = async function () {
+ const checks = [];
+ async function check(name, operation) {
+ try { checks.push({ name, passed: await operation() === true }); }
+ catch (error) { checks.push({ name, passed: false, error: String(error) }); }
+ }
+ function collect(readable) {
+ const reader = readable.getReader();
+ const result = { chunks: [], error: undefined };
+ result.done = (async () => {
+ try {
+ while (true) {
+ const item = await reader.read();
+ if (item.done) return;
+ result.chunks.push(Array.from(item.value));
+ }
+ } catch (error) { result.error = error; }
+ })();
+ return result;
+ }
+ const inputs = [
+ "", "abc", "\u0000\u20ac\u4e2d", "A\ud83d\udc99B",
+ "\ud800", "\udfff", "\ud800\udc00", "\udbff\udfff",
+ "\ud800A\udc00", "\ud800\udbff\udc00", "\udbff\udfff\ud800",
+ "\ud800\ud800\udc00\udfff",
+ ];
+ for (const input of ["abc", "\ud83d\udc99"]) {
+ await check(`whole input ${input}`, async () => {
+ const encoder = new TextEncoderStream();
+ const result = collect(encoder.readable);
+ const writer = encoder.writable.getWriter();
+ await writer.write(input);
+ await writer.close();
+ await result.done;
+ return result.error === undefined && JSON.stringify(result.chunks) ===
+ JSON.stringify([Array.from(new TextEncoder().encode(input))]);
+ });
+ }
+ for (const [index, input] of inputs.entries()) {
+ const expected = Array.from(new TextEncoder().encode(input));
+ for (let split = 0; split <= input.length; split++) {
+ await check(`partition ${index} at ${split}`, async () => {
+ const parts = [input.slice(0, split), "", input.slice(split), ""];
+ const source = new ReadableStream({ start(controller) {
+ for (const part of parts) controller.enqueue(part);
+ controller.close();
+ } });
+ const result = collect(source.pipeThrough(new TextEncoderStream()));
+ await result.done;
+ return result.error === undefined && result.chunks.every(chunk => chunk.length > 0) &&
+ JSON.stringify(result.chunks.flat()) === JSON.stringify(expected);
+ });
+ }
+ }
+ for (const end of ["close", "pair", "abort", "throw"]) {
+ await check(`pending leading surrogate: ${end}`, async () => {
+ const encoder = new TextEncoderStream();
+ const result = collect(encoder.readable);
+ const writer = encoder.writable.getWriter();
+ writer.closed.catch(() => {});
+ await writer.write("\ud83d");
+ await writer.write("");
+ const held = result.chunks.length === 0;
+ const sentinel = {};
+ if (end === "pair") {
+ await writer.write("\udc99");
+ await writer.close();
+ } else if (end === "close") {
+ await writer.close();
+ } else if (end === "abort") {
+ await writer.abort(sentinel);
+ } else {
+ try {
+ await writer.write({ toString() { throw sentinel; } });
+ return false;
+ } catch (error) { if (error !== sentinel) return false; }
+ }
+ await result.done;
+ const expected = end === "close" ? [[0xef, 0xbf, 0xbd]] :
+ end === "pair" ? [[0xf0, 0x9f, 0x92, 0x99]] : [];
+ return held && JSON.stringify(result.chunks) === JSON.stringify(expected) &&
+ result.error === (end === "abort" || end === "throw" ? sentinel : undefined);
+ });
+ }
+ await check("DOMString conversion preserves a surrogate pair split across objects", async () => {
+ const encoder = new TextEncoderStream();
+ const result = collect(encoder.readable);
+ const writer = encoder.writable.getWriter();
+ let calls = 0;
+ for (const value of ["\ud83d", "", "\udc99"]) {
+ await writer.write({ toString() { calls++; return value; } });
+ }
+ await writer.close();
+ await result.done;
+ return calls === 3 && result.error === undefined &&
+ JSON.stringify(result.chunks) === JSON.stringify([[0xf0, 0x9f, 0x92, 0x99]]);
+ });
+ await check("interleaved encoders have separate pending surrogates", async () => {
+ const left = new TextEncoderStream(), right = new TextEncoderStream();
+ const a = collect(left.readable), b = collect(right.readable);
+ const aw = left.writable.getWriter(), bw = right.writable.getWriter();
+ await aw.write("\ud83d");
+ await bw.write("A");
+ await aw.write("\udc99");
+ await Promise.all([aw.close(), bw.close(), a.done, b.done]);
+ return a.error === undefined && b.error === undefined &&
+ JSON.stringify(a.chunks) === JSON.stringify([[0xf0, 0x9f, 0x92, 0x99]]) &&
+ JSON.stringify(b.chunks) === JSON.stringify([[65]]);
+ });
+ return { checks };
+};
diff --git a/moli-streams/src/transform.rs b/moli-streams/src/transform.rs
index 7a5a5b086..0db767246 100644
--- a/moli-streams/src/transform.rs
+++ b/moli-streams/src/transform.rs
@@ -42,8 +42,9 @@ impl TransformMode {
pub const fn flush_algorithm(self) -> TransformFlushAlgorithm {
match self {
Self::Callback => TransformFlushAlgorithm::Callback,
+ Self::TextEncoder => TransformFlushAlgorithm::TextEncoder,
Self::TextDecoder => TransformFlushAlgorithm::TextDecoder,
- Self::Identity | Self::TextEncoder => TransformFlushAlgorithm::None,
+ Self::Identity => TransformFlushAlgorithm::None,
}
}
@@ -70,6 +71,7 @@ pub enum TransformWriteAlgorithm {
pub enum TransformFlushAlgorithm {
None,
Callback,
+ TextEncoder,
TextDecoder,
}
@@ -561,7 +563,7 @@ mod tests {
true,
TransformMode::TextEncoder,
TransformWriteAlgorithm::TextEncoder,
- TransformFlushAlgorithm::None,
+ TransformFlushAlgorithm::TextEncoder,
TransformCancelAlgorithm::None,
),
(