Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(resource): integrate shared disk-backed response storage
ldm018a9f45be2bb
diff --git a/Cargo.lock b/Cargo.lockindex b97087d36..7ce6816ab 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2880,8 +2880,10 @@ dependencies = ["http-auth","moli-browser-profile","moli-cookie-jar",+ "moli-disk-pool","moli-dom","moli-fetch",+ "moli-parkable-image","moli-web-mime","regex","serde",@@ -3078,6 +3080,7 @@ dependencies = ["moli-cookie-jar","moli-crypto","moli-css-parse",+ "moli-disk-pool","moli-dom","moli-encoding","moli-encoding-detector",@@ -3098,6 +3101,7 @@ dependencies = ["moli-owner-queue","moli-page-types","moli-paint",+ "moli-parkable-image","moli-parser","moli-protocol-cdp","moli-script",diff --git a/moli-page-types/Cargo.toml b/moli-page-types/Cargo.tomlindex 03070faf5..13f6cfcd6 100644--- a/moli-page-types/Cargo.toml+++ b/moli-page-types/Cargo.toml@@ -5,14 +5,16 @@ version = "0.1.0"edition = "2024"[features]-test-support = []+test-support = ["moli-disk-pool/test-support"][dependencies]bitflags = "2.11.0"moli-browser-profile = { path = "../moli-browser-profile" }moli-cookie-jar = { path = "../moli-cookie-jar" }moli-dom = { path = "../moli-dom" }+moli-disk-pool = { path = "../moli-disk-pool" }moli-fetch = { path = "../moli-fetch" }+moli-parkable-image = { path = "../moli-parkable-image" }moli-web-mime = { path = "../moli-web-mime" }http-auth = { version = "0.1.10", default-features = false }regex = "1.12.3"@@ -20,5 +22,8 @@ serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.145"url = "2.5.7"+[dev-dependencies]+moli-disk-pool = { path = "../moli-disk-pool", features = ["test-support"] }+[lints]workspace = truediff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rsindex cca2ce969..a94496da8 100644--- a/moli-page-types/src/lib.rs+++ b/moli-page-types/src/lib.rs@@ -15,19 +15,12 @@ mod renderer_transport_memory;use std::{borrow::Cow,collections::{BTreeMap, BTreeSet},- fs::{self, File, OpenOptions},- io::{self, Read, Seek, Write},- path::{Path, PathBuf},- sync::{- Arc,- atomic::{AtomicU64, Ordering},- },- time::{SystemTime, UNIX_EPOCH},+ io::{self, Write},+ sync::Arc,};-#[cfg(unix)]-use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};-+use moli_disk_pool::{DiskData, DiskPool};+use moli_parkable_image::ParkableImage;use regex::Regex;use serde::{Deserialize, Serialize};use serde_json::Value;@@ -43,7 +36,6 @@ use moli_fetch::{use moli_web_mime::is_json_module_mime;const SUBRESOURCE_RESPONSE_BODY_MEMORY_LIMIT: usize = 1024 * 1024;-static NEXT_SUBRESOURCE_RESPONSE_BODY_SPOOL_ID: AtomicU64 = AtomicU64::new(1);pub use document_activity::DocumentActivity;pub use inspector_identity::{@@ -367,6 +359,20 @@ impl NavigationResponse {self.body_bytes().to_vec()}+ /// Moves the complete materialized payload out of this response.+ ///+ /// The response head remains usable and the response body becomes empty.+ /// This is intended for handing large payloads to an owning backing store+ /// without briefly retaining both the response buffer and a clone.+ pub fn take_body_bytes(&mut self) -> Vec<u8> {+ std::mem::replace(+ &mut self.body,+ ResponseBody::materialized_text(String::new(), Vec::new()),+ )+ .try_into_materialized_bytes()+ .expect("NavigationResponse body should remain materialized")+ }+pub fn materialized_body(&self) -> ResponseBody {self.body.clone_materialized()@@ -1482,24 +1488,73 @@ pub struct SubresourceResponseBody {#[derive(Debug)]enum SubresourceResponseBodyInner {Memory(Vec<u8>),- File { path: PathBuf, len: usize },+ Disk {+ storage: PooledSubresourceResponseBody,+ },+ ParkableImage(ParkableImage),}-impl SubresourceResponseBodyInner {- fn in_memory_bytes(&self) -> Option<&[u8]> {- match self {- Self::Memory(bytes) => Some(bytes),- Self::File { .. } => None,+#[derive(Debug)]+struct PooledSubresourceResponseBody {+ chunks: Vec<DiskData>,+ trailing_bytes: Box<[u8]>,+ len: usize,+}++impl PooledSubresourceResponseBody {+ fn read_range(&self, offset: usize, max_len: usize) -> io::Result<Vec<u8>> {+ if offset >= self.len || max_len == 0 {+ return Ok(Vec::new());}+ let len = max_len.min(self.len - offset);+ let mut bytes = vec![0; len];+ self.read_exact_at(offset, &mut bytes)?;+ Ok(bytes)}-}-impl Drop for SubresourceResponseBodyInner {- fn drop(&mut self) {- if let Self::File { path, .. } = self {- let _ = fs::remove_file(path);+ fn read_exact_at(&self, mut offset: usize, mut buffer: &mut [u8]) -> io::Result<()> {+ for chunk in &self.chunks {+ if buffer.is_empty() {+ break;+ }+ if offset >= chunk.len() {+ offset -= chunk.len();+ continue;+ }+ let len = buffer.len().min(chunk.len() - offset);+ chunk.read_exact_at(offset, &mut buffer[..len])?;+ buffer = &mut buffer[len..];+ offset = 0;+ }++ if !buffer.is_empty() {+ let trailing = self.trailing_bytes.get(offset..).ok_or_else(|| {+ io::Error::new(+ io::ErrorKind::UnexpectedEof,+ "pooled resource body ended before its recorded length",+ )+ })?;+ let len = buffer.len().min(trailing.len());+ buffer[..len].copy_from_slice(&trailing[..len]);+ buffer = &mut buffer[len..];+ }++ if buffer.is_empty() {+ Ok(())+ } else {+ Err(io::Error::new(+ io::ErrorKind::UnexpectedEof,+ "pooled resource body ended before its recorded length",+ ))}}++ fn write_to<W: Write>(&self, writer: &mut W) -> io::Result<()> {+ for chunk in &self.chunks {+ chunk.write_to(writer)?;+ }+ writer.write_all(&self.trailing_bytes)+ }}impl PartialEq for SubresourceResponseBody {@@ -1507,13 +1562,17 @@ impl PartialEq for SubresourceResponseBody {if Arc::ptr_eq(&self.inner, &other.inner) {return true;}- let Some(left_bytes) = self.inner.in_memory_bytes() else {- return false;- };- let Some(right_bytes) = other.inner.in_memory_bytes() else {- return false;- };- left_bytes == right_bytes+ match (self.inner.as_ref(), other.inner.as_ref()) {+ (+ SubresourceResponseBodyInner::Memory(left),+ SubresourceResponseBodyInner::Memory(right),+ ) => left == right,+ (+ SubresourceResponseBodyInner::ParkableImage(left),+ SubresourceResponseBodyInner::ParkableImage(right),+ ) => left.shares_storage_with(right),+ _ => false,+ }}}@@ -1524,9 +1583,9 @@ pub struct SubresourceResponseBodyWriter {memory_limit: usize,len: usize,memory: Vec<u8>,- file: Option<File>,- path: Option<PathBuf>,- spool_failed: bool,+ disk_pool: Option<DiskPool>,+ disk_chunks: Vec<DiskData>,+ disk_write_failed: bool,}impl Default for SubresourceResponseBodyWriter {@@ -1537,120 +1596,103 @@ impl Default for SubresourceResponseBodyWriter {impl SubresourceResponseBodyWriter {pub fn new(memory_limit: usize) -> Self {+ Self::with_memory_limit_and_disk_pool(memory_limit, None)+ }++ /// Creates a writer using the default in-memory threshold and a shared+ /// browser resource disk pool when one is available.+ pub fn with_disk_pool(disk_pool: Option<DiskPool>) -> Self {+ Self::with_memory_limit_and_disk_pool(SUBRESOURCE_RESPONSE_BODY_MEMORY_LIMIT, disk_pool)+ }++ pub fn with_memory_limit_and_disk_pool(+ memory_limit: usize,+ disk_pool: Option<DiskPool>,+ ) -> Self {Self {memory_limit,len: 0,memory: Vec::new(),- file: None,- path: None,- spool_failed: false,+ disk_pool,+ disk_chunks: Vec::new(),+ disk_write_failed: false,}}- pub fn append(&mut self, bytes: &[u8]) {+ pub fn append(&mut self, mut bytes: &[u8]) {if bytes.is_empty() {return;}- if self.file.is_some() {- if let Some(file) = self.file.as_mut()- && file.write_all(bytes).is_ok()- {- self.len = self.len.saturating_add(bytes.len());- return;- }- self.abandon_file_to_memory();- self.spool_failed = true;- }- if self.spool_failed || self.len.saturating_add(bytes.len()) <= self.memory_limit {+ self.len = self.len.saturating_add(bytes.len());+ if self.disk_write_failed || self.disk_pool.is_none() {self.memory.extend_from_slice(bytes);- self.len = self.len.saturating_add(bytes.len());return;}- if self.ensure_file().is_ok()- && let Some(file) = self.file.as_mut()- && file.write_all(bytes).is_ok()++ if self.disk_chunks.is_empty()+ && self.memory.len().saturating_add(bytes.len()) <= self.memory_limit{- self.len = self.len.saturating_add(bytes.len());+ self.memory.extend_from_slice(bytes);return;}- // Spooling is an optimization for CDP bookkeeping. If private temp-file- // creation fails, keep the protocol-correct body in memory rather than- // dropping captured bytes.- self.spool_failed = true;- self.abandon_file_to_memory();- self.memory.extend_from_slice(bytes);- self.len = self.len.saturating_add(bytes.len());- }- pub fn finish(mut self) -> SubresourceResponseBody {- if let Some(mut file) = self.file.take() {- if file.flush().is_ok() {- let path = self- .path- .take()- .expect("subresource body spool path should be set");- return SubresourceResponseBody {- inner: Arc::new(SubresourceResponseBodyInner::File {- path,- len: self.len,- }),- };+ let chunk_size = self.memory_limit.max(1);+ while !bytes.is_empty() {+ if self.memory.len() == chunk_size && !self.flush_memory_to_disk() {+ self.disk_write_failed = true;+ self.memory.extend_from_slice(bytes);+ return;}- self.file = Some(file);- self.abandon_file_to_memory();- }- if let Some(path) = self.path.take() {- let _ = fs::remove_file(path);- }- SubresourceResponseBody::from_bytes(std::mem::take(&mut self.memory))- }-- fn ensure_file(&mut self) -> io::Result<()> {- if self.file.is_some() {- return Ok(());- }- let path = unique_subresource_response_body_spool_path()?;- let mut options = OpenOptions::new();- options.create_new(true).read(true).write(true);- configure_secure_subresource_response_body_spool_file_options(&mut options);- let mut file = options.open(&path)?;- if !self.memory.is_empty() {- if let Err(error) = file.write_all(&self.memory) {- drop(file);- let _ = fs::remove_file(&path);- return Err(error);+ let available = chunk_size - self.memory.len();+ let consumed = available.min(bytes.len());+ self.memory.extend_from_slice(&bytes[..consumed]);+ bytes = &bytes[consumed..];+ if self.memory.len() == chunk_size && !self.flush_memory_to_disk() {+ self.disk_write_failed = true;+ self.memory.extend_from_slice(bytes);+ return;}- self.memory.clear();}- self.path = Some(path);- self.file = Some(file);- Ok(())}- fn abandon_file_to_memory(&mut self) {- let _ = self.file.take();- if let Some(path) = self.path.take() {- if let Ok(mut bytes) = fs::read(&path) {- bytes.extend_from_slice(&self.memory);- self.memory = bytes;+ pub fn finish(mut self) -> SubresourceResponseBody {+ if !self.disk_chunks.is_empty() {+ if !self.memory.is_empty() && !self.disk_write_failed {+ let _ = self.flush_memory_to_disk();}- let _ = fs::remove_file(path);+ let stored_len = self+ .disk_chunks+ .iter()+ .map(DiskData::len)+ .sum::<usize>()+ .saturating_add(self.memory.len());+ debug_assert_eq!(stored_len, self.len);+ return SubresourceResponseBody {+ inner: Arc::new(SubresourceResponseBodyInner::Disk {+ storage: PooledSubresourceResponseBody {+ chunks: std::mem::take(&mut self.disk_chunks),+ trailing_bytes: std::mem::take(&mut self.memory).into_boxed_slice(),+ len: self.len,+ },+ }),+ };}- self.len = self.memory.len();- }-- #[cfg(feature = "test-support")]- pub fn replace_spool_path_for_test(&mut self, replacement: PathBuf) -> Option<PathBuf> {- self.path.replace(replacement)+ SubresourceResponseBody::from_bytes(std::mem::take(&mut self.memory))}-}-impl Drop for SubresourceResponseBodyWriter {- fn drop(&mut self) {- let _ = self.file.take();- if let Some(path) = self.path.take() {- let _ = fs::remove_file(path);+ fn flush_memory_to_disk(&mut self) -> bool {+ if self.memory.is_empty() {+ return true;}+ let Some(pool) = self.disk_pool.as_ref() else {+ return false;+ };+ let Ok(Some(chunk)) = pool.store(&self.memory) else {+ return false;+ };+ self.disk_chunks.push(chunk);+ self.memory.clear();+ true}}@@ -1661,6 +1703,14 @@ impl SubresourceResponseBody {}}+ /// Builds a response body that shares the encoded image backing.+ /// Cloning this body does not clone bytes.+ pub fn from_parkable_image(image: ParkableImage) -> Self {+ Self {+ inner: Arc::new(SubresourceResponseBodyInner::ParkableImage(image)),+ }+ }+/// Copies the exact bytes from a materialized fetch response into the/// renderer-neutral subresource body carrier.pub fn from_fetch_response(response: &Response) -> Self {@@ -1686,7 +1736,10 @@ impl SubresourceResponseBody {pub fn try_bytes(&self) -> io::Result<Cow<'_, [u8]>> {match self.inner.as_ref() {SubresourceResponseBodyInner::Memory(bytes) => Ok(Cow::Borrowed(bytes)),- SubresourceResponseBodyInner::File { .. } => self.materialize_bytes().map(Cow::Owned),+ SubresourceResponseBodyInner::Disk { .. } => self.materialize_bytes().map(Cow::Owned),+ SubresourceResponseBodyInner::ParkableImage(image) => image+ .snapshot()+ .map(|snapshot| Cow::Owned(snapshot.to_vec())),}}@@ -1735,15 +1788,11 @@ impl SubresourceResponseBody {SubresourceResponseBodyInner::Memory(bytes) => {Ok(bytes.get(offset..).map(<[u8]>::to_vec).unwrap_or_default())}- SubresourceResponseBodyInner::File { path, len, .. } => {- if offset >= *len {- return Ok(Vec::new());- }- let mut file = File::open(path)?;- file.seek(io::SeekFrom::Start(offset as u64))?;- let mut bytes = Vec::with_capacity(len.saturating_sub(offset));- file.read_to_end(&mut bytes)?;- Ok(bytes)+ SubresourceResponseBodyInner::Disk { storage, .. } => {+ storage.read_range(offset, storage.len.saturating_sub(offset))+ }+ SubresourceResponseBodyInner::ParkableImage(image) => {+ image.read_range(offset, image.len().saturating_sub(offset))}}}@@ -1760,34 +1809,19 @@ impl SubresourceResponseBody {let len = remaining.len().min(max_len);Ok(remaining[..len].to_vec())}- SubresourceResponseBodyInner::File { path, len, .. } => {- if offset >= *len {- return Ok(Vec::new());- }- let mut file = File::open(path)?;- file.seek(io::SeekFrom::Start(offset as u64))?;- let mut chunk = vec![0; max_len.min(len.saturating_sub(offset))];- let read = file.read(&mut chunk)?;- chunk.truncate(read);- Ok(chunk)+ SubresourceResponseBodyInner::Disk { storage, .. } => {+ storage.read_range(offset, max_len)}+ SubresourceResponseBodyInner::ParkableImage(image) => image.read_range(offset, max_len),}}pub fn write_bytes_to<W: Write>(&self, writer: &mut W) -> io::Result<()> {match self.inner.as_ref() {SubresourceResponseBodyInner::Memory(bytes) => writer.write_all(bytes),- SubresourceResponseBodyInner::File { path, .. } => {- let mut file = File::open(path)?;- let mut buffer = [0; 64 * 1024];- loop {- let read = file.read(&mut buffer)?;- if read == 0 {- break;- }- writer.write_all(&buffer[..read])?;- }- Ok(())+ SubresourceResponseBodyInner::Disk { storage, .. } => storage.write_to(writer),+ SubresourceResponseBodyInner::ParkableImage(image) => {+ writer.write_all(&image.snapshot()?)}}}@@ -1795,7 +1829,8 @@ impl SubresourceResponseBody {pub fn len(&self) -> usize {match self.inner.as_ref() {SubresourceResponseBodyInner::Memory(bytes) => bytes.len(),- SubresourceResponseBodyInner::File { len, .. } => *len,+ SubresourceResponseBodyInner::Disk { storage, .. } => storage.len,+ SubresourceResponseBodyInner::ParkableImage(image) => image.len(),}}@@ -1804,42 +1839,6 @@ impl SubresourceResponseBody {}}-fn unique_subresource_response_body_spool_path() -> io::Result<PathBuf> {- let root = std::env::temp_dir().join("moli-subresource-body-spool");- create_secure_subresource_response_body_spool_root(&root)?;- let id = NEXT_SUBRESOURCE_RESPONSE_BODY_SPOOL_ID.fetch_add(1, Ordering::Relaxed);- let nanos = SystemTime::now()- .duration_since(UNIX_EPOCH)- .map(|duration| duration.as_nanos())- .unwrap_or_default();- Ok(root.join(format!(- "subresource-body-{}-{id}-{nanos}.bin",- std::process::id()- )))-}--#[cfg(unix)]-fn create_secure_subresource_response_body_spool_root(root: &Path) -> io::Result<()> {- let mut builder = fs::DirBuilder::new();- builder.recursive(true).mode(0o700);- builder.create(root)?;- fs::set_permissions(root, fs::Permissions::from_mode(0o700))?;- Ok(())-}--#[cfg(not(unix))]-fn create_secure_subresource_response_body_spool_root(root: &Path) -> io::Result<()> {- fs::create_dir_all(root)-}--#[cfg(unix)]-fn configure_secure_subresource_response_body_spool_file_options(options: &mut OpenOptions) {- options.mode(0o600);-}--#[cfg(not(unix))]-fn configure_secure_subresource_response_body_spool_file_options(_options: &mut OpenOptions) {}-#[derive(Debug, Clone, Default)]pub struct SubresourceResponseWaitCriteria {pub url_contains: Option<String>,@@ -3505,6 +3504,13 @@ mod tests {Url::parse(&format!("https://example.test{path}")).expect("test URL should parse")}+ fn pooled_body_writer(memory_limit: usize, pool: &DiskPool) -> SubresourceResponseBodyWriter {+ SubresourceResponseBodyWriter::with_memory_limit_and_disk_pool(+ memory_limit,+ Some(pool.clone()),+ )+ }+#[test]fn navigation_response_byte_parts_transfer_exact_storage() {let template =@@ -4022,7 +4028,8 @@ mod tests {#[test]fn subresource_response_body_writer_keeps_small_body_in_memory() {- let mut writer = SubresourceResponseBodyWriter::new(16);+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(16, &pool);writer.append(b"hello");writer.append(b" world");let body = writer.finish();@@ -4030,53 +4037,158 @@ mod tests {assert_eq!(body.len(), 11);assert_eq!(body.try_bytes().unwrap().as_ref(), b"hello world");assert_eq!(body.clone_body_bytes(), b"hello world");+ assert_eq!(pool.diagnostics().disk_footprint_bytes, 0);+ }++ #[test]+ fn navigation_response_body_can_move_into_an_external_backing_without_cloning() {+ let mut response = NavigationResponse::from_head_and_body(+ ResponseHead {+ final_url: test_url("/image.png"),+ status: 200,+ headers: vec![("Content-Type".to_owned(), "image/png".to_owned())],+ request_cookie_report: None,+ cookie_set_reports: Vec::new(),+ redirected: false,+ redirect_chain: Vec::new(),+ from_cache: false,+ negotiated_http_version: None,+ },+ String::new(),+ b"encoded-image".to_vec(),+ );++ assert_eq!(response.take_body_bytes(), b"encoded-image");+ assert!(response.body_bytes().is_empty());+ assert_eq!(response.status, 200);+ assert_eq!(response.final_url, test_url("/image.png"));}#[test]- fn subresource_response_body_writer_spools_large_body_and_materializes_on_demand() {- let mut writer = SubresourceResponseBodyWriter::new(4);+ fn subresource_response_body_writer_pools_large_body_and_materializes_on_demand() {+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(4, &pool);writer.append(b"hello");writer.append(b" world");let body = writer.finish();assert_eq!(body.len(), 11);+ assert_eq!(pool.diagnostics().disk_footprint_bytes, 11);assert_eq!(body.try_bytes().unwrap().as_ref(), b"hello world");let mut copied = Vec::new();body.write_bytes_to(&mut copied)- .expect("spooled body should stream-copy into writer");+ .expect("pooled body should stream-copy into writer");assert_eq!(copied, b"hello world");}#[test]- fn subresource_response_body_reads_memory_and_spooled_chunks_by_offset() {+ fn pooled_response_body_releases_the_empty_staging_allocation() {+ let memory_limit = 1024 * 1024;+ let bytes = vec![7; memory_limit + 1];+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(memory_limit, &pool);+ writer.append(&bytes);+ let body = writer.finish();++ assert_eq!(body.len(), bytes.len());+ assert_eq!(body.materialize_bytes().unwrap(), bytes);+ assert!(+ body.renderer_transport_retained_memory_bytes() < memory_limit,+ "a disk-backed body must not retain its full staging allocation"+ );+ }++ #[test]+ fn subresource_response_body_reads_memory_and_pooled_chunks_by_offset() {let memory = SubresourceResponseBody::from_bytes(b"hello world".to_vec());assert_eq!(memory.read_chunk(6, 3).unwrap(), b"wor");assert_eq!(memory.materialize_bytes_from(6).unwrap(), b"world");assert_eq!(memory.diagnostic_clone_body_bytes_from(6), b"world");- let mut writer = SubresourceResponseBodyWriter::new(4);+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(4, &pool);writer.append(b"hello");writer.append(b" world");- let spooled = writer.finish();- assert_eq!(spooled.read_chunk(6, 3).unwrap(), b"wor");- assert_eq!(spooled.materialize_bytes_from(6).unwrap(), b"world");- assert_eq!(spooled.diagnostic_clone_body_bytes_from(6), b"world");+ let pooled = writer.finish();+ assert_eq!(pooled.read_chunk(3, 5).unwrap(), b"lo wo");+ assert_eq!(pooled.read_chunk(6, 3).unwrap(), b"wor");+ assert_eq!(pooled.materialize_bytes_from(6).unwrap(), b"world");+ assert_eq!(pooled.diagnostic_clone_body_bytes_from(6), b"world");}#[test]- fn subresource_response_body_fallible_materialize_reports_missing_spool_file() {- let missing_path = std::env::temp_dir().join(format!(- "moli-missing-subresource-body-{}",- std::process::id()- ));- let _ = fs::remove_file(&missing_path);- let body = SubresourceResponseBody {- inner: Arc::new(SubresourceResponseBodyInner::File {- path: missing_path,- len: 5,- }),- };+ fn subresource_response_bodies_share_pool_and_release_extents_on_last_drop() {+ let pool = DiskPool::new(None).unwrap();+ let mut first_writer = pooled_body_writer(4, &pool);+ first_writer.append(b"abcdefgh");+ let first = first_writer.finish();+ let mut second_writer = pooled_body_writer(4, &pool);+ second_writer.append(b"ijklmnop");+ let second = second_writer.finish();++ assert_eq!(pool.diagnostics().disk_footprint_bytes, 16);+ assert_eq!(pool.diagnostics().free_bytes, 0);+ let first_clone = first.clone();+ drop(first);+ assert_eq!(+ pool.diagnostics().free_bytes,+ 0,+ "the body Arc should own its pooled extents"+ );+ drop(first_clone);+ assert_eq!(pool.diagnostics().free_bytes, 8);+ drop(second);+ assert_eq!(pool.diagnostics().free_bytes, 16);+ assert_eq!(pool.diagnostics().free_chunk_count, 1);+ }++ #[test]+ fn parkable_image_body_shares_one_extent_and_unparks_on_demand() {+ let pool = DiskPool::new(None).unwrap();+ let manager = moli_parkable_image::ParkableImageManager::new(+ Some(pool.clone()),+ moli_parkable_image::ParkableImagePolicy {+ min_size_to_park: 1,+ parking_delay: std::time::Duration::ZERO,+ reader_release_delay: std::time::Duration::ZERO,+ },+ );+ let encoded = manager.from_frozen_bytes(b"encoded image".to_vec());+ let body = SubresourceResponseBody::from_parkable_image(encoded.clone());++ manager.force_park_images();+ assert_eq!(manager.diagnostics().parked_count, 1);+ assert_eq!(pool.diagnostics().disk_footprint_bytes, 13);+ assert_eq!(body.read_chunk(8, 5).unwrap(), b"image");+ assert_eq!(body.materialize_bytes().unwrap(), b"encoded image");+ assert_eq!(pool.diagnostics().disk_footprint_bytes, 13);++ drop(encoded);+ assert_eq!(pool.diagnostics().free_bytes, 0);+ drop(body);+ assert_eq!(pool.diagnostics().free_bytes, 13);+ }++ #[test]+ fn subresource_response_body_keeps_order_when_pool_reaches_capacity() {+ let pool = DiskPool::new(Some(4)).unwrap();+ let mut writer = pooled_body_writer(4, &pool);+ writer.append(b"hello world");+ let body = writer.finish();++ assert_eq!(pool.diagnostics().disk_footprint_bytes, 4);+ assert_eq!(body.materialize_bytes().unwrap(), b"hello world");+ assert_eq!(body.read_chunk(3, 6).unwrap(), b"lo wor");+ }++ #[test]+ fn subresource_response_body_fallible_materialize_reports_pool_corruption() {+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(2, &pool);+ writer.append(b"hello");+ let body = writer.finish();+ pool.truncate_file_for_test(0).unwrap();assert!(body.materialize_bytes().is_err());assert!(body.materialize_bytes_from(2).is_err());@@ -4253,28 +4365,14 @@ mod tests {#[test]fn subresource_response_body_partial_eq_does_not_treat_unreadable_sources_as_empty() {- let missing_path_a = std::env::temp_dir().join(format!(- "moli-missing-subresource-body-eq-a-{}",- std::process::id()- ));- let missing_path_b = std::env::temp_dir().join(format!(- "moli-missing-subresource-body-eq-b-{}",- std::process::id()- ));- let _ = fs::remove_file(&missing_path_a);- let _ = fs::remove_file(&missing_path_b);- let body_a = SubresourceResponseBody {- inner: Arc::new(SubresourceResponseBodyInner::File {- path: missing_path_a,- len: 5,- }),- };- let body_b = SubresourceResponseBody {- inner: Arc::new(SubresourceResponseBodyInner::File {- path: missing_path_b,- len: 5,- }),- };+ let pool = DiskPool::new(None).unwrap();+ let mut writer_a = pooled_body_writer(2, &pool);+ writer_a.append(b"first");+ let body_a = writer_a.finish();+ let mut writer_b = pooled_body_writer(2, &pool);+ writer_b.append(b"other");+ let body_b = writer_b.finish();+ pool.truncate_file_for_test(0).unwrap();assert_eq!(body_a, body_a.clone());assert_ne!(body_a, body_b);@@ -4282,33 +4380,35 @@ mod tests {}#[test]- fn subresource_response_body_partial_eq_keeps_file_backed_equality_io_free() {- let mut left_writer = SubresourceResponseBodyWriter::new(2);+ fn subresource_response_body_partial_eq_keeps_disk_backed_equality_io_free() {+ let pool = DiskPool::new(None).unwrap();+ let mut left_writer = pooled_body_writer(2, &pool);left_writer.append(b"same");let left = left_writer.finish();- let mut right_writer = SubresourceResponseBodyWriter::new(2);+ let mut right_writer = pooled_body_writer(2, &pool);right_writer.append(b"same");let right = right_writer.finish();assert_ne!(left, right,- "ordinary equality should not read separate file-backed sources"+ "ordinary equality should not read separate disk-backed sources");assert!(left.try_byte_eq(&right)- .expect("explicit byte equality should read spooled bodies"),+ .expect("explicit byte equality should read pooled bodies"),"callers that need content equality must opt in to source I/O");}#[test]fn subresource_network_record_response_body_byte_equality_is_explicit() {- let mut left_writer = SubresourceResponseBodyWriter::new(2);+ let pool = DiskPool::new(None).unwrap();+ let mut left_writer = pooled_body_writer(2, &pool);left_writer.append(b"same");let left_body = left_writer.finish();- let mut right_writer = SubresourceResponseBodyWriter::new(2);+ let mut right_writer = pooled_body_writer(2, &pool);right_writer.append(b"same");let right_body = right_writer.finish();@@ -4352,14 +4452,15 @@ mod tests {assert!(left_record.try_response_body_byte_eq(&right_record)- .expect("explicit record body equality should read spooled bodies"),+ .expect("explicit record body equality should read pooled bodies"),"callers that need record body content equality must opt in");}#[test]- fn subresource_response_body_preserves_invalid_bytes_when_spooled() {- let mut writer = SubresourceResponseBodyWriter::new(2);+ fn subresource_response_body_preserves_invalid_bytes_when_pooled() {+ let pool = DiskPool::new(None).unwrap();+ let mut writer = pooled_body_writer(2, &pool);writer.append(b"hi ");writer.append(&[0xff, b'!']);let body = writer.finish();diff --git a/moli-page-types/src/renderer_transport_memory.rs b/moli-page-types/src/renderer_transport_memory.rsindex 05bab0592..9454b067e 100644--- a/moli-page-types/src/renderer_transport_memory.rs+++ b/moli-page-types/src/renderer_transport_memory.rs@@ -1,7 +1,7 @@//! Conservative retained-memory estimates for renderer output admission.//!//! These estimates intentionally live beside the renderer-neutral payload-//! definitions, where private capacities and file-backed response state are+//! definitions, where private capacities and disk-backed response state are//! visible. They do not model serialized CDP JSON; protocol projection has not//! happened while these values are queued.@@ -127,10 +127,14 @@ impl SubresourceResponseBody {SubresourceResponseBodyInner::Memory(bytes) => bytes.capacity().saturating_add(std::mem::size_of::<SubresourceResponseBodyInner>()),- SubresourceResponseBodyInner::File { path, .. } => path- .as_os_str()- .len()- .saturating_mul(2)+ SubresourceResponseBodyInner::Disk { storage } => storage+ .chunks+ .capacity()+ .saturating_mul(std::mem::size_of::<DiskData>())+ .saturating_add(storage.trailing_bytes.len())+ .saturating_add(std::mem::size_of::<SubresourceResponseBodyInner>()),+ SubresourceResponseBodyInner::ParkableImage(image) => image+ .retained_memory_bytes().saturating_add(std::mem::size_of::<SubresourceResponseBodyInner>()),}}diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.tomlindex be320e876..e85c61aa2 100644--- a/moli-renderer-v8/Cargo.toml+++ b/moli-renderer-v8/Cargo.toml@@ -39,8 +39,10 @@ moli-browser-profile = { path = "../moli-browser-profile" }moli-crypto = { path = "../moli-crypto" }moli-css-parse = { path = "../moli-css-parse" }moli-dom = { path = "../moli-dom" }+moli-disk-pool = { path = "../moli-disk-pool" }moli-parser = { path = "../moli-parser" }moli-page-types = { path = "../moli-page-types" }+moli-parkable-image = { path = "../moli-parkable-image" }moli-protocol-cdp = { path = "../moli-protocol-cdp" }moli-selector = { path = "../moli-selector" }moli-shared-worker = { path = "../moli-shared-worker" }diff --git a/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs b/moli-renderer-v8/src/context_bootstrap/performance_runtime.rsindex fb4f8712e..b9c381d10 100644--- a/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs+++ b/moli-renderer-v8/src/context_bootstrap/performance_runtime.rs@@ -227,14 +227,29 @@ impl ResourcePerformanceEntry {start_unix_millis: Option<f64>,response: &crate::protocol_types::NavigationResponse,) -> Self {- let body_size = response.body_bytes().len() as f64;+ Self::from_network_response_with_body_size(+ name,+ initiator_type,+ start_unix_millis,+ response,+ response.body_bytes().len(),+ )+ }++ pub(crate) fn from_network_response_with_body_size(+ name: impl Into<String>,+ initiator_type: &'static str,+ start_unix_millis: Option<f64>,+ response: &crate::protocol_types::NavigationResponse,+ body_size: usize,+ ) -> Self {Self::from_response_parts(name,initiator_type,start_unix_millis,response.status,&response.headers,- body_size,+ body_size as f64,)}diff --git a/moli-renderer-v8/src/native_bridge/context_host/image_resources/decode.rs b/moli-renderer-v8/src/native_bridge/context_host/image_resources/decode.rsindex f9969eeff..11a5aab1a 100644--- a/moli-renderer-v8/src/native_bridge/context_host/image_resources/decode.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/image_resources/decode.rs@@ -1,5 +1,6 @@use std::{collections::HashMap, sync::Arc};+use moli_parkable_image::ParkableImage;use parking_lot::Mutex;use super::{@@ -45,6 +46,7 @@ impl Default for ImageDecodeCoordinator {pub(super) enum ImageDecodeQueueError {JobLimit,EncodedByteLimit,+ EncodedSource,}pub(super) struct ReadyDecodedImage {@@ -86,7 +88,7 @@ impl ImageDecodeCoordinator {identity: ImageResourceRequestIdentity,target: WindowDocumentTaskTarget,metadata: ImageDecodeMetadata,- encoded: &[u8],+ encoded: ParkableImage,) -> Result<(), ImageDecodeQueueError> {let coordinator = self.clone();let task_id = RendererPageImageLoadEventTaskId::new(identity.element, identity.sequence);@@ -119,7 +121,7 @@ impl ImageDecodeCoordinator {store: CssImageResourceStore,identity: CssImageResourceRequestIdentity,metadata: ImageDecodeMetadata,- encoded: &[u8],+ encoded: ParkableImage,) -> Result<(), ImageDecodeQueueError> {self.submit_job(runner,@@ -152,7 +154,7 @@ impl ImageDecodeCoordinator {&self,runner: RendererResourceTaskRunner,metadata: ImageDecodeMetadata,- encoded: &[u8],+ encoded: ParkableImage,complete: impl FnOnce(ImageDecodeResult) + Send + 'static,) -> Result<(), ImageDecodeQueueError> {self.submit_job(runner, metadata, encoded, move |result, _job_permit| {@@ -164,21 +166,24 @@ impl ImageDecodeCoordinator {&self,runner: RendererResourceTaskRunner,metadata: ImageDecodeMetadata,- encoded: &[u8],+ encoded: ParkableImage,complete: impl FnOnce(ImageDecodeResult, ImageDecodeJobPermit) + Send + 'static,) -> Result<(), ImageDecodeQueueError> {- let job_permit =- self.inner- .budget- .admit_job(encoded.len())- .map_err(|error| match error {- ImageResourceBudgetError::JobLimit => ImageDecodeQueueError::JobLimit,- ImageResourceBudgetError::EncodedByteLimit- | ImageResourceBudgetError::DecodedByteLimit => {- ImageDecodeQueueError::EncodedByteLimit- }- })?;- let encoded = encoded.to_vec();+ let encoded_len = encoded.len();+ let job_permit = self+ .inner+ .budget+ .admit_job(encoded_len)+ .map_err(|error| match error {+ ImageResourceBudgetError::JobLimit => ImageDecodeQueueError::JobLimit,+ ImageResourceBudgetError::EncodedByteLimit+ | ImageResourceBudgetError::DecodedByteLimit => {+ ImageDecodeQueueError::EncodedByteLimit+ }+ })?;+ let snapshot = encoded+ .snapshot()+ .map_err(|_| ImageDecodeQueueError::EncodedSource)?;let coordinator = self.clone();runner.spawn(async move {let Ok(_concurrency) = coordinator.inner.concurrency.clone().acquire_owned().await@@ -187,28 +192,31 @@ impl ImageDecodeCoordinator {};let budget = coordinator.inner.budget.clone();let decoded = tokio::task::spawn_blocking(move || {- let retained_bytes = metadata- .retained_byte_len(encoded.len())- .ok_or_else(|| "decoded image byte estimate overflowed".to_owned())?;- let decoded_bytes_permit =- budget.reserve_decoded(retained_bytes).map_err(|error| {- format!("decoded image budget rejected the resource: {error:?}")- })?;- let content = match metadata {- ImageDecodeMetadata::Raster(metadata) => {- let decoded =- moli_image::decode_raster_image_with_metadata(&encoded, metadata)- .map_err(|error| error.to_string())?;- DecodedImageContent::Raster(Arc::new(decoded.image))- }- ImageDecodeMetadata::Svg(metadata) => {- let decoded =- moli_image::decode_svg_image_with_metadata(&encoded, metadata)- .map_err(|error| error.to_string())?;- DecodedImageContent::Svg(Arc::new(decoded))- }- };- Ok::<_, String>(ReadyDecodedImage {+ let result = (|| {+ let retained_bytes = metadata+ .retained_byte_len(encoded_len)+ .ok_or_else(|| "decoded image byte estimate overflowed".to_owned())?;+ let decoded_bytes_permit =+ budget.reserve_decoded(retained_bytes).map_err(|error| {+ format!("decoded image budget rejected the resource: {error:?}")+ })?;+ let content = match metadata {+ ImageDecodeMetadata::Raster(metadata) => {+ let decoded =+ moli_image::decode_raster_image_with_metadata(&snapshot, metadata)+ .map_err(|error| error.to_string())?;+ DecodedImageContent::Raster(Arc::new(decoded.image))+ }+ ImageDecodeMetadata::Svg(metadata) => {+ let decoded =+ moli_image::decode_svg_image_with_metadata(&snapshot, metadata)+ .map_err(|error| error.to_string())?;+ DecodedImageContent::Svg(Arc::new(decoded))+ }+ };+ Ok::<_, String>((content, decoded_bytes_permit))+ })();+ result.map(|(content, decoded_bytes_permit)| ReadyDecodedImage {content,decoded_bytes_permit,})@@ -264,3 +272,54 @@ impl ImageDecodeCoordinator {self.inner.completions.lock().remove(&task_id).is_some()}}++#[cfg(test)]+mod tests {+ use std::time::Duration;++ use moli_disk_pool::DiskPool;+ use moli_parkable_image::{ParkableImageManager, ParkableImagePolicy};++ use super::*;++ #[tokio::test(flavor = "multi_thread")]+ async fn decode_releases_its_snapshot_for_manager_parking() {+ let pixels = moli_image::RgbaImage::try_new(1, 1, vec![255, 0, 0, 255]).unwrap();+ let bytes = moli_image::encode_png(&pixels).unwrap().bytes;+ let metadata = moli_image::probe_raster_image(&bytes).unwrap();+ let pool = DiskPool::new(None).unwrap();+ let manager = ParkableImageManager::new(+ Some(pool.clone()),+ ParkableImagePolicy {+ min_size_to_park: 1,+ parking_delay: Duration::ZERO,+ reader_release_delay: Duration::ZERO,+ },+ );+ let encoded = manager.from_frozen_bytes(bytes.clone());+ let coordinator = ImageDecodeCoordinator::default();+ let runner = RendererResourceTaskRunner::from_current_tokio().unwrap();+ let (complete_tx, complete_rx) = tokio::sync::oneshot::channel();++ coordinator+ .submit_preload(+ runner,+ ImageDecodeMetadata::Raster(metadata),+ encoded.clone(),+ move |result| {+ let _ = complete_tx.send(result);+ },+ )+ .unwrap();+ let result = tokio::time::timeout(Duration::from_secs(5), complete_rx)+ .await+ .expect("image decode should complete")+ .expect("decode callback should send its result");+ assert!(matches!(result, ImageDecodeResult::Ready(_)));+ manager.force_park_images();+ assert_eq!(manager.diagnostics().parked_count, 1);+ assert_eq!(manager.diagnostics().retained_memory_bytes, 0);+ assert_eq!(pool.diagnostics().disk_footprint_bytes, bytes.len() as u64);+ assert_eq!(encoded.snapshot().unwrap().as_ref(), bytes);+ }+}diff --git a/moli-renderer-v8/src/native_bridge/context_host/image_resources/mod.rs b/moli-renderer-v8/src/native_bridge/context_host/image_resources/mod.rsindex 629b23318..bbf542d65 100644--- a/moli-renderer-v8/src/native_bridge/context_host/image_resources/mod.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/image_resources/mod.rs@@ -4,6 +4,8 @@ mod decode;mod preload;mod state;+use moli_parkable_image::ParkableImage;+use crate::{document_runtime::DomHandle,frame_owner_model::FrameDocumentTaskOwner,@@ -250,7 +252,7 @@ impl super::JsContextHost {&mut self,identity: &CssImageResourceRequestIdentity,descriptor: Option<ImageResponseDescriptor>,- encoded: &[u8],+ encoded: ParkableImage,) -> bool {let Some(descriptor) = descriptor else {return self.image_resources.css.fail(identity);@@ -262,6 +264,7 @@ impl super::JsContextHost {{return false;}+ let encoded_len = encoded.len();let submission = self.document_resource_loader_for_owner(identity.document_owner).map(|loader| loader.task_runner())@@ -279,7 +282,7 @@ impl super::JsContextHost {tracing::debug!(document = identity.document_handle.index(),request_id = identity.request_id,- encoded_bytes = encoded.len(),+ encoded_bytes = encoded_len,"queued bounded CSS image resource decode");return true;@@ -328,7 +331,7 @@ impl super::JsContextHost {element: DomHandle,sequence: super::ImageLoadEventId,descriptor: Option<ImageResponseDescriptor>,- encoded: &[u8],+ encoded: ParkableImage,) -> ImageResponseCompletion {self.complete_pending_image_response_if_matches(element,@@ -346,7 +349,7 @@ impl super::JsContextHost {sequence: super::ImageLoadEventId,internal_id: u64,descriptor: Option<ImageResponseDescriptor>,- encoded: &[u8],+ encoded: ParkableImage,) -> ImageResponseCompletion {self.complete_pending_image_response_if_matches(element,@@ -365,7 +368,7 @@ impl super::JsContextHost {internal_id: Option<u64>,source: super::PendingImageLoadTerminalSource,descriptor: Option<ImageResponseDescriptor>,- encoded: &[u8],+ encoded: ParkableImage,) -> ImageResponseCompletion {let Some(pending) = self.pending_image_load_event(element) else {return ImageResponseCompletion::Ignored;@@ -435,6 +438,7 @@ impl super::JsContextHost {{return ImageResponseCompletion::Ignored;}+ let encoded_len = encoded.len();let submission = loader.map(|loader| loader.task_runner()).ok_or(decode::ImageDecodeQueueError::JobLimit)@@ -456,7 +460,7 @@ impl super::JsContextHost {tracing::debug!(image = element.index(),sequence = sequence.get(),- encoded_bytes = encoded.len(),+ encoded_bytes = encoded_len,"queued bounded image resource decode");return ImageResponseCompletion::Accepted { followup: None };diff --git a/moli-renderer-v8/src/native_bridge/context_host/image_resources/preload.rs b/moli-renderer-v8/src/native_bridge/context_host/image_resources/preload.rsindex 3ccc15f1f..36949d503 100644--- a/moli-renderer-v8/src/native_bridge/context_host/image_resources/preload.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/image_resources/preload.rs@@ -69,6 +69,7 @@ struct ScannedImagePreloadLoadState {pub(crate) struct ScannedImagePreloadOutcome {network_result: SharedNavigationResponseResult,+ encoded: Option<moli_parkable_image::ParkableImage>,_resource: Option<Arc<ReadyImageResource>>,_retained_body_permit: Option<ScannedImagePreloadBodyPermit>,}@@ -77,6 +78,10 @@ impl ScannedImagePreloadOutcome {pub(crate) fn network_result(&self) -> &SharedNavigationResponseResult {&self.network_result}++ pub(crate) fn encoded(&self) -> Option<moli_parkable_image::ParkableImage> {+ self.encoded.clone()+ }}struct ScannedImagePreloadBodyPermit {@@ -258,16 +263,13 @@ impl SharedScannedImagePreloadLoad {&self,runner: RendererResourceTaskRunner,network_result: SharedNavigationResponseResult,+ encoded: Option<moli_parkable_image::ParkableImage>,response_is_decode_eligible: bool,) {if self.try_outcome().is_some() {return;}- let encoded_bytes = network_result- .as_ref()- .as_ref()- .map(|response| response.body_bytes().len())- .unwrap_or(0);+ let encoded_bytes = encoded.as_ref().map_or(0, |encoded| encoded.len());let retained_body_permit = if encoded_bytes == 0 {None} else {@@ -289,32 +291,37 @@ impl SharedScannedImagePreloadLoad {.as_ref().as_ref().ok()- .and_then(crate::network_host::image_response_descriptor)+ .zip(encoded.as_ref())+ .and_then(|(response, encoded)| {+ crate::network_host::image_response_descriptor_from_parkable(+ response, encoded,+ )+ })}).flatten();let Some(descriptor) = descriptor else {self.finish(ScannedImagePreloadOutcome {network_result,+ encoded,_resource: None,_retained_body_permit: retained_body_permit,});return;};- let encoded_source = network_result.clone();- let encoded = encoded_source- .as_ref()- .as_ref()- .expect("a decoded preload descriptor requires a response")- .body_bytes();+ let encoded = encoded.expect("a decoded preload descriptor requires encoded bytes");let load = self.clone();- let completion_state = Arc::new(Mutex::new(Some((network_result, retained_body_permit))));+ let completion_state = Arc::new(Mutex::new(Some((+ network_result,+ encoded.clone(),+ retained_body_permit,+ ))));let completion_state_for_decode = completion_state.clone();let result = self.inner.decode.submit_preload(runner,descriptor.decode_metadata,encoded,move |decode_result| {- let Some((network_result, retained_body_permit)) =+ let Some((network_result, encoded, retained_body_permit)) =completion_state_for_decode.lock().take()else {return;@@ -339,6 +346,7 @@ impl SharedScannedImagePreloadLoad {.insert(load.inner.identity.request_key.clone(), &resource);load.finish(ScannedImagePreloadOutcome {network_result,+ encoded: Some(encoded),_resource: Some(resource),_retained_body_permit: retained_body_permit,});@@ -353,6 +361,7 @@ impl SharedScannedImagePreloadLoad {);load.finish(ScannedImagePreloadOutcome {network_result,+ encoded: Some(encoded),_resource: None,_retained_body_permit: retained_body_permit,});@@ -361,12 +370,13 @@ impl SharedScannedImagePreloadLoad {},);if result.is_err() {- let (network_result, retained_body_permit) = completion_state+ let (network_result, encoded, retained_body_permit) = completion_state.lock().take().expect("rejected preload decode must retain its completion state");self.finish(ScannedImagePreloadOutcome {network_result,+ encoded: Some(encoded),_resource: None,_retained_body_permit: retained_body_permit,});@@ -506,6 +516,7 @@ mod tests {claimed.finish_network_result(RendererResourceTaskRunner::for_test(),Arc::new(Err("network failed".to_owned())),+ None,false,);let outcome = claimed@@ -559,15 +570,13 @@ mod tests {vec![("Content-Type".to_owned(), "image/png".to_owned())],String::new(),);- let response = crate::protocol_types::NavigationResponse::from_head_and_body(- response.head(),- String::from_utf8_lossy(&encoded.bytes).into_owned(),- encoded.bytes,- );+ let encoded =+ moli_parkable_image::ParkableImageManager::default().from_frozen_bytes(encoded.bytes);load.finish_network_result(RendererResourceTaskRunner::for_test(),Arc::new(Ok(response)),+ Some(encoded),true,);let outcome = tokio::time::timeout(std::time::Duration::from_secs(2), load.wait_outcome())@@ -632,15 +641,13 @@ mod tests {vec![("Content-Type".to_owned(), "image/png".to_owned())],String::new(),);- let response = crate::protocol_types::NavigationResponse::from_head_and_body(- response.head(),- String::new(),- body_bytes,- );+ let encoded =+ moli_parkable_image::ParkableImageManager::default().from_frozen_bytes(body_bytes);load.finish_network_result(RendererResourceTaskRunner::for_test(),Arc::new(Ok(response)),+ Some(encoded),false,);let outcome = loaddiff --git a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rsindex ef473fb31..638c8798e 100644--- a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs@@ -292,6 +292,29 @@ impl JsContextHost {resource_type,request_initiator_type,result,+ None,+ );+ self.record_subresource_network(record);+ }++ pub(crate) fn record_get_subresource_network_result_with_body_and_initiator(+ &mut self,+ frame_id: Option<String>,+ document_url: url::Url,+ request_url: url::Url,+ resource_type: SubresourceResourceType,+ request_initiator_type: SubresourceRequestInitiatorType,+ result: &std::result::Result<crate::protocol_types::NavigationResponse, String>,+ response_body: SubresourceResponseBody,+ ) {+ let record = Self::get_subresource_network_record_with_initiator(+ frame_id,+ document_url,+ request_url,+ resource_type,+ request_initiator_type,+ result,+ Some(response_body),);self.record_subresource_network(record);}@@ -314,6 +337,7 @@ impl JsContextHost {resource_type,request_initiator_type,result,+ None,);self.push_network_output_item(ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record,@@ -327,6 +351,7 @@ impl JsContextHost {resource_type: SubresourceResourceType,request_initiator_type: SubresourceRequestInitiatorType,result: &std::result::Result<crate::protocol_types::NavigationResponse, String>,+ response_body: Option<SubresourceResponseBody>,) -> SubresourceNetworkRecord {match result {Ok(response) => SubresourceNetworkRecord::success_with_body(@@ -342,7 +367,8 @@ impl JsContextHost {response.final_url.clone(),response.status,response.headers.clone(),- SubresourceResponseBody::from_navigation_response(response),+ response_body+ .unwrap_or_else(|| SubresourceResponseBody::from_navigation_response(response)),response.cookie_set_reports.clone(),).with_request_initiator_type(request_initiator_type)diff --git a/moli-renderer-v8/src/native_bridge/element/images/src.rs b/moli-renderer-v8/src/native_bridge/element/images/src.rsindex 3ba4c69ff..ce9873cba 100644--- a/moli-renderer-v8/src/native_bridge/element/images/src.rs+++ b/moli-renderer-v8/src/native_bridge/element/images/src.rs@@ -229,13 +229,15 @@ fn queue_image_load_event(followup,);}- Ok(crate::network_host::ImageElementResourceFetchStart::Local { response }) => {- let descriptor = crate::network_host::image_response_descriptor(&response);+ Ok(crate::network_host::ImageElementResourceFetchStart::Local {+ descriptor,+ encoded,+ }) => {let completion = runtime.complete_pending_image_load_local_response_if_matches(handle,pending.id(),descriptor,- response.body_bytes(),+ encoded,);queue_image_load_terminal_followup_if_ready(runtime_ptr,diff --git a/moli-renderer-v8/src/network/backend/diagnostics.rs b/moli-renderer-v8/src/network/backend/diagnostics.rsindex 259c39944..596652bb2 100644--- a/moli-renderer-v8/src/network/backend/diagnostics.rs+++ b/moli-renderer-v8/src/network/backend/diagnostics.rs@@ -1,11 +1,63 @@use super::SharedMemoryResourceCacheDiagnostics;use crate::network::loads::DetachedKeepaliveLoadDiagnostics;+/// Snapshot of the browser-context scoped resource disk pool.+#[derive(Debug, Clone, Copy, serde::Serialize, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub struct BrowserResourceDiskPoolDiagnostics {+ pub may_write: bool,+ pub disk_footprint_bytes: u64,+ pub free_bytes: usize,+ pub free_chunk_count: usize,+}++impl From<moli_disk_pool::DiskPoolDiagnostics> for BrowserResourceDiskPoolDiagnostics {+ fn from(diagnostics: moli_disk_pool::DiskPoolDiagnostics) -> Self {+ Self {+ may_write: diagnostics.may_write,+ disk_footprint_bytes: diagnostics.disk_footprint_bytes,+ free_bytes: diagnostics.free_bytes,+ free_chunk_count: diagnostics.free_chunk_count,+ }+ }+}++/// Snapshot of encoded image bytes managed by the browser-runtime image pool.+#[derive(Debug, Clone, Copy, serde::Serialize, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub struct BrowserParkableImageDiagnostics {+ pub image_count: usize,+ pub resident_count: usize,+ pub parking_count: usize,+ pub resident_with_disk_backup_count: usize,+ pub parked_count: usize,+ pub retained_memory_bytes: usize,+ pub retained_disk_bytes: usize,+}++impl From<moli_parkable_image::ParkableImageManagerDiagnostics>+ for BrowserParkableImageDiagnostics+{+ fn from(diagnostics: moli_parkable_image::ParkableImageManagerDiagnostics) -> Self {+ Self {+ image_count: diagnostics.image_count,+ resident_count: diagnostics.resident_count,+ parking_count: diagnostics.parking_count,+ resident_with_disk_backup_count: diagnostics.resident_with_disk_backup_count,+ parked_count: diagnostics.parked_count,+ retained_memory_bytes: diagnostics.retained_memory_bytes,+ retained_disk_bytes: diagnostics.retained_disk_bytes,+ }+ }+}+/// Snapshot of one browser-context scoped renderer resource runtime.#[derive(Debug, Clone, Copy, serde::Serialize, PartialEq, Eq)]#[serde(rename_all = "camelCase")]pub struct BrowserResourceRuntimeDiagnostics {pub runtime_id: u64,pub memory_cache: SharedMemoryResourceCacheDiagnostics,+ pub disk_pool: Option<BrowserResourceDiskPoolDiagnostics>,+ pub parkable_images: BrowserParkableImageDiagnostics,pub(crate) detached_keepalive_loads: DetachedKeepaliveLoadDiagnostics,}diff --git a/moli-renderer-v8/src/network/backend/mod.rs b/moli-renderer-v8/src/network/backend/mod.rsindex 6811c3ce1..2029c6af9 100644--- a/moli-renderer-v8/src/network/backend/mod.rs+++ b/moli-renderer-v8/src/network/backend/mod.rs@@ -8,7 +8,10 @@ mod diagnostics;mod memory_cache;mod runtime;-pub use diagnostics::BrowserResourceRuntimeDiagnostics;+pub use diagnostics::{+ BrowserParkableImageDiagnostics, BrowserResourceDiskPoolDiagnostics,+ BrowserResourceRuntimeDiagnostics,+};pub use memory_cache::SharedMemoryResourceCacheDiagnostics;pub use runtime::{BrowserResourceRuntime, BrowserResourceRuntimeOwner, BrowserResourceRuntimeOwnerRegistrar,diff --git a/moli-renderer-v8/src/network/backend/runtime.rs b/moli-renderer-v8/src/network/backend/runtime.rsindex 4849b7049..5e5d37ddf 100644--- a/moli-renderer-v8/src/network/backend/runtime.rs+++ b/moli-renderer-v8/src/network/backend/runtime.rs@@ -5,15 +5,18 @@ use std::{rc::{Rc, Weak as RcWeak},sync::{Arc, Weak as ArcWeak,- atomic::{AtomicU64, Ordering},+ atomic::{AtomicBool, AtomicU64, Ordering},},};use moli_cookie_jar::SharedBrowserCookieStore;+use moli_disk_pool::DiskPool;use moli_fetch::{FetchClient, FetchClientHandle, FetchConfig, FetchRuntimeJoinReport};+use moli_parkable_image::{ParkableImageManager, ParkableImagePolicy};use parking_lot::{Mutex, RwLock};use super::{+ BrowserParkableImageDiagnostics, BrowserResourceDiskPoolDiagnostics,BrowserResourceRuntimeDiagnostics, SharedMemoryResourceCacheDiagnostics,memory_cache::SharedMemoryResourceCache,};@@ -127,6 +130,10 @@ struct BrowserResourceRuntimeInner {owner_root_id: AtomicU64,client: FetchClientHandle,memory_cache: Mutex<SharedMemoryResourceCache>,+ disk_pool: Option<DiskPool>,+ parkable_images: ParkableImageManager,+ parkable_image_schedule_notify: Arc<tokio::sync::Notify>,+ parkable_image_scheduler_started: AtomicBool,detached_keepalive_loads: DetachedKeepaliveLoadRegistry,}@@ -142,12 +149,24 @@ impl BrowserResourceRuntimeOwner {let runtime_id = NEXT_BROWSER_RESOURCE_RUNTIME_ID.fetch_add(1, Ordering::Relaxed).saturating_add(1);+ let disk_pool = DiskPool::new(None).ok();+ let parkable_image_schedule_notify = Arc::new(tokio::sync::Notify::new());+ let schedule_notify = Arc::clone(&parkable_image_schedule_notify);+ let parkable_images = ParkableImageManager::new_with_schedule_wakeup(+ disk_pool.clone(),+ ParkableImagePolicy::default(),+ move || schedule_notify.notify_one(),+ );let runtime = BrowserResourceRuntime {inner: Arc::new(BrowserResourceRuntimeInner {id: runtime_id,owner_root_id: AtomicU64::new(0),client: fetch_owner.handle(),memory_cache: Mutex::new(SharedMemoryResourceCache::default()),+ parkable_images,+ disk_pool,+ parkable_image_schedule_notify,+ parkable_image_scheduler_started: AtomicBool::new(false),detached_keepalive_loads: DetachedKeepaliveLoadRegistry::default(),}),};@@ -207,10 +226,49 @@ impl BrowserResourceRuntime {self.inner.memory_cache.lock().diagnostics()}+ pub fn disk_pool(&self) -> Option<DiskPool> {+ self.inner.disk_pool.clone()+ }++ pub fn disk_pool_diagnostics(&self) -> Option<BrowserResourceDiskPoolDiagnostics> {+ self.inner+ .disk_pool+ .as_ref()+ .map(|pool| pool.diagnostics().into())+ }++ pub fn parkable_image_manager(&self) -> ParkableImageManager {+ self.inner.parkable_images.clone()+ }++ pub fn parkable_image_diagnostics(&self) -> BrowserParkableImageDiagnostics {+ self.inner.parkable_images.diagnostics().into()+ }++ pub(crate) fn ensure_parkable_image_scheduler(+ &self,+ runner: &crate::network::RendererResourceTaskRunner,+ ) {+ if self+ .inner+ .parkable_image_scheduler_started+ .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)+ .is_err()+ {+ return;+ }++ let weak_runtime = Arc::downgrade(&self.inner);+ let schedule_notify = Arc::clone(&self.inner.parkable_image_schedule_notify);+ runner.spawn(run_parkable_image_scheduler(weak_runtime, schedule_notify));+ }+pub fn diagnostics(&self) -> BrowserResourceRuntimeDiagnostics {BrowserResourceRuntimeDiagnostics {runtime_id: self.runtime_id_for_diagnostics(),memory_cache: self.memory_cache_diagnostics(),+ disk_pool: self.disk_pool_diagnostics(),+ parkable_images: self.parkable_image_diagnostics(),detached_keepalive_loads: self.detached_keepalive_diagnostics(),}}@@ -254,6 +312,52 @@ impl BrowserResourceRuntime {}}+async fn run_parkable_image_scheduler(+ weak_runtime: ArcWeak<BrowserResourceRuntimeInner>,+ schedule_notify: Arc<tokio::sync::Notify>,+) {+ loop {+ // Arm the notification before reading the deadline so a concurrent+ // state transition cannot be lost between inspection and waiting.+ let notified = schedule_notify.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();++ let Some(runtime) = weak_runtime.upgrade() else {+ return;+ };+ let next_deadline = runtime.parkable_images.next_parking_deadline();+ drop(runtime);++ match next_deadline {+ Some(deadline) => {+ let sleep = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline));+ tokio::pin!(sleep);+ tokio::select! {+ () = notified.as_mut() => continue,+ () = sleep.as_mut() => {}+ }+ }+ None => {+ notified.await;+ continue;+ }+ }++ let Some(runtime) = weak_runtime.upgrade() else {+ return;+ };+ let manager = runtime.parkable_images.clone();+ drop(runtime);+ if tokio::task::spawn_blocking(move || manager.park_images())+ .await+ .is_err()+ {+ return;+ }+ }+}+impl BrowserResourceRuntimeOwner {pub fn request_shutdown(&self) {self.fetch_owner.request_shutdown();@@ -546,6 +650,7 @@ impl Drop for BrowserResourceRuntimeOwnerSet {impl Drop for BrowserResourceRuntimeInner {fn drop(&mut self) {+ self.parkable_image_schedule_notify.notify_one();self.client.request_shutdown();}}@@ -595,6 +700,32 @@ mod tests {BrowserResourceRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store())}+ #[tokio::test(flavor = "multi_thread")]+ async fn parkable_image_scheduler_runs_after_the_last_reader_release_grace() -> Result<()> {+ let (root, binding) = BrowserResourceRuntimeOwnerRoot::new(registration());+ let runtime = binding.current();+ let runner = crate::network::RendererResourceTaskRunner::from_current_tokio()?;+ runtime.ensure_parkable_image_scheduler(&runner);+ let manager = runtime.parkable_image_manager();+ let image = manager.from_frozen_bytes(vec![7; 4096]);++ let snapshot = image.snapshot()?;+ drop(snapshot);+ timeout(Duration::from_secs(5), async {+ while manager.diagnostics().parked_count == 0 {+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ })+ .await+ .expect("the reader-release deadline should wake the image scheduler");++ assert_eq!(manager.diagnostics().resident_count, 0);+ assert_eq!(manager.diagnostics().parked_count, 1);+ drop((image, manager, runtime, binding));+ root.shutdown_and_join();+ Ok(())+ }+#[test]fn retired_owner_reaps_after_last_external_runtime_lease_drops() {let (root, binding) = BrowserResourceRuntimeOwnerRoot::new(registration());diff --git a/moli-renderer-v8/src/network/mod.rs b/moli-renderer-v8/src/network/mod.rsindex ccf1c431d..bcba300d2 100644--- a/moli-renderer-v8/src/network/mod.rs+++ b/moli-renderer-v8/src/network/mod.rs@@ -18,7 +18,8 @@ mod request_client;mod task_runner;pub use backend::{- BrowserResourceRuntime, BrowserResourceRuntimeDiagnostics, BrowserResourceRuntimeOwner,+ BrowserParkableImageDiagnostics, BrowserResourceDiskPoolDiagnostics, BrowserResourceRuntime,+ BrowserResourceRuntimeDiagnostics, BrowserResourceRuntimeOwner,BrowserResourceRuntimeOwnerRegistrar, BrowserResourceRuntimeOwnerRegistration,SharedMemoryResourceCacheDiagnostics,};diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rsindex 8a34f9234..38044b467 100644--- a/moli-renderer-v8/src/network/request_client.rs+++ b/moli-renderer-v8/src/network/request_client.rs@@ -9,10 +9,12 @@ use anyhow::{Context, Result, anyhow};use moli_cookie_jar::{BrowserCookieFacadeContext, SharedBrowserCookieStore, new_shared_browser_cookie_store,};+use moli_disk_pool::DiskPool;use moli_fetch::{FetchCancelHandle, FetchConfig, NetworkFetchResult, RawResponse, Request, Response,ResponseHead, StreamingRawResponse,};+use moli_parkable_image::ParkableImageManager;use crate::{protocol_types::OptionalResourceFetchMask, types::SubresourceResourceType};@@ -71,6 +73,19 @@ impl ResourceRequestClient {self.resource_runtime = resource_runtime;}+ pub fn disk_pool(&self) -> Option<DiskPool> {+ self.resource_runtime.disk_pool()+ }++ pub(crate) fn parkable_image_manager(+ &self,+ runner: &crate::network::RendererResourceTaskRunner,+ ) -> ParkableImageManager {+ self.resource_runtime+ .ensure_parkable_image_scheduler(runner);+ self.resource_runtime.parkable_image_manager()+ }+pub fn shares_resource_runtime_with(&self, other: &Self) -> bool {self.resource_runtime.shares_state_with(&other.resource_runtime)diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rsindex 9884cc279..10a89ebeb 100644--- a/moli-renderer-v8/src/network/request_client/tests.rs+++ b/moli-renderer-v8/src/network/request_client/tests.rs@@ -29,7 +29,7 @@ use super::{};use crate::network::{BrowserResourceRuntimeOwner, BrowserResourceRuntimeOwnerRoot};use crate::protocol_types::OptionalResourceFetchMask;-use crate::types::SubresourceResourceType;+use crate::types::{SubresourceResourceType, SubresourceResponseBodyWriter};// These transport/cache tests use browser navigation semantics; origin policy// tests below construct their own explicit origin.@@ -64,6 +64,65 @@ fn loader_clones_share_one_browser_resource_runtime() {);}+#[test]+fn loader_clones_share_one_resource_disk_pool() {+ let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");+ let clone = loader.clone();+ let pool = loader+ .disk_pool()+ .expect("resource runtime should create one disk pool");+ let mut writer =+ SubresourceResponseBodyWriter::with_memory_limit_and_disk_pool(2, clone.disk_pool());+ writer.append(b"shared");+ let body = writer.finish();++ assert_eq!(body.materialize_bytes().unwrap(), b"shared");+ assert_eq!(pool.diagnostics().disk_footprint_bytes, 6);+ assert_eq!(+ loader+ .resource_runtime_diagnostics()+ .disk_pool+ .expect("disk pool diagnostics should be present")+ .disk_footprint_bytes,+ 6+ );+ drop(body);+ assert_eq!(pool.diagnostics().free_bytes, 6);+}++#[test]+fn loader_clones_share_one_parkable_image_manager() {+ let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");+ let clone = loader.clone();+ let runner = crate::network::RendererResourceTaskRunner::for_test();+ let image = loader+ .parkable_image_manager(&runner)+ .from_frozen_bytes(vec![7; 2 * 1024]);++ assert_eq!(+ clone+ .resource_runtime_diagnostics()+ .parkable_images+ .image_count,+ 1+ );+ assert_eq!(+ clone+ .resource_runtime_diagnostics()+ .parkable_images+ .retained_memory_bytes,+ 2 * 1024+ );+ drop(image);+ assert_eq!(+ loader+ .resource_runtime_diagnostics()+ .parkable_images+ .image_count,+ 0+ );+}+#[test]fn page_loader_fork_shares_backend_but_isolates_mutable_policy() {let parent_owner = ResourceRequestClient::new(&FetchConfig::default()).expect("parent loader");@@ -133,6 +192,47 @@ fn independently_created_loaders_use_isolated_browser_resource_runtimes() {left.resource_runtime_diagnostics().runtime_id,right.resource_runtime_diagnostics().runtime_id);++ let runner = crate::network::RendererResourceTaskRunner::for_test();+ let _left_image = left+ .parkable_image_manager(&runner)+ .from_frozen_bytes(vec![1; 1024]);+ assert_eq!(+ left.resource_runtime_diagnostics()+ .parkable_images+ .image_count,+ 1+ );+ assert_eq!(+ right+ .resource_runtime_diagnostics()+ .parkable_images+ .image_count,+ 0+ );++ let left_data = left+ .disk_pool()+ .expect("left runtime should have a disk pool")+ .store(b"left")+ .unwrap()+ .unwrap();+ assert_eq!(+ left.resource_runtime_diagnostics()+ .disk_pool+ .unwrap()+ .disk_footprint_bytes,+ 4+ );+ assert_eq!(+ right+ .resource_runtime_diagnostics()+ .disk_pool+ .unwrap()+ .disk_footprint_bytes,+ 0+ );+ drop(left_data);}fn unique_test_cache_dir() -> PathBuf {diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rsindex 721aa70f0..55f9cf470 100644--- a/moli-renderer-v8/src/network_host.rs+++ b/moli-renderer-v8/src/network_host.rs@@ -28,7 +28,7 @@ use moli_webapi_declare::WebApiObject;use crate::network::ResourceRequestClient;pub(crate) use self::async_fetch::{- browser_request_needs_manual_preflight_redirects,+ browser_request_needs_manual_preflight_redirects, collect_image_response_into_parkable,fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata,fetch_browser_subresource_with_preflight_and_network_metadata,fetch_browser_subresource_with_preflight_headers,@@ -112,8 +112,9 @@ pub(crate) use self::headers::headers_constructor_callback;pub(crate) use self::headers::install_headers_template_bindings;pub(crate) use self::headers::{HeadersGuard, filter_headers_for_guard};pub(crate) use self::image::{- ImageElementResourceFetchStart, ScannedImagePreloadStart, image_response_descriptor,- start_image_element_resource_fetch, start_scanned_image_preload,+ ImageElementResourceFetchStart, ScannedImagePreloadStart,+ image_response_descriptor_from_parkable, start_image_element_resource_fetch,+ start_scanned_image_preload,};pub(in crate::network_host) use self::js_values::{defined_object_string_property, v8_json_parse};pub(crate) use self::media::{diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rsindex 4cc4459b8..ccd1948b2 100644--- a/moli-renderer-v8/src/network_host/async_fetch.rs+++ b/moli-renderer-v8/src/network_host/async_fetch.rs@@ -492,6 +492,11 @@ pub(crate) fn spawn_async_subresource_fetch(request_headers: moli_fetch::RequestHeaders,request_body: Option<String>,) {+ let parkable_image_manager = matches!(+ request.browser_request_metadata(),+ Some(BrowserRequestMetadata::Image)+ )+ .then(|| loader.parkable_image_manager(&task_runner));task_runner.spawn(async move {let preflight_observer =CorsPreflightNetworkObserver::new(completion_tx.clone(), network_context);@@ -510,6 +515,7 @@ pub(crate) fn spawn_async_subresource_fetch()) && request.follow_redirects&& request.request_mode != RequestMode::NoCors;+ let can_collect_image_body = parkable_image_manager.is_some() && request.follow_redirects;if moli_trace::cdp_runtime_trace_enabled() {tracing::info!(target: "moli_cdp_nav_timing",@@ -522,9 +528,35 @@ pub(crate) fn spawn_async_subresource_fetch(auth_requires_buffered_transport,requires_manual_preflight_redirects,can_stream_subresource_body,+ can_collect_image_body,stage = "async_subresource_transport_selected",);}+ if !auth_requires_buffered_transport && can_collect_image_body {+ let result = fetch_browser_image_into_parkable(+ &loader,+ request,+ cancel_handle,+ preflight_request_headers,+ Some(&preflight_observer),+ parkable_image_manager+ .expect("an image transport selection must retain its parkable manager"),+ )+ .await;+ let _ = completion_tx.send_async_subresource(AsyncSubresourceFetchCompletion {+ internal_id,+ request_url,+ request_method,+ request_headers,+ request_body,+ response_status_text: None,+ skip_fetch_security_validation: false,+ response_filter: None,+ network_error_text: None,+ result: crate::types::AsyncSubresourceFetchResult::from_image_result(result),+ });+ return;+ }if auth_requires_buffered_transport || !can_stream_subresource_body {let result = fetch_browser_subresource_with_preflight_headers_and_observer(loader,@@ -551,7 +583,7 @@ pub(crate) fn spawn_async_subresource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result,+ result: result.into(),});return;}@@ -585,12 +617,79 @@ pub(crate) fn spawn_async_subresource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err(error),+ result: crate::types::AsyncSubresourceFetchResult::Failure(error),});}});}+async fn fetch_browser_image_into_parkable(+ loader: &ResourceRequestClient,+ request: Request,+ cancel_handle: Option<FetchCancelHandle>,+ preflight_request_headers: Vec<(String, String)>,+ preflight_observer: Option<&CorsPreflightNetworkObserver>,+ manager: moli_parkable_image::ParkableImageManager,+) -> Result<+ (+ crate::protocol_types::NavigationResponse,+ moli_parkable_image::ParkableImage,+ ),+ String,+> {+ let requires_manual_preflight_redirects =+ browser_request_needs_manual_preflight_redirects(&request, &preflight_request_headers);+ let observed = if requires_manual_preflight_redirects {+ fetch_browser_subresource_raw_stream_with_manual_preflight_redirects(+ loader,+ request,+ cancel_handle,+ preflight_request_headers,+ preflight_observer,+ )+ .await?+ } else {+ fetch_browser_subresource_raw_stream_with_preflight_headers_and_observer(+ loader,+ request,+ cancel_handle,+ preflight_request_headers,+ preflight_observer,+ )+ .await?+ };+ collect_image_response_into_parkable(observed, manager).await+}++pub(crate) async fn collect_image_response_into_parkable(+ observed: NetworkFetchResult<StreamingRawResponse>,+ manager: moli_parkable_image::ParkableImageManager,+) -> Result<+ (+ crate::protocol_types::NavigationResponse,+ moli_parkable_image::ParkableImage,+ ),+ String,+> {+ let (mut response, request_observation) = observed.into_parts();+ let head = response.head();+ let mut encoded = Vec::new();+ while let Some(bytes) = response.next_chunk().await {+ encoded.extend_from_slice(&bytes);+ }+ response.finish().await.map_err(format_network_error)?;+ let encoded = manager.from_frozen_bytes(encoded);+ let response = crate::protocol_types::NavigationResponse::from_head_and_body(+ head,+ String::new(),+ Vec::new(),+ )+ .with_network_request_headers(+ request_observation.map(|observation| observation.into_headers()),+ );+ Ok((response, encoded))+}+async fn fetch_browser_subresource_streaming_with_preflight_headers(completion_tx: RendererResourceCompletionSender,loader: ResourceRequestClient,@@ -1192,6 +1291,87 @@ mod tests {anyhow::bail!("timed out waiting for async subresource event")}+ #[tokio::test(flavor = "multi_thread")]+ async fn image_body_is_received_directly_into_one_parkable_image() -> Result<()> {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let addr = listener.local_addr()?;+ let server = tokio::spawn(async move {+ let (mut stream, _) = listener.accept().await.unwrap();+ let request = read_http_request_text(&mut stream).await.unwrap();+ assert!(request.starts_with("GET /image.png HTTP/1.1"));+ stream+ .write_all(+ concat!(+ "HTTP/1.1 200 OK\r\n",+ "Content-Type: image/png\r\n",+ "Content-Length: 9\r\n",+ "Connection: close\r\n",+ "\r\n",+ "first"+ )+ .as_bytes(),+ )+ .await+ .unwrap();+ tokio::task::yield_now().await;+ stream.write_all(b"tail").await.unwrap();+ });++ let mut queue = RendererResourceCompletionTestHarness::new();+ let loader_owner = ResourceRequestClient::new(&FetchConfig::default())?;+ let loader = loader_owner.handle();+ let request_url = Url::parse(&format!("http://{addr}/image.png"))?;+ let document_url = Url::parse(&format!("http://{addr}/page"))?;+ let request = Request::get(request_url.as_str())?+ .with_initiator_url(&document_url)+ .with_request_origin(moli_url::WebOrigin::from_url(&document_url))+ .with_request_mode(RequestMode::NoCors)+ .with_redirect_mode(RequestRedirectMode::Follow)+ .with_browser_request_metadata(BrowserRequestMetadata::Image);++ spawn_async_subresource_fetch(+ crate::network::RendererResourceTaskRunner::from_current_tokio()?,+ queue.sender(),+ loader,+ request,+ Some(FetchCancelHandle::new()),+ Vec::new(),+ 75,+ AsyncSubresourceNetworkContext {+ frame_id: None,+ request_origin: moli_url::WebOrigin::from_url(&document_url),+ document_url,+ resource_type: SubresourceResourceType::Image,+ policy_context: Default::default(),+ },+ request_url,+ "GET".to_owned(),+ Vec::new(),+ None,+ );++ let AsyncSubresourceFetchEvent::Completion(completion) =+ next_async_subresource_event(&mut queue).await?+ else {+ anyhow::bail!("image transport must emit one buffered terminal completion");+ };+ assert_eq!(completion.internal_id, 75);+ let response = completion+ .result+ .as_ref()+ .map_err(|error| anyhow::anyhow!(error.clone()))?;+ assert_eq!(response.status, 200);+ assert!(response.body_bytes().is_empty());+ let encoded = completion+ .result+ .encoded()+ .expect("image completion must carry its encoded backing");+ assert_eq!(encoded.snapshot()?.as_ref(), b"firsttail");++ server.await?;+ Ok(())+ }+#[tokio::test(flavor = "multi_thread")]async fn service_worker_redirect_chain_prefixes_streaming_network_fallback() -> Result<()> {let listener = TcpListener::bind("127.0.0.1:0").await?;diff --git a/moli-renderer-v8/src/network_host/csp_reports.rs b/moli-renderer-v8/src/network_host/csp_reports.rsindex cb7a45128..75d71b484 100644--- a/moli-renderer-v8/src/network_host/csp_reports.rs+++ b/moli-renderer-v8/src/network_host/csp_reports.rs@@ -346,7 +346,7 @@ fn dispatch_service_worker_content_security_policy_report(skip_fetch_security_validation: true,response_filter: Default::default(),network_error_text: None,- result: Err("service worker csp report fetch dispatch failed".to_owned()),+ result: Err("service worker csp report fetch dispatch failed".to_owned()).into(),});true}diff --git a/moli-renderer-v8/src/network_host/event_source/request.rs b/moli-renderer-v8/src/network_host/event_source/request.rsindex e87b3d3b1..ed90f5a7c 100644--- a/moli-renderer-v8/src/network_host/event_source/request.rs+++ b/moli-renderer-v8/src/network_host/event_source/request.rs@@ -258,7 +258,7 @@ pub(crate) fn start_event_source_request<'s>(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Ok(response.into()),+ result: Ok(response.into()).into(),},);return;@@ -445,7 +445,7 @@ fn dispatch_service_worker_event_source<'s>(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker EventSource dispatch failed".to_owned()),+ result: Err("service worker EventSource dispatch failed".to_owned()).into(),},);}diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rsindex a1ca1ff50..d74efbb4b 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs@@ -128,7 +128,7 @@ pub(super) fn dispatch_service_worker_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker fetch dispatch failed".to_owned()),+ result: Err("service worker fetch dispatch failed".to_owned()).into(),});Ok(Some(internal_id))}diff --git a/moli-renderer-v8/src/network_host/image.rs b/moli-renderer-v8/src/network_host/image.rsindex a3732097c..e9a59c65d 100644--- a/moli-renderer-v8/src/network_host/image.rs+++ b/moli-renderer-v8/src/network_host/image.rs@@ -7,8 +7,8 @@ use crate::service_worker_runtime::{service_worker_fetch_request_metadata,};use crate::types::{- AsyncSubresourceFetchCompletion, AsyncSubresourceNetworkContext, ImageRequestCorsMode,- ImageRequestKey, PendingSubresourceFetchInfo, SubresourceResourceType,+ AsyncSubresourceFetchCompletion, AsyncSubresourceFetchResult, AsyncSubresourceNetworkContext,+ ImageRequestCorsMode, ImageRequestKey, PendingSubresourceFetchInfo, SubresourceResourceType,};use moli_fetch::{BrowserRequestMetadata, FetchCancelHandle, FetchPriorityHint, RequestCredentialsMode,@@ -18,7 +18,8 @@ use moli_fetch::{#[derive(Debug)]pub(crate) enum ImageElementResourceFetchStart {Local {- response: Box<crate::protocol_types::NavigationResponse>,+ descriptor: Option<crate::native_bridge::ImageResponseDescriptor>,+ encoded: moli_parkable_image::ParkableImage,},Failed,PolicySkipped,@@ -57,6 +58,10 @@ pub(crate) fn start_scanned_image_preload(};let document_url = environment.document_url;let request_origin = environment.request_origin;+ let task_runner = resource_loader.task_runner();+ let parkable_images = resource_loader+ .request_client()+ .parkable_image_manager(&task_runner);let client_id = host.service_worker_client_id_for_subresource_owner(OwnerDispatchScope::Top);if host.service_worker_controller_for_fetch(client_id, &document_url, &request_url)@@ -98,15 +103,15 @@ pub(crate) fn start_scanned_image_preload(.with_fetch_priority_hint(fetch_priority),Err(error) => {load.finish_network_result(- resource_loader.task_runner(),+ task_runner,std::sync::Arc::new(Err(error.to_string())),+ None,false,);return ScannedImagePreloadStart::Admitted;}};let loader = resource_loader.request_client().clone();- let task_runner = resource_loader.task_runner();let decode_runner = task_runner.clone();let cancel_handle = load.cancel_handle();task_runner.spawn(async move {@@ -115,34 +120,28 @@ pub(crate) fn start_scanned_image_preload(.await{Ok(observed) => {- let (response, request_observation) = observed.into_parts();- response- .into_materialized_raw_response()+ crate::network_host::collect_image_response_into_parkable(observed, parkable_images).await- .map(|response| {- let (head, body) = response.into_parts();- let body_bytes = body- .try_into_materialized_bytes()- .expect("materialized raw image response must retain exact bytes");- crate::protocol_types::NavigationResponse::from_head_and_body(- head,- String::new(),- body_bytes,- )- .with_network_request_headers(- request_observation.map(|observation| observation.into_headers()),- )- })- .map_err(|error| format!("scanned image preload body failed: {error:#}"))+ .map_err(|error| format!("scanned image preload body failed: {error}"))}Err(error) => Err(format!("scanned image preload failed: {error:#}")),};+ let (result, encoded) = match result {+ Ok((response, encoded)) => (Ok(response), Some(encoded)),+ Err(error) => (Err(error), None),+ };let response_is_decode_eligible = result.as_ref().is_ok_and(|response| {+ let Some(encoded) = encoded.as_ref() else {+ return false;+ };+ let Ok(snapshot) = encoded.snapshot() else {+ return false;+ };response.redirect_chain.is_empty()&& validate_fetch_response_security_policy_with_body(&request_origin,&response.head(),- response.body_bytes(),+ &snapshot,RequestMode::NoCors,RequestCredentialsMode::Include,policy_context,@@ -152,6 +151,7 @@ pub(crate) fn start_scanned_image_preload(load.finish_network_result(decode_runner,std::sync::Arc::new(result),+ encoded,response_is_decode_eligible,);});@@ -215,18 +215,29 @@ pub(crate) fn start_image_element_resource_fetch(let fetch_priority = FetchPriorityHint::from_attribute(element.attribute("fetchpriority"));if let Some(response) = local_url_response(&request_url) {- let response: crate::protocol_types::NavigationResponse = response.into();+ let mut response: crate::protocol_types::NavigationResponse = response.into();+ let manager = resource_loader.as_ref().map_or_else(+ moli_parkable_image::ParkableImageManager::default,+ |loader| {+ let runner = loader.task_runner();+ loader.request_client().parkable_image_manager(&runner)+ },+ );+ let encoded = manager.from_frozen_bytes(response.take_body_bytes());+ let descriptor = image_response_descriptor_from_parkable(&response, &encoded);let result = Ok(response);- host.record_get_subresource_network_result_with_initiator(+ host.record_get_subresource_network_result_with_body_and_initiator(frame_id,document_url,request_url,SubresourceResourceType::Image,request_initiator_type,&result,+ crate::types::SubresourceResponseBody::from_parkable_image(encoded.clone()),);- return result.map(|response| ImageElementResourceFetchStart::Local {- response: Box::new(response),+ return result.map(|_| ImageElementResourceFetchStart::Local {+ descriptor,+ encoded,});}@@ -364,7 +375,9 @@ pub(crate) fn start_image_element_resource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker image fetch dispatch failed".to_owned()),+ result: AsyncSubresourceFetchResult::Failure(+ "service worker image fetch dispatch failed".to_owned(),+ ),},);}@@ -386,7 +399,10 @@ pub(crate) fn start_image_element_resource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: outcome.network_result().as_ref().clone(),+ result: AsyncSubresourceFetchResult::from_image_parts(+ outcome.network_result().as_ref().clone(),+ outcome.encoded(),+ ),});});return Ok(ImageElementResourceFetchStart::Pending);@@ -415,15 +431,24 @@ pub(crate) fn start_image_element_resource_fetch(Ok(ImageElementResourceFetchStart::Pending)}-pub(crate) fn image_response_descriptor(+pub(crate) fn image_response_descriptor_from_parkable(response: &crate::protocol_types::NavigationResponse,+ encoded: &moli_parkable_image::ParkableImage,+) -> Option<crate::native_bridge::ImageResponseDescriptor> {+ let snapshot = encoded.snapshot().ok()?;+ image_response_descriptor_from_bytes(response.status, &response.headers, &snapshot)+}++fn image_response_descriptor_from_bytes(+ status: u16,+ headers: &[(String, String)],+ body: &[u8],) -> Option<crate::native_bridge::ImageResponseDescriptor> {- if !image_response_status_is_successful(response.status) {+ if !image_response_status_is_successful(status) {return None;}- let body = response.body_bytes();let computed_mime_type = moli_web_mime::computed_response_mime_type(- &response.headers,+ headers,moli_web_mime::MimeSniffingContext::Image,body,);diff --git a/moli-renderer-v8/src/network_host/media.rs b/moli-renderer-v8/src/network_host/media.rsindex 821561d0b..e5fd04513 100644--- a/moli-renderer-v8/src/network_host/media.rs+++ b/moli-renderer-v8/src/network_host/media.rs@@ -198,7 +198,7 @@ pub(crate) fn start_media_element_resource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker media fetch dispatch failed".to_owned()),+ result: Err("service worker media fetch dispatch failed".to_owned()).into(),},);}diff --git a/moli-renderer-v8/src/network_host/stylesheet_subresource.rs b/moli-renderer-v8/src/network_host/stylesheet_subresource.rsindex e10c3dc84..0d766a7eb 100644--- a/moli-renderer-v8/src/network_host/stylesheet_subresource.rs+++ b/moli-renderer-v8/src/network_host/stylesheet_subresource.rs@@ -100,13 +100,33 @@ pub(crate) fn start_stylesheet_subresource_fetch(};if let Some(response) = local_url_response(&request_url) {- let response: crate::protocol_types::NavigationResponse = response.into();+ let mut response: crate::protocol_types::NavigationResponse = response.into();+ let encoded = (resource_kind == StylesheetLoadBlockingResourceKind::Image).then(|| {+ let manager = host+ .document_resource_loader_for_owner(binding.owner())+ .map_or_else(+ moli_parkable_image::ParkableImageManager::default,+ |loader| {+ let runner = loader.task_runner();+ loader.request_client().parkable_image_manager(&runner)+ },+ );+ manager.from_frozen_bytes(response.take_body_bytes())+ });if let Some(identity) = css_image.as_ref() {- let descriptor = image_response_descriptor(&response);+ let descriptor = crate::network_host::image_response_descriptor_from_parkable(+ &response,+ encoded+ .as_ref()+ .expect("a local CSS image must have parkable bytes"),+ );let _ = host.complete_stylesheet_css_image_response(identity,descriptor,- response.body_bytes(),+ encoded+ .as_ref()+ .expect("a local CSS image must have parkable bytes")+ .clone(),);}let terminal = if is_main_web_font {@@ -121,14 +141,27 @@ pub(crate) fn start_stylesheet_subresource_fetch(} else {StylesheetSubresourceFetchStart::Settled};- host.record_get_subresource_network_result_with_initiator(- frame_id,- document_url,- request_url,- resource_type,- SubresourceRequestInitiatorType::Css,- &Ok(response),- );+ let result = Ok(response);+ if let Some(encoded) = encoded {+ host.record_get_subresource_network_result_with_body_and_initiator(+ frame_id,+ document_url,+ request_url,+ resource_type,+ SubresourceRequestInitiatorType::Css,+ &result,+ crate::types::SubresourceResponseBody::from_parkable_image(encoded),+ );+ } else {+ host.record_get_subresource_network_result_with_initiator(+ frame_id,+ document_url,+ request_url,+ resource_type,+ SubresourceRequestInitiatorType::Css,+ &result,+ );+ }host.settle_stylesheet_subresource_load_delay(binding);return Ok(terminal);}@@ -241,7 +274,8 @@ pub(crate) fn start_stylesheet_subresource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker stylesheet subresource dispatch failed".to_owned()),+ result: Err("service worker stylesheet subresource dispatch failed".to_owned())+ .into(),},);}diff --git a/moli-renderer-v8/src/network_host/text_track.rs b/moli-renderer-v8/src/network_host/text_track.rsindex 50c21e200..be6fe549e 100644--- a/moli-renderer-v8/src/network_host/text_track.rs+++ b/moli-renderer-v8/src/network_host/text_track.rs@@ -193,7 +193,8 @@ pub(crate) fn start_text_track_resource_fetch(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker text-track fetch dispatch failed".to_owned()),+ result: Err("service worker text-track fetch dispatch failed".to_owned())+ .into(),},);}diff --git a/moli-renderer-v8/src/network_host/xhr/send/paths.rs b/moli-renderer-v8/src/network_host/xhr/send/paths.rsindex 56a0698d3..68de82b65 100644--- a/moli-renderer-v8/src/network_host/xhr/send/paths.rs+++ b/moli-renderer-v8/src/network_host/xhr/send/paths.rs@@ -142,7 +142,7 @@ pub(super) fn dispatch_service_worker_xhr(skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("service worker xhr dispatch failed".to_owned()),+ result: Err("service worker xhr dispatch failed".to_owned()).into(),});Some(internal_id)}diff --git a/moli-renderer-v8/src/page_task_queue.rs b/moli-renderer-v8/src/page_task_queue.rsindex 66f0c2f2d..59183a75e 100644--- a/moli-renderer-v8/src/page_task_queue.rs+++ b/moli-renderer-v8/src/page_task_queue.rs@@ -853,7 +853,7 @@ mod tests {skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("test failure".to_owned()),+ result: Err("test failure".to_owned()).into(),}}diff --git a/moli-renderer-v8/src/page_task_queue/resource_completions.rs b/moli-renderer-v8/src/page_task_queue/resource_completions.rsindex 92abed54c..87de7745b 100644--- a/moli-renderer-v8/src/page_task_queue/resource_completions.rs+++ b/moli-renderer-v8/src/page_task_queue/resource_completions.rs@@ -835,7 +835,7 @@ mod tests {skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("test".to_owned()),+ result: Err("test".to_owned()).into(),}}diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs b/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rsindex 7a3de92fd..c7ba3204e 100644--- a/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs+++ b/moli-renderer-v8/src/runtime/page_vm/tests/async_subresource_completion.rs@@ -47,7 +47,7 @@ fn failed_fetch_completion(internal_id: u64, request_url: &Url) -> AsyncSubresouskip_fetch_security_validation: false,response_filter: None,network_error_text: Some("typed test failure".to_owned()),- result: Err("typed test failure".to_owned()),+ result: Err("typed test failure".to_owned()).into(),}))}diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch.rsindex c0ddce0d4..46401aabc 100644--- a/moli-renderer-v8/src/script_vm/subresource_fetch.rs+++ b/moli-renderer-v8/src/script_vm/subresource_fetch.rs@@ -20,17 +20,18 @@ use crate::runtime::{};use crate::types::{AsyncSubresourceFetchCompletion, AsyncSubresourceFetchEvent,- AsyncSubresourceFetchResponseFilter, ChildBlockingStylesheetLoadCompletion,- ChildClassicScriptLoadCompletion, ChildDocumentLoadCompletion,- ChildModuleDependencyFetchCompletion, ChildModulepreloadFetchCompletion,- ChildParserModuleRootFetchCompletion, DedicatedWorkerId, NetworkBodySourceId,- PendingSubresourceAuthInfo, PendingSubresourceAuthState, PendingSubresourceContinuation,- PendingSubresourceContinueEvent, PendingSubresourceContinueOutcome,- PendingSubresourceFetchInfo, PendingSubresourceFetchState, PendingSubresourceResponseInfo,- PendingSubresourceResponseState, PopupClassicScriptLoadCompletion, PopupDocumentLoadCompletion,- RunningSubresourceFetchState, StreamingSubresourceFetchState, SubresourceNetworkRecord,- SubresourceNetworkRequestHandle, SubresourceRequestInitiatorType, SubresourceResourceType,- SubresourceResponseBody, SubresourceResponseBodyWriter,+ AsyncSubresourceFetchResponseFilter, AsyncSubresourceFetchResult,+ ChildBlockingStylesheetLoadCompletion, ChildClassicScriptLoadCompletion,+ ChildDocumentLoadCompletion, ChildModuleDependencyFetchCompletion,+ ChildModulepreloadFetchCompletion, ChildParserModuleRootFetchCompletion, DedicatedWorkerId,+ NetworkBodySourceId, PendingSubresourceAuthInfo, PendingSubresourceAuthState,+ PendingSubresourceContinuation, PendingSubresourceContinueEvent,+ PendingSubresourceContinueOutcome, PendingSubresourceFetchInfo, PendingSubresourceFetchState,+ PendingSubresourceResponseInfo, PendingSubresourceResponseState,+ PopupClassicScriptLoadCompletion, PopupDocumentLoadCompletion, RunningSubresourceFetchState,+ StreamingSubresourceFetchState, SubresourceNetworkRecord, SubresourceNetworkRequestHandle,+ SubresourceRequestInitiatorType, SubresourceResourceType, SubresourceResponseBody,+ SubresourceResponseBodyWriter,};use crate::util::v8_string;@@ -263,7 +264,10 @@ fn apply_media_subresource_terminal(}enum ImageSubresourceTerminal<'a> {- Response(&'a crate::protocol_types::NavigationResponse),+ Response {+ response: &'a crate::protocol_types::NavigationResponse,+ encoded: &'a moli_parkable_image::ParkableImage,+ },Failure,}@@ -273,12 +277,13 @@ impl ImageSubresourceTerminal<'_> {request_url: &url::Url,) -> crate::context_bootstrap::ResourcePerformanceEntry {match self {- Self::Response(response) => {- crate::context_bootstrap::ResourcePerformanceEntry::from_network_response(+ Self::Response { response, encoded } => {+ crate::context_bootstrap::ResourcePerformanceEntry::from_network_response_with_body_size(request_url.as_str(),"img",None,response,+ encoded.len(),)}Self::Failure => {@@ -302,8 +307,9 @@ fn apply_image_subresource_terminal(terminal: ImageSubresourceTerminal<'_>,) {let (accepted, followup) = match &terminal {- ImageSubresourceTerminal::Response(response) => {- let descriptor = crate::network_host::image_response_descriptor(response);+ ImageSubresourceTerminal::Response { response, encoded } => {+ let descriptor =+ crate::network_host::image_response_descriptor_from_parkable(response, encoded);let completion = context_host.borrow_mut().complete_pending_image_load_network_response_if_matches(@@ -311,7 +317,7 @@ fn apply_image_subresource_terminal(sequence,internal_id,descriptor,- response.body_bytes(),+ (*encoded).clone(),);(completion.accepted(), completion.followup())}@@ -1605,7 +1611,7 @@ impl ScriptVm {skip_fetch_security_validation: true,response_filter: Default::default(),network_error_text: None,- result: Err("service worker csp report fetch dispatch failed".to_owned()),+ result: Err("service worker csp report fetch dispatch failed".to_owned()).into(),});Ok(None)}@@ -3536,7 +3542,7 @@ impl ScriptVm {fn resolve_pending_subresource_fetch_body(&mut self,- mut pending: PendingSubresourceFetchState,+ pending: PendingSubresourceFetchState,request_url: Url,request_method: String,request_headers: moli_fetch::RequestHeaders,@@ -3547,6 +3553,41 @@ impl ScriptVm {network_error_text: Option<String>,result: std::result::Result<crate::protocol_types::NavigationResponse, String>,) -> Result<AsyncSubresourceFetchBodyActivity> {+ self.resolve_pending_subresource_fetch_completion(+ pending,+ request_url,+ request_method,+ request_headers,+ request_body,+ response_status_text,+ skip_fetch_security_validation,+ response_filter,+ network_error_text,+ result.into(),+ )+ }++ #[allow(clippy::too_many_arguments)]+ fn resolve_pending_subresource_fetch_completion(+ &mut self,+ mut pending: PendingSubresourceFetchState,+ request_url: Url,+ request_method: String,+ request_headers: Vec<(String, String)>,+ request_body: Option<String>,+ response_status_text: Option<String>,+ skip_fetch_security_validation: bool,+ response_filter: Option<AsyncSubresourceFetchResponseFilter>,+ network_error_text: Option<String>,+ completion_result: AsyncSubresourceFetchResult,+ ) -> Result<AsyncSubresourceFetchBodyActivity> {+ let (supplied_parkable_image, result) = match completion_result {+ AsyncSubresourceFetchResult::Response(response) => (None, Ok(response)),+ AsyncSubresourceFetchResult::Image { response, encoded } => {+ (Some(encoded), Ok(response))+ }+ AsyncSubresourceFetchResult::Failure(error) => (None, Err(error)),+ };let trace_started = moli_trace::cdp_runtime_trace_enabled().then(Instant::now);let trace_fields = async_subresource_trace_fields_for_pending("completion",@@ -3657,10 +3698,21 @@ impl ScriptVm {| SubresourceResourceType::Video| SubresourceResourceType::Xhr) {+ let supplied_snapshot = supplied_parkable_image+ .as_ref()+ .map(|image| {+ image.snapshot().map_err(|error| {+ format!("failed to read image response bytes: {error}")+ })+ })+ .transpose()?;+ let response_body = supplied_snapshot+ .as_ref()+ .map_or_else(|| response.body_bytes(), AsRef::as_ref);let validation = crate::network_host::validate_fetch_response_security_policy_with_body_classified(&pending.request_origin,&response.head(),- response.body_bytes(),+ response_body,pending.request_mode,pending.credentials_mode,pending.policy_context,@@ -3685,8 +3737,20 @@ impl ScriptVm {let request_initiator_type = pending.continuation.request_initiator_type();match result {- Ok(response) => {+ Ok(mut response) => {let response_status = response.status;+ let parkable_image = (!opaque_response_blocked+ && pending.info.resource_type == SubresourceResourceType::Image)+ .then(|| {+ supplied_parkable_image.clone().unwrap_or_else(|| {+ let runner = pending.load.task_runner();+ pending+ .load+ .request_client()+ .parkable_image_manager(&runner)+ .from_frozen_bytes(response.take_body_bytes())+ })+ });let request_cookie_report = response.request_cookie_report.clone()@@ -3720,7 +3784,12 @@ impl ScriptVm {response.final_url.clone(),response.status,response.headers.clone(),- SubresourceResponseBody::from_navigation_response(&response),+ parkable_image.as_ref().map_or_else(+ || SubresourceResponseBody::from_navigation_response(&response),+ |image| {+ SubresourceResponseBody::from_parkable_image(image.clone())+ },+ ),response.cookie_set_reports.clone(),).with_from_cache(response.from_cache)@@ -3820,15 +3889,23 @@ impl ScriptVm {image_handle,sequence,..- } => apply_image_subresource_terminal(- scope,- &context_host,- image_handle,- sequence,- pending.info.internal_id,- &pending.info.url,- ImageSubresourceTerminal::Response(&observable_response),- ),+ } => {+ let encoded = parkable_image+ .as_ref()+ .expect("an accepted image response must be parkable");+ apply_image_subresource_terminal(+ scope,+ &context_host,+ image_handle,+ sequence,+ pending.info.internal_id,+ &pending.info.url,+ ImageSubresourceTerminal::Response {+ response: &observable_response,+ encoded,+ },+ )+ }PendingSubresourceContinuation::Media {media_handle,sequence,@@ -3862,16 +3939,21 @@ impl ScriptVm {css_image,} => {if let Some(identity) = css_image.as_ref() {- let descriptor =- crate::network_host::image_response_descriptor(- &observable_response,- );+ let descriptor = crate::network_host::image_response_descriptor_from_parkable(+ &observable_response,+ parkable_image+ .as_ref()+ .expect("a CSS image response must be parkable"),+ );let _ = context_host.borrow_mut().complete_stylesheet_css_image_response(identity,descriptor,- observable_response.body_bytes(),+ parkable_image+ .as_ref()+ .expect("a CSS image response must be parkable")+ .clone(),);}if binding.child_handle().is_none() {@@ -4092,7 +4174,7 @@ impl ScriptVm {skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result,+ result: result.into(),});});}@@ -4342,7 +4424,7 @@ impl ScriptVm {completion.skip_fetch_security_validation,completion.response_filter,completion.network_error_text,- completion.result,+ completion.result.into_result(),);trace_async_subresource_stage("async_subresource_complete_done",@@ -4374,7 +4456,7 @@ impl ScriptVm {trace_fields,trace_started,);- let activity = self.resolve_pending_subresource_fetch_body(+ let activity = self.resolve_pending_subresource_fetch_completion(pending,completion.request_url,completion.request_method,@@ -4515,6 +4597,7 @@ impl ScriptVm {let accepted_document = pending.execution_context.window_document_network_only_identity();+ let disk_pool = pending.load.request_client().disk_pool();self._context_host.borrow_mut().record_streaming_subresource_fetch(StreamingSubresourceFetchState {@@ -4527,7 +4610,7 @@ impl ScriptVm {body_source_id: started.body_source_id,head: started.head,network_request_headers: started.network_request_headers,- body_writer: SubresourceResponseBodyWriter::default(),+ body_writer: SubresourceResponseBodyWriter::with_disk_pool(disk_pool),event_source_parser: None,xhr_response: None,});@@ -4899,6 +4982,7 @@ impl ScriptVm {if let PendingSubresourceContinuation::Xhr(xhr) = &pending.continuation {let xhr = v8::Local::new(scope, xhr);let pending_owner = pending.execution_context.dispatch_scope();+ let disk_pool = pending.load.request_client().disk_pool();let xhr_response = crate::types::XhrStreamingResponseState::new(&observable_head.headers,);@@ -4914,7 +4998,7 @@ impl ScriptVm {body_source_id: started.body_source_id,head: started.head.clone(),network_request_headers: started.network_request_headers.clone(),- body_writer: SubresourceResponseBodyWriter::default(),+ body_writer: SubresourceResponseBodyWriter::with_disk_pool(disk_pool),event_source_parser: None,xhr_response: Some(xhr_response),});@@ -5055,6 +5139,7 @@ impl ScriptVm {}let pending_owner = pending.execution_context.dispatch_scope();+ let disk_pool = pending.load.request_client().disk_pool();self._context_host.borrow_mut().record_streaming_subresource_fetch(StreamingSubresourceFetchState {response_filter: started.response_filter,@@ -5066,7 +5151,7 @@ impl ScriptVm {body_source_id: started.body_source_id,head: started.head.clone(),network_request_headers: started.network_request_headers.clone(),- body_writer: SubresourceResponseBodyWriter::default(),+ body_writer: SubresourceResponseBodyWriter::with_disk_pool(disk_pool),event_source_parser,xhr_response: None,},diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rsindex 620cee54b..7de54b9fd 100644--- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs+++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs@@ -2227,7 +2227,7 @@ __swNonStreamFailurePromise.then(skip_fetch_security_validation: false,response_filter: None,network_error_text: Some(crate::network_host::FAILED_ERROR_TEXT.to_owned()),- result: Err(rejection_message.to_owned()),+ result: Err(rejection_message.to_owned()).into(),}).expect("async subresource failure should settle pending fetch");@@ -3167,16 +3167,18 @@ async fn streaming_xhr_materialization_failure_errors_body_source_before_close()let request_url = Url::parse("https://xhr-streaming-materialize-error.test/data").expect("request URL should parse");- let mut body_writer = crate::types::SubresourceResponseBodyWriter::new(1);+ let disk_pool = load_client+ .disk_pool()+ .expect("streaming XHR materialization test should have a disk pool");+ let mut body_writer =+ crate::types::SubresourceResponseBodyWriter::with_memory_limit_and_disk_pool(+ 1,+ Some(disk_pool.clone()),+ );body_writer.append(b"hello");- let missing_path = std::env::temp_dir().join(format!(- "moli-missing-streaming-xhr-body-{}-{body_source_id}",- std::process::id()- ));- let _ = std::fs::remove_file(&missing_path);- let original_spool_path = body_writer- .replace_spool_path_for_test(missing_path)- .expect("body writer should have spooled to a temp file");+ disk_pool+ .truncate_file_for_test(0)+ .expect("test should truncate the pooled body file");let context_ptr: *const v8::Global<v8::Context> = &vm.page_default_context as *const _;let context_host = vm._context_host.clone();@@ -3264,7 +3266,6 @@ async fn streaming_xhr_materialization_failure_errors_body_source_before_close()vm.finish_streaming_async_subresource_fetch(internal_id, body_source_id, Ok(())).expect("streaming XHR finish should surface a network failure, not panic");- let _ = std::fs::remove_file(original_spool_path);let records: Vec<_> = vm._context_hostdiff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rsindex d0edbfb0f..12f88ba21 100644--- a/moli-renderer-v8/src/script_vm/tests/mod.rs+++ b/moli-renderer-v8/src/script_vm/tests/mod.rs@@ -2365,7 +2365,7 @@ async fn child_navigation_retires_local_window_owned_xhr() {skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("stale retired XHR completion".to_owned()),+ result: Err("stale retired XHR completion".to_owned()).into(),}).expect("late completion for retired XHR should be harmless");let stale_open = vm@@ -2815,7 +2815,7 @@ fn main_document_open_fetch_redirect_uses_source_document_csp_report_context() {skip_fetch_security_validation: true,response_filter: None,network_error_text: None,- result: Ok(redirected_fetch_response(&request_url, final_url)),+ result: Ok(redirected_fetch_response(&request_url, final_url)).into(),}).expect("source-owned Fetch redirect should complete in the preserved LocalWindow");@@ -2982,7 +2982,7 @@ async fn child_navigation_aborts_fetch_and_detaches_keepalive() {skip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("stale retired Fetch completion".to_owned()),+ result: Err("stale retired Fetch completion".to_owned()).into(),}).expect("late ordinary Fetch completion should be harmless");let _ = vm.take_network_output();@@ -3002,7 +3002,8 @@ async fn child_navigation_aborts_fetch_and_detaches_keepalive() {200,vec![("content-type".to_owned(), "text/plain".to_owned())],"keepalive completed".to_owned(),- )),+ ))+ .into(),}).expect("detached keepalive completion should remain observable without V8");assert!(@@ -3144,7 +3145,8 @@ async fn detached_keepalive_redirect_reports_source_document_csp_without_v8() {result: Ok(redirected_fetch_response(&report_only_request,report_only_final,- )),+ ))+ .into(),}).expect("detached report-only keepalive should complete without V8");let enforce_final = Url::parse("https://enforce-redirect-target.test/final").unwrap();@@ -3158,7 +3160,7 @@ async fn detached_keepalive_redirect_reports_source_document_csp_without_v8() {skip_fetch_security_validation: true,response_filter: None,network_error_text: None,- result: Ok(redirected_fetch_response(&enforce_request, enforce_final)),+ result: Ok(redirected_fetch_response(&enforce_request, enforce_final)).into(),}).expect("detached enforcing keepalive should fail without entering V8");@@ -3454,7 +3456,8 @@ fn main_document_open_preserves_accepted_beacon_without_rebind() {204,Vec::new(),String::new(),- )),+ ))+ .into(),}).expect("accepted main Beacon should complete without entering V8");assert!(@@ -3718,7 +3721,8 @@ fn main_document_open_preserves_accepted_csp_report_but_rejects_stale_owner_reus204,Vec::new(),String::new(),- )),+ ))+ .into(),}).expect("accepted main CSP report should complete without entering V8");assert!(@@ -12763,7 +12767,7 @@ async fn main_image_source_restart_cancels_exact_request_and_drops_stale_terminaskip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("stale cancelled image completion".to_owned()),+ result: Err("stale cancelled image completion".to_owned()).into(),}).expect("stale cancelled image completion should be harmless");wait_for_image_load_event_executor_test_task(&mut vm, "replacement image decode completion")@@ -13432,7 +13436,7 @@ async fn main_media_source_restart_cancels_exact_network_request_and_stale_termiskip_fetch_security_validation: false,response_filter: None,network_error_text: None,- result: Err("stale cancelled media completion".to_owned()),+ result: Err("stale cancelled media completion".to_owned()).into(),}).expect("stale cancelled media completion should be harmless");diff --git a/moli-renderer-v8/src/script_vm/tests/window_execution_context.rs b/moli-renderer-v8/src/script_vm/tests/window_execution_context.rsindex 4c25bd757..cb805eb3c 100644--- a/moli-renderer-v8/src/script_vm/tests/window_execution_context.rs+++ b/moli-renderer-v8/src/script_vm/tests/window_execution_context.rs@@ -524,7 +524,8 @@ fn borrowed_fetch_uses_receiver_realm_and_keeps_reaction_realm_independent() {200,vec![("content-type".to_owned(), "text/plain".to_owned())],"borrowed completion".to_owned(),- )),+ ))+ .into(),}).expect("borrowed Fetch completion should enter the receiver Promise realm");vm.eval("0")diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rsindex 0adc80e1c..55112497b 100644--- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs+++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs@@ -644,7 +644,7 @@ impl ServiceWorkerRuntimeService {skip_fetch_security_validation: true,response_filter,network_error_text: None,- result: Ok(navigation_response),+ result: Ok(navigation_response).into(),});}@@ -697,7 +697,7 @@ impl ServiceWorkerRuntimeService {skip_fetch_security_validation: false,response_filter: None,network_error_text,- result: Err(message),+ result: Err(message).into(),});}}diff --git a/moli-renderer-v8/src/types.rs b/moli-renderer-v8/src/types.rsindex e1546efb6..3f5598bb5 100644--- a/moli-renderer-v8/src/types.rs+++ b/moli-renderer-v8/src/types.rs@@ -596,7 +596,105 @@ pub(super) struct AsyncSubresourceFetchCompletion {pub(super) skip_fetch_security_validation: bool,pub(super) response_filter: Option<AsyncSubresourceFetchResponseFilter>,pub(super) network_error_text: Option<String>,- pub(super) result: std::result::Result<NavigationResponse, String>,+ pub(super) result: AsyncSubresourceFetchResult,+}++#[derive(Debug)]+pub(super) enum AsyncSubresourceFetchResult {+ Response(NavigationResponse),+ Image {+ response: NavigationResponse,+ encoded: moli_parkable_image::ParkableImage,+ },+ Failure(String),+}++impl AsyncSubresourceFetchResult {+ pub(super) fn from_image_result(+ result: std::result::Result<+ (NavigationResponse, moli_parkable_image::ParkableImage),+ String,+ >,+ ) -> Self {+ match result {+ Ok((response, encoded)) => Self::Image { response, encoded },+ Err(error) => Self::Failure(error),+ }+ }++ pub(super) fn from_image_parts(+ result: std::result::Result<NavigationResponse, String>,+ encoded: Option<moli_parkable_image::ParkableImage>,+ ) -> Self {+ match (result, encoded) {+ (Ok(response), Some(encoded)) => Self::Image { response, encoded },+ (Ok(response), None) => Self::Response(response),+ (Err(error), _) => Self::Failure(error),+ }+ }++ pub(super) fn into_result(self) -> std::result::Result<NavigationResponse, String> {+ match self {+ Self::Response(response) | Self::Image { response, .. } => Ok(response),+ Self::Failure(error) => Err(error),+ }+ }++ #[cfg(test)]+ pub(super) fn as_ref(&self) -> std::result::Result<&NavigationResponse, &String> {+ match self {+ Self::Response(response) | Self::Image { response, .. } => Ok(response),+ Self::Failure(error) => Err(error),+ }+ }++ #[cfg(test)]+ pub(super) fn encoded(&self) -> Option<&moli_parkable_image::ParkableImage> {+ match self {+ Self::Image { encoded, .. } => Some(encoded),+ Self::Response(_) | Self::Failure(_) => None,+ }+ }++ #[cfg(test)]+ pub(super) fn is_ok(&self) -> bool {+ !matches!(self, Self::Failure(_))+ }++ #[cfg(test)]+ pub(super) fn err(self) -> Option<String> {+ match self {+ Self::Failure(error) => Some(error),+ Self::Response(_) | Self::Image { .. } => None,+ }+ }++ #[cfg(test)]+ pub(super) fn expect(self, message: &str) -> NavigationResponse {+ match self {+ Self::Response(response) | Self::Image { response, .. } => response,+ Self::Failure(error) => panic!("{message}: {error:?}"),+ }+ }++ #[cfg(test)]+ pub(super) fn expect_err(self, message: &str) -> String {+ match self {+ Self::Failure(error) => error,+ Self::Response(response) | Self::Image { response, .. } => {+ panic!("{message}: {response:?}")+ }+ }+ }+}++impl From<std::result::Result<NavigationResponse, String>> for AsyncSubresourceFetchResult {+ fn from(result: std::result::Result<NavigationResponse, String>) -> Self {+ match result {+ Ok(response) => Self::Response(response),+ Err(error) => Self::Failure(error),+ }+ }}#[derive(Clone, Copy, Debug, Eq, PartialEq)]diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rsindex 56df94812..8728dece9 100644--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs@@ -240,7 +240,7 @@ pub(in crate::worker) fn spawn_worker_fetch_network(}} else if !allow_headers_first {// Worker fetch still resolves after the full response, but ordinary- // transfers can keep their body in the same chunked/spooled carrier that+ // transfers can keep their body in the same chunked/pooled carrier that// CDP Network capture uses instead of forcing a single buffered Response.match fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata(&loader,@@ -254,7 +254,10 @@ pub(in crate::worker) fn spawn_worker_fetch_network(let (mut response, network_request_headers) =worker_network_result_parts(observed);let head = response.head();- let mut body_writer = SubresourceResponseBodyWriter::default();+ let mut body_writer =+ SubresourceResponseBodyWriter::with_disk_pool(+ loader.disk_pool(),+ );while let Some(chunk) = response.next_chunk().await {body_writer.append(&chunk);}@@ -292,7 +295,10 @@ pub(in crate::worker) fn spawn_worker_fetch_network(network_request_headers,},));- let mut body_writer = SubresourceResponseBodyWriter::default();+ let mut body_writer =+ SubresourceResponseBodyWriter::with_disk_pool(+ loader.disk_pool(),+ );while let Some(chunk) = response.next_chunk().await {body_writer.append(&chunk);let _ = completion_tx.send(WorkerFetchEvent::StreamingChunk(@@ -580,7 +586,7 @@ pub(in crate::worker) fn spawn_worker_xhr_network(}Ok(request) => {// Ordinary worker XHR can keep the network/cache path streaming- // and spool large bodies until the XHR DONE boundary.+ // and pool large bodies until the XHR DONE boundary.match fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata(&loader,request,@@ -593,7 +599,9 @@ pub(in crate::worker) fn spawn_worker_xhr_network(let (mut response, network_request_headers) =worker_network_result_parts(observed);let head = response.head();- let mut body_writer = SubresourceResponseBodyWriter::default();+ let mut body_writer = SubresourceResponseBodyWriter::with_disk_pool(+ loader.disk_pool(),+ );while let Some(chunk) = response.next_chunk().await {body_writer.append(&chunk);}