Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

53.7 KB
Raw Blame
mod fetch_deadline;
mod lifecycle_fetch;
mod navigation_engine;
pub mod storage_partition;

pub use crate::config::BrowserConfig;

use crate::{
    network::{DocumentFetchContextSeed, NavigationResourceLoader, ResourceRequestClient},
    page::{DocumentStartScript, Page},
    renderer::{JsRuntime, RendererOwnerCommand, materialize_page_created_reply},
    selector::QueryEngine,
};
use anyhow::Result;
use anyhow::{Context, anyhow};
use moli_cookie_jar::StoredCookie;
use moli_fetch::{
    FetchCancelHandle, RawResponse, Request, StreamingRawResponse, ensure_http_status_success,
};
use moli_page_types::NavigationResponse;
use moli_renderer_v8::network::{
    BrowserResourceRuntime, BrowserResourceRuntimeOwner, PageNetworkPolicy,
};
use moli_url::is_about_blank as is_about_blank_url;
use moli_web_mime::response_headers_indicate_raw_document;
use std::fmt;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use std::time::Instant;
use std::{rc::Rc, sync::Arc};
use storage_partition::StoragePartitionState;
use tokio::sync::oneshot;
use tracing::{debug, info, warn};
use url::Url;

pub use crate::renderer::ExternalRawDocumentBodyStream;
pub use crate::renderer::PageVmInitStage;
pub use crate::renderer::RendererReplyBoundary;
pub use fetch_deadline::{FetchDeadline, FetchReadinessTimeout, FetchTimeoutPhase};
pub use moli_renderer_v8::{
    DetachedParserScriptFetchContinuation, RendererBrowserContextRuntime,
    RendererBrowserContextRuntimeOwner, RendererBrowserContextRuntimeOwnerAccess,
    RendererDocumentPreparationTarget, RendererLifecycleDecider, RendererLifecycleDecision,
    RendererLifecycleSnapshot, RendererPageReplacementTarget, RendererPageReservationToken,
    RendererReservedServiceWorkerClient, RendererServiceWorkerMainResourceFetch,
    RendererSharedWorkerRuntimeDiagnostics,
};
pub use navigation_engine::{
    BuiltDocumentPage, CommittedDocumentPageReplacement, CommittedDocumentResourceSource,
    NavigationEngine, NavigationPageStorageHandles, NavigationResourceStorageHandles,
    NavigationRuntimeConfig, PendingBuiltDocumentPage, PreparedDocumentPage,
    PreparedDocumentPageCommitConfiguration, PreparedDocumentPageCommitPermit,
};

static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);

/// Controls whether a document fetch may consume a raw/download response.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RawDocumentFetchPolicy {
    /// Read the response body and return [`FetchedDocument::Raw`].
    Materialize,
    /// Return response metadata with an empty body and cancel the download.
    /// Callers can use the metadata to select an error without reading a body
    /// that their output format cannot consume.
    MetadataOnly,
    /// Read the body only for statuses below the threshold. Otherwise return
    /// response metadata with an empty body and cancel the download.
    MaterializeBelowStatus(u16),
    /// Require a renderable Page, rejecting raw responses from their headers.
    RequirePage,
}

/// Typed rejection produced by [`RawDocumentFetchPolicy::RequirePage`].
#[derive(Debug)]
pub struct RawDocumentPageRequired;

impl fmt::Display for RawDocumentPageRequired {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str("raw non-HTML document cannot be returned through the Page fetch API")
    }
}

impl std::error::Error for RawDocumentPageRequired {}

impl RawDocumentFetchPolicy {
    fn should_materialize_body(self, status: u16) -> Result<bool> {
        match self {
            Self::Materialize => Ok(true),
            Self::MetadataOnly => Ok(false),
            Self::MaterializeBelowStatus(threshold) => Ok(status < threshold),
            Self::RequirePage => Err(anyhow::Error::new(RawDocumentPageRequired)),
        }
    }
}

fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool {
    let expected = match wait_until {
        RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle",
        RenderedDomWaitUntil::DomStable => "timed out waiting for domstable",
        RenderedDomWaitUntil::DomContentLoaded
        | RenderedDomWaitUntil::Load
        | RenderedDomWaitUntil::Done => return false,
    };

    error.chain().any(|cause| cause.to_string() == expected)
}

#[derive(Debug, Clone, Default)]
struct AutomationController;

#[derive(Debug, Clone, Default)]
struct WebApiRegistry;

#[derive(Clone)]
pub struct Browser {
    config: BrowserConfig,
    js_runtime: JsRuntime,
    resource_runtime: BrowserResourceRuntime,
    page_network_policy: PageNetworkPolicy,
    partition: Arc<StoragePartitionState>,
    _web_api_registry: WebApiRegistry,
    _selector_engine: QueryEngine,
    _automation: AutomationController,
    // Declared last so each Browser clone releases its request-side handles
    // before the final shared, thread-affine lifetime owner tears down.
    _lifetime_owner: Rc<BrowserLifetimeOwner>,
}

struct BrowserLifetimeOwner {
    js_runtime: Option<JsRuntime>,
    browser_context_owner: moli_renderer_v8::RendererBrowserContextRuntimeOwner,
    partition: Arc<StoragePartitionState>,
}

impl std::fmt::Debug for Browser {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Browser")
            .field("config", &self.config)
            .field(
                "partition_indexed_db_manager_strong_count",
                &std::sync::Arc::strong_count(self.partition.indexed_db_manager()),
            )
            .field("js_runtime", &self.js_runtime)
            .field("resource_runtime", &self.resource_runtime)
            .field("partition_persistence", &self.partition.persistence())
            .field("partition_id", &self.partition.id())
            .field("_web_api_registry", &self._web_api_registry)
            .field("_selector_engine", &self._selector_engine)
            .field("_automation", &self._automation)
            .finish()
    }
}

impl Drop for BrowserLifetimeOwner {
    fn drop(&mut self) {
        debug!("terminating browser renderer producers");
        if let Some(js_runtime) = self.js_runtime.take() {
            js_runtime.terminate_resource_producers_for_owner_shutdown();
            // Do not leave RenderRuntimeOwner to implicit field drop after
            // network join/storage flush. Releasing it here establishes the
            // renderer -> network -> persistent-storage teardown order.
            drop(js_runtime);
        }
        self.browser_context_owner.shutdown_and_join();
        if let Err(error) = self.partition.flush() {
            warn!(
                error = %error,
                "failed to flush browser storage partition"
            );
        }
        debug!("browser renderer and resource owners joined");
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RenderedDomWaitUntil {
    DomContentLoaded,
    Load,
    NetworkIdle,
    DomStable,
    Done,
}

impl RenderedDomWaitUntil {
    /// The concrete lifecycle boundary at which a live `Page` can first be
    /// handed back to the host. Network-idle and DOM-stable are observations
    /// made on that live Page, after Load and DCL respectively.
    fn base_stage(self) -> PageVmInitStage {
        match self {
            Self::DomContentLoaded | Self::DomStable => PageVmInitStage::DomContentLoaded,
            Self::Load | Self::NetworkIdle | Self::Done => PageVmInitStage::Load,
        }
    }

    fn has_best_effort_page_wait(self) -> bool {
        matches!(self, Self::NetworkIdle | Self::DomStable)
    }
}

fn fetch_timeout_phase_for_stage(stage: PageVmInitStage) -> FetchTimeoutPhase {
    match stage {
        PageVmInitStage::DomContentLoaded => FetchTimeoutPhase::WaitingForDomContentLoaded,
        PageVmInitStage::Load => FetchTimeoutPhase::WaitingForLoad,
    }
}

#[derive(Debug)]
pub struct RawDocument {
    response: RawResponse,
}

impl RawDocument {
    fn from_response(response: RawResponse) -> Self {
        Self { response }
    }

    pub fn final_url(&self) -> &Url {
        &self.response.final_url
    }

    pub fn status(&self) -> u16 {
        self.response.status
    }

    pub fn headers(&self) -> &[(String, Vec<u8>)] {
        &self.response.headers
    }

    pub fn navigation_redirect_chain(&self) -> &[moli_fetch::RedirectInfo] {
        &self.response.redirect_chain
    }

    pub fn body_bytes(&self) -> &[u8] {
        self.response.body_bytes()
    }
}

#[derive(Debug)]
pub enum FetchedDocument {
    Page(Page),
    Raw(Box<RawDocument>),
}

// Browser's public page-loading API starts a new navigation without a caller
// Document. An HTTP request supplied here enters browser semantics with that
// opaque initiator; renderer subresources must already carry their environment.
fn initial_navigation_request(request: Request) -> Request {
    if request.request_origin().is_none() {
        request
            .with_request_origin(moli_url::WebOrigin::Opaque)
            .with_request_mode(moli_fetch::RequestMode::Navigate)
    } else {
        request
    }
}

impl Browser {
    fn resource_request_client(&self) -> ResourceRequestClient {
        ResourceRequestClient::from_browser_resource_runtime_with_page_network_policy(
            self.resource_runtime.clone(),
            self.page_network_policy.clone(),
        )
    }

    pub fn new(mut config: BrowserConfig) -> Result<Self> {
        let partition = Arc::new(StoragePartitionState::open(config.profile_dir())?);
        if config.fetch().http_cache_dir().is_none()
            && let Some(http_cache_root) = partition.http_cache_root()
        {
            config
                .fetch_mut()
                .set_http_cache_dir(Some(http_cache_root.display().to_string()));
        }
        let resource_runtime =
            BrowserResourceRuntimeOwner::new(config.fetch(), partition.cookie_store());
        let page_network_policy = PageNetworkPolicy::new(
            config.optional_resource_fetch_mask(),
            config.subframe_loading_enabled(),
            config.author_styles_disabled(),
        );
        // A script-disabled CLI fetch must not execute site-owned JavaScript
        // through a previously registered Service Worker before the new
        // Document's own execution policy can take effect.
        page_network_policy.set_bypass_service_worker(config.script_execution_disabled());
        let browser_context_owner =
            RendererBrowserContextRuntime::new_owned_with_service_worker_resource_store_and_browser_resource_runtime(
                partition.service_worker_resource_store(),
                resource_runtime,
                config.debugger_enabled(),
            );
        let browser_context_access = browser_context_owner.owner_access();
        let resource_runtime = browser_context_access
            .current_browser_resource_runtime()
            .map_err(|error| anyhow!("browser context resource owner unavailable: {error}"))?;
        let js_runtime =
            JsRuntime::initialize_with_browser_context_owner_access(&browser_context_access)?;
        js_runtime
            .renderer_owner_handle()
            .configure_layout_configuration(config.layout_configuration())?;
        let lifetime_owner = Rc::new(BrowserLifetimeOwner {
            js_runtime: Some(js_runtime.clone()),
            browser_context_owner,
            partition: Arc::clone(&partition),
        });
        Ok(Self {
            js_runtime,
            resource_runtime,
            page_network_policy,
            partition,
            _web_api_registry: WebApiRegistry,
            _selector_engine: QueryEngine,
            _automation: AutomationController,
            config,
            _lifetime_owner: lifetime_owner,
        })
    }

    pub fn session(&self) -> Session {
        Session {
            id: NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed),
            browser: self.clone(),
        }
    }

    pub fn cookies(&self) -> Result<Vec<StoredCookie>> {
        self.partition.cookies()
    }

    pub fn clear_indexed_db_origin(&self, origin: &str) -> std::result::Result<(), String> {
        moli_renderer_v8::clear_indexed_db_origin(self.partition.indexed_db_manager(), origin)
    }

    pub fn import_cookies(&self, cookies: impl IntoIterator<Item = StoredCookie>) -> Result<usize> {
        self.partition.import_cookies(cookies)
    }

    pub async fn fetch(&self, raw_url: &str) -> Result<Page> {
        self.fetch_request(Request::get(raw_url)?).await
    }

    pub async fn fetch_with_wait_until(
        &self,
        raw_url: &str,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
    ) -> Result<Page> {
        self.fetch_request_with_wait_until(Request::get(raw_url)?, wait_until, timeout)
            .await
    }

    pub async fn fetch_request(&self, request: Request) -> Result<Page> {
        self.fetch_internal(request, PageVmInitStage::Load).await
    }

    pub async fn fetch_request_with_wait_until(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
    ) -> Result<Page> {
        self.fetch_page_with_wait_until_deadline(
            request,
            wait_until,
            FetchDeadline::new(timeout)?,
            false,
        )
        .await
    }

    pub async fn fetch_allow_http_error(&self, raw_url: &str) -> Result<Page> {
        self.fetch_request_allow_http_error(Request::get(raw_url)?)
            .await
    }

    pub async fn fetch_allow_http_error_with_wait_until(
        &self,
        raw_url: &str,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
    ) -> Result<Page> {
        self.fetch_request_allow_http_error_with_wait_until(
            Request::get(raw_url)?,
            wait_until,
            timeout,
        )
        .await
    }

    pub async fn fetch_request_allow_http_error(&self, request: Request) -> Result<Page> {
        self.fetch_allow_http_error_internal(request, PageVmInitStage::Load)
            .await
    }

    pub async fn fetch_request_document_allow_http_error(
        &self,
        request: Request,
    ) -> Result<FetchedDocument> {
        self.fetch_document_allow_http_error_internal(
            request,
            PageVmInitStage::Load,
            RendererReplyBoundary::Stage,
        )
        .await
    }

    pub async fn fetch_request_allow_http_error_with_wait_until(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
    ) -> Result<Page> {
        self.fetch_page_with_wait_until_deadline(
            request,
            wait_until,
            FetchDeadline::new(timeout)?,
            true,
        )
        .await
    }

    async fn fetch_page_with_wait_until_deadline(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        deadline: FetchDeadline,
        allow_http_error: bool,
    ) -> Result<Page> {
        let mut page = match self
            .fetch_document_to_base_stage(
                request,
                wait_until,
                deadline,
                RendererReplyBoundary::Stage,
                None,
                RawDocumentFetchPolicy::RequirePage,
            )
            .await?
        {
            FetchedDocument::Page(page) => page,
            FetchedDocument::Raw(_) => unreachable!(
                "the Page fetch path rejects raw documents before body materialization"
            ),
        };

        self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)
            .await?;
        if !allow_http_error {
            ensure_http_status_success(page.final_url().as_str(), page.status(), false)?;
        }
        Ok(page)
    }

    pub async fn fetch_request_document_allow_http_error_with_wait_until(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
    ) -> Result<FetchedDocument> {
        let deadline = FetchDeadline::new(timeout)?;
        self.fetch_request_document_allow_http_error_with_wait_until_deadline(
            request,
            wait_until,
            deadline,
            RawDocumentFetchPolicy::Materialize,
        )
        .await
    }

    /// Fetches a document to `wait_until` without starting a new timeout
    /// budget. Callers can reuse `deadline` for later readiness conditions and
    /// choose whether raw responses are materialized or rejected from their
    /// headers.
    pub async fn fetch_request_document_allow_http_error_with_wait_until_deadline(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        deadline: FetchDeadline,
        raw_document_policy: RawDocumentFetchPolicy,
    ) -> Result<FetchedDocument> {
        let fetched = self
            .fetch_document_to_base_stage(
                request,
                wait_until,
                deadline,
                RendererReplyBoundary::Stage,
                None,
                raw_document_policy,
            )
            .await?;

        match fetched {
            FetchedDocument::Page(mut page) => {
                self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline)
                    .await?;
                Ok(FetchedDocument::Page(page))
            }
            FetchedDocument::Raw(raw) => Ok(FetchedDocument::Raw(raw)),
        }
    }

    /// Reaches the concrete lifecycle boundary that makes a live Page
    /// available. For NetworkIdle and DomStable this is only the Load or DCL
    /// base stage; their best-effort observation runs outside materialization.
    async fn fetch_document_to_base_stage(
        &self,
        request: Request,
        wait_until: RenderedDomWaitUntil,
        deadline: FetchDeadline,
        reply_boundary: RendererReplyBoundary,
        lifecycle_decider: Option<RendererLifecycleDecider>,
        raw_document_policy: RawDocumentFetchPolicy,
    ) -> Result<FetchedDocument> {
        let request = initial_navigation_request(request);
        let timeout = deadline.timeout();
        let stage = wait_until.base_stage();

        let raw_url = request.url.as_str().to_owned();
        debug!(
            url = %raw_url,
            wait_until = ?wait_until,
            timeout_ms = timeout.as_millis(),
            stage = ?stage,
            "starting fetch_document_allow_http_error_with_wait_until deadline"
        );
        // This deadline is strict: if the response or base DCL/Load stage does
        // not arrive in time, there is no Page on which best-effort readiness
        // can operate. Only the later NetworkIdle/DomStable observation may
        // soften an expired deadline.
        let outer_deadline = deadline.at();
        let requested_url = request.url.clone();
        if is_about_blank_url(&requested_url) {
            return self
                .fetch_document_wait_timeout(
                    &raw_url,
                    wait_until,
                    timeout,
                    stage,
                    outer_deadline,
                    || fetch_timeout_phase_for_stage(stage),
                    self.materialize_static_html_page(
                        &raw_url,
                        requested_url,
                        stage,
                        reply_boundary,
                        lifecycle_decider,
                        String::new(),
                    ),
                )
                .await
                .map(FetchedDocument::Page);
        }
        let navigation_loader = NavigationResourceLoader::new(
            self.resource_request_client(),
            requested_url.clone(),
            moli_renderer_v8::network::RendererResourceTaskRunner::from_current_tokio()?,
        );
        let service_worker_fetch = self
            .fetch_document_wait_timeout(
                &raw_url,
                wait_until,
                timeout,
                stage,
                outer_deadline,
                || FetchTimeoutPhase::WaitingForResponseHeaders,
                self.fetch_service_worker_main_resource_for_navigation(
                    &request,
                    &navigation_loader,
                ),
            )
            .await?;
        let RendererServiceWorkerMainResourceFetch {
            reserved_client,
            response,
        } = service_worker_fetch;
        if let Some(response) = response {
            navigation_loader.note_service_worker_response_ready()?;
            if response_headers_indicate_raw_document(&response.headers) {
                let body = if raw_document_policy.should_materialize_body(response.status)? {
                    response.clone_body_bytes()
                } else {
                    Vec::new()
                };
                let raw_response = RawResponse::from_head_and_body(response.head(), body);
                return Ok(FetchedDocument::Raw(Box::new(RawDocument::from_response(
                    raw_response,
                ))));
            }
            let document_fetch_context_seed =
                navigation_loader.commit(response.final_url.clone())?;
            let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();
            let timeout_progress = page_creation_progress.clone();
            return self
                .fetch_document_wait_timeout(
                    &raw_url,
                    wait_until,
                    timeout,
                    stage,
                    outer_deadline,
                    move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),
                    self.materialize_streaming_raw_response_page_with_page_creation_progress(
                        &raw_url,
                        requested_url,
                        stage,
                        reply_boundary,
                        lifecycle_decider,
                        streaming_raw_response_from_navigation_response(response)?,
                        document_fetch_context_seed,
                        reserved_client,
                        page_creation_progress,
                    ),
                )
                .await
                .map(FetchedDocument::Page);
        }
        let response = self
            .fetch_document_wait_timeout(
                &raw_url,
                wait_until,
                timeout,
                stage,
                outer_deadline,
                || FetchTimeoutPhase::WaitingForResponseHeaders,
                navigation_loader.fetch_raw_stream(request),
            )
            .await?;
        if response_headers_indicate_raw_document(&response.headers) {
            if !raw_document_policy.should_materialize_body(response.status)? {
                let raw_response = RawResponse::from_head_and_body(response.head(), Vec::new());
                return Ok(FetchedDocument::Raw(Box::new(RawDocument::from_response(
                    raw_response,
                ))));
            }
            // Binary/download-like main resources have no DOM lifecycle
            // milestones, but materializing a RawDocument still consumes the
            // caller-owned absolute fetch-readiness budget.
            return self
                .fetch_document_wait_timeout(
                    &raw_url,
                    wait_until,
                    timeout,
                    stage,
                    outer_deadline,
                    || FetchTimeoutPhase::StreamingMainBody,
                    self.materialize_streaming_raw_response_raw_document(response),
                )
                .await
                .map(|raw| FetchedDocument::Raw(Box::new(raw)));
        }

        let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
        let page_creation_progress = moli_renderer_v8::RendererPageCreationProgress::new();
        let timeout_progress = page_creation_progress.clone();
        self.fetch_document_wait_timeout(
            &raw_url,
            wait_until,
            timeout,
            stage,
            outer_deadline,
            move || FetchTimeoutPhase::from_renderer_phase(timeout_progress.phase()),
            self.materialize_streaming_raw_response_page_with_page_creation_progress(
                &raw_url,
                requested_url,
                stage,
                reply_boundary,
                lifecycle_decider,
                response,
                document_fetch_context_seed,
                reserved_client,
                page_creation_progress,
            ),
        )
        .await
        .map(FetchedDocument::Page)
    }

    pub async fn wait_for_page_delay(&self, page: &mut Page, delay: Duration) -> Result<()> {
        if delay.is_zero() {
            return Ok(());
        }

        let deadline = Instant::now()
            .checked_add(delay)
            .unwrap_or_else(Instant::now);
        loop {
            let ms_to_next_timeout = page.ms_to_next_timeout().await?;

            let now = Instant::now();
            if now >= deadline {
                break;
            }

            let remaining = deadline.saturating_duration_since(now);
            let sleep_for = ms_to_next_timeout
                .map(Duration::from_millis)
                .unwrap_or(Duration::from_millis(50))
                .min(Duration::from_millis(50))
                .min(remaining);
            if sleep_for.is_zero() {
                continue;
            }
            tokio::time::sleep(sleep_for).await;
        }

        Ok(())
    }

    pub async fn wait_for_page_network_idle(
        &self,
        page: &mut Page,
        timeout: Duration,
    ) -> Result<()> {
        page.wait_for_network_idle(&self.resource_request_client(), timeout)
            .await
    }

    pub async fn wait_for_page_dom_stable(&self, page: &mut Page, timeout: Duration) -> Result<()> {
        page.wait_for_dom_stable(&self.resource_request_client(), timeout)
            .await
    }

    pub async fn wait_for_selector(
        &self,
        page: &mut Page,
        selector: &str,
        timeout: Duration,
    ) -> Result<crate::page::RendererDocumentQuerySelectorNode> {
        page.wait_for_selector(&self.resource_request_client(), selector, timeout)
            .await
    }

    /// Waits for a selector using the unspent portion of `deadline`.
    pub async fn wait_for_selector_with_deadline(
        &self,
        page: &mut Page,
        selector: &str,
        deadline: FetchDeadline,
    ) -> Result<crate::page::RendererDocumentQuerySelectorNode> {
        let loader = self.resource_request_client();
        // The absolute deadline is the timeout authority. The renderer keeps
        // its own relative timeout only as a safety bound; giving both layers
        // the same remaining duration would make their timers race and could
        // discard the typed fetch phase when the renderer wins.
        let command_timeout = deadline.timeout();
        deadline
            .wait(
                FetchTimeoutPhase::WaitingForSelector,
                page.wait_for_selector(&loader, selector, command_timeout),
            )
            .await
    }

    /// Waits until `expression` is truthy. When it evaluates to a function,
    /// the function is invoked without arguments as the predicate on each
    /// polling turn; a returned Promise is awaited.
    pub async fn wait_for_script_truthy(
        &self,
        page: &mut Page,
        expression: &str,
        timeout: Duration,
    ) -> Result<()> {
        page.wait_for_script_truthy(&self.resource_request_client(), expression, timeout)
            .await
    }

    /// Waits for a truthy script result using the unspent portion of
    /// `deadline`. Function results are invoked as zero-argument predicates,
    /// and returned Promises are awaited.
    pub async fn wait_for_script_truthy_with_deadline(
        &self,
        page: &mut Page,
        expression: &str,
        deadline: FetchDeadline,
    ) -> Result<()> {
        let loader = self.resource_request_client();
        let command_timeout = deadline.timeout();
        deadline
            .wait(
                FetchTimeoutPhase::WaitingForScript,
                page.wait_for_script_truthy(&loader, expression, command_timeout),
            )
            .await
    }

    pub async fn wait_for_subresource_response(
        &self,
        page: &mut Page,
        criteria: crate::page::SubresourceResponseWaitCriteria,
        timeout: Duration,
    ) -> Result<()> {
        page.wait_for_subresource_response(&self.resource_request_client(), criteria, timeout)
            .await
    }

    /// Waits for a matching response using the unspent portion of `deadline`.
    pub async fn wait_for_subresource_response_with_deadline(
        &self,
        page: &mut Page,
        criteria: crate::page::SubresourceResponseWaitCriteria,
        deadline: FetchDeadline,
    ) -> Result<()> {
        let loader = self.resource_request_client();
        let command_timeout = deadline.timeout();
        deadline
            .wait(
                FetchTimeoutPhase::WaitingForSubresourceResponse,
                page.wait_for_subresource_response(&loader, criteria, command_timeout),
            )
            .await
    }

    /// Observes NetworkIdle or DomStable with the unspent part of `deadline`.
    ///
    /// A timeout in this observation is deliberately best-effort: the Page
    /// has already reached its required Load/DCL base stage, so the current
    /// snapshot is returned with a warning. Errors unrelated to timeout still
    /// fail the fetch. Concrete lifecycle modes are no-ops here.
    pub async fn wait_for_page_readiness_with_deadline(
        &self,
        page: &mut Page,
        wait_until: RenderedDomWaitUntil,
        deadline: FetchDeadline,
    ) -> Result<()> {
        if !wait_until.has_best_effort_page_wait() {
            return Ok(());
        }

        let loader = self.resource_request_client();
        let remaining = deadline.remaining();
        let result = match wait_until {
            RenderedDomWaitUntil::NetworkIdle => {
                tokio::time::timeout_at(
                    deadline.at(),
                    page.wait_for_network_idle(&loader, remaining),
                )
                .await
            }
            RenderedDomWaitUntil::DomStable => {
                tokio::time::timeout_at(deadline.at(), page.wait_for_dom_stable(&loader, remaining))
                    .await
            }
            RenderedDomWaitUntil::DomContentLoaded
            | RenderedDomWaitUntil::Load
            | RenderedDomWaitUntil::Done => unreachable!(
                "concrete lifecycle modes returned before starting a best-effort Page wait"
            ),
        };

        let timeout_error = match result {
            Ok(Ok(())) => return Ok(()),
            Ok(Err(error)) if is_fetch_readiness_timeout(&error, wait_until) => error,
            Ok(Err(error)) => {
                return Err(error).with_context(|| {
                    anyhow!("failed while waiting for page readiness {wait_until:?}")
                });
            }
            Err(error) => anyhow::Error::new(error),
        };

        warn!(
            page_id = page.page_id(),
            url = %page.requested_url(),
            final_url = %page.final_url(),
            wait_until = ?wait_until,
            timeout_ms = deadline.timeout().as_millis(),
            remaining_ms = remaining.as_millis(),
            error = %timeout_error,
            "fetch readiness wait timed out; returning best-effort page"
        );
        Ok(())
    }

    async fn fetch_internal(&self, request: Request, stage: PageVmInitStage) -> Result<Page> {
        let request = initial_navigation_request(request);
        let raw_url = request.url.as_str().to_owned();
        let requested_url = request.url.clone();
        if is_about_blank_url(&requested_url) {
            return self
                .materialize_static_html_page(
                    &raw_url,
                    requested_url,
                    stage,
                    RendererReplyBoundary::Stage,
                    None,
                    String::new(),
                )
                .await;
        }
        let navigation_loader = NavigationResourceLoader::new(
            self.resource_request_client(),
            requested_url.clone(),
            moli_renderer_v8::network::RendererResourceTaskRunner::from_current_tokio()?,
        );
        let service_worker_fetch = self
            .fetch_service_worker_main_resource_for_navigation(&request, &navigation_loader)
            .await?;
        let RendererServiceWorkerMainResourceFetch {
            reserved_client,
            response,
        } = service_worker_fetch;
        if let Some(response) = response {
            navigation_loader.note_service_worker_response_ready()?;
            let document_fetch_context_seed =
                navigation_loader.commit(response.final_url.clone())?;
            let response = streaming_raw_response_from_navigation_response(response)?;
            let page = self
                .materialize_streaming_raw_response_page(
                    &raw_url,
                    requested_url,
                    stage,
                    RendererReplyBoundary::Stage,
                    None,
                    response,
                    document_fetch_context_seed,
                    reserved_client,
                )
                .await?;
            ensure_http_status_success(page.final_url().as_str(), page.status(), false)?;
            return Ok(page);
        }
        let response = navigation_loader.fetch_raw_stream(request).await?;
        let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
        let page = self
            .materialize_streaming_raw_response_page(
                &raw_url,
                requested_url,
                stage,
                RendererReplyBoundary::Stage,
                None,
                response,
                document_fetch_context_seed,
                reserved_client,
            )
            .await?;
        ensure_http_status_success(page.final_url().as_str(), page.status(), false)?;
        Ok(page)
    }

    async fn fetch_allow_http_error_internal(
        &self,
        request: Request,
        stage: PageVmInitStage,
    ) -> Result<Page> {
        let request = initial_navigation_request(request);
        let raw_url = request.url.as_str().to_owned();
        let requested_url = request.url.clone();
        if is_about_blank_url(&requested_url) {
            return self
                .materialize_static_html_page(
                    &raw_url,
                    requested_url,
                    stage,
                    RendererReplyBoundary::Stage,
                    None,
                    String::new(),
                )
                .await;
        }
        let navigation_loader = NavigationResourceLoader::new(
            self.resource_request_client(),
            requested_url.clone(),
            moli_renderer_v8::network::RendererResourceTaskRunner::from_current_tokio()?,
        );
        let service_worker_fetch = self
            .fetch_service_worker_main_resource_for_navigation(&request, &navigation_loader)
            .await?;
        let RendererServiceWorkerMainResourceFetch {
            reserved_client,
            response,
        } = service_worker_fetch;
        if let Some(response) = response {
            navigation_loader.note_service_worker_response_ready()?;
            let document_fetch_context_seed =
                navigation_loader.commit(response.final_url.clone())?;
            return self
                .materialize_streaming_raw_response_page(
                    &raw_url,
                    requested_url,
                    stage,
                    RendererReplyBoundary::Stage,
                    None,
                    streaming_raw_response_from_navigation_response(response)?,
                    document_fetch_context_seed,
                    reserved_client,
                )
                .await;
        }
        let response = navigation_loader.fetch_raw_stream(request).await?;
        let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
        self.materialize_streaming_raw_response_page(
            &raw_url,
            requested_url,
            stage,
            RendererReplyBoundary::Stage,
            None,
            response,
            document_fetch_context_seed,
            reserved_client,
        )
        .await
    }

    async fn fetch_document_allow_http_error_internal(
        &self,
        request: Request,
        stage: PageVmInitStage,
        reply_boundary: RendererReplyBoundary,
    ) -> Result<FetchedDocument> {
        let request = initial_navigation_request(request);
        let raw_url = request.url.as_str().to_owned();
        let requested_url = request.url.clone();
        if is_about_blank_url(&requested_url) {
            return self
                .materialize_static_html_page(
                    &raw_url,
                    requested_url,
                    stage,
                    reply_boundary,
                    None,
                    String::new(),
                )
                .await
                .map(FetchedDocument::Page);
        }
        let navigation_loader = NavigationResourceLoader::new(
            self.resource_request_client(),
            requested_url.clone(),
            moli_renderer_v8::network::RendererResourceTaskRunner::from_current_tokio()?,
        );
        let service_worker_fetch = self
            .fetch_service_worker_main_resource_for_navigation(&request, &navigation_loader)
            .await?;
        let RendererServiceWorkerMainResourceFetch {
            reserved_client,
            response,
        } = service_worker_fetch;
        if let Some(response) = response {
            navigation_loader.note_service_worker_response_ready()?;
            let document_fetch_context_seed =
                navigation_loader.commit(response.final_url.clone())?;
            if response_headers_indicate_raw_document(&response.headers) {
                let raw_response =
                    RawResponse::from_head_and_body(response.head(), response.clone_body_bytes());
                return Ok(FetchedDocument::Raw(Box::new(RawDocument::from_response(
                    raw_response,
                ))));
            }
            return self
                .materialize_streaming_raw_response_page(
                    &raw_url,
                    requested_url,
                    stage,
                    reply_boundary,
                    None,
                    streaming_raw_response_from_navigation_response(response)?,
                    document_fetch_context_seed,
                    reserved_client,
                )
                .await
                .map(FetchedDocument::Page);
        }
        let response = navigation_loader.fetch_raw_stream(request).await?;
        let document_fetch_context_seed = navigation_loader.commit(response.final_url.clone())?;
        if response_headers_indicate_raw_document(&response.headers) {
            return self
                .materialize_streaming_raw_response_raw_document(response)
                .await
                .map(|raw| FetchedDocument::Raw(Box::new(raw)));
        }
        self.materialize_streaming_raw_response_page(
            &raw_url,
            requested_url,
            stage,
            reply_boundary,
            None,
            response,
            document_fetch_context_seed,
            reserved_client,
        )
        .await
        .map(FetchedDocument::Page)
    }

    async fn fetch_service_worker_main_resource_for_navigation(
        &self,
        request: &Request,
        navigation_loader: &NavigationResourceLoader,
    ) -> Result<RendererServiceWorkerMainResourceFetch> {
        self.js_runtime
            .browser_context_runtime()
            .fetch_service_worker_main_resource_for_navigation(request, navigation_loader)
            .await
    }

    async fn fetch_document_wait_timeout<T, F, P>(
        &self,
        raw_url: &str,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
        stage: PageVmInitStage,
        deadline: tokio::time::Instant,
        timeout_phase: P,
        future: F,
    ) -> Result<T>
    where
        F: std::future::Future<Output = Result<T>>,
        P: FnOnce() -> FetchTimeoutPhase,
    {
        match tokio::time::timeout_at(deadline, future).await {
            Ok(result) => result,
            Err(_) => Err(self.fetch_document_wait_timeout_error(
                raw_url,
                wait_until,
                timeout,
                stage,
                timeout_phase(),
            )),
        }
    }

    fn fetch_document_wait_timeout_error(
        &self,
        raw_url: &str,
        wait_until: RenderedDomWaitUntil,
        timeout: Duration,
        stage: PageVmInitStage,
        phase: FetchTimeoutPhase,
    ) -> anyhow::Error {
        warn!(
            url = %raw_url,
            wait_until = ?wait_until,
            timeout_ms = timeout.as_millis(),
            stage = ?stage,
            timeout_phase = %phase,
            "fetch_document_allow_http_error_with_wait_until timed out"
        );
        anyhow::Error::new(FetchReadinessTimeout::new(timeout, phase))
    }

    async fn materialize_static_html_page(
        &self,
        raw_url: &str,
        requested_url: Url,
        stage: PageVmInitStage,
        reply_boundary: RendererReplyBoundary,
        lifecycle_decider: Option<RendererLifecycleDecider>,
        response_body: String,
    ) -> Result<Page> {
        let started = Instant::now();
        debug!(
            url = %raw_url,
            stage = ?stage,
            "starting static html fetch_internal"
        );

        let document_start_scripts = self
            .config
            .document_start_scripts()
            .iter()
            .cloned()
            .map(|source| DocumentStartScript {
                registry_key: None,
                devtools_session: None,
                source,
                world_name: None,
                has_bidi_channel_argument: false,
                bidi_channel_handoffs: Vec::new(),
            })
            .collect::<Vec<_>>();
        let renderer_owner = self.js_runtime.renderer_owner_handle();
        let mut create_page_request = renderer_owner.build_create_html_page_request(
            requested_url.clone(),
            None,
            false,
            0,
            200,
            Vec::new(),
            &self.resource_request_client(),
            moli_renderer_v8::RendererWebStorageHandles::new(
                self.partition.web_storage_store(),
                self.partition.session_storage_store(),
            ),
            requested_url,
            response_body,
            document_start_scripts,
            vec![],
            vec![],
            Default::default(),
            false,
            Vec::new(),
            false,
            None,
            stage,
        );
        create_page_request
            .document_settings
            .script_execution_disabled = self.config.script_execution_disabled();
        create_page_request.wpt_extensions_enabled = self.config.wpt_extensions_enabled();
        create_page_request.document_settings.debugger_enabled = self.config.debugger_enabled();
        create_page_request.reply_boundary = reply_boundary;
        create_page_request.lifecycle_decider = lifecycle_decider;
        create_page_request.indexed_db_manager = Some(self.partition.weak_indexed_db_manager());
        create_page_request.storage_bucket_store = Some(self.partition.storage_bucket_store());
        let reply = renderer_owner
            .dispatch_command(RendererOwnerCommand::CreateHtmlPage(create_page_request))
            .await?;
        let page = materialize_page_created_reply(&renderer_owner, reply)?;
        info!(
            page_id = page.page_id(),
            url = %page.requested_url(),
            elapsed_ms = started.elapsed().as_millis(),
            "static html fetch_internal completed"
        );
        Ok(page)
    }

    async fn materialize_streaming_raw_response_page(
        &self,
        raw_url: &str,
        requested_url: Url,
        stage: PageVmInitStage,
        reply_boundary: RendererReplyBoundary,
        lifecycle_decider: Option<RendererLifecycleDecider>,
        response: StreamingRawResponse,
        document_fetch_context_seed: DocumentFetchContextSeed,
        reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,
    ) -> Result<Page> {
        self.materialize_streaming_raw_response_page_with_page_creation_progress(
            raw_url,
            requested_url,
            stage,
            reply_boundary,
            lifecycle_decider,
            response,
            document_fetch_context_seed,
            reserved_service_worker_client,
            moli_renderer_v8::RendererPageCreationProgress::new(),
        )
        .await
    }

    async fn materialize_streaming_raw_response_page_with_page_creation_progress(
        &self,
        raw_url: &str,
        requested_url: Url,
        stage: PageVmInitStage,
        reply_boundary: RendererReplyBoundary,
        lifecycle_decider: Option<RendererLifecycleDecider>,
        response: StreamingRawResponse,
        document_fetch_context_seed: DocumentFetchContextSeed,
        reserved_service_worker_client: Option<RendererReservedServiceWorkerClient>,
        page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,
    ) -> Result<Page> {
        let started = Instant::now();
        debug!(
            url = %raw_url,
            stage = ?stage,
            "starting raw streaming fetch_internal"
        );

        let final_url = response.final_url.clone();
        if document_fetch_context_seed.final_url() != &final_url {
            return Err(anyhow!(
                "navigation commit seed final URL `{}` does not match response `{final_url}`",
                document_fetch_context_seed.final_url()
            ));
        }
        let page_loader =
            ResourceRequestClient::from_browser_resource_runtime_with_page_network_policy(
                document_fetch_context_seed.browser_resource_runtime(),
                document_fetch_context_seed.page_network_policy(),
            );
        let response_status = response.status;
        let response_headers = response.headers.clone();
        let redirected = response.redirected;
        let redirect_count = response.redirect_chain.len();
        let navigation_redirect_chain = response
            .redirect_chain
            .iter()
            .cloned()
            .map(Into::into)
            .collect();
        let raw_body =
            external_raw_document_body_from_streaming_response_with_page_creation_progress(
                response,
                page_creation_progress,
            );
        let document_start_scripts = self
            .config
            .document_start_scripts()
            .iter()
            .cloned()
            .map(|source| DocumentStartScript {
                registry_key: None,
                devtools_session: None,
                source,
                world_name: None,
                has_bidi_channel_argument: false,
                bidi_channel_handoffs: Vec::new(),
            })
            .collect::<Vec<_>>();

        let (
            handle,
            page_state,
            _page_creation_diagnostics,
            page_creation_artifacts,
            pending_download,
        ) = self
            .js_runtime
            .create_streaming_raw_page_from_external_body(
                requested_url,
                final_url,
                None,
                redirected,
                redirect_count,
                navigation_redirect_chain,
                response_status,
                response_headers,
                &page_loader,
                moli_renderer_v8::RendererWebStorageHandles::new(
                    self.partition.web_storage_store(),
                    self.partition.session_storage_store(),
                ),
                raw_body,
                self.config.wpt_extensions_enabled(),
                stage,
                reply_boundary,
                moli_renderer_v8::RendererTopLevelNavigationDispatch::FollowInStandaloneAdapter,
                moli_renderer_v8::RendererNavigationReplyPolicy::FollowBeforeReply,
                reserved_service_worker_client,
                lifecycle_decider,
                moli_renderer_v8::RendererDocumentOptions {
                    indexed_db_manager: Some(self.partition.weak_indexed_db_manager()),
                    storage_bucket_store: Some(self.partition.storage_bucket_store()),
                    document_start_scripts,
                    document_settings: moli_page_types::DocumentSettings {
                        script_execution_disabled: self.config.script_execution_disabled(),
                        debugger_enabled: self.config.debugger_enabled(),
                        ..Default::default()
                    },
                    ..Default::default()
                },
            )
            .await?;
        if pending_download.is_some() {
            return Err(anyhow!(
                "raw streaming page creation produced a pending download for `{raw_url}`"
            ));
        }
        let page = Page::from_attached_handle_with_creation_artifacts(
            handle,
            page_state,
            page_creation_artifacts,
        );
        info!(
            page_id = page.page_id(),
            url = %page.requested_url(),
            elapsed_ms = started.elapsed().as_millis(),
            "raw streaming fetch_internal completed"
        );
        Ok(page)
    }

    async fn materialize_streaming_raw_response_raw_document(
        &self,
        response: StreamingRawResponse,
    ) -> Result<RawDocument> {
        let started = Instant::now();
        let final_url = response.final_url.clone();
        let status = response.status;
        let raw_response = response.into_materialized_raw_response().await?;
        info!(
            url = %final_url,
            status,
            body_bytes = raw_response.body_bytes().len(),
            elapsed_ms = started.elapsed().as_millis(),
            "raw document fetch_internal completed"
        );
        Ok(RawDocument::from_response(raw_response))
    }

    pub fn config(&self) -> &BrowserConfig {
        &self.config
    }
}

fn external_raw_document_body_from_streaming_response_with_page_creation_progress(
    response: StreamingRawResponse,
    page_creation_progress: moli_renderer_v8::RendererPageCreationProgress,
) -> ExternalRawDocumentBodyStream {
    external_raw_document_body_from_streaming_response_with_observer_and_progress(
        response,
        None,
        Some(page_creation_progress),
    )
}

#[cfg(test)]
fn external_raw_document_body_from_streaming_response_with_body_eof_observer(
    response: StreamingRawResponse,
    body_eof_observer: Option<oneshot::Sender<()>>,
) -> ExternalRawDocumentBodyStream {
    external_raw_document_body_from_streaming_response_with_observer_and_progress(
        response,
        body_eof_observer,
        None,
    )
}

fn external_raw_document_body_from_streaming_response_with_observer_and_progress(
    mut response: StreamingRawResponse,
    mut body_eof_observer: Option<oneshot::Sender<()>>,
    page_creation_progress: Option<moli_renderer_v8::RendererPageCreationProgress>,
) -> ExternalRawDocumentBodyStream {
    let (completion_tx, completion_rx) = oneshot::channel();
    let (body_tx, body_stream) = match page_creation_progress {
        Some(progress) => ExternalRawDocumentBodyStream::channel_with_page_creation_progress(
            completion_rx,
            progress,
        ),
        None => ExternalRawDocumentBodyStream::channel(completion_rx),
    };
    tokio::spawn(async move {
        let result = async {
            loop {
                let chunk = tokio::select! {
                    // The renderer may discard the prepared/raw Document
                    // while the network body is stalled. Observe that
                    // receiver closure concurrently with upstream input so
                    // dropping the bridge deterministically drops (and
                    // cancels) the real StreamingRawResponse.
                    _ = body_tx.closed() => return Ok(()),
                    chunk = response.next_chunk() => chunk,
                };
                let Some(chunk) = chunk else {
                    if let Some(observer) = body_eof_observer.take() {
                        let _ = observer.send(());
                    }
                    break;
                };
                if body_tx.send(chunk).await.is_err() {
                    return Ok(());
                }
            }
            tokio::select! {
                // Upstream body EOF does not imply that terminal completion
                // has arrived. Dropping the external body in that interval
                // must still cancel/drop the response and release its exact
                // resource-runtime lease.
                _ = body_tx.closed() => Ok(()),
                completion = response.finish() => completion,
            }
        }
        .await;
        let _ = completion_tx.send(result);
    });
    body_stream
}

fn streaming_raw_response_from_navigation_response(
    response: NavigationResponse,
) -> Result<StreamingRawResponse> {
    let head = response.head();
    let body = response.clone_body_bytes();
    let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
    if !body.is_empty() {
        body_tx
            .send(body)
            .map_err(|_| anyhow!("failed to enqueue service worker main resource body"))?;
    }
    drop(body_tx);
    let (completion_tx, completion_rx) = oneshot::channel();
    let _ = completion_tx.send(Ok(()));
    Ok(StreamingRawResponse::new_with_head(
        head,
        body_rx,
        FetchCancelHandle::new(),
        completion_rx,
    ))
}

#[derive(Debug, Clone)]
pub struct Session {
    id: u64,
    browser: Browser,
}

impl Session {
    pub fn id(&self) -> u64 {
        self.id
    }

    pub async fn open(&self, raw_url: &str) -> Result<Page> {
        self.browser.fetch(raw_url).await
    }
}

#[cfg(test)]
mod tests;